npm - mailwoman - Versions diffs - 0.2.0 - Mend

mailwoman 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1375) hide show

package/server/static/index.html ADDED Viewed

@@ -0,0 +1,279 @@
+<!doctype html>
+<html lang="en" ng-app="demo">
+	<head>
+		<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" />
+		<script src="https://code.jquery.com/jquery-3.0.0.js"></script>
+		<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" />
+		<link href="https://fonts.googleapis.com/css?family=Inconsolata" rel="stylesheet" />
+		<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.8.3/underscore-min.js"></script>
+		<script src="https://unpkg.com/ansi_up@4.0.4/ansi_up.js"></script>
+		<style>
+			#table,
+			#table tbody,
+			#table tbody tr,
+			#table tbody tr td,
+			#table tbody tr th {
+				margin: 0;
+				padding: 0;
+				font-size: 20px;
+				font-family: "Inconsolata", monospace;
+			}
+			#table {
+				border-collapse: separate;
+				border-spacing: 0.1em 0.4em;
+			}
+			.kb {
+				padding: 0.1em 0.4em !important;
+				border: 1px solid #ccc;
+				/* font-size:11px; */
+				/* font-family:Arial,Helvetica,sans-serif; */
+				background-color: #f7f7f7;
+				color: #333;
+				-moz-box-shadow:
+					0 1px 0px rgba(0, 0, 0, 0.2),
+					0 0 0 2px #ffffff inset;
+				-webkit-box-shadow:
+					0 1px 0px rgba(0, 0, 0, 0.2),
+					0 0 0 2px #ffffff inset;
+				box-shadow:
+					0 1px 0px rgba(0, 0, 0, 0.2),
+					0 0 0 2px #ffffff inset;
+				-moz-border-radius: 3px;
+				-webkit-border-radius: 3px;
+				border-radius: 3px;
+				/* display:inline-block; */
+				margin: 0 0.1em;
+				text-shadow: 0 1px 0 #fff;
+				/* line-height:1.4; */
+				white-space: nowrap;
+			}
+			#table tbody tr th,
+			#table tbody tr td {
+				text-align: center;
+				text-overflow: clip;
+			}
+			#table tbody tr td {
+				font-size: 10px;
+			}
+			.score {
+				width: 80px;
+				display: block;
+				text-align: left;
+				background-color: #dedede;
+			}
+			.span {
+				background-color: rgb(223, 223, 223);
+			}
+			.span-street {
+				background-color: rgb(238, 193, 95);
+			}
+			.span-housenumber {
+				background-color: rgb(230, 134, 222);
+			}
+			.span-postcode {
+				background-color: rgb(134, 195, 230);
+			}
+			#debug {
+				white-space: pre-wrap;
+				font-family: monospace;
+			}
+			#debug span {
+				color: lightgrey;
+				display: inline-block;
+			}
+		</style>
+		<script>
+			function exec(args) {
+				$("#input").empty()
+				parse(args, function (results) {
+					results = results || []
+					// render results
+					render(results)
+				})
+			}
+			function render(res) {
+				$("#input").empty()
+				$("#solutions").empty()
+				$("#table").empty()
+				$("#debug").empty()
+				// $('#input').html(JSON.stringify(res.input))
+				// $('#solutions').html(JSON.stringify(res.solutions))
+				var body = $("<tbody>")
+				// headings
+				var row = $("<tr>")
+				row.append($("<th>").addClass("score").text(""))
+				for (let characterIndex = res.input.start; characterIndex < res.input.end; characterIndex++) {
+					const cell = $("<th>")
+					let char = res.input.body.charAt(characterIndex)
+					if (char.trim().length > 0) {
+						cell.addClass("kb")
+					} else {
+						char = String.fromCharCode(160)
+					}
+					cell.text(char)
+					row.append(cell)
+				}
+				body.append(row)
+				// solutions
+				res.solutions.forEach(function (s) {
+					row = $("<tr>")
+					row.append(
+						$("<th>")
+							.addClass("score")
+							.text((s.score * 100).toFixed(1) + "%")
+					)
+					for (let i = res.input.start; i < res.input.end; i++) {
+						const cell = $("<td>")
+						var spanStart = false
+						var classification
+						for (var j = 0; j < s.classifications.length; j++) {
+							if (s.classifications[j].start === i) {
+								spanStart = true
+								classification = s.classifications[j]
+							}
+						}
+						if (spanStart) {
+							var covers = classification.end - classification.start
+							cell.attr("colspan", covers)
+							cell.addClass("span")
+							cell.addClass("span-" + classification.label)
+							cell.text(classification.label)
+							i += covers - 1
+						} else {
+							cell.text(String.fromCharCode(160))
+						}
+						row.append(cell)
+					}
+					body.append(row)
+				})
+				$("#table").append(body)
+				const debugHtml = new AnsiUp().ansi_to_html(res.debug)
+				$("#debug").html(debugHtml)
+			}
+			function request(url, params, cb) {
+				$.ajax({
+					url,
+					method: "GET",
+					data: params,
+					headers: { Accept: "application/json" },
+				}).done(cb)
+			}
+			function parse(args, cb) {
+				saveText(args.text)
+				request(
+					"/parser/parse",
+					{
+						...args,
+						debug: true,
+					},
+					cb
+				)
+			}
+			function saveText(text) {
+				if (window.localStorage) {
+					window.localStorage.setItem("text", text)
+				}
+			}
+			function loadText() {
+				let hash = window.location.hash
+				if (hash && hash.length) {
+					if (hash[0] === "#") {
+						hash = hash.slice(1)
+					}
+					if (hash.length) {
+						return decodeURIComponent(hash)
+					}
+				}
+				if (window.localStorage) {
+					var text = window.localStorage.getItem("text")
+					if (text) {
+						return text
+					}
+				}
+				// default text
+				return "1389a County Road 42 IA"
+			}
+			$(document).ready(function () {
+				var liveMode = _.debounce(function () {
+					// if (e.which === 13) { return false; } // not for enter key
+					var args = { text: $("#text").val() }
+					exec(args)
+					// update location hash
+					window.location.hash = encodeURIComponent(args.text)
+					return false
+				}, 200)
+				// button / keyup event handlers
+				$("#search").on("submit", liveMode)
+				$("#search").on("keyup", liveMode)
+				$("#go").on("click", function () {
+					$("#search").submit()
+					return false
+				})
+				$("#text").val(loadText())
+				liveMode({})
+			})
+		</script>
+	</head>
+	<body>
+		<div style="margin: 20px">
+			<form id="search" action="">
+				<div class="row">
+					<div class="col-md-6">
+						<div class="input-group">
+							<input id="text" type="text" class="form-control" placeholder="Search for..." />
+							<span class="input-group-btn">
+								<button id="go" class="btn btn-default" type="button">Parse!</button>
+							</span>
+						</div>
+						<ul id="input" class="btn-group" role="group" style="margin-top: 10px; list-style: none"></ul>
+						<ul id="solutions" class="list-group" style="margin-top: 10px"></ul>
+						<table id="table"></table>
+						<div id="debug"></div>
+					</div>
+				</div>
+			</form>
+		</div>
+	</body>
+</html>

package/server/types/locals.d.ts ADDED Viewed

@@ -0,0 +1,19 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { AddressParser } from "mailwoman"
+declare global {
+	namespace Express {
+		interface Locals {
+			parser: {
+				address: AddressParser
+			}
+		}
+	}
+}
+export {}

package/solver/ExclusiveCartesianSolver.ts ADDED Viewed

@@ -0,0 +1,74 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { Tokenizer } from "../tokenization/Tokenizer.js"
+import Solution from "./Solution.js"
+import HashMapSolver from "./super/HashMapSolver.js"
+const MAX_RECURSION = 10
+const MAX_SOLUTIONS = 50000
+class ExclusiveCartesianSolver extends HashMapSolver {
+	solve(tokenizer: Tokenizer) {
+		const map = this.generateHashMap(tokenizer, false, true)
+		const solutions = Array.from(map.values()).reverse()
+		const exclusiveSolutions = ExclusiveCartesianSolver.exclusiveCartesian(...solutions)
+		tokenizer.solutions = tokenizer.solutions.concat(exclusiveSolutions)
+	}
+	// compute the unique cartesian product
+	// (all permutations of non-overlapping tokens from different classifications)
+	static exclusiveCartesian(...args: Solution[]): Solution[] {
+		let r: Solution[] = []
+		const max = args.length - 1
+		if (!args.length) {
+			return r
+		}
+		const helper = (currentSolution: Solution, solutionIndex: number) => {
+			const referencedSolution = args[solutionIndex]!
+			for (let pairIndex = 0, l = referencedSolution.pair.length; pairIndex < l; pairIndex++) {
+				const copy = currentSolution.copy() // clone solution
+				if (referencedSolution.pair[pairIndex]?.span.body.length) {
+					copy.pair.push(referencedSolution.pair[pairIndex]!)
+				}
+				if (solutionIndex === max) {
+					if (copy.pair.length && r.length < MAX_SOLUTIONS) {
+						r.push(copy)
+					}
+				} else if (solutionIndex < MAX_RECURSION) {
+					helper(copy, solutionIndex + 1)
+				}
+			}
+		}
+		helper(new Solution(), 0)
+		// reverse order
+		r = r.reverse()
+		// do not add a pair where the span intersects an existing pair
+		r = r.filter((s) => {
+			return !s.pair.some((p1, i1) => {
+				return s.pair.some((p2, i2) => {
+					if (i2 <= i1) {
+						return false
+					}
+					return p1.span.intersects(p2.span)
+				})
+			})
+		})
+		return r
+	}
+}
+export default ExclusiveCartesianSolver

package/solver/HouseNumberPositionPenalty.ts ADDED Viewed

@@ -0,0 +1,80 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import HouseNumberClassification from "../classification/HouseNumberClassification.js"
+import StreetClassification from "../classification/StreetClassification.js"
+import { Tokenizer } from "../tokenization/Tokenizer.js"
+import { Solver } from "./super/BaseSolver.js"
+const basePenalty = 0.05
+// https://github.com/pelias/api/blob/master/middleware/localNamingConventions.js
+const numberLastLangs = new Map<string, number>([
+	["de", basePenalty],
+	["sl", basePenalty],
+	["pl", basePenalty],
+	["bs", basePenalty],
+	["hr", basePenalty],
+	["nl", basePenalty],
+	["cs", basePenalty],
+	["da", basePenalty],
+	// Guatemala & Honduras do not flip their house numbers
+	["es", basePenalty / 2],
+	["fi", basePenalty],
+	["el", basePenalty],
+	["is", basePenalty],
+	["it", basePenalty],
+	["nb", basePenalty],
+	["pt", basePenalty],
+	["sv", basePenalty],
+	["sk", basePenalty],
+	["tr", basePenalty],
+	["ro", basePenalty],
+	["hu", basePenalty],
+])
+// const numberFirstLangs = {
+// 	en: basePenalty,
+// 	fr: basePenalty / 2, // Switzerland and Andorre has some french streets
+// }
+const numberFirstLangs = new Map<string, number>([
+	["en", basePenalty],
+	["fr", basePenalty / 2], // Switzerland and Andorre has some french streets
+])
+class HouseNumberPositionPenalty implements Solver {
+	solve(tokenizer: Tokenizer) {
+		tokenizer.solutions.forEach((s) => {
+			const housenumber = s.pair.find((p) => p.classification.constructor === HouseNumberClassification)
+			const street = s.pair.find((p) => p.classification.constructor === StreetClassification)
+			// Do nothing if there is no street/housenumber or no meta in street classification
+			if (!housenumber || !street || !street.classification.meta || !street.classification.meta.langs) {
+				return
+			}
+			const langs = Object.keys(street.classification.meta.langs)
+			// For now, we don't supports multi-lang entries
+			if (langs.length !== 1 || langs[0] === "all") {
+				return
+			}
+			const lang = langs[0]!
+			const numberLastLangMatch = numberLastLangs.get(lang)
+			const numberFirstMatch = numberFirstLangs.get(lang)
+			// Check if the number should be in last position (after street) or first position (before street)
+			if (numberLastLangMatch && housenumber.span.start < street.span.start) {
+				s.penalty += numberLastLangMatch
+			} else if (numberFirstMatch && street.span.start < housenumber.span.start) {
+				s.penalty += numberFirstMatch
+			}
+		})
+	}
+}
+export default HouseNumberPositionPenalty

package/solver/InvalidSolutionFilter.ts ADDED Viewed

@@ -0,0 +1,34 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { Tokenizer } from "../tokenization/Tokenizer.js"
+class InvalidSolutionFilter {
+	patterns: string[][]
+	constructor(patterns: string | string[][]) {
+		this.patterns = Array.isArray(patterns) ? patterns : []
+		this.patterns.map((p) => p.sort()) // sort alphabetically
+	}
+	solve(tokenizer: Tokenizer) {
+		tokenizer.solutions = tokenizer.solutions.filter((solution) => {
+			// sort alphabetically
+			const classifications = solution.pair.map((p) => p.classification.constructor.name).sort()
+			return !this.patterns.some((p) => {
+				if (classifications.length !== p.length) {
+					return false
+				}
+				return classifications.every((_, i) => classifications[i] === p[i])
+			})
+		})
+	}
+}
+export default InvalidSolutionFilter

package/solver/LeadingAreaDeclassifier.ts ADDED Viewed

@@ -0,0 +1,41 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { Tokenizer } from "../tokenization/Tokenizer.js"
+const NETURAL_CLASSIFICATIONS = ["PostcodeClassification"]
+const ADMIN_CLASSIFICATIONS = ["LocalityClassification", "RegionClassification", "CountryClassification"]
+class LeadingAreaDeclassifier {
+	solve(tokenizer: Tokenizer) {
+		for (const solution of tokenizer.solutions) {
+			// record the position of the last non-admin cursor position
+			let lastNonAdminCursorPosition = 0
+			for (const solutionPair of solution.pair) {
+				const isAdmin = ADMIN_CLASSIFICATIONS.some((c) => solutionPair.classification.constructor.name === c)
+				const isNeut = NETURAL_CLASSIFICATIONS.some((c) => solutionPair.classification.constructor.name === c)
+				if (!isAdmin && !isNeut) {
+					lastNonAdminCursorPosition = solutionPair.span.end
+				}
+			}
+			solution.pair = solution.pair.filter((p) => {
+				const isAdmin = ADMIN_CLASSIFICATIONS.some((c) => p.classification.constructor.name === c)
+				if (isAdmin && p.span.end < lastNonAdminCursorPosition) {
+					return false
+				}
+				return true
+			})
+		}
+		tokenizer.solutions.sort((a, b) => b.score - a.score) // sort results by score desc
+		tokenizer.solutions.forEach((s) => s.pair.sort((a, b) => a.span.start - b.span.start)) // sort by span start
+	}
+}
+export default LeadingAreaDeclassifier

package/solver/MultiStreetSolver.ts ADDED Viewed

@@ -0,0 +1,142 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+import { Tokenizer } from "../tokenization/Tokenizer.js"
+import HashMapSolver from "./super/HashMapSolver.js"
+// classifications which are more granular than StreetClassification
+// should not be included in intersection solutions.
+const MORE_GRANULAR_THAN_STREET = [
+	"HouseNumberClassification",
+	"LevelClassification",
+	"LevelTypeClassification",
+	"UnitClassification",
+	"UnitTypeClassification",
+	"VenueClassification",
+]
+/**
+ * If a 'multistreet' classification was detected then add a new solution which covers all streets
+ * included.
+ *
+ * The ExclusiveCartesianSolver ensures that each public classification can exist only once per
+ * solution (ie. we can't have two postalcodes).
+ *
+ * Intersections are the only exception to this rule, so rather than modifying the
+ * ExclusiveCartesianSolver we use this solver to create the missing intersection solutions (those
+ * with 2x 'street' labels).
+ *
+ * One of the challenges is that there could be many different interpretations of the admin tokens,
+ * so we need to ensure that each of those permutations is also represented by a distinct
+ * intersection solution.
+ *
+ * The solver works by iterating over existing solutions looking for any which identified a street,
+ * it then clones that solution, removes any tokens less granular than street and attempts to add
+ * the new street token in its place.
+ *
+ * Care is taken to ensure that the resulting solution does not contain tokens in overlapping
+ * positions.
+ */
+class MultiStreetSolver extends HashMapSolver {
+	solve(tokenizer: Tokenizer) {
+		const map = this.generateHashMap(tokenizer, true)
+		const multistreet = map.get("multistreet")
+		const street = map.get("street")
+		// sanity checking
+		if (!multistreet || multistreet.pair.length < 1) {
+			return
+		}
+		if (!street || street.pair.length < 2) {
+			return
+		}
+		// only currently consider one multistreet parse (for simplicity)
+		// @todo: there may be some rare cases where we detect more than one?
+		const multi = multistreet.pair[0]
+		if (!multi) return
+		// generate a list of streets which intersect the multistreet
+		const streets = street.pair.filter((s) => s.span.intersects(multi.span))
+		if (streets.length < 2) {
+			return
+		}
+		// generate a list of candidate solutions which could potentially be
+		// cloned to generate new intersection solutions
+		let candidates = tokenizer.solutions.filter((solution) => {
+			// candidate solution must contain a street and also that street
+			// must intersect the multistreet classification.
+			return solution.pair.some(
+				(s) => s.classification.constructor.name === "StreetClassification" && s.span.intersects(multi.span)
+			)
+		})
+		// truncate the candidates by making a copy of the current solution and removing all solution
+		// pairs which came before the street and also any pairs less granular than street
+		// (such as venue, housenumber etc.)
+		candidates = candidates.map((solution) => {
+			// find the street solution pair (there should be exactly one)
+			const candidateStreet = solution.pair.find((s) => s.classification.constructor.name === "StreetClassification")!
+			// remove some pairs from the solution
+			const truncated = solution.copy()
+			truncated.pair = truncated.pair.filter((s) => {
+				return (
+					s.span.start >= candidateStreet.span.start &&
+					!MORE_GRANULAR_THAN_STREET.includes(s.classification.constructor.name)
+				)
+			})
+			return truncated
+		})
+		// the truncation step above can generate duplicate solutions so a 'content hash'
+		// is generated in order to deduplicate them.
+		// note: this is purely a performance optimization as it generates fewer candidates
+		candidates = uniqBy(candidates, (truncated) => {
+			return truncated.pair.map((p) => `${p.classification.label}:${p.span.norm}`).join("_")
+		})
+		// iterate over candidates and generate new intersection solutions
+		candidates.forEach((truncated) => {
+			// find all street classsifications which intersect the 'multistreet' span
+			// and also do not overlap an existing pair in this solution.
+			streets.forEach((candidateStreet) => {
+				if (truncated.pair.every((p) => !p.span.intersects(candidateStreet.span))) {
+					// make a copy of the truncated solution and add the additional street
+					const intersection = truncated.copy()
+					intersection.pair.push(candidateStreet)
+					// append this solution
+					tokenizer.solutions.push(intersection)
+				}
+			})
+		})
+	}
+}
+export default MultiStreetSolver
+function uniqBy<T>(arr: T[], predicate: (o: T) => any): T[] {
+	const pickedObjects = arr
+		.filter(Boolean)
+		.reduce((map, item) => {
+			const key = predicate(item)
+			if (!key) return map
+			return map.has(key) ? map : map.set(key, item)
+		}, new Map())
+		.values()
+	return Array.from(pickedObjects)
+}