mailwoman 0.2.0

package/classifier/DirectionalClassifier.test.ts

@@ -0,0 +1,190 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import { DirectionalClassification } from "mailwoman/classification"
+import test from "tape"
+import { Span } from "../tokenization/Span.js"
+import DirectionalClassifier from "./DirectionalClassifier.js"
+
+const classifier = new DirectionalClassifier()
+
+test("contains numerals: honours contains.numerals boolean", (t) => {
+	const span = new Span("example")
+	span.contains.numerals = true
+	classifier.each(span)
+	t.deepEqual(span.classifications, {})
+	t.end()
+})
+
+type DirectionalTestCaseRecord = {
+	valid: string[]
+	invalid: string[]
+}
+
+const testCasesByLangauge = new Map<string, DirectionalTestCaseRecord>([
+	[
+		"English",
+		{
+			valid: [
+				"north",
+				"n",
+				"n.",
+				"south",
+				"s",
+				"s.",
+				"east",
+				"e",
+				"e.",
+				"west",
+				"w",
+				"w.",
+				"northeast",
+				"ne",
+				"ne.",
+				"southeast",
+				"se",
+				"se.",
+				"northwest",
+				"nw",
+				"nw.",
+				"southwest",
+				"sw",
+				"sw.",
+				"lower",
+				"lwr",
+				"upper",
+				"upr",
+				"middle",
+				"mdl",
+				"centre",
+				"center",
+				"ctr",
+				"central",
+				"ctrl",
+			],
+
+			invalid: ["northsouth", "ns", "ns.", "westeast", "we", "we."],
+		},
+	],
+	[
+		"Spanish",
+		{
+			valid: [
+				"norte",
+				"n",
+				"n.",
+				"sur",
+				"s",
+				"s.",
+				"este",
+				"e",
+				"e.",
+				"oeste",
+				"w",
+				"w.",
+				"noreste",
+				"ne",
+				"ne.",
+				"sureste",
+				"se",
+				"se.",
+				"noroeste",
+				"nw",
+				"nw.",
+				"suroeste",
+				"sw",
+				"sw.",
+			],
+
+			invalid: ["norsur", "ns", "ns.", "oesteeste", "we", "we."],
+		},
+	],
+
+	[
+		"German",
+		{
+			valid: [
+				"nord",
+				"n",
+				"n.",
+				"süd",
+				"s",
+				"s.",
+				"ost",
+				"o",
+				"o.",
+				"west",
+				"w",
+				"w.",
+				"nordost",
+				"no",
+				"no.",
+				"südost",
+				"so",
+				"so.",
+				"nordwest",
+				"nw",
+				"nw.",
+				"südwest",
+				"sw",
+				"sw.",
+			],
+
+			invalid: ["nordsüd", "ns", "ns.", "westost", "wo", "wo."],
+		},
+	],
+	[
+		"French",
+		{
+			valid: [
+				"nord",
+				"n",
+				"n.",
+				"sud",
+				"s",
+				"s.",
+				"est",
+				"e",
+				"e.",
+				"ouest",
+				"o",
+				"o.",
+				"nord est",
+				"ne",
+				"ne.",
+				"sud est",
+				"se",
+				"se.",
+				"nord ouest",
+				"no",
+				"no.",
+				"sud ouest",
+				"so",
+				"so.",
+			],
+
+			invalid: ["nordsud", "ns", "ns.", "ouestest", "oe", "oe."],
+		},
+	],
+])
+
+for (const [language, cases] of testCasesByLangauge) {
+	for (const token of cases.valid) {
+		test(`${language}: ${token}`, (t) => {
+			const span = classifier.classify(token)
+			t.deepEqual(span.classifications, { DirectionalClassification: new DirectionalClassification(1.0) })
+			t.end()
+		})
+	}
+
+	for (const token of cases.invalid) {
+		test(`${language}: ${token}`, (t) => {
+			const span = classifier.classify(token)
+			t.deepEqual(span.classifications, {})
+			t.end()
+		})
+	}
+}

package/classifier/DirectionalClassifier.ts

@@ -0,0 +1,47 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import DirectionalClassification from "../classification/DirectionalClassification.js"
+import * as libpostal from "../resources/libpostal/libpostal.js"
+import { Span } from "../tokenization/Span.js"
+import { WordClassifier } from "./super/WordClassifier.js"
+
+// dictionaries sourced from the libpostal project
+// see: https://github.com/openvenues/libpostal
+
+// const languages = libpostal.languages
+
+// optionally control which languages are included
+// note: reducing the languages will have a considerable performance benefit
+const languages = ["en", "es", "de", "fr", "nl", "nb"] as const
+
+class DirectionalClassifier extends WordClassifier {
+	public index: libpostal.PostalIndex = {}
+
+	constructor() {
+		super()
+
+		libpostal.load(this.index, languages, "directionals.txt")
+	}
+
+	each(span: Span) {
+		// skip spans which contain numbers
+		if (span.contains.numerals) {
+			return
+		}
+
+		// use an inverted index for full token matching as it's O(1)
+		if (Object.hasOwn(this.index, span.norm)) {
+			span.classify(new DirectionalClassification(1))
+
+			// try again for abbreviations denoted by a period such as 'n.'
+		} else if (span.norm.slice(-1) === "." && Object.hasOwn(this.index, span.norm.slice(0, -1))) {
+			span.classify(new DirectionalClassification(1))
+		}
+	}
+}
+
+export default DirectionalClassifier

package/classifier/GivenNameClassifier.test.ts

@@ -0,0 +1,32 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import { GivenNameClassification } from "mailwoman/classification"
+import test from "tape"
+import { Span } from "../tokenization/Span.js"
+import GivenNameClassifier from "./GivenNameClassifier.js"
+
+const classifier = new GivenNameClassifier()
+
+test("contains numerals: honours contains.numerals boolean", (t) => {
+	const span = new Span("example")
+	span.contains.numerals = true
+	classifier.each(span)
+	t.deepEqual(span.classifications, {})
+	t.end()
+})
+
+const valid = ["Anderson Luís", "Peter"]
+
+valid.forEach((token) => {
+	test(`classify: ${token}`, (t) => {
+		const span = classifier.classify(token)
+		t.deepEqual(span.classifications, {
+			GivenNameClassification: new GivenNameClassification(1.0),
+		})
+		t.end()
+	})
+})

package/classifier/GivenNameClassifier.ts

@@ -0,0 +1,40 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import GivenNameClassification from "../classification/GivenNameClassification.js"
+import * as libpostal from "../resources/libpostal/libpostal.js"
+import { Span } from "../tokenization/Span.js"
+import PhraseClassifier from "./super/PhraseClassifier.js"
+
+// dictionaries sourced from the libpostal project
+// see: https://github.com/openvenues/libpostal
+
+class GivenNameClassifier extends PhraseClassifier {
+	public index: libpostal.PostalIndex = {}
+
+	constructor() {
+		super()
+
+		libpostal.load(this.index, ["all"], "given_names.txt", {
+			lowercase: true,
+			minlength: 3, // prevent very short names being indexed
+		})
+	}
+
+	each(span: Span) {
+		// skip spans which contain numbers
+		if (span.contains.numerals) {
+			return
+		}
+
+		// use an inverted index for full token matching as it's O(1)
+		if (Object.hasOwn(this.index, span.norm)) {
+			span.classify(new GivenNameClassification(1))
+		}
+	}
+}
+
+export default GivenNameClassifier

package/classifier/HouseNumberClassifier.test.ts

@@ -0,0 +1,164 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import { HouseNumberClassification } from "mailwoman/classification"
+import test from "tape"
+import { Span } from "../tokenization/Span.js"
+import HouseNumberClassifier from "./HouseNumberClassifier.js"
+
+const classifier = new HouseNumberClassifier()
+
+test("contains numerals: honours contains.numerals boolean", (t) => {
+	const span = new Span("100")
+	span.contains.numerals = false
+	classifier.each(span)
+	t.deepEqual(span.classifications, {})
+	t.end()
+})
+
+test("numeric: single digit", (t) => {
+	const span = classifier.classify("1")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("numeric: two digits", (t) => {
+	const span = classifier.classify("12")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("numeric: three digits", (t) => {
+	const span = classifier.classify("123")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("numeric: four digits", (t) => {
+	const span = classifier.classify("1234")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(0.9) })
+	t.end()
+})
+
+test("numeric: five digits", (t) => {
+	const span = classifier.classify("12345")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(0.2) })
+	t.end()
+})
+
+test("numeric: six digits", (t) => {
+	const span = classifier.classify("123456")
+	t.deepEqual(span.classifications, {})
+	t.end()
+})
+
+test("letter suffix: single digit", (t) => {
+	const span = classifier.classify("1A")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("letter suffix: two digits", (t) => {
+	const span = classifier.classify("12b")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("letter suffix: three digits", (t) => {
+	const span = classifier.classify("123C")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("letter suffix: four digits", (t) => {
+	const span = classifier.classify("1234d")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(0.9) })
+	t.end()
+})
+
+test("letter suffix: five digits", (t) => {
+	const span = classifier.classify("12345E")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(0.2) })
+	t.end()
+})
+
+test("letter suffix: six digits", (t) => {
+	const span = classifier.classify("123456f")
+	t.deepEqual(span.classifications, {})
+	t.end()
+})
+
+test("letter suffix: Cyrillic", (t) => {
+	const span = classifier.classify("15в")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("letter suffix: Cyrillic", (t) => {
+	const span = classifier.classify("15б")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("hyphenated: 10-19", (t) => {
+	const span = classifier.classify("10-19")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("hyphenated: 10-19a", (t) => {
+	const span = classifier.classify("10-19a")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("hyphenated: 10-19B", (t) => {
+	const span = classifier.classify("10-19B")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("forward slash: 1/135", (t) => {
+	const span = classifier.classify("1/135")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("forward slash: 1a/135", (t) => {
+	const span = classifier.classify("1a/135")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("forward slash: 1B/125", (t) => {
+	const span = classifier.classify("1B/125")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("misc: 6N23", (t) => {
+	const span = classifier.classify("6N23")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("misc: W350N5337", (t) => {
+	const span = classifier.classify("W350N5337")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+test("misc: N453", (t) => {
+	const span = classifier.classify("N453")
+	t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+	t.end()
+})
+
+// test('misc: д. 1, кв. 1', (t) => {
+//   let s = classify('д. 1, кв. 1')
+//   t.deepEqual(span.classifications, { HouseNumberClassification: new HouseNumberClassification(1.0) })
+//   t.end()
+// })

package/classifier/HouseNumberClassifier.ts

@@ -0,0 +1,56 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import HouseNumberClassification from "../classification/HouseNumberClassification.js"
+import { Span } from "../tokenization/Span.js"
+import { WordClassifier } from "./super/WordClassifier.js"
+
+// copied from: https://github.com/mapbox/carmen/blob/5489f0e67a4f31280ae1b9d091952c97280b83e7/lib/text-processing/termops.js#L269-L290
+// note: \u0400-\u04FF represents a-z in the Cyrillic alphabet
+
+class HouseNumberClassifier extends WordClassifier {
+	each(span: Span) {
+		// skip spans which do not contain numbers
+		if (!span.contains.numerals) {
+			return
+		}
+
+		if (
+			/^\d{1,5}[a-zA-Z\u0400-\u04FF]?$/.test(span.body) || // 10 or 10a Style
+			/^(\d{1,5})-(\d{1,5})[a-zA-Z\u0400-\u04FF]?$/.test(span.body) || // 10-19 or 10-19a Style
+			/^(\d{1,5})[a-zA-Z\u0400-\u04FF]?\/(\d{1,5})$/.test(span.body) || // 1/135 or 1b/135 Style
+			/^(\d{1,5})([nsewNSEW])(\d{1,5})[a-zA-Z]?$/.test(span.body) || // 6N23 Style (ie Kane County, IL)
+			/^([nsewNSEW])(\d{1,5})([nsewNSEW]\d{1,5})?$/.test(span.body) // W350N5337 or N453 Style (ie Waukesha County, WI)
+			// /^\d{1,5}(к\d{1,5})?(с\d{1,5})?$/.test(span.body) // Russian style including korpus (cyrillic к) and stroenie (cyrillic с)
+		) {
+			let confidence = 1
+			const prev = span.graph.findOne("prev")
+
+			if (prev) {
+				// Housenumber must not be preceded by unit type
+				if (Object.hasOwn(prev.classifications, "LevelTypeClassification")) {
+					return
+				}
+
+				if (Object.hasOwn(prev.classifications, "UnitTypeClassification")) {
+					return
+				}
+			}
+
+			// it's possible to have 5 digit housenumbers
+			// but they are fairly uncommon
+			if (/^\d{5}/.test(span.norm)) {
+				confidence = 0.2
+			} else if (/^\d{4}/.test(span.norm)) {
+				confidence = 0.9
+			}
+
+			span.classify(new HouseNumberClassification(confidence))
+		}
+	}
+}
+
+export default HouseNumberClassifier

package/classifier/IntersectionClassifier.ts

@@ -0,0 +1,47 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import IntersectionClassification from "../classification/IntersectionClassification.js"
+import { Span } from "../tokenization/Span.js"
+import PhraseClassifier from "./super/PhraseClassifier.js"
+
+class IntersectionClassifier extends PhraseClassifier {
+	public index = new Map<string, boolean>([
+		["&", true],
+		["and", true],
+		["und", true],
+		["@", true],
+		["at", true],
+		["con", true],
+		["an der ecke von", true],
+	])
+
+	each(span: Span) {
+		// skip spans which contain numbers
+		if (span.contains.numerals) {
+			return
+		}
+
+		// do not classify tokens with tokens missing before or afterwards
+		const firstChild = span.graph.findOne("child:first") || span
+		const prev = firstChild.graph.findOne("prev")
+		const next = firstChild.graph.findOne("next")
+		if (!prev || !next) {
+			return
+		}
+
+		// use an inverted index for full token matching as it's O(1)
+		if (this.index.has(span.norm)) {
+			// classify phrase
+			span.classify(new IntersectionClassification(1))
+
+			// classify child spans
+			span.graph.findAll("child").forEach((c) => c.classify(new IntersectionClassification(1)))
+		}
+	}
+}
+
+export default IntersectionClassifier

package/classifier/LevelClassifier.ts

@@ -0,0 +1,48 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import LevelClassification from "../classification/LevelClassification.js"
+import { LevelTypeClassification } from "../classification/LevelTypeClassification.js"
+import { Span } from "../tokenization/Span.js"
+import { ClassifyInput } from "./super/BaseClassifier.js"
+import { WordClassifier } from "./super/WordClassifier.js"
+
+const combinedFloorRegexp = /^\d{1,2}$/
+
+export class LevelClassifier extends WordClassifier {
+	each(span: Span) {
+		const prev = span.graph.findOne("prev")
+		const hasPrevLevelToken = prev?.classifications
+			? Object.hasOwn(prev.classifications, "LevelTypeClassification")
+			: false
+
+		// If the previous token in a level word, like floor, fl, or floor.
+		if (hasPrevLevelToken && combinedFloorRegexp.test(span.body)) {
+			span.classify(new LevelClassification(1))
+		}
+
+		// // A token that starts with a '#' and is not the first token in the query
+		// // and matches our regexp is always labeled as a level
+		// if (span.body[0] === "FLOOR" && prev && combinedFloorRegexp.test(span.body)) {
+		// 	span.classify(new LevelClassification(1))
+		// }
+	}
+
+	override classify(input: ClassifyInput, prev?: ClassifyInput): Span {
+		const span = Span.from(input)
+
+		if (prev) {
+			const p = Span.from(prev)
+
+			p.classify(new LevelTypeClassification(1.0))
+			span.graph.add("prev", p)
+		}
+
+		this.each(span)
+
+		return span
+	}
+}

package/classifier/LevelTypeClassifier.test.ts

@@ -0,0 +1,22 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import { LevelTypeClassification } from "mailwoman/classification"
+import test from "tape"
+import { LevelTypeClassifier } from "./LevelTypeClassifier.js"
+
+const classifier = new LevelTypeClassifier()
+
+for (const token of ["fl", "floor"]) {
+	test(`english level types: ${token}`, (t) => {
+		const span = classifier.classify(token)
+
+		t.deepEqual(span.classifications, {
+			LevelTypeClassification: new LevelTypeClassification(1, {}),
+		})
+		t.end()
+	})
+}

package/classifier/LevelTypeClassifier.ts

@@ -0,0 +1,32 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import { LevelTypeClassification } from "../classification/LevelTypeClassification.js"
+import * as libpostal from "../resources/libpostal/libpostal.js"
+import { Span } from "../tokenization/Span.js"
+import { WordClassifier } from "./super/WordClassifier.js"
+
+export class LevelTypeClassifier extends WordClassifier {
+	public index: libpostal.PostalIndex = {}
+
+	constructor() {
+		super()
+
+		libpostal.load(this.index, ["en"], "level_types_numbered.txt")
+	}
+
+	each(span: Span) {
+		// skip spans which contain numbers
+		if (span.contains.numerals) {
+			return
+		}
+
+		// use an inverted index for full token matching as it's O(1)
+		if (Object.hasOwn(this.index, span.norm)) {
+			span.classify(new LevelTypeClassification(1.0))
+		}
+	}
+}