mailwoman 0.2.0

package/resources/libpostal/libpostal.test.ts

@@ -0,0 +1,325 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import test from "tape"
+import * as libpostal from "./libpostal.js"
+
+test("generatePlurals: pluralize english tokens", (t) => {
+	const index = {
+		cat: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		cat: { langs: { en: true } },
+		cats: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: pluralize mixed eng/xxx language tokens", (t) => {
+	const index = {
+		cat: { langs: { en: true, fr: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		cat: { langs: { en: true, fr: true } },
+		cats: { langs: { en: true } }, // not assigned to fr
+	})
+	t.end()
+})
+
+test("generatePlurals: ignore non-english tokens", (t) => {
+	const index = {
+		cat: { langs: { fr: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		cat: { langs: { fr: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - functional", (t) => {
+	const index = {
+		cat: { langs: { en: true } },
+		dog: { langs: { en: true } },
+		dogs: { langs: { en: true } }, // already plural
+		fish: { langs: { en: true } }, // same word singular/plural in English
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		cat: { langs: { en: true } },
+		dog: { langs: { en: true } },
+		dogs: { langs: { en: true } },
+		fish: { langs: { en: true } },
+		cats: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - identical singular plural", (t) => {
+	const index = {
+		bison: { langs: { en: true } },
+		buffalo: { langs: { en: true } },
+		deer: { langs: { en: true } },
+		fish: { langs: { en: true } },
+		moose: { langs: { en: true } },
+		pike: { langs: { en: true } },
+		plankton: { langs: { en: true } },
+		salmon: { langs: { en: true } },
+		sheep: { langs: { en: true } },
+		swine: { langs: { en: true } },
+		trout: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		bison: { langs: { en: true } },
+		buffalo: { langs: { en: true } },
+		deer: { langs: { en: true } },
+		fish: { langs: { en: true } },
+		moose: { langs: { en: true } },
+		pike: { langs: { en: true } },
+		plankton: { langs: { en: true } },
+		salmon: { langs: { en: true } },
+		sheep: { langs: { en: true } },
+		swine: { langs: { en: true } },
+		trout: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - sibilant sound", (t) => {
+	const index = {
+		kiss: { langs: { en: true } },
+		phase: { langs: { en: true } },
+		dish: { langs: { en: true } },
+		massage: { langs: { en: true } },
+		witch: { langs: { en: true } },
+		judge: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		kiss: { langs: { en: true } },
+		phase: { langs: { en: true } },
+		dish: { langs: { en: true } },
+		massage: { langs: { en: true } },
+		witch: { langs: { en: true } },
+		judge: { langs: { en: true } },
+
+		kisses: { langs: { en: true } },
+		phases: { langs: { en: true } },
+		dishes: { langs: { en: true } },
+		massages: { langs: { en: true } },
+		witches: { langs: { en: true } },
+		judges: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - voiceless consonant", (t) => {
+	const index = {
+		lap: { langs: { en: true } },
+		cat: { langs: { en: true } },
+		clock: { langs: { en: true } },
+		cuff: { langs: { en: true } },
+		death: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		lap: { langs: { en: true } },
+		cat: { langs: { en: true } },
+		clock: { langs: { en: true } },
+		cuff: { langs: { en: true } },
+		death: { langs: { en: true } },
+
+		laps: { langs: { en: true } },
+		cats: { langs: { en: true } },
+		clocks: { langs: { en: true } },
+		cuffs: { langs: { en: true } },
+		deaths: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - regular plural", (t) => {
+	const index = {
+		boy: { langs: { en: true } },
+		girl: { langs: { en: true } },
+		chair: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		boy: { langs: { en: true } },
+		girl: { langs: { en: true } },
+		chair: { langs: { en: true } },
+
+		boys: { langs: { en: true } },
+		girls: { langs: { en: true } },
+		chairs: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - nouns ending in -o", (t) => {
+	const index = {
+		hero: { langs: { en: true } },
+		potato: { langs: { en: true } },
+		volcano: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		hero: { langs: { en: true } },
+		potato: { langs: { en: true } },
+		volcano: { langs: { en: true } },
+
+		heroes: { langs: { en: true } },
+		potatoes: { langs: { en: true } },
+		volcanoes: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - nouns ending in -o (Italian loanwords)", (t) => {
+	const index = {
+		canto: { langs: { en: true } },
+		hetero: { langs: { en: true } },
+		photo: { langs: { en: true } },
+		zero: { langs: { en: true } },
+		piano: { langs: { en: true } },
+		portico: { langs: { en: true } },
+		pro: { langs: { en: true } },
+		quarto: { langs: { en: true } },
+		kimono: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		canto: { langs: { en: true } },
+		hetero: { langs: { en: true } },
+		photo: { langs: { en: true } },
+		zero: { langs: { en: true } },
+		piano: { langs: { en: true } },
+		portico: { langs: { en: true } },
+		pro: { langs: { en: true } },
+		quarto: { langs: { en: true } },
+		kimono: { langs: { en: true } },
+
+		cantos: { langs: { en: true } },
+		heteros: { langs: { en: true } },
+		photos: { langs: { en: true } },
+		zeros: { langs: { en: true } },
+		pianos: { langs: { en: true } },
+		porticos: { langs: { en: true } },
+		pros: { langs: { en: true } },
+		quartos: { langs: { en: true } },
+		kimonos: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - nouns ending in -y", (t) => {
+	const index = {
+		cherry: { langs: { en: true } },
+		lady: { langs: { en: true } },
+		sky: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		cherry: { langs: { en: true } },
+		lady: { langs: { en: true } },
+		sky: { langs: { en: true } },
+
+		cherries: { langs: { en: true } },
+		ladies: { langs: { en: true } },
+		skies: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - nouns ending in -quy", (t) => {
+	const index = {
+		soliloquy: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		soliloquy: { langs: { en: true } },
+
+		soliloquies: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - voiceless fricatives", (t) => {
+	const index = {
+		bath: { langs: { en: true } },
+		mouth: { langs: { en: true } },
+		calf: { langs: { en: true } },
+		leaf: { langs: { en: true } },
+		knife: { langs: { en: true } },
+		life: { langs: { en: true } },
+		house: { langs: { en: true } },
+		moth: { langs: { en: true } },
+		proof: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		bath: { langs: { en: true } },
+		mouth: { langs: { en: true } },
+		calf: { langs: { en: true } },
+		leaf: { langs: { en: true } },
+		knife: { langs: { en: true } },
+		life: { langs: { en: true } },
+		house: { langs: { en: true } },
+		moth: { langs: { en: true } },
+		proof: { langs: { en: true } },
+
+		baths: { langs: { en: true } },
+		mouths: { langs: { en: true } },
+		calves: { langs: { en: true } },
+		leaves: { langs: { en: true } },
+		knives: { langs: { en: true } },
+		lives: { langs: { en: true } },
+		houses: { langs: { en: true } },
+		moths: { langs: { en: true } },
+		proofs: { langs: { en: true } },
+	})
+	t.end()
+})
+
+test("generatePlurals: english - nouns ending in -f", (t) => {
+	const index = {
+		dwarf: { langs: { en: true } },
+		hoof: { langs: { en: true } },
+		elf: { langs: { en: true } },
+		turf: { langs: { en: true } },
+	}
+	libpostal.generatePlurals(index)
+
+	t.deepEqual(index, {
+		dwarf: { langs: { en: true } },
+		hoof: { langs: { en: true } },
+		elf: { langs: { en: true } },
+		turf: { langs: { en: true } },
+
+		dwarves: { langs: { en: true } },
+		hooves: { langs: { en: true } },
+		elves: { langs: { en: true } },
+		turfs: { langs: { en: true } },
+	})
+	t.end()
+})

package/resources/libpostal/libpostal.ts

@@ -0,0 +1,106 @@
+/**
+ * @copyright OpenISP, Inc.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import { resourceDictionaryPathBuilder } from "mailwoman/sdk/repo"
+import { existsSync, readdirSync, readFileSync } from "node:fs"
+import * as path from "node:path"
+import pluralize from "pluralize"
+import * as custom from "../custom/custom.js"
+import { normalizeCell, NormalizeCellOptions } from "../helper.js"
+import * as pelias from "../pelias/pelias.js"
+
+const dictPath = resourceDictionaryPathBuilder("libpostal")
+// console.log("Dictionary path:", dictPath)
+
+export const languages = new Set(readdirSync(dictPath).filter((p) => !p.includes(".")))
+
+// console.log(`Found ${languages.size} languages:`, languages)
+
+export interface Cell {
+	langs: Record<string, boolean>
+}
+
+export type PostalIndex = Record<string, Cell>
+
+/**
+ * Load a dictionary file.
+ */
+export function load(index: PostalIndex, langs: Iterable<string>, filename: string, options?: NormalizeCellOptions) {
+	const add = _add(index, options)
+	const remove = _remove(index, options)
+
+	// console.debug("Loading dictionary file:", filename)
+
+	for (const lang of langs) {
+		const filepath = dictPath(lang, filename).toString()
+		// console.debug("Loading dictionary file:", filepath)
+
+		if (!existsSync(filepath)) {
+			// console.debug("Dictionary file not found:", filepath)
+			continue
+		}
+
+		const dict = readFileSync(filepath, "utf8")
+
+		dict.split("\n").forEach((row) => {
+			row.split("|").forEach((cell) => add(lang, cell))
+		})
+	}
+
+	for (const lang of langs) {
+		pelias.load(path.join("libpostal", lang, filename), (value) => add(lang, value), remove)
+	}
+
+	for (const lang of langs) {
+		custom.load(path.join("libpostal", lang, filename), (value) => add(lang, value), remove)
+	}
+}
+
+type Adder = (lang: string, cell: string) => void
+
+/**
+ * Create an adder function.
+ */
+function _add(index: PostalIndex, options?: NormalizeCellOptions): Adder {
+	return (lang, cell) => {
+		const value = normalizeCell(cell, options)
+		if (value && value.length) {
+			index[value] = index[value] || { langs: {} }
+			index[value].langs[lang] = true
+		}
+	}
+}
+
+type Remover = (cell: string) => void
+
+/**
+ * Create a remover function.
+ */
+function _remove(index: PostalIndex, options?: NormalizeCellOptions): Remover {
+	return (cell) => {
+		const value = normalizeCell(cell, options)
+
+		if (value && value.length) {
+			delete index[value]
+		}
+	}
+}
+
+// This functionality is only currently available for English
+// see: https://github.com/plurals/pluralize
+// @todo: find similar libraries which cover other languages
+/**
+ * Generate plurals for the index.
+ */
+export function generatePlurals(index: PostalIndex) {
+	for (const [cell, value] of Object.entries(index)) {
+		if (!value.langs.en) continue
+
+		const plural = pluralize(cell)
+		index[plural] = index[plural] || { langs: {} }
+		index[plural].langs.en = true
+	}
+}

package/resources/libpostal/resources-download.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+git clone https://github.com/openvenues/libpostal.git /tmp/libpostal
+
+rm -rf ./dictionaries
+cp -r /tmp/libpostal/resources/dictionaries .
+
+rm -rf /tmp/libpostal

package/resources/pelias/dictionaries/libpostal/af/personal_titles.txt

@@ -0,0 +1,2 @@
+!kort|k
+!korte|kte

package/resources/pelias/dictionaries/libpostal/all/surnames.txt

@@ -0,0 +1 @@
+!street

package/resources/pelias/dictionaries/libpostal/de/concatenated_suffixes_separable.txt

@@ -0,0 +1,3 @@
+str
+park
+kanal

package/resources/pelias/dictionaries/libpostal/de/place_names.txt

@@ -0,0 +1 @@
+!ma

package/resources/pelias/dictionaries/libpostal/de/street_types.txt

@@ -0,0 +1,2 @@
+# abbrviation for 'chaussee' conflicts with Swiss countrycode
+!ch

package/resources/pelias/dictionaries/libpostal/en/place_names.txt

@@ -0,0 +1,7 @@
+temple
+cathedral
+stop
+!dist
+building
+field
+!parish

package/resources/pelias/dictionaries/libpostal/en/road_types.txt

@@ -0,0 +1,18 @@
+county highway|ch|c.h.|c.h|c h|c.hw|c hw|co.hw|co hw|cty.hw|cty hw|c.hgwy|c hgwy|co.hgwy|co hgwy|cty.hgwy|cty hgwy|c.hway|c hway|co.hway|co hway|cty.hway|cty hway|c.hwy|c hwy|co.hwy|co hwy|cty.hwy|cty hwy|c.hi|c hi|co.hi|co hi|cty.hi|cty hi
+county road|cr|c.r.|c.r|c r|co.r|co r|c.rd|c rd|co.rd|co rd|cty.r|cty r|cty.rd|cty rd
+county route|cr|c.r.|c.r|c r|co.r|co r|c.rt|c rt|co.rt|co rt|cty.r|cty r|cty.rt|cty rt|c.rte|c rte|co.rte|co rte|cty.rte|cty rte|county touring route
+highroad|hrd|high road|hird|hi.rd|hi rd
+highway|hgwy|hw|hway|hwy|hi|hwye|hywy
+road|rd|ro|r|roa|raod
+route|rt|rte
+state highway|s.highway|s highway|st.highway|st highway|sh|s.h.|s.h|s h|st.h|st h|s.hw|s hw|st.hw|st hw|s.hwy|s hwy|shwy|s.hgwy|s hgwy|st.hgwy|st hgwy|s.hway|s hway|st.hway|st hway|s.hwy|s hwy|st.hwy|st hwy|s.hi|s hi|st.hi|st hi|statehighway
+state road|sr|stateroad|s.r.|s.r|s r|s.road|s road|st.road|st road|staterd|srd|s.rd|s rd|state rd|strd|st.rd|st rd
+state route|sr|stateroute|s.r.|s.r|s r|s.route|s route|st.route|st route|statert|srt|s.rt|s rt|srte|s.rte|s rte|state rt|state rte|strt|strte|st.rt|st rt|st.rte|st rte
+township highway|th|t.h.|t.h|t h|twp.h|twp h|tshp.h|tshp h|t.hw|t hw|twp.hw|twp hw|tshp.hw|tshp hw|t.hgwy|t hgwy|twp.hgwy|twp hgwy|tshp.hgwy|tshp hgwy|t.hway|t hway|twp.hway|twp hway|tshp.hway|tshp hway|t.hwy|t hwy|twp.hwy|twp hwy|tshp.hwy|tshp hwy|t.hi|t hi|twp.hi|twp hi|tshp.hi|tshp hi
+township road|tr|t.r.|t.r|t r|t rd|t.rd|trd|twpr|twp.r|twp r|twp.rd|twp rd|tshp.r|tshp r|tshp.rd|tshp rd|township rd|tp rd
+township route|tr|t.r.|t.r|t r|t rt|t.rt|trt|t.rte|t rte|twpr|twp.r|twp r|twp.rt|twp rt|twp.rte|twp rte|tshp.r|tshp r|tshp.rt|tshp rt|tshp.rte|tshp rte
+interstate highway|interstate|i|ih|i h
+
+# https://en.wikipedia.org/wiki/Farm-to-market_road
+farm road|fm
+ranch road|rm

package/resources/pelias/dictionaries/libpostal/en/street_names.txt

@@ -0,0 +1,2 @@
+# bad abbreviation for the very uncommon suffix "Artery"
+!art

package/resources/pelias/dictionaries/libpostal/en/street_types.txt

@@ -0,0 +1,16 @@
+# 183 Vista Paku, Pauanui, 3579, New Zealand
+paku
+
+# Rushendon Furlong, Pitstone, England
+furlong
+
+# bad abbreviation for the very uncommon suffix "Artery"
+!art
+
+# 1384 Cambridge beltway, Cambridge, MD 21613, USA
+beltway
+
+# https://github.com/pelias/parser/issues/140
+!broadway|bdwy|bway|bwy|brdway
+!esplanade|esp|espl
+market

package/resources/pelias/dictionaries/libpostal/en/unit_types_numbered.txt

@@ -0,0 +1,3 @@
+!shop|shp
+!stop
+!store|stor

package/resources/pelias/dictionaries/libpostal/es/street_types.txt

@@ -0,0 +1,2 @@
+# conflicts with US state abbreviation
+!ca

package/resources/pelias/dictionaries/libpostal/fr/personal_titles.txt

@@ -0,0 +1,4 @@
+l'amiral
+adjudant|l'adjudant
+empereur|l'empereur
+inspecteur|l'inspecteur

package/resources/pelias/dictionaries/libpostal/fr/street_types.txt

@@ -0,0 +1,5 @@
+cité|cite
+cités|cites
+
+# https://github.com/pelias/parser/pull/141#issuecomment-895230721
+!esplanades|esps

package/resources/pelias/dictionaries/libpostal/it/street_types.txt

@@ -0,0 +1,2 @@
+# this Italian contracted form of Androna causes issues in English
+!and

package/resources/pelias/dictionaries/libpostal/ko/street_types.txt

@@ -0,0 +1 @@
+!ga

package/resources/pelias/dictionaries/libpostal/nb/concatenated_suffixes_inseparable.txt

@@ -0,0 +1,40 @@
+plass|pl.|p.
+haugen
+skogen|skog
+gården|gård
+åsen
+haugen
+skogen|skog
+gården|gård
+åsen
+gård|gården
+holm|holmen
+haugen|Haugane
+krysset
+skogen|skog
+grend|grenda
+flat|flaten
+våg|vågen
+strand|stranda|stranden
+vik|vika
+platå
+stad
+terrasse
+hage|hagen
+øya
+lund|lunden
+holm|holmen
+berg|berget
+landa|lande|landet
+dal|dalen
+sund
+bakke|bakken
+høgda|høgde
+borg|borgen
+gard|garden|garda
+mark|marka
+voll|vollen
+sjø|sjøen
+lie
+sti|stien
+tangen

package/resources/pelias/dictionaries/libpostal/nb/directionals.txt

@@ -0,0 +1,8 @@
+øvre|ovre
+øvste|ovste
+nedre
+midtre
+ytre
+indre
+sør|sor|s
+sørre|sorre

package/resources/pelias/dictionaries/libpostal/nb/street_types.txt

@@ -0,0 +1,36 @@
+plass
+haugen
+skogen|skog
+gården|gård
+åsen
+gård|gården
+holm|holmen
+haugen|Haugane
+krysset
+skogen|skog
+grend|grenda
+flat|flaten
+våg|vågen
+strand|stranda|stranden
+vik|vika
+platå
+stad
+terrasse
+hage|hagen|hager
+øya
+lund|lunden
+holm|holmen
+berg|berget
+landa|lande|landet
+dal|dalen
+sund
+bakke|bakken
+høgda|høgde
+borg|borgen
+gard|garden|garda
+mark|marka
+voll|vollen
+sjø|sjøen
+lie
+sti|stien
+tangen

package/resources/pelias/dictionaries/libpostal/nl/concatenated_suffixes_inseparable.txt

@@ -0,0 +1 @@
+burg|brg|bg

package/resources/pelias/dictionaries/libpostal/nl/concatenated_suffixes_separable.txt

@@ -0,0 +1,8 @@
+baan
+daal
+dijk
+dreef
+!plain|pln.
+plein|pln
+plantsoen|plnts
+singel|sngl

package/resources/pelias/dictionaries/libpostal/nl/directionals.txt

@@ -0,0 +1,4 @@
+noordzijde|nz|n.z.|n.z|n z
+oostzijde|oz|o.z.|o.z|o z
+westzijde|wz|w.z.|w.z|w z
+zuidzijde|zz|z.z.|z.z|z z

package/resources/pelias/dictionaries/libpostal/nl/personal_suffixes.txt

@@ -0,0 +1,4 @@
+junior|jr|jnr
+senior|sr|snr
+# st is not a personal suffix in Dutch
+!st

package/resources/pelias/dictionaries/libpostal/nl/personal_titles.txt

@@ -0,0 +1,43 @@
+!kort|k
+!korte|kte
+aalmoezenier
+admiraal|adm
+bisschop|biss
+#typo in LibPostal resource
+!burgermeester|burg|bgm
+burgemeester|burg|bgm
+commissaris|comm
+deken|dkn
+directeur|dir
+frater|fr
+graaf
+gravin
+goeverneur|goev
+gouverneur|gouv
+heer|hr
+jonker|jkr
+juffrouw|juffr
+hertog|htg
+kanunnik|kan
+kapelaan|kap
+kapitein|kapt
+keizer
+luitenant generaal|lt gen
+!mevrouw|mevr
+mevrouw|mevr|mw
+madame|mad
+majoor|maj
+notaris|not
+overste|ov
+pater|ptr
+prelaat|prlt
+rector|rect
+schepen|sch
+schout|sch
+schout bij nacht|sbn
+secretaris|secr
+sekretaris|sekr
+veldmaarschalk|veldm
+vicaris|vic
+wethouder|weth
+zusters|zr

package/resources/pelias/dictionaries/libpostal/pl/place_names.txt

@@ -0,0 +1 @@
+wojsko|wojska|wojsk