@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/tools/sub/venue/head-nouns.ts

@@ -0,0 +1,208 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * @file Head-noun derivation for the sub-venue lexicon — proposing the addressed form of a designator
+ * from the encyclopaedic label a vocabulary source provides.
+ *
+ * Everything derived lands `curated: false`, because a derivation is a hypothesis a locale's own data confirms or kills.
+ */
+
+import { isPresent } from "@mailwoman/core/objects"
+
+import type { SubVenueSurface } from "#subvenue/table"
+
+/**
+ * Diacritic-flattened ascii fold, for comparing a Slavic or Turkish inflection against its Latin root.
+ *
+ * Deliberately `\p{Diacritic}` rather than `@mailwoman/normalize/fold`'s
+ * `stripCombiningMarks` (`\p{M}`), because the two diverge outside plain diacritics
+ * and {@link HEAD_NOUN_PREFIX_FLOOR} was calibrated against this fold.
+ */
+function asciiFold(text: string): string {
+	return text
+		.normalize("NFD")
+		.replaceAll(/\p{Diacritic}/gu, "")
+		.toLowerCase()
+}
+
+/**
+ * How many leading characters two ascii-folded forms must share for one to count as the other's inflection.
+ *
+ * Five, or the id's own length when shorter.
+ * A limit of six loses Spanish `satélite`.
+ * A limit of four admits Italian `campo`, which means field.
+ */
+const HEAD_NOUN_PREFIX_FLOOR = 5
+
+/**
+ * The shortest substring a non-Latin head-noun candidate may be.
+ *
+ * Two, because `楼` by itself means "building" and would fire on every Chinese building name.
+ */
+const NON_LATIN_HEAD_MIN_LENGTH = 2
+
+/**
+ * How many head-noun candidates one non-Latin record+language group may contribute.
+ *
+ * Six, capped because the substring lattice of a nine-character label is large
+ * and no entry past the sixth attests more than two surfaces.
+ */
+const NON_LATIN_HEAD_CANDIDATE_CAP = 6
+
+/**
+ * The scripts an ascii-folded prefix comparison against a Latin designator id can work on.
+ */
+const LATIN_PHRASE = /^[\p{Script=Latin}\d\s\p{P}]+$/u
+
+/**
+ * The scripts the shared-substring derivation may run on: Han, Hiragana, Katakana, Hangul.
+ *
+ * Narrower than a general non-Latin filter.
+ * A wider run produced unusable fragments in six scripts: Cyrillic, Greek, Arabic, Thai, Burmese or Tamil.
+ * No available extract attests those surfaces.
+ */
+const SHARED_SUBSTRING_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u
+
+/**
+ * Derive the head noun of every multi-part surface, so `terminal aeroportuaria`
+ * contributes the form anyone actually writes on an envelope.
+ *
+ * Latin script uses a cognate test against the designator's own canonical id.
+ * Han and Kana use a shared substring, because a token split finds no boundary there.
+ * Everything derived lands `curated: false`.
+ */
+export function deriveHeadNounSurfaces(surfaces: readonly SubVenueSurface[]): SubVenueSurface[] {
+	const derived = new Map<string, SubVenueSurface>()
+	const seen = new Set(surfaces.map((s) => `${s.phrase}\0${s.recordID}\0${s.lang}`))
+
+	const emit = (phrase: string, from: SubVenueSurface): void => {
+		if (phrase === from.phrase) return
+
+		const key = `${phrase}\0${from.recordID}\0${from.lang}`
+
+		if (seen.has(key) || derived.has(key)) return
+
+		derived.set(key, {
+			phrase,
+			recordID: from.recordID,
+			recordKind: from.recordKind,
+			lang: from.lang,
+			region: "",
+			source: "derived:head-noun",
+			curated: false,
+			observations: 0,
+			context: {},
+		})
+	}
+
+	for (const surface of surfaces) {
+		if (!LATIN_PHRASE.test(surface.phrase)) continue
+
+		const parts = surface.phrase.split(/[^\p{L}\p{N}]+/u).filter(isPresent)
+
+		if (parts.length < 2) continue
+
+		const root = asciiFold(surface.recordID)
+		const floor = Math.min(HEAD_NOUN_PREFIX_FLOOR, root.length)
+
+		for (const part of parts) {
+			const folded = asciiFold(part)
+
+			if (folded.length >= floor && commonPrefixLength(folded, root) >= floor) {
+				emit(part, surface)
+			}
+		}
+	}
+
+	const groups = new Map<string, Set<string>>()
+
+	for (const surface of surfaces) {
+		if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase)) continue
+
+		// Group `zh`, `zh-cn`, and `zh-hant` as writing systems for one vocabulary.
+		// The simplified/traditional pair supports a shared substring.
+		const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]!}`
+		const pool = groups.get(key) ?? new Set<string>()
+		pool.add(surface.phrase)
+		groups.set(key, pool)
+	}
+
+	const candidatesByGroup = new Map<string, string[]>()
+
+	for (const [key, pool] of groups) {
+		if (pool.size < 2) continue
+		candidatesByGroup.set(key, sharedSubstringCandidates(pool))
+	}
+
+	for (const surface of surfaces) {
+		if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase)) continue
+
+		const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]!}`
+
+		for (const candidate of candidatesByGroup.get(key) ?? []) {
+			if (surface.phrase.includes(candidate)) {
+				emit(candidate, surface)
+			}
+		}
+	}
+
+	return [...derived.values()]
+}
+
+function commonPrefixLength(a: string, b: string): number {
+	const limit = Math.min(a.length, b.length)
+	let i = 0
+
+	while (i < limit && a[i] === b[i]) {
+		i++
+	}
+
+	return i
+}
+
+/**
+ * Substrings occurring in at least two distinct members of `pool`, ranked by that count
+ * then length, capped at {@link NON_LATIN_HEAD_CANDIDATE_CAP}.
+ *
+ * A candidate never spans whitespace.
+ * Where a pool member has spaces, the candidate must be a whole token.
+ *
+ * This keeps Korean `공항 터미널` from contributing a fragment across the space.
+ * Maximal candidates only: one contained in a longer candidate produced by the same
+ * number of surfaces is dropped, since counting can never separate the two.
+ */
+function sharedSubstringCandidates(pool: ReadonlySet<string>): string[] {
+	const phrases = [...pool]
+	const spaced = phrases.some((phrase) => /\s/u.test(phrase))
+	const tokens = spaced ? new Set(phrases.flatMap((phrase) => phrase.split(/\s+/u).filter(isPresent))) : null
+	const counts = new Map<string, number>()
+
+	for (const phrase of phrases) {
+		const local = new Set<string>()
+
+		for (let length = NON_LATIN_HEAD_MIN_LENGTH; length <= phrase.length; length++) {
+			for (let start = 0; start + length <= phrase.length; start++) {
+				const candidate = phrase.slice(start, start + length)
+
+				if (/\s/u.test(candidate)) continue
+				local.add(candidate)
+			}
+		}
+
+		for (const candidate of local) {
+			counts.set(candidate, (counts.get(candidate) ?? 0) + 1)
+		}
+	}
+
+	const kept = [...counts].filter(([candidate, count]) => count >= 2 && (!tokens || tokens.has(candidate)))
+
+	return kept
+		.filter(([candidate, count]) =>
+			kept.every(([other, otherCount]) => other === candidate || otherCount !== count || !other.includes(candidate))
+		)
+		.toSorted((a, b) => b[1] - a[1] || b[0].length - a[0].length || a[0].localeCompare(b[0]))
+		.slice(0, NON_LATIN_HEAD_CANDIDATE_CAP)
+		.map(([candidate]) => candidate)
+}

package/lib/tools/sub/venue/lexicon.ts

@@ -0,0 +1,394 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file Builds the sub-venue designator lexicon from Wikidata, OSM and Overture fetch outputs.
+ */
+
+import { readLocalJSONFile, statPath } from "@mailwoman/core/fs/readers"
+import { writeLocalFile } from "@mailwoman/core/fs/writers"
+import { prettyJSON } from "@mailwoman/core/json"
+import { isoDate } from "@mailwoman/core/utils"
+import { basename, PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { extractAttestedPhrases, readSubVenueJSONL, type SubVenueHarvestRow } from "#subvenue/harvest"
+import { SUBVENUE_PROMOTIONS, type SubVenuePromotion } from "#subvenue/promotions"
+import { buildSurfaceIndex } from "#subvenue/surfaces"
+import {
+	CONCEPT_QIDS,
+	type IdentifierShape,
+	LexiconTier,
+	PROPOSED_DESIGNATORS,
+	SHIPPED_DESIGNATOR_SEED,
+	SHIPPED_MODIFIER_SEED,
+	type SubVenueDesignator,
+	type SubVenueLexiconSource,
+	type SubVenueLexiconTable,
+	type SubVenueModifier,
+	type SubVenueSurface,
+	SUBVENUE_LEXICON_VERSION,
+} from "#subvenue/table"
+import { deriveHeadNounSurfaces } from "#tools/sub/venue/head-nouns"
+import { surfacesFromWikidata } from "#tools/sub/venue/wikidata"
+
+export * from "#subvenue/harvest"
+export * from "#tools/sub/venue/head-nouns"
+export * from "#subvenue/surfaces"
+export * from "#subvenue/table"
+export * from "#tools/sub/venue/wikidata"
+
+/**
+ * Returns a copy of the surfaces with `curated: true` set on each surface that a promotion matches.
+ *
+ * A promotion matches a surface with the same record id and phrase whose language
+ * is the promotion's language or `und`.
+ * An extract surface must also come from the promotion's region.
+ *
+ * A surface without a region, such as a Wikidata label, matches only when no rejection
+ * exists for the same designator, phrase and language in any region.
+ * For example, `pier` is promoted for en-GB and rejected for en-US,
+ * so the region-free English `pier` stays uncurated.
+ *
+ * Rejections never mark a surface.
+ * They stay in the table as a record of the decision.
+ */
+export function applyPromotions(
+	surfaces: readonly SubVenueSurface[],
+	promotions: readonly SubVenuePromotion[]
+): SubVenueSurface[] {
+	const language = (locale: string): string => locale.split("-")[0]!
+	const region = (locale: string): string => locale.split("-")[1] ?? ""
+
+	const rejectedLanguages = new Set(
+		promotions
+			.filter((promotion) => promotion.decision === "reject")
+			.map((promotion) => `${promotion.designatorID} ${promotion.phrase} ${language(promotion.locale)}`)
+	)
+
+	const promoted = promotions.filter((promotion) => promotion.decision === "promote")
+
+	return surfaces.map((surface) => {
+		if (surface.curated) return surface
+
+		const hit = promoted.some((promotion) => {
+			if (promotion.designatorID !== surface.recordID || promotion.phrase !== surface.phrase) return false
+
+			const lang = language(promotion.locale)
+
+			if (surface.lang !== lang && surface.lang !== "und") return false
+
+			if (surface.region === region(promotion.locale)) return true
+
+			return surface.region === "" && !rejectedLanguages.has(`${promotion.designatorID} ${promotion.phrase} ${lang}`)
+		})
+
+		return hit ? { ...surface, curated: true } : surface
+	})
+}
+
+/**
+ * One harvest input with the source and region stamped on its surfaces.
+ */
+export interface SubVenueHarvest {
+	rows: readonly SubVenueHarvestRow[]
+	source?: string
+	region?: string
+}
+
+/**
+ * The parsed inputs of {@link buildSubVenueLexicon}, which never reads the filesystem.
+ */
+export interface BuildSubVenueLexiconInput {
+	/**
+	 * The raw `designator-labels.json` SPARQL response, or `null` to build the seed-only table.
+	 */
+	wikidata: unknown | null
+	/**
+	 * The harvest inputs in contribution order.
+	 *
+	 * Each harvest can match only phrases that exist before it runs, including those from earlier harvests.
+	 */
+	harvests: readonly SubVenueHarvest[]
+	/**
+	 * Provenance rows that the caller copies from the fetch manifests.
+	 */
+	sources: readonly SubVenueLexiconSource[]
+	/**
+	 * Defaults to {@link SUBVENUE_PROMOTIONS}.
+	 *
+	 * An empty array builds the uncurated table.
+	 */
+	promotions?: readonly SubVenuePromotion[]
+}
+
+/**
+ * Builds the lexicon table deterministically, so identical inputs produce identical output.
+ *
+ * Three ordering rules apply:
+ *
+ * 1. Seed surfaces come first, so `terminal` indexes to the `terminal` designator
+ *    instead of a Wikidata alias.
+ * 2. Head nouns are derived after Wikidata and before the harvests, because a harvest
+ *    can find only phrases such as `ターミナル` that already exist as surfaces.
+ * 3. Promotions apply last, so they can curate a surface from any source.
+ */
+export function buildSubVenueLexicon(input: BuildSubVenueLexiconInput): SubVenueLexiconTable {
+	const designators: SubVenueDesignator[] = SHIPPED_DESIGNATOR_SEED.map((seed) => ({
+		id: seed.id,
+		tier: LexiconTier.SubVenue,
+		modifierEligible: seed.modifierEligible,
+		shipped: true,
+		provenance: [...seed.provenance],
+	}))
+
+	const byID = new Map(designators.map((d) => [d.id, d]))
+
+	for (const proposed of PROPOSED_DESIGNATORS) {
+		const existing = byID.get(proposed.id)
+
+		if (existing) {
+			existing.provenance = [...new Set([...existing.provenance, ...proposed.provenance])]
+
+			continue
+		}
+
+		const record: SubVenueDesignator = {
+			id: proposed.id,
+			tier: proposed.tier,
+			modifierEligible: false,
+			shipped: false,
+			provenance: [...proposed.provenance],
+		}
+
+		designators.push(record)
+		byID.set(record.id, record)
+	}
+
+	// A designator records its Wikidata concept id even when the concept contributes no surface.
+	for (const [id, qid] of Object.entries(CONCEPT_QIDS)) {
+		const record = byID.get(id)
+
+		if (record) {
+			record.provenance = [...new Set([...record.provenance, `wikidata:${qid}`])]
+		}
+	}
+
+	const modifiers: SubVenueModifier[] = SHIPPED_MODIFIER_SEED.map((id) => ({
+		id,
+		shipped: true,
+		provenance: ["codex:directionals"],
+	}))
+
+	const surfaces: SubVenueSurface[] = [
+		...designators.map((d): SubVenueSurface => ({
+			phrase: d.id,
+			recordID: d.id,
+			recordKind: "designator",
+			lang: "en",
+			region: "",
+			source: "seed",
+			// Shipped English designators count as curated.
+			curated: d.shipped,
+			observations: 0,
+			context: {},
+		})),
+		...modifiers.map((m): SubVenueSurface => ({
+			phrase: m.id,
+			recordID: m.id,
+			recordKind: "modifier",
+			lang: "en",
+			region: "",
+			source: "seed",
+			curated: true,
+			observations: 0,
+			context: {},
+		})),
+	]
+
+	if (input.wikidata) {
+		surfaces.push(...surfacesFromWikidata(input.wikidata))
+	}
+
+	surfaces.push(...deriveHeadNounSurfaces(surfaces))
+
+	const identifierShapes: IdentifierShape[] = []
+
+	for (const harvest of input.harvests) {
+		const attested = extractAttestedPhrases(harvest.rows, buildSurfaceIndex(surfaces), {
+			source: harvest.source,
+			region: harvest.region,
+		})
+
+		surfaces.push(...attested.surfaces)
+		identifierShapes.push(...attested.identifierShapes)
+	}
+
+	const promotions = [...(input.promotions ?? SUBVENUE_PROMOTIONS)]
+	const curated = applyPromotions(surfaces, promotions)
+
+	// Every array is sorted so that the output is stable.
+	designators.sort((a, b) => a.id.localeCompare(b.id))
+	modifiers.sort((a, b) => a.id.localeCompare(b.id))
+
+	curated.sort(
+		(a, b) =>
+			a.phrase.localeCompare(b.phrase) ||
+			a.recordID.localeCompare(b.recordID) ||
+			a.lang.localeCompare(b.lang) ||
+			a.region.localeCompare(b.region) ||
+			a.source.localeCompare(b.source)
+	)
+
+	return {
+		version: SUBVENUE_LEXICON_VERSION,
+		sources: input.sources.toSorted((a, b) => a.id.localeCompare(b.id)),
+		designators,
+		modifiers,
+		surfaces: curated,
+		identifierShapes: identifierShapes.toSorted(
+			(a, b) =>
+				a.designatorID.localeCompare(b.designatorID) ||
+				a.region.localeCompare(b.region) ||
+				a.shape.localeCompare(b.shape)
+		),
+		promotions: promotions.toSorted(
+			(a, b) =>
+				a.designatorID.localeCompare(b.designatorID) ||
+				a.locale.localeCompare(b.locale) ||
+				a.phrase.localeCompare(b.phrase)
+		),
+	}
+}
+
+/**
+ * Serializes the table as pretty-printed JSON.
+ *
+ * Run `oxfmt` over the written file before committing, because committed JSON must match oxfmt's output.
+ */
+export function serializeSubVenueLexicon(table: SubVenueLexiconTable): string {
+	return prettyJSON(table)
+}
+
+/**
+ * The fields of the Wikidata fetch manifest that the lexicon copies into `sources[]`.
+ */
+interface WikidataFetchManifest {
+	endpoint?: string
+	license?: string
+	downloaded_at?: string
+	files?: Array<{ filename?: string; rows?: number }>
+}
+
+/**
+ * One OSM extract to harvest, with the ISO country code of its rows.
+ */
+export interface SubVenueExtractInput {
+	path: string
+	region: string
+}
+
+/**
+ * Options for {@link generateSubVenueLexicon}.
+ */
+export interface GenerateSubVenueLexiconOptions {
+	/**
+	 * The output directory of `mailwoman corpus fetch wikidata-subvenue`.
+	 *
+	 * Without it, the function builds the seed-only table.
+	 */
+	wikidataDir?: PathBuilderLike
+	/**
+	 * OSM extract JSONL files, one per region.
+	 */
+	extracts?: readonly SubVenueExtractInput[]
+	/**
+	 * Overture rows that the caller already read with `readOvertureSubVenues`.
+	 *
+	 * The caller opens `poi.db`, so this function does not depend on the database.
+	 */
+	overtureRows?: readonly (SubVenueHarvestRow & { country: string })[]
+	/**
+	 * The `poi.db` layer vintage.
+	 * The builder uses it only when `overtureRows` is non-empty.
+	 */
+	overtureVintage?: string
+	/**
+	 * The path of the written table.
+	 */
+	outPath: string
+}
+
+/**
+ * Reads the fetch outputs, builds the table with {@link buildSubVenueLexicon}, and writes it.
+ *
+ * Run `oxfmt` over `outPath` afterwards, because committed JSON must match oxfmt's output.
+ */
+export async function generateSubVenueLexicon(options: GenerateSubVenueLexiconOptions): Promise<SubVenueLexiconTable> {
+	const sources: SubVenueLexiconSource[] = []
+	const harvests: SubVenueHarvest[] = []
+	let wikidata: unknown = null
+
+	if (options.wikidataDir) {
+		const wikidataDir = PathBuilder.from(options.wikidataDir)
+
+		wikidata = await readLocalJSONFile<unknown>(wikidataDir("designator-labels.json"))
+
+		const manifest = await readLocalJSONFile<WikidataFetchManifest>(wikidataDir("MANIFEST.json"))
+
+		const labelFile = manifest.files?.find((f) => f.filename === "designator-labels.json")
+
+		sources.push({
+			id: "wikidata",
+			origin: manifest.endpoint ?? "https://query.wikidata.org/sparql",
+			license: manifest.license ?? "CC0",
+			// The table stores only the date so that a re-fetch on the same day produces no diff.
+			retrieved: (manifest.downloaded_at ?? "").slice(0, 10),
+			rows: labelFile?.rows ?? 0,
+		})
+	}
+
+	for (const extract of options.extracts ?? []) {
+		const rows = await readSubVenueJSONL(extract.path)
+
+		harvests.push({ rows, source: "osm", region: extract.region })
+
+		sources.push({
+			id: `osm:${extract.region.toLowerCase()}`,
+			// The origin records the Geofabrik region name, because a local path would leak the data root.
+			origin: `OpenStreetMap via Geofabrik (${basename(extract.path, ".jsonl")})`,
+			license: "ODbL (OpenStreetMap)",
+			// The extract is a local build output, so its mtime is when its rows were produced.
+			retrieved: isoDate((await statPath(extract.path)).mtime),
+			rows: rows.length,
+		})
+	}
+
+	if (options.overtureRows?.length) {
+		// Promotions match on region, so the function harvests Overture rows one country at a time.
+		const byCountry = new Map<string, SubVenueHarvestRow[]>()
+
+		for (const row of options.overtureRows) {
+			const bucket = byCountry.get(row.country) ?? []
+			bucket.push(row)
+			byCountry.set(row.country, bucket)
+		}
+
+		for (const [country, rows] of [...byCountry].toSorted((a, b) => a[0].localeCompare(b[0]))) {
+			harvests.push({ rows, source: "overture", region: country })
+		}
+
+		sources.push({
+			id: "overture",
+			origin: "Overture Maps Foundation places, via the poi.db spatial layer",
+			license: "CDLA-Permissive-2.0",
+			retrieved: options.overtureVintage ?? "",
+			rows: options.overtureRows.length,
+		})
+	}
+
+	const table = buildSubVenueLexicon({ wikidata, harvests, sources })
+	await writeLocalFile(serializeSubVenueLexicon(table), options.outPath)
+
+	return table
+}

package/lib/tools/sub/venue/wikidata.ts

@@ -0,0 +1,88 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file The Wikidata leg of the sub-venue lexicon — turning the designator-label sparql payload into
+ *   surfaces.
+ *
+ *   Reads an already-parsed payload rather than issuing a query: the acquisition lives in
+ *   `fetch/wikidata-subvenue.ts`, and keeping the conversion a pure function of parsed input is what
+ *   lets the builder stay deterministic and testable with no network and no fixture on disk.
+ *
+ *   Wikidata gives a concept name per language rather than a designator as addressed. Q849706's Spanish label
+ *   is `terminal aeroportuaria`; the addressed form is `Terminal`. So every surface produced here lands
+ *   `curated: false`, and the head-noun derivation is what proposes the addressed form from it.
+ */
+
+import { normalizeSurface } from "#subvenue/surfaces"
+import { CONCEPT_QIDS, type SubVenueSurface } from "#subvenue/table"
+
+/**
+ * The sparql results envelope, narrowed to the columns the designator-label query produces.
+ */
+interface SPARQLBinding {
+	item?: { value: string }
+	lang?: { value: string }
+	label?: { value: string }
+	kind?: { value: string }
+}
+
+interface SPARQLEnvelope {
+	results?: { bindings?: SPARQLBinding[] }
+}
+
+/**
+ * Turn the Wikidata designator-label payload into surfaces.
+ *
+ * Wikidata sometimes returns an untagged literal.
+ * The builder drops that row when its language tag is empty, when the QID maps to no
+ * designator in {@link CONCEPT_QIDS}, or when the normalized phrase is empty.
+ *
+ * Everything that survives lands `curated: false` — see the module docstring.
+ */
+export function surfacesFromWikidata(
+	payload: unknown,
+	conceptQIDs: Readonly<Record<string, string>> = CONCEPT_QIDS
+): SubVenueSurface[] {
+	const byQID = new Map(Object.entries(conceptQIDs).map(([id, qid]) => [qid, id]))
+	const envelope = payload as SPARQLEnvelope
+	const seen = new Set<string>()
+	const out: SubVenueSurface[] = []
+
+	for (const binding of envelope.results?.bindings ?? []) {
+		const qid = binding.item?.value?.split("/").pop()
+		const recordID = qid ? byQID.get(qid) : undefined
+		const lang = binding.lang?.value
+		const raw = binding.label?.value
+
+		if (!recordID || !lang || !raw) continue
+
+		const phrase = normalizeSurface(raw)
+
+		if (!phrase) continue
+
+		const source = binding.kind?.value === "alt" ? "wikidata:alt" : "wikidata:label"
+		// A concept can use the same string as a label and an alias.
+		// Dialect subtags can also repeat a phrase (`zh`, `zh-cn`, `zh-hans` all use 航站楼).
+		// Key the dedupe on the tuple that identifies a row.
+		const key = `${phrase}\0${recordID}\0${lang}\0${source}`
+
+		if (seen.has(key)) continue
+		seen.add(key)
+
+		out.push({
+			phrase,
+			recordID,
+			recordKind: "designator",
+			lang,
+			region: "",
+			source,
+			curated: false,
+			observations: 0,
+			context: {},
+		})
+	}
+
+	return out
+}