@mailwoman/corpus 9.4.0 → 10.1.0

package/out/adapters/cordis/adapter.js

@@ -0,0 +1,616 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `cordis`: the organization files CORDIS publishes for the participants in the EU's research
+ * framework programmes, read for the address each participating organization registered.
+ *
+ * Input is the publisher's own CSV archives, one per programme, as
+ * `https://cordis.europa.eu/data/cordis-<programme>projects-csv.zip` serves them for FP7
+ * (`fp7`), Horizon 2020 (`h2020`) and Horizon Europe (`HORIZON`).
+ * Each archive holds an `organization.csv`: `;`-separated, every field double-quoted, one row per participation.
+ * An organizationtherefore appears once for every project it took part in.
+ * The adapter streams the member out of the archive through `readZipEntry` and `CSVSpliterator`,
+ * so no extracted copy sits on disk.
+ * `opts.inputPath` is one archive, one extracted `organization.csv`, or a directory of archives.
+ * `fetchCORDIS` writes the directory form.
+ *
+ * Measured on the archives fetched 2026-10-03, the three members hold 420,811 participation rows and
+ * 78,842 distinct published records, and the adapter writes 57,930 rows over 158 countries.
+ *
+ * ## The columns read
+ *
+ * `name`, `shortName`, `street`, `postCode`, `city` and `country` reach a row or decide whether
+ * one is written. `organisationID`, `activityType`, `SME`, `vatNumber` and `nutsCode` were read while
+ * deciding the rules below and reach no row.
+ *
+ * ## Organizations, and no natural person
+ *
+ * A corpus holds the addresses of organizations and no address of an identifiable natural person.
+ * CORDIS publishes no legal-form column.
+ * `activityType` takes the five values `HES`, `PRC`, `REC`, `OTH` and `PUB`, plus an empty value on
+ * older rows, and no value marks a natural person.
+ * A natural person still enters the participant register: a sole trader, or a researcher contracted
+ * in their own name, is registered under their own name, mostly with `activityType` `PRC`.
+ * Measured over the three archives, the names of those registrations take three forms, and
+ * {@linkcode naturalPersonForm} refuses each:
+ *
+ * 1. **Every word is a personal name.** `FISK ADRIAN`, `SCHWARZ KAI-UWE WOLFGANG`, `Kerstin
+ *    Minnich`: two to four words, each a given name or surname in libpostal's `all` dictionaries,
+ *    at least one a given name. 194 records.
+ * 2. **The participant register's `SURNAME Given` casing.** `BUCKENHUSKES Herbert Johannes`: an
+ *    upper-case run followed by title-case words that are all personal names, one of them a given
+ *    name, for a surname the dictionary lacks. 5 records.
+ * 3. **A sole-trader legal form.** German `e.K.`, `e.Kfm.` and `e.Kfr.`, and `Inh.` (owner), in the
+ *    name or the short name, as in `Melina Bucher e.Kfr.`. 3 records.
+ *
+ * Neither name rule fires on a name holding a word that is a country's name, so `ERICSSON FRANCE` and
+ * `IDEMIA Public Security France` are admitted.
+ *
+ * The check is a refusal rule rather than a classifier. A sole trader trading under a business name
+ * that holds no personal name passes it. Some organizations are refused: `JOHN COCKERILL` by the first
+ * rule, and `UAB Nando` and `US Forest Service` among the 5 the casing rule refuses. The count each
+ * rule refuses is in the `dropped` map. The natural person a reader would expect first,
+ * `CORDOVA Trey Christian`, publishes no street, postcode or city at all and is refused as
+ * {@linkcode CORDISRefusal.LocalityAbsent}.
+ *
+ * The publisher's legal notice states the same limit from its side: reuse may require clearing
+ * additional rights "if a specific content depicts identifiable private individuals".
+ *
+ * ## Country codes
+ *
+ * CORDIS writes EU country codes, and those differ from ISO 3166-1 in two places: `EL` is Greece and
+ * `UK` is the United Kingdom. Both are mapped, to `GR` and `GB`. Any other value that is not an ISO
+ * alpha-2 code refuses the row: `XK` (Kosovo, a user-assigned code, 34 records), `ZZ` (1), an empty
+ * value (323), and `DE;HU` (1), two countries in one field.
+ *
+ * A country whose codex `layoutForCountry` is `null` has no rendering to write the row in.
+ * The row is refused with {@linkcode CORDISRefusal.CountryNoLayout} rather than rendered in another
+ * country's order: 353 records over 37 countries, Ghana's 77 the most.
+ *
+ * ## The street line
+ *
+ * `street` holds the whole street line as the participant typed it, usually but not always in the
+ * country's own order: `THE WELLCOME TRUST LIMITED` writes `EUSTON ROAD 215 GIBBS BUILDING`, a
+ * number-last line in a number-first country. The house number is therefore split off only on the
+ * side the country's order puts it, and {@linkcode streetOrderForCountry} reads that side from codex's
+ * `house-number-precedes-street` claim. A country whose observations disagree, or state no side,
+ * has no unambiguous side, and its lines are kept only where they hold no digit.
+ *
+ * **A street line holding a digit the split could not place refuses the row**, as `it-anac` does, and
+ * so does a line whose remainder still holds a digit once the number is split off. `C GRAN DE GRACIA
+ * NUM 1, PLANTA 4, PUERTA 3` would otherwise label `3`, the door, as the house number, and `TRIERSTRAAT
+ * 49 STRATENPLAN BUS 7` would label the box. The cost is the street whose name holds a digit, `PLACE DU
+ * 20 AOUT 7`, which is refused with the rest. These two rules refuse 14,438 and 4,646 records,
+ * the largest share of the 20,911 refused, led by France, Great Britain and Spain.
+ *
+ * A civic-number marker between the street and the number (`NR`, `NO`, `NUM`, `Nº`) is dropped, so the
+ * street is `HAUPTSTRASSE` rather than `HAUPTSTRASSE NR.`. A kilometer point, `CTRA SANT LLORENC DE
+ * MORUNYS KM, 2`, is refused rather than read as a house number.
+ *
+ * ## Placeholders and the postcode
+ *
+ * A value holding no letter and no digit (`-`, `.`), or one of `N/A`, `NA`, `none`, `Not applicable`,
+ * is treated as absent. A postcode with no digit (`EIRE`, `Clare`, `POSTFACH`) and a postcode of
+ * zeroes alone are dropped from the row rather than refusing it, and counted as `component:postcode:*`.
+ * A postcode written with its country prefix, `LV-1063` or `LT-01103`, is kept as published: Latvia
+ * and Lithuania write that prefix in their official form.
+ *
+ * A layout with no slot for a published component leaves it out, and the row is still written:
+ * the United Arab Emirates' layout has no locality line, and Japan's Latin layout none below the
+ * prefecture. Each is counted as `component:<tag>:unplaced`.
+ *
+ * ## Deduplication and identity
+ *
+ * An organizationappears once per project, across three programmes. The adapter reads each distinct
+ * published record once, keyed on the five fields that reach a row, and writes each distinct rendered
+ * address once. A record whose address changed between programmes contributes one row per version.
+ * The row id is content-addressed over the aligned components.
+ *
+ * ## License
+ *
+ * The CORDIS legal notice at `https://cordis.europa.eu/about/legal`, read 2026-10-03, states:
+ * "Unless otherwise noted (e.g. in individual copyright notices), the reuse of the editorial content on
+ * this website owned by the EU is authorized under the Creative Commons Attribution 4.0 International
+ * (CC BY 4.0) licence." The same notice records that "The Commission's reuse policy is implemented by
+ * Commission Decision 2011/833/EU of 12 December 2011 on the reuse of Commission documents", and the
+ * data.europa.eu records for `cordisfp7projects`, `cordish2020projects` and
+ * `cordis-eu-research-projects-under-horizon-europe-2021-2027` label each CSV distribution with that
+ * decision (`COM_REUSE`). CC BY requires credit and a statement that changes were made, so every row
+ * records the license and {@linkcode CORDIS_ATTRIBUTION} is the credit the model card prints.
+ *
+ * The adapter honors `opts.limit`, `opts.signal` and `opts.country`.
+ */
+import { ConventionClaimID, ObservationStance } from "@mailwoman/codex/address/convention-claims";
+import { formatAddressRow, scriptOfComponents } from "@mailwoman/codex/address/format";
+import { conventionClaimForCountry, layoutForCountry } from "@mailwoman/codex/address/layouts";
+import { CountryISO2 } from "@mailwoman/codex/country/codes";
+import { matchCountry } from "@mailwoman/codex/country/country";
+import { readZipEntry } from "@mailwoman/core/fs/zip";
+import { PathBuilder } from "path-ts";
+import { CSVSpliterator } from "spliterator";
+import { Globerator } from "spliterator/node/fs";
+import { loadLibpostalDictionary } from "#adapters/libpostal-dictionary";
+import { stableSourceID } from "#adapters/source-id";
+import { splitStreetLine, splitTrailingStreetLine } from "#adapters/street-line";
+import { SourceRegister } from "#registers";
+import { AddressRole, countDropped, SurfaceOrigin, } from "#types";
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const CORDIS_ADAPTER_ID = "cordis";
+/**
+ * The license the CORDIS legal notice states for content the EU owns.
+ */
+export const CORDIS_LICENSE = "CC-BY-4.0";
+/**
+ * The credit CC BY 4.0 obliges.
+ *
+ * The model card prints it beside the license and a statement that the data was changed.
+ */
+export const CORDIS_ATTRIBUTION = "© European Union, CORDIS (Publications Office of the European Union)";
+/**
+ * The archive member that holds the participants in every programme's CSV zip.
+ */
+export const CORDIS_ORGANIZATION_MEMBER = "organization.csv";
+/**
+ * The columns the adapter indexes by the publisher's own spelling.
+ *
+ * Checked against the first record, so a renamed column raises instead of
+ * reaching every row as an empty string.
+ */
+export const CORDIS_REQUIRED_COLUMNS = [
+    "organisationID",
+    "name",
+    "shortName",
+    "activityType",
+    "street",
+    "postCode",
+    "city",
+    "country",
+];
+/**
+ * The two places CORDIS's EU country codes differ from ISO 3166-1 alpha-2.
+ */
+export const CORDIS_COUNTRY_ALIASES = {
+    EL: "GR",
+    UK: "GB",
+};
+const ISO_ALPHA2 = new Set(Object.values(CountryISO2));
+/**
+ * Why a published record did not become a row.
+ */
+export const CORDISRefusal = {
+    /**
+     * `country` is not an ISO 3166-1 alpha-2 code once `EL` and `UK` are mapped.
+     */
+    CountryNotISO: "country-not-iso",
+    /**
+     * Codex has no layout for the country, so no rendering states where each component goes.
+     */
+    CountryNoLayout: "country-no-layout",
+    /**
+     * Every word of the name is a personal name.
+     */
+    NamePersonalWords: "name-personal-words",
+    /**
+     * The name takes the participant register's `SURNAME Given` casing.
+     */
+    NamePersonalCasing: "name-personal-casing",
+    /**
+     * The name or short name holds a sole-trader legal form.
+     */
+    NameSoleTrader: "name-sole-trader",
+    /**
+     * The record states no city.
+     */
+    LocalityAbsent: "locality-absent",
+    /**
+     * The street line holds a digit and no house number could be split off on the country's side.
+     */
+    StreetNumberUnplaced: "street-number-unplaced",
+    /**
+     * A house number was split off, and the street that remains still holds a digit.
+     */
+    StreetDigitInRemainder: "street-digit-in-remainder",
+    /**
+     * Beside the locality, the record holds neither a street nor a postcode.
+     */
+    ComponentsTooFew: "components-too-few",
+    /**
+     * The components do not render in the country's layout.
+     */
+    Unrenderable: "unrenderable",
+};
+/**
+ * Load libpostal's language-independent given-name and surname dictionaries.
+ */
+export async function loadPersonNameLexicon() {
+    const [givenNames, surnames] = await Promise.all([
+        loadLibpostalDictionary("all", "given_names.txt"),
+        loadLibpostalDictionary("all", "surnames.txt"),
+    ]);
+    return { givenNames, surnames };
+}
+/**
+ * Which side of the street name a country writes the house number on.
+ */
+export const StreetOrder = {
+    NumberFirst: "number-first",
+    NumberLast: "number-last",
+};
+const streetOrderMemo = new Map();
+/**
+ * The side a country writes its house number on, or `null` where codex's sources disagree or state none.
+ *
+ * Read from the `house-number-precedes-street` claim, which collects libaddressinput's `fmt` and `lfmt`,
+ * the OpenCage street order, the board's hand-authored layout and a retrieved UPU S42 template.
+ * A side is returned only when at least one source states it and none states the other.
+ *
+ * Hong Kong's Chinese-script `fmt` writes the number after the street
+ * and its Latin `lfmt` before it, so Hong Kong has no side.
+ */
+export function streetOrderForCountry(country) {
+    const memo = streetOrderMemo.get(country);
+    if (memo !== undefined)
+        return memo;
+    const claim = conventionClaimForCountry(ConventionClaimID.HouseNumberPrecedesStreet, country);
+    const stances = claim?.observations.map((observation) => observation.stance) ?? [];
+    const supports = stances.includes(ObservationStance.Supports);
+    const contradicts = stances.includes(ObservationStance.Contradicts);
+    let order = null;
+    if (supports && !contradicts) {
+        order = StreetOrder.NumberFirst;
+    }
+    else if (contradicts && !supports) {
+        order = StreetOrder.NumberLast;
+    }
+    streetOrderMemo.set(country, order);
+    return order;
+}
+/**
+ * The values the publisher writes where it holds none, compared with whitespace removed and case folded.
+ */
+const PLACEHOLDER_VALUES = new Set(["na", "n/a", "n.a.", "none", "notapplicable", "null", "nil"]);
+/**
+ * Whether a published value is a placeholder for an absent one, or holds no letter and no digit.
+ */
+export function isCORDISPlaceholder(value) {
+    const trimmed = value.trim();
+    if (!/[\p{L}\d]/u.test(trimmed))
+        return true;
+    return PLACEHOLDER_VALUES.has(trimmed.replaceAll(/\s+/gu, "").toLowerCase());
+}
+/**
+ * A published field's value with inner whitespace collapsed, or an empty string
+ * where the publisher states none.
+ */
+function fieldValue(value) {
+    const trimmed = (value ?? "").replaceAll(/\s+/gu, " ").trim();
+    return isCORDISPlaceholder(trimmed) ? "" : trimmed;
+}
+/**
+ * A sole-trader legal form: German `e.K.`, `e.Kfm.`, `e.Kfr.`, or `Inh.` naming the owner.
+ *
+ * Case-sensitive, because the legal form is written with a lower-case `e`,
+ * and `ACADEMICIAN E. K. FIODOROV` holds the same letters as two initials.
+ */
+const SOLE_TRADER_MARKER = /(?<![\p{L}.])e\.\s?K(?:fm|fr)?\.(?!\p{L})|(?<!\p{L})(?:Inh|INH)\.(?=\s)/u;
+/**
+ * One word of a personal name: letters, with internal hyphens and apostrophes.
+ */
+const NAME_WORD = /^\p{L}+(?:[-'’]\p{L}+)*$/u;
+/**
+ * A word written in capitals throughout, two letters or more.
+ */
+const UPPER_WORD = /^\p{Lu}[\p{Lu}'’-]+$/u;
+/**
+ * A word written with one leading capital and lower case after it, two letters or more.
+ */
+const TITLE_WORD = /^\p{Lu}\p{Ll}+(?:[-'’]\p{Lu}?\p{Ll}+)*$/u;
+/**
+ * The most words a personal name is read with.
+ *
+ * Four covers a surname with three given names, `SUNDQVIST MARIE CHARLOTTE HELENA`,
+ * and keeps the dictionary rule off longer organization names.
+ */
+const MAX_PERSONAL_NAME_WORDS = 4;
+/**
+ * The shortest word read as a country name.
+ *
+ * `matchCountry` resolves two- and three-letter codes as well as names, and a two-letter
+ * word such as `AS` is a Norwegian legal form rather than a market.
+ */
+const MIN_COUNTRY_WORD_LENGTH = 4;
+/**
+ * The words a name check reads: two to {@link MAX_PERSONAL_NAME_WORDS},
+ * each a {@link NAME_WORD}, or `null` for a name outside that shape.
+ */
+function nameWords(name) {
+    const words = name.split(/\s+/u).filter((word) => word !== "");
+    if (words.length < 2 || words.length > MAX_PERSONAL_NAME_WORDS)
+        return null;
+    return words.every((word) => NAME_WORD.test(word)) ? words : null;
+}
+/**
+ * The personal-name form a participant's name takes, or `null` where it takes none.
+ *
+ * Exported so a test and a measurement can read the rule the adapter applies rather than a copy of it.
+ */
+export function naturalPersonForm(name, shortName, lexicon) {
+    if (SOLE_TRADER_MARKER.test(name) || SOLE_TRADER_MARKER.test(shortName))
+        return CORDISRefusal.NameSoleTrader;
+    const words = nameWords(name.trim());
+    if (!words)
+        return null;
+    const isGiven = (part) => lexicon.givenNames.has(part.toLowerCase());
+    const isPersonal = (part) => isGiven(part) || lexicon.surnames.has(part.toLowerCase());
+    // A word naming a country is an organization's market rather than a person's name.
+    const namesCountry = (word) => word.length >= MIN_COUNTRY_WORD_LENGTH && matchCountry(word) !== null;
+    if (words.some(namesCountry))
+        return null;
+    const parts = words.flatMap((word) => word.split(/-/u));
+    if (parts.some(isGiven) && parts.every(isPersonal))
+        return CORDISRefusal.NamePersonalWords;
+    // The casing rule reads the upper-case run as a surname the dictionary may lack,
+    // so it asks only the title-case words to be personal names, one of them a given name.
+    const firstTitle = words.findIndex((word) => !UPPER_WORD.test(word));
+    if (firstTitle > 0) {
+        const tail = words.slice(firstTitle);
+        const tailParts = tail.flatMap((word) => word.split(/-/u));
+        if (tail.every((word) => TITLE_WORD.test(word)) && tailParts.some(isGiven) && tailParts.every(isPersonal)) {
+            return CORDISRefusal.NamePersonalCasing;
+        }
+    }
+    return null;
+}
+/**
+ * A civic-number marker left at the end of the street once the number is split off.
+ */
+const TRAILING_CIVIC_MARKER = /[\s,]+(?:nr|no|n|num|núm|nº|n°)\.?$/iu;
+/**
+ * A kilometer marker followed by its number, as in `CTRA SANT LLORENC DE MORUNYS KM, 2`.
+ */
+const KILOMETRE_POINT = /(?<!\p{L})km\.?[\s,]*\d/iu;
+/**
+ * A leading number joined to the street by a comma, `12, MG Road`,
+ * which {@linkcode splitStreetLine} does not read.
+ */
+const LEADING_NUMBER_COMMA = /^(\d+[\p{L}]?),\s*/u;
+/**
+ * The street line split on the side the country writes its number, or the refusal the line meets.
+ *
+ * `order` is `null` for a country with no unambiguous side, and such a line
+ * is kept only where it holds no digit.
+ */
+export function splitCORDISStreetLine(line, order) {
+    const trimmed = line
+        .replaceAll(/\s+/gu, " ")
+        .replace(/[\s,.;]+$/u, "")
+        .trim();
+    if (!trimmed)
+        return null;
+    // A kilometer point identifies a place on a road rather than a premise, so its number is never a house number.
+    if (KILOMETRE_POINT.test(trimmed))
+        return { refused: CORDISRefusal.StreetNumberUnplaced };
+    let split = { street: trimmed };
+    if (order === StreetOrder.NumberFirst) {
+        split = splitStreetLine(trimmed.replace(LEADING_NUMBER_COMMA, "$1 "));
+    }
+    else if (order === StreetOrder.NumberLast) {
+        split = splitTrailingStreetLine(trimmed);
+    }
+    if (!split)
+        return null;
+    if (!split.house_number) {
+        return /\d/u.test(split.street) ? { refused: CORDISRefusal.StreetNumberUnplaced } : split;
+    }
+    const street = split.street.replace(TRAILING_CIVIC_MARKER, "").trim();
+    if (!street || /\d/u.test(street))
+        return { refused: CORDISRefusal.StreetDigitInRemainder };
+    return { house_number: split.house_number, street };
+}
+/**
+ * The ISO 3166-1 alpha-2 code for a CORDIS `country` value, or `null` for one that maps to none.
+ */
+export function cordisCountryCode(value) {
+    const code = (value ?? "").trim().toUpperCase();
+    const mapped = CORDIS_COUNTRY_ALIASES[code] ?? code;
+    return ISO_ALPHA2.has(mapped) ? mapped : null;
+}
+/**
+ * Read one published record into a row, or report why it yielded none.
+ *
+ * Exported because the refusal reasons are the measurement this adapter is checked by.
+ */
+export function readCORDISRecord(record, lexicon) {
+    const country = cordisCountryCode(record.country);
+    if (!country)
+        return { refused: CORDISRefusal.CountryNotISO, country: null };
+    if (!layoutForCountry(country))
+        return { refused: CORDISRefusal.CountryNoLayout, country };
+    const name = fieldValue(record.name);
+    const personal = naturalPersonForm(name, fieldValue(record.shortName), lexicon);
+    if (personal)
+        return { refused: personal, country };
+    const locality = fieldValue(record.city);
+    if (!locality)
+        return { refused: CORDISRefusal.LocalityAbsent, country };
+    const discarded = [];
+    const components = {};
+    if (/\p{L}/u.test(name)) {
+        components.venue = name;
+    }
+    const streetLine = fieldValue(record.street);
+    if (streetLine) {
+        const split = splitCORDISStreetLine(streetLine, streetOrderForCountry(country));
+        if (split && "refused" in split)
+            return { refused: split.refused, country };
+        if (split?.house_number) {
+            components.house_number = split.house_number;
+        }
+        if (split) {
+            components.street = split.street;
+        }
+    }
+    const postcode = fieldValue(record.postCode);
+    if (postcode && !/\d/u.test(postcode)) {
+        discarded.push("component:postcode:no-digit");
+    }
+    else if (postcode && /^[0\s-]+$/u.test(postcode)) {
+        discarded.push("component:postcode:zeroes");
+    }
+    else if (postcode) {
+        components.postcode = postcode;
+    }
+    if (!components.street && !components.postcode)
+        return { refused: CORDISRefusal.ComponentsTooFew, country };
+    components.locality = locality;
+    // The script is passed rather than left to the formatter, which keeps a country's local
+    // order whenever its Latin order has fewer slots: `Weixing Road` and `Changchun` would
+    // otherwise print run together in China's Han-script line, with no separator between them.
+    const rendered = formatAddressRow(components, country, { singleLine: true, script: scriptOfComponents(components) });
+    if (!rendered)
+        return { refused: CORDISRefusal.Unrenderable, country };
+    const { raw, components: aligned } = rendered;
+    // Checked again on what the layout printed: Burkina Faso's layout has no postcode slot,
+    // so `MINISTERE DE LA SANTE` with postcode `7003` and no street prints as a name and a city.
+    if (!aligned.street && !aligned.postcode)
+        return { refused: CORDISRefusal.ComponentsTooFew, country };
+    // A layout with no slot for a component leaves it out of `raw`, as the United Arab Emirates'
+    // leaves the locality, and the count records how often a published value went unwritten.
+    for (const tag of rendered.unplaced) {
+        discarded.push(`component:${tag}:unplaced`);
+    }
+    return {
+        admitted: {
+            raw,
+            components: aligned,
+            country,
+            source: CORDIS_ADAPTER_ID,
+            source_id: stableSourceID(CORDIS_ADAPTER_ID, aligned),
+            corpus_version: "",
+            license: CORDIS_LICENSE,
+        },
+        discarded,
+    };
+}
+/**
+ * Every distinct published record under `inputPath`, read once each.
+ *
+ * A record is distinct on the five fields that reach a row, so an organization that took part in many
+ * projects with one address is read once, and one whose address changed is read once per version.
+ * `onDuplicate` is called once for every participation row a record already read covers.
+ *
+ * The adapter and the measurement both read through this, so the counts a
+ * measurement reports are the adapter's own.
+ */
+export async function* readCORDISRecords(inputPath, lexicon, options = {}) {
+    const seen = new Set();
+    for (const source of await cordisSources(inputPath)) {
+        if (options.signal?.aborted)
+            return;
+        const records = CSVSpliterator.fromAsync(source.archive ? readZipEntry(source.path, CORDIS_ORGANIZATION_MEMBER) : source.path, { normalizeKeys: false, columnDelimiter: ";" });
+        let first = true;
+        for await (const record of records) {
+            if (options.signal?.aborted)
+                return;
+            if (first) {
+                assertCORDISColumns(record, source.path);
+                first = false;
+            }
+            const key = [record.name, record.street, record.postCode, record.city, record.country]
+                .map((value) => (value ?? "").replaceAll(/\s+/gu, " ").trim())
+                .join("\u001F");
+            if (seen.has(key)) {
+                options.onDuplicate?.();
+                continue;
+            }
+            seen.add(key);
+            yield { record, reading: readCORDISRecord(record, lexicon) };
+        }
+    }
+}
+/**
+ * Raise when a record lacks a column the adapter indexes, naming the file and the missing columns.
+ */
+function assertCORDISColumns(record, path) {
+    const missing = CORDIS_REQUIRED_COLUMNS.filter((column) => !(column in record));
+    if (missing.length) {
+        throw new Error(`${CORDIS_ADAPTER_ID} adapter: ${PathBuilder.from(path).toString()} has no column ${missing.join(", ")}`);
+    }
+}
+/**
+ * Every input under `inputPath`: one archive, one extracted CSV, or a directory of archives.
+ *
+ * A directory is read in sorted name order, so a rerun meets the records in the
+ * same order and keeps the same first copy.
+ * A directory holding no archive raises rather than yielding zero rows.
+ */
+async function cordisSources(inputPath) {
+    const path = PathBuilder.from(inputPath);
+    const basename = path.basename().toLowerCase();
+    if (basename.endsWith(".zip"))
+        return [{ path, archive: true }];
+    if (basename.endsWith(".csv"))
+        return [{ path, archive: false }];
+    const names = await Globerator.from("*.zip", { cwd: path.toString(), absolute: false }).toSorted();
+    if (!names.length) {
+        throw new Error(`${CORDIS_ADAPTER_ID} adapter: ${path.toString()} holds no .zip archive`);
+    }
+    return names.map((name) => ({ path: path(name), archive: true }));
+}
+export function createCORDISAdapter() {
+    return {
+        id: CORDIS_ADAPTER_ID,
+        defaultLicense: CORDIS_LICENSE,
+        addressRole: AddressRole.RegisteredOffice,
+        register: SourceRegister.CORDISParticipants,
+        surface: SurfaceOrigin.Rendered,
+        description: "CORDIS organization files (FP7, Horizon 2020, Horizon Europe): the registered addresses of EU research-programme participants.",
+        async *rows(opts) {
+            const requested = opts.country ? opts.country.trim().toUpperCase() : undefined;
+            const lexicon = await loadPersonNameLexicon();
+            const written = new Set();
+            let emitted = 0;
+            const readings = readCORDISRecords(opts.inputPath, lexicon, {
+                signal: opts.signal,
+                onDuplicate: () => countDropped(opts, "row:duplicate-record"),
+            });
+            for await (const { reading } of readings) {
+                if (opts.signal?.aborted)
+                    break;
+                if (opts.limit !== undefined && emitted >= opts.limit)
+                    break;
+                if ("refused" in reading) {
+                    if (!requested || reading.country === requested) {
+                        countDropped(opts, `row:${reading.refused}`);
+                    }
+                    continue;
+                }
+                const row = reading.admitted;
+                if (requested && row.country !== requested)
+                    continue;
+                for (const key of reading.discarded) {
+                    countDropped(opts, key);
+                }
+                const key = `${row.country}\u001E${row.raw}`;
+                if (written.has(key)) {
+                    countDropped(opts, "row:duplicate-address");
+                    continue;
+                }
+                written.add(key);
+                yield row;
+                emitted++;
+            }
+        },
+    };
+}
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const cordisAdapter = createCORDISAdapter();
+//# sourceMappingURL=adapter.js.map

package/out/adapters/cordis/adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../lib/adapters/cordis/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2HG;AAEH,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,4CAA4C,CAAA;AACjG,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAA;AACtF,OAAO,EAAE,yBAAyB,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAA;AAC9F,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAA;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAA;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAwB,MAAM,uBAAuB,CAAA;AACtG,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EACN,WAAW,EAIX,YAAY,EACZ,aAAa,GACb,MAAM,QAAQ,CAAA;AAEf;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,QAAQ,CAAA;AAEzC;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,WAAW,CAAA;AAEzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,sEAAsE,CAAA;AAExG;;GAEG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,kBAAkB,CAAA;AAE5D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAsB;IACzD,gBAAgB;IAChB,MAAM;IACN,WAAW;IACX,cAAc;IACd,QAAQ;IACR,UAAU;IACV,MAAM;IACN,SAAS;CACT,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAqC;IACvE,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;CACR,CAAA;AAED,MAAM,UAAU,GAAwB,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAA;AAE3E;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B;;OAEG;IACH,aAAa,EAAE,iBAAiB;IAChC;;OAEG;IACH,eAAe,EAAE,mBAAmB;IACpC;;OAEG;IACH,iBAAiB,EAAE,qBAAqB;IACxC;;OAEG;IACH,kBAAkB,EAAE,sBAAsB;IAC1C;;OAEG;IACH,cAAc,EAAE,kBAAkB;IAClC;;OAEG;IACH,cAAc,EAAE,iBAAiB;IACjC;;OAEG;IACH,oBAAoB,EAAE,wBAAwB;IAC9C;;OAEG;IACH,sBAAsB,EAAE,2BAA2B;IACnD;;OAEG;IACH,gBAAgB,EAAE,oBAAoB;IACtC;;OAEG;IACH,YAAY,EAAE,cAAc;CACnB,CAAA;AAyCV;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IAC1C,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAChD,uBAAuB,CAAC,KAAK,EAAE,iBAAiB,CAAC;QACjD,uBAAuB,CAAC,KAAK,EAAE,cAAc,CAAC;KAC9C,CAAC,CAAA;IAEF,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAA;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IAC1B,WAAW,EAAE,cAAc;IAC3B,UAAU,EAAE,aAAa;CAChB,CAAA;AAOV,MAAM,eAAe,GAAG,IAAI,GAAG,EAA8B,CAAA;AAE7D;;;;;;;;;GASG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACpD,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAEzC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAA;IAEnC,MAAM,KAAK,GAAG,yBAAyB,CAAC,iBAAiB,CAAC,yBAAyB,EAAE,OAAO,CAAC,CAAA;IAC7F,MAAM,OAAO,GAAG,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAClF,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAA;IAC7D,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAA;IAEnE,IAAI,KAAK,GAAuB,IAAI,CAAA;IAEpC,IAAI,QAAQ,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9B,KAAK,GAAG,WAAW,CAAC,WAAW,CAAA;IAChC,CAAC;SAAM,IAAI,WAAW,IAAI,CAAC,QAAQ,EAAE,CAAC;QACrC,KAAK,GAAG,WAAW,CAAC,UAAU,CAAA;IAC/B,CAAC;IAED,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IAEnC,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,MAAM,kBAAkB,GAAwB,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAA;AAEtH;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAChD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAE5B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAA;IAE5C,OAAO,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;AAC7E,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,KAAgC;IACnD,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAE7D,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAA;AACnD,CAAC;AAED;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,0EAA0E,CAAA;AAErG;;GAEG;AACH,MAAM,SAAS,GAAG,2BAA2B,CAAA;AAE7C;;GAEG;AACH,MAAM,UAAU,GAAG,uBAAuB,CAAA;AAE1C;;GAEG;AACH,MAAM,UAAU,GAAG,0CAA0C,CAAA;AAE7D;;;;;GAKG;AACH,MAAM,uBAAuB,GAAG,CAAC,CAAA;AAEjC;;;;;GAKG;AACH,MAAM,uBAAuB,GAAG,CAAC,CAAA;AAEjC;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAY;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAA;IAE9D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,uBAAuB;QAAE,OAAO,IAAI,CAAA;IAE3E,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAA;AAClE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,SAAiB,EAAE,OAA0B;IAC5F,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC;QAAE,OAAO,aAAa,CAAC,cAAc,CAAA;IAE5G,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;IAEpC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,OAAO,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;IACrF,MAAM,UAAU,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;IACvG,mFAAmF;IACnF,MAAM,YAAY,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,uBAAuB,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,IAAI,CAAA;IAErH,IAAI,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;IAEvD,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC;QAAE,OAAO,aAAa,CAAC,iBAAiB,CAAA;IAE1F,iFAAiF;IACjF,uFAAuF;IACvF,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAEpE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAE1D,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3G,OAAO,aAAa,CAAC,kBAAkB,CAAA;QACxC,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,qBAAqB,GAAG,uCAAuC,CAAA;AAErE;;GAEG;AACH,MAAM,eAAe,GAAG,2BAA2B,CAAA;AAEnD;;;GAGG;AACH,MAAM,oBAAoB,GAAG,qBAAqB,CAAA;AAElD;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CACpC,IAAY,EACZ,KAAyB;IAEzB,MAAM,OAAO,GAAG,IAAI;SAClB,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;SACzB,IAAI,EAAE,CAAA;IAER,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzB,+GAA+G;IAC/G,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,oBAAoB,EAAE,CAAA;IAEzF,IAAI,KAAK,GAA2B,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;IAEvD,IAAI,KAAK,KAAK,WAAW,CAAC,WAAW,EAAE,CAAC;QACvC,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC,CAAA;IACtE,CAAC;SAAM,IAAI,KAAK,KAAK,WAAW,CAAC,UAAU,EAAE,CAAC;QAC7C,KAAK,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAA;IACzC,CAAC;IAED,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;IAC1F,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAErE,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,sBAAsB,EAAE,CAAA;IAE3F,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,MAAM,EAAE,CAAA;AACpD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAgC;IACjE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC/C,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;IAEnD,OAAO,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;AAC9C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAgC,EAAE,OAA0B;IAC5F,MAAM,OAAO,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAEjD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAE5E,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,eAAe,EAAE,OAAO,EAAE,CAAA;IAE1F,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACpC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,CAAA;IAE/E,IAAI,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAA;IAEnD,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAExC,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,cAAc,EAAE,OAAO,EAAE,CAAA;IAExE,MAAM,SAAS,GAAa,EAAE,CAAA;IAC9B,MAAM,UAAU,GAA+B,EAAE,CAAA;IAEjD,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,UAAU,CAAC,KAAK,GAAG,IAAI,CAAA;IACxB,CAAC;IAED,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAE5C,IAAI,UAAU,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,qBAAqB,CAAC,UAAU,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAA;QAE/E,IAAI,KAAK,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,CAAA;QAE3E,IAAI,KAAK,EAAE,YAAY,EAAE,CAAC;YACzB,UAAU,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAA;QAC7C,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACX,UAAU,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAA;QACjC,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAE5C,IAAI,QAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,SAAS,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;IAC9C,CAAC;SAAM,IAAI,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,SAAS,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA;IAC5C,CAAC;SAAM,IAAI,QAAQ,EAAE,CAAC;QACrB,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC/B,CAAC;IAED,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,gBAAgB,EAAE,OAAO,EAAE,CAAA;IAE3G,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAE9B,wFAAwF;IACxF,uFAAuF;IACvF,2FAA2F;IAC3F,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAEpH,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,YAAY,EAAE,OAAO,EAAE,CAAA;IAEtE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAA;IAE7C,wFAAwF;IACxF,6FAA6F;IAC7F,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,aAAa,CAAC,gBAAgB,EAAE,OAAO,EAAE,CAAA;IAErG,6FAA6F;IAC7F,yFAAyF;IACzF,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACrC,SAAS,CAAC,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO;QACN,QAAQ,EAAE;YACT,GAAG;YACH,UAAU,EAAE,OAAO;YACnB,OAAO;YACP,MAAM,EAAE,iBAAiB;YACzB,SAAS,EAAE,cAAc,CAAC,iBAAiB,EAAE,OAAO,CAAC;YACrD,cAAc,EAAE,EAAE;YAClB,OAAO,EAAE,cAAc;SACvB;QACD,SAAS;KACT,CAAA;AACF,CAAC;AAUD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,iBAAiB,CACvC,SAA0B,EAC1B,OAA0B,EAC1B,OAAO,GAAuD,EAAE;IAEhE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,MAAM,MAAM,IAAI,MAAM,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;QACrD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAEnC,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CACvC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EACpF,EAAE,aAAa,EAAE,KAAK,EAAE,eAAe,EAAE,GAAG,EAAE,CAC9C,CAAA;QAED,IAAI,KAAK,GAAG,IAAI,CAAA;QAEhB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YACpC,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,OAAM;YAEnC,IAAI,KAAK,EAAE,CAAC;gBACX,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;gBACxC,KAAK,GAAG,KAAK,CAAA;YACd,CAAC;YAED,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC;iBACpF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;iBAC7D,IAAI,CAAC,QAAQ,CAAC,CAAA;YAEhB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO,CAAC,WAAW,EAAE,EAAE,CAAA;gBAEvB,SAAQ;YACT,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEb,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAA;QAC7D,CAAC;IACF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAgC,EAAE,IAAqB;IACnF,MAAM,OAAO,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAA;IAE/E,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CACd,GAAG,iBAAiB,aAAa,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,kBAAkB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxG,CAAA;IACF,CAAC;AACF,CAAC;AAOD;;;;;;GAMG;AACH,KAAK,UAAU,aAAa,CAAC,SAA0B;IACtD,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAA;IAE9C,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IAE/D,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAElG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,GAAG,iBAAiB,aAAa,IAAI,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAA;IAC1F,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;AAClE,CAAC;AAED,MAAM,UAAU,mBAAmB;IAClC,OAAO;QACN,EAAE,EAAE,iBAAiB;QACrB,cAAc,EAAE,cAAc;QAC9B,WAAW,EAAE,WAAW,CAAC,gBAAgB;QACzC,QAAQ,EAAE,cAAc,CAAC,kBAAkB;QAC3C,OAAO,EAAE,aAAa,CAAC,QAAQ;QAC/B,WAAW,EACV,gIAAgI;QAEjI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;YAC9E,MAAM,OAAO,GAAG,MAAM,qBAAqB,EAAE,CAAA;YAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAA;YAEjC,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE;gBAC3D,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,WAAW,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,sBAAsB,CAAC;aAC7D,CAAC,CAAA;YAEF,IAAI,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC1C,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAK;gBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;oBAC1B,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;wBACjD,YAAY,CAAC,IAAI,EAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;oBAC7C,CAAC;oBAED,SAAQ;gBACT,CAAC;gBAED,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAA;gBAE5B,IAAI,SAAS,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;oBAAE,SAAQ;gBAEpD,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;oBACrC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;gBACxB,CAAC;gBAED,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,OAAO,SAAS,GAAG,CAAC,GAAG,EAAE,CAAA;gBAE5C,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACtB,YAAY,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAA;oBAE3C,SAAQ;gBACT,CAAC;gBAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBAEhB,MAAM,GAAG,CAAA;gBAET,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,mBAAmB,EAAE,CAAA"}

package/out/adapters/dedup-key.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The key that decides two rows describe the same address.
+ */
+import type { CanonicalRow } from "#types";
+/**
+ * Canonical dedup key for a row.
+ *
+ * Two rows that share this key are duplicates and the first one wins.
+ * The key combines `country` with the sorted `components` dictionary and the row's
+ * `raw` text, lower-cased with whitespace collapsed.
+ *
+ * License and provenance fields stay out of the key, so two adapters emitting
+ * the same address produce the same key.
+ * That alone does not deduplicate the corpus across adapters: `runAdapter` holds its key store as a local
+ * and discards it when the adapter returns, so each adapter is deduplicated only against itself.
+ *
+ * A cross-adapter pass would hold every adapter's keys at once and would have to choose
+ * which adapter's copy of an address survives.
+ * That choice sets the row's `source`, `license` and `register`, which makes it a
+ * corpus-level decision the runner does not take today.
+ *
+ * An augmented row is never deduplicated against the row it was fanned from, because
+ * `recipe.recipe` joins the key when present and each augmentation variant therefore survives.
+ */
+export declare function canonicalDedupKey(row: CanonicalRow): string;
+//# sourceMappingURL=dedup-key.d.ts.map