@mailwoman/corpus 9.4.0 → 10.1.0

package/out/ro/adapters/retea-scolara/adapter.js

@@ -0,0 +1,209 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `ro-retea-scolara`: Romania's school network (`Rețea școlară`), the Ministry of Education's list of every
+ * education unit for a school year, read for each unit's postal address.
+ *
+ * Input is the publisher's own workbook, one sheet named `Export`, as data.gov.ro serves it for the
+ * dataset `69392a50-5750-4a54-90c1-d461de44da6d`. Three rows precede the header. The adapter finds the
+ * header by its `Strada` cell rather than by position, because the preamble is a generation timestamp
+ * whose length the publisher does not fix.
+ *
+ * ## Which fields make the address
+ *
+ * The sheet splits the address into `Strada`, `Numar`, `Cod postal` and `Localitate unitate`, so the
+ * adapter splits no street line. Measured over the 2025–2026 edition's 18,022 units:
+ *
+ * - `Strada` is the street name as the school writes it, usually without the street-type word:
+ *   `REPUBLICII`, `PRINCIPALA`, `Piața 1848`. 198 values begin with `STR.` and 247 with `CALEA`.
+ *   The value is copied unchanged, because a street type the publisher did not write is not evidence.
+ * - `Numar` holds the house number. `FN` (`fără număr`, "without number") and `-` state that the premise
+ *   has none, on 601 and 180 units. Both are read as absent. A range such as `12-14` is a house number.
+ * - `Cod postal` holds a six-digit Romanian postcode on 17,484 units. A value of any other length is
+ *   left out of the row rather than refusing the row, because the address is complete without it.
+ * - `Judet PJ` holds the county as a two-letter code. The row carries no region, because libaddressinput's
+ *   Romanian layout places the county between postcode and locality, and this reader has no source for
+ *   the county name in the form an address prints it.
+ *
+ * `Denumire lunga unitate`, the unit's full name, is the venue.
+ *
+ * ## Organizations only
+ *
+ * Every unit is a school, kindergarten, children's club, county inspectorate or similar public body, and
+ * the sheet carries no natural person's address. The venue is the institution's name.
+ *
+ * ## Identity and license
+ *
+ * The row id is content-addressed over the aligned components. A school and its kindergarten often share
+ * one address under two names, so they are two rows.
+ *
+ * data.gov.ro publishes the dataset under `CC-BY-4.0`, the license its CKAN record names for the
+ * publisher `Ministerul Educației`. CC BY 4.0 requires credit, a link to the license and a statement that
+ * changes were made, so every row records the license and {@linkcode RO_RETEA_SCOLARA_ATTRIBUTION} is the
+ * credit the model card carries.
+ */
+import { formatAddressRow } from "@mailwoman/codex/address/format";
+import { XLSXSpliterator } from "spliterator";
+import { UnsupportedCountryError } from "#adapters/errors";
+import { stableSourceID } from "#adapters/source-id";
+import { SourceRegister } from "#registers";
+import { AddressRole, countDropped, SurfaceOrigin, } from "#types";
+/**
+ * Registry id for this adapter, stamped into every row it emits.
+ */
+export const RO_RETEA_SCOLARA_ADAPTER_ID = "ro-retea-scolara";
+/**
+ * The one jurisdiction this adapter emits.
+ */
+export const RO_RETEA_SCOLARA_COUNTRIES = ["RO"];
+/**
+ * The license data.gov.ro names for the dataset.
+ */
+export const RO_RETEA_SCOLARA_LICENSE = "CC-BY-4.0";
+/**
+ * The credit CC BY 4.0 obliges.
+ */
+export const RO_RETEA_SCOLARA_ATTRIBUTION = "Ministerul Educației, Rețea școlară, via data.gov.ro";
+/**
+ * Why a unit did not become a row.
+ */
+export const ReteaScolaraRefusal = {
+    /**
+     * The unit states no locality.
+     */
+    LocalityAbsent: "row:locality-absent",
+    /**
+     * The unit states neither a street nor a house number, so the locality alone would be a place name.
+     */
+    PremiseAbsent: "row:premise-absent",
+    /**
+     * The components do not render as a Romanian address line.
+     */
+    Unrenderable: "row:unrenderable",
+    /**
+     * Another unit already produced this exact row.
+     */
+    Duplicate: "row:duplicate",
+};
+const NO_NUMBER = new Set(["FN", "F.N.", "F/N", "-", "0", "SN", "S/N"]);
+const NUMBER_MARKER = /^(?:NR|NUM|NUMAR)\.?\s*/iu;
+const POSTCODE = /^\d{6}$/u;
+function cell(unit, header) {
+    return String(unit[header] ?? "")
+        .replaceAll(/\s+/gu, " ")
+        .trim();
+}
+/**
+ * The house number a `Numar` cell states, or an empty string where it states none.
+ */
+export function readHouseNumber(value) {
+    const stripped = value.replace(NUMBER_MARKER, "").trim();
+    return NO_NUMBER.has(stripped.toUpperCase()) ? "" : stripped;
+}
+/**
+ * Reads one unit into a row or the reason it is refused.
+ */
+export function readReteaScolaraUnit(unit) {
+    const locality = cell(unit, "Localitate unitate");
+    if (!locality)
+        return { refused: ReteaScolaraRefusal.LocalityAbsent };
+    const street = cell(unit, "Strada");
+    const houseNumber = readHouseNumber(cell(unit, "Numar"));
+    if (!street && !houseNumber)
+        return { refused: ReteaScolaraRefusal.PremiseAbsent };
+    const postcode = cell(unit, "Cod postal");
+    const venue = cell(unit, "Denumire lunga unitate") || cell(unit, "Denumire scurta unitate");
+    const components = {};
+    if (venue) {
+        components.venue = venue;
+    }
+    if (street) {
+        components.street = street;
+    }
+    if (houseNumber) {
+        components.house_number = houseNumber;
+    }
+    if (POSTCODE.test(postcode)) {
+        components.postcode = postcode;
+    }
+    components.locality = locality;
+    const rendered = formatAddressRow(components, "RO", { singleLine: true });
+    if (!rendered)
+        return { refused: ReteaScolaraRefusal.Unrenderable };
+    return {
+        admitted: {
+            raw: rendered.raw,
+            components: rendered.components,
+            country: "RO",
+            locale: "ro-RO",
+            source: RO_RETEA_SCOLARA_ADAPTER_ID,
+            source_id: stableSourceID(RO_RETEA_SCOLARA_ADAPTER_ID, rendered.components),
+            corpus_version: "",
+            license: RO_RETEA_SCOLARA_LICENSE,
+        },
+    };
+}
+/**
+ * The sheet's units keyed by its header, skipping the preamble above the row that holds `Strada`.
+ */
+export async function* readReteaScolaraUnits(inputPath) {
+    let header = null;
+    for await (const row of XLSXSpliterator.fromAsync(inputPath.toString(), {
+        mode: "array",
+        header: false,
+    })) {
+        if (!header) {
+            if (row.some((value) => value === "Strada")) {
+                header = row.map((value) => String(value ?? "").trim());
+            }
+            continue;
+        }
+        if (!row.some((value) => value !== null && value !== ""))
+            continue;
+        yield Object.fromEntries(header.map((name, index) => [name, row[index]]));
+    }
+    if (!header)
+        throw new Error(`${RO_RETEA_SCOLARA_ADAPTER_ID}: no header row with a "Strada" cell in ${inputPath}`);
+}
+export function createReteaScolaraAdapter() {
+    return {
+        id: RO_RETEA_SCOLARA_ADAPTER_ID,
+        defaultLicense: RO_RETEA_SCOLARA_LICENSE,
+        addressRole: AddressRole.Facility,
+        register: SourceRegister.ReteaScolara,
+        surface: SurfaceOrigin.Rendered,
+        description: "Romania's school network (Rețea școlară): the postal address of every education unit.",
+        async *rows(opts) {
+            if (opts.country && !RO_RETEA_SCOLARA_COUNTRIES.includes(opts.country)) {
+                throw new UnsupportedCountryError(RO_RETEA_SCOLARA_ADAPTER_ID, RO_RETEA_SCOLARA_COUNTRIES, opts.country);
+            }
+            const seen = new Set();
+            let emitted = 0;
+            for await (const unit of readReteaScolaraUnits(opts.inputPath)) {
+                if (opts.signal?.aborted)
+                    break;
+                if (opts.limit !== undefined && emitted >= opts.limit)
+                    break;
+                const reading = readReteaScolaraUnit(unit);
+                if (!("admitted" in reading)) {
+                    countDropped(opts, reading.refused);
+                    continue;
+                }
+                if (seen.has(reading.admitted.source_id)) {
+                    countDropped(opts, ReteaScolaraRefusal.Duplicate);
+                    continue;
+                }
+                seen.add(reading.admitted.source_id);
+                yield reading.admitted;
+                emitted++;
+            }
+        },
+    };
+}
+/**
+ * The registered `ro-retea-scolara` adapter instance, built by {@linkcode createReteaScolaraAdapter}.
+ */
+export const reteaScolaraAdapter = createReteaScolaraAdapter();
+//# sourceMappingURL=adapter.js.map

package/out/ro/adapters/retea-scolara/adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../lib/ro/adapters/retea-scolara/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAA;AAElE,OAAO,EAAE,eAAe,EAAsB,MAAM,aAAa,CAAA;AAEjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EACN,WAAW,EAIX,YAAY,EACZ,aAAa,GACb,MAAM,QAAQ,CAAA;AAEf;;GAEG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,kBAAkB,CAAA;AAE7D;;GAEG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAsB,CAAC,IAAI,CAAC,CAAA;AAEnE;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,WAAW,CAAA;AAEnD;;GAEG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,sDAAsD,CAAA;AAElG;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAClC;;OAEG;IACH,cAAc,EAAE,qBAAqB;IACrC;;OAEG;IACH,aAAa,EAAE,oBAAoB;IACnC;;OAEG;IACH,YAAY,EAAE,kBAAkB;IAChC;;OAEG;IACH,SAAS,EAAE,eAAe;CACjB,CAAA;AAWV,MAAM,SAAS,GAAwB,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAA;AAE5F,MAAM,aAAa,GAAG,2BAA2B,CAAA;AAEjD,MAAM,QAAQ,GAAG,UAAU,CAAA;AAE3B,SAAS,IAAI,CAAC,IAAsB,EAAE,MAAc;IACnD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;SAC/B,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,IAAI,EAAE,CAAA;AACT,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC5C,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAExD,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAA;AAC7D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAsB;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAA;IAEjD,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,CAAC,cAAc,EAAE,CAAA;IAErE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IACnC,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;IAExD,IAAI,CAAC,MAAM,IAAI,CAAC,WAAW;QAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,CAAC,aAAa,EAAE,CAAA;IAElF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAA;IAE3F,MAAM,UAAU,GAA+B,EAAE,CAAA;IAEjD,IAAI,KAAK,EAAE,CAAC;QACX,UAAU,CAAC,KAAK,GAAG,KAAK,CAAA;IACzB,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACZ,UAAU,CAAC,MAAM,GAAG,MAAM,CAAA;IAC3B,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QACjB,UAAU,CAAC,YAAY,GAAG,WAAW,CAAA;IACtC,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC/B,CAAC;IAED,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAE9B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;IAEzE,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,CAAC,YAAY,EAAE,CAAA;IAEnE,OAAO;QACN,QAAQ,EAAE;YACT,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,OAAO;YACf,MAAM,EAAE,2BAA2B;YACnC,SAAS,EAAE,cAAc,CAAC,2BAA2B,EAAE,QAAQ,CAAC,UAAU,CAAC;YAC3E,cAAc,EAAE,EAAE;YAClB,OAAO,EAAE,wBAAwB;SACjC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,qBAAqB,CAAC,SAA0B;IACtE,IAAI,MAAM,GAAoB,IAAI,CAAA;IAElC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAAkB,SAAS,CAAC,QAAQ,EAAE,EAAE;QACxF,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,KAAK;KACb,CAAC,EAAE,CAAC;QACJ,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,QAAQ,CAAC,EAAE,CAAC;gBAC7C,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YACxD,CAAC;YAED,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YAAE,SAAQ;QAElE,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1E,CAAC;IAED,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,2BAA2B,2CAA2C,SAAS,EAAE,CAAC,CAAA;AACnH,CAAC;AAED,MAAM,UAAU,yBAAyB;IACxC,OAAO;QACN,EAAE,EAAE,2BAA2B;QAC/B,cAAc,EAAE,wBAAwB;QACxC,WAAW,EAAE,WAAW,CAAC,QAAQ;QACjC,QAAQ,EAAE,cAAc,CAAC,YAAY;QACrC,OAAO,EAAE,aAAa,CAAC,QAAQ;QAC/B,WAAW,EAAE,uFAAuF;QAEpG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACxE,MAAM,IAAI,uBAAuB,CAAC,2BAA2B,EAAE,0BAA0B,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YACzG,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;YAC9B,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChE,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAK;gBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAA;gBAE1C,IAAI,CAAC,CAAC,UAAU,IAAI,OAAO,CAAC,EAAE,CAAC;oBAC9B,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;oBAEnC,SAAQ;gBACT,CAAC;gBAED,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC1C,YAAY,CAAC,IAAI,EAAE,mBAAmB,CAAC,SAAS,CAAC,CAAA;oBAEjD,SAAQ;gBACT,CAAC;gBAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;gBACpC,MAAM,OAAO,CAAC,QAAQ,CAAA;gBAEtB,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,yBAAyB,EAAE,CAAA"}

package/out/ro/tools/fetch/retea-scolara.d.ts

@@ -0,0 +1,69 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Downloads Romania's school network workbook for the `ro-retea-scolara` adapter.
+ *
+ * The Ministry of Education publishes a new workbook each school year under one data.gov.ro dataset,
+ * and each upload has a new resource URL. The fetcher therefore asks the portal's CKAN API for the
+ * dataset's resources and downloads the newest XLSX, rather than holding a URL that the next upload
+ * retires. The manifest records the resource URL, the CKAN `last_modified` and the license id the
+ * portal states, so a corpus row traces to one upload.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { type PathBuilderLike } from "path-ts";
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download";
+/**
+ * The data.gov.ro dataset that holds every school year's workbook.
+ */
+export declare const RO_RETEA_SCOLARA_DATASET_ID = "69392a50-5750-4a54-90c1-d461de44da6d";
+/**
+ * The CKAN call that lists the dataset's resources and its license.
+ */
+export declare const RO_RETEA_SCOLARA_PACKAGE_URL = "https://data.gov.ro/api/3/action/package_show?id=69392a50-5750-4a54-90c1-d461de44da6d";
+/**
+ * The file name the adapter reads, whatever the upload's own name.
+ */
+export declare const RO_RETEA_SCOLARA_FILENAME = "retea-scolara.xlsx";
+/**
+ * One CKAN resource, as `package_show` lists it.
+ */
+export interface CKANResource {
+    url: string;
+    format?: string | null;
+    name?: string | null;
+    last_modified?: string | null;
+    created?: string | null;
+}
+export interface ReteaScolaraManifest extends SourceManifest {
+    resource_name: string | null;
+    last_modified: string | null;
+    license_id: string | null;
+}
+/**
+ * The newest XLSX resource, by its `last_modified` or else its `created` date.
+ *
+ * @throws When the dataset lists no XLSX resource.
+ */
+export declare function newestXLSXResource(resources: readonly CKANResource[]): CKANResource;
+export interface DownloadReteaScolaraOptions {
+    outputDir: PathBuilderLike;
+    force?: boolean;
+    retries?: number;
+    retryDelayMs?: number;
+    signal?: AbortSignal;
+    report?: (line: string) => void;
+}
+export declare function downloadReteaScolara(client: Pick<APIClient, "fetch">, options: DownloadReteaScolaraOptions): Promise<FetchSummary>;
+/**
+ * The workbook path the adapter reads under a fetch root.
+ */
+export declare function reteaScolaraInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike;
+export interface FetchReteaScolaraOptions extends BaseFetchOptions {
+    force?: boolean;
+    retries?: number;
+    signal?: AbortSignal;
+}
+export declare function fetchReteaScolara(options: FetchReteaScolaraOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=retea-scolara.d.ts.map

package/out/ro/tools/fetch/retea-scolara.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retea-scolara.d.ts","sourceRoot":"","sources":["../../../../lib/ro/tools/fetch/retea-scolara.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAK/C,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAG3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAG3F;;GAEG;AACH,eAAO,MAAM,2BAA2B,yCAAyC,CAAA;AAEjF;;GAEG;AACH,eAAO,MAAM,4BAA4B,0FAAoF,CAAA;AAE7H;;GAEG;AACH,eAAO,MAAM,yBAAyB,uBAAuB,CAAA;AAI7D;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAOD,MAAM,WAAW,oBAAqB,SAAQ,cAAc;IAC3D,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;CACzB;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,SAAS,YAAY,EAAE,GAAG,YAAY,CASnF;AAED,MAAM,WAAW,2BAA2B;IAC3C,SAAS,EAAE,eAAe,CAAA;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED,wBAAsB,oBAAoB,CACzC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,2BAA2B,GAClC,OAAO,CAAC,YAAY,CAAC,CAgEvB;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC,GAAG,eAAe,CAE3F;AAED,MAAM,WAAW,wBAAyB,SAAQ,gBAAgB;IACjE,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED,wBAAsB,iBAAiB,CACtC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAYvB"}

package/out/ro/tools/fetch/retea-scolara.js

@@ -0,0 +1,117 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Downloads Romania's school network workbook for the `ro-retea-scolara` adapter.
+ *
+ * The Ministry of Education publishes a new workbook each school year under one data.gov.ro dataset,
+ * and each upload has a new resource URL. The fetcher therefore asks the portal's CKAN API for the
+ * dataset's resources and downloads the newest XLSX, rather than holding a URL that the next upload
+ * retires. The manifest records the resource URL, the CKAN `last_modified` and the license id the
+ * portal states, so a corpus row traces to one upload.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { tryStat } from "@mailwoman/core/fs/readers";
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { PathBuilder } from "path-ts";
+import { RO_RETEA_SCOLARA_ADAPTER_ID, RO_RETEA_SCOLARA_LICENSE } from "#ro/adapters/retea-scolara/adapter";
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download";
+/**
+ * The data.gov.ro dataset that holds every school year's workbook.
+ */
+export const RO_RETEA_SCOLARA_DATASET_ID = "69392a50-5750-4a54-90c1-d461de44da6d";
+/**
+ * The CKAN call that lists the dataset's resources and its license.
+ */
+export const RO_RETEA_SCOLARA_PACKAGE_URL = `https://data.gov.ro/api/3/action/package_show?id=${RO_RETEA_SCOLARA_DATASET_ID}`;
+/**
+ * The file name the adapter reads, whatever the upload's own name.
+ */
+export const RO_RETEA_SCOLARA_FILENAME = "retea-scolara.xlsx";
+const SLUG = RO_RETEA_SCOLARA_ADAPTER_ID;
+/**
+ * The newest XLSX resource, by its `last_modified` or else its `created` date.
+ *
+ * @throws When the dataset lists no XLSX resource.
+ */
+export function newestXLSXResource(resources) {
+    const dated = resources
+        .filter((resource) => (resource.format ?? "").toUpperCase() === "XLSX")
+        .map((resource) => ({ resource, at: resource.last_modified ?? resource.created ?? "" }))
+        .toSorted((a, b) => b.at.localeCompare(a.at));
+    if (!dated.length)
+        throw new Error(`${SLUG}: dataset ${RO_RETEA_SCOLARA_DATASET_ID} lists no XLSX resource`);
+    return dated[0].resource;
+}
+export async function downloadReteaScolara(client, options) {
+    const { report } = options;
+    const destDir = PathBuilder.from(options.outputDir);
+    const workbookPath = destDir(RO_RETEA_SCOLARA_FILENAME);
+    const manifestPath = destDir("MANIFEST.json");
+    await makeDirectories(destDir);
+    report?.(`=== ${SLUG}`);
+    const response = await client.fetch({
+        url: RO_RETEA_SCOLARA_PACKAGE_URL,
+        signal: options.signal,
+    });
+    const pkg = response.data.result;
+    const resource = newestXLSXResource(pkg.resources ?? []);
+    const lastModified = resource.last_modified ?? resource.created ?? null;
+    if (pkg.license_id !== RO_RETEA_SCOLARA_LICENSE) {
+        report?.(`  the portal now states license ${pkg.license_id ?? "none"}, not ${RO_RETEA_SCOLARA_LICENSE}`);
+    }
+    const recorded = await readManifest(manifestPath);
+    const stat = await tryStat(workbookPath);
+    if (!options.force &&
+        recorded &&
+        stat &&
+        recorded.source_url === resource.url &&
+        recorded.last_modified === lastModified &&
+        recorded.bytes === stat.size) {
+        report?.(`  present, and the portal lists the same upload`);
+        return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
+    }
+    const { bytes } = await downloadToFile({
+        url: resource.url,
+        dest: workbookPath,
+        retries: options.retries,
+        retryDelayMs: options.retryDelayMs,
+        report,
+    });
+    const sha256 = await sha256File(workbookPath);
+    report?.(`  ✓ ${resource.name ?? resource.url} ${ByteFormatter.formatIEC(bytes)} sha256=${sha256}`);
+    const manifest = {
+        source_url: resource.url,
+        filename: RO_RETEA_SCOLARA_FILENAME,
+        bytes,
+        sha256,
+        downloaded_at: new Date().toISOString(),
+        resource_name: resource.name ?? null,
+        last_modified: lastModified,
+        license_id: pkg.license_id ?? null,
+    };
+    await writeManifest(manifestPath, manifest);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+/**
+ * The workbook path the adapter reads under a fetch root.
+ */
+export function reteaScolaraInputPath(outRoot) {
+    return outRoot(SLUG, RO_RETEA_SCOLARA_FILENAME);
+}
+export async function fetchReteaScolara(options, report) {
+    await using client = new APIClient({ displayName: SLUG, retry: true });
+    // Awaited rather than returned, so the client is disposed after the download finishes.
+    return await downloadReteaScolara(client, {
+        outputDir: options.outRoot(SLUG),
+        force: options.force,
+        retries: options.retries,
+        retryDelayMs: options.retryDelayMs,
+        signal: options.signal,
+        report,
+    });
+}
+//# sourceMappingURL=retea-scolara.js.map

package/out/ro/tools/fetch/retea-scolara.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retea-scolara.js","sourceRoot":"","sources":["../../../../lib/ro/tools/fetch/retea-scolara.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D,OAAO,EAAE,2BAA2B,EAAE,wBAAwB,EAAE,MAAM,oCAAoC,CAAA;AAE1G,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEnF;;GAEG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,sCAAsC,CAAA;AAEjF;;GAEG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,oDAAoD,2BAA2B,EAAE,CAAA;AAE7H;;GAEG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,oBAAoB,CAAA;AAE7D,MAAM,IAAI,GAAG,2BAA2B,CAAA;AAwBxC;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAkC;IACpE,MAAM,KAAK,GAAG,SAAS;SACrB,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;SACtE,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;SACvF,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9C,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,aAAa,2BAA2B,yBAAyB,CAAC,CAAA;IAE5G,OAAO,KAAK,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAA;AAC1B,CAAC;AAWD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,MAAgC,EAChC,OAAoC;IAEpC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAC1B,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,YAAY,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAAA;IACvD,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IAE7C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAA0B;QAC5D,GAAG,EAAE,4BAA4B;QACjC,MAAM,EAAE,OAAO,CAAC,MAAM;KACtB,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAA;IAChC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAA;IACxD,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC,OAAO,IAAI,IAAI,CAAA;IAEvE,IAAI,GAAG,CAAC,UAAU,KAAK,wBAAwB,EAAE,CAAC;QACjD,MAAM,EAAE,CAAC,mCAAmC,GAAG,CAAC,UAAU,IAAI,MAAM,SAAS,wBAAwB,EAAE,CAAC,CAAA;IACzG,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAuB,YAAY,CAAC,CAAA;IACvE,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,CAAA;IAExC,IACC,CAAC,OAAO,CAAC,KAAK;QACd,QAAQ;QACR,IAAI;QACJ,QAAQ,CAAC,UAAU,KAAK,QAAQ,CAAC,GAAG;QACpC,QAAQ,CAAC,aAAa,KAAK,YAAY;QACvC,QAAQ,CAAC,KAAK,KAAK,IAAI,CAAC,IAAI,EAC3B,CAAC;QACF,MAAM,EAAE,CAAC,iDAAiD,CAAC,CAAA;QAE3D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAC9D,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,IAAI,EAAE,YAAY;QAClB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,CAAA;IAE7C,MAAM,EAAE,CAAC,OAAO,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,GAAG,IAAI,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,MAAM,EAAE,CAAC,CAAA;IAEnG,MAAM,QAAQ,GAAyB;QACtC,UAAU,EAAE,QAAQ,CAAC,GAAG;QACxB,QAAQ,EAAE,yBAAyB;QACnC,KAAK;QACL,MAAM;QACN,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,aAAa,EAAE,QAAQ,CAAC,IAAI,IAAI,IAAI;QACpC,aAAa,EAAE,YAAY;QAC3B,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,IAAI;KAClC,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAoC;IACzE,OAAO,OAAO,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAA;AAChD,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,OAAiC,EACjC,MAA+B;IAE/B,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtE,uFAAuF;IACvF,OAAO,MAAM,oBAAoB,CAAC,MAAM,EAAE;QACzC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;KACN,CAAC,CAAA;AACH,CAAC"}

package/out/runner.d.ts

@@ -3,91 +3,114 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Adapter runner — drives a `CorpusAdapter` to completion and writes intermediate JSONL + a
- *   per-slice manifest.
- *
- *   Output layout under `outputDir`:
- *
- *   ```
- *   <outputDir>/<adapter.id>/
- *   canonical.jsonl       # one row per line, in emission order
- *   MANIFEST.json         # adapter id, version, row count, sha256, license, started_at, ended_at
- * ```
- *
- *   The runner is responsible for everything an adapter is **not** responsible for:
- *
- *   - Stamping `corpus_version` on every row (adapters must NOT set it).
- *   - Applying `canonicalDedupKey` and skipping duplicates.
- *   - Streaming sha256 over JSONL bytes so the manifest checksum doesn't require a re-read.
- *   - Honoring backpressure on the output write stream.
- *   - Counting + emitting periodic progress to an optional callback.
- *   - Honoring `signal` (delegates to adapter's iteration boundary).
- *
- *   The runner does NOT perform alignment, tokenization, synthesis, or slicing into Parquet. Those
- *   steps run later, consuming the JSONL slices this writes.
+ *   Adapter runner — drives a `CorpusAdapter` to completion and writes `canonical.jsonl` plus a
+ *   `MANIFEST.json` per adapter. It does not align, tokenize, synthesize, or write Parquet.
  */
 import { type WriteStream } from "@mailwoman/core/fs/streams";
 import { type PathBuilderLike } from "path-ts";
-import { type AdapterRegistry } from "#adapters/utils";
+import type { AdapterRegistry } from "#adapters/registry";
 import type { AdapterOptions, CorpusAdapter } from "#types";
 /**
  * Snapshot of the runner's state, emitted on every progress tick.
  */
 export interface RunnerProgress {
-    /**
-     * Adapter being driven.
-     */
     adapterID: string;
     /**
-     * Total rows the adapter has yielded (before dedup).
+     * Total rows the adapter has yielded, before dedup.
      */
     yielded: number;
-    /**
-     * Rows actually written to JSONL (after dedup).
-     */
     written: number;
-    /**
-     * Bytes written to JSONL so far.
-     */
     bytes: number;
-    /**
-     * Wall-clock milliseconds since the run started.
-     */
     elapsed_ms: number;
 }
 /**
- * Per-invocation options for `runAdapter`.
+ * Base-2 logarithm of the fingerprint table's slot count.
+ *
+ * 2^27 slots hold 93,952,409 keys before the load limit.
+ * The largest measured adapter has 57,570,829 distinct keys.
+ * The table occupies 2.0 GiB outside the V8 heap.
  */
-export interface RunAdapterOptions {
+export declare const DEFAULT_DEDUP_SLOTS_LOG2 = 27;
+/**
+ * How many distinct dedup keys a `Set`-backed run holds before it stops adding new ones.
+ *
+ * A V8 `Set` refuses a 16,777,216th entry outright, so this cap was a real bound rather than a preference.
+ * It applies only to a run that opts out of the fingerprint table
+ * through {@linkcode RunAdapterOptions.dedupMaxSize}.
+ */
+export declare const DEFAULT_DEDUP_MAX_SIZE = 10000000;
+/**
+ * Where a run held its dedup keys.
+ */
+export declare const DedupStore: {
     /**
-     * Adapter to drive.
+     * 128-bit fingerprints in a flat `Uint32Array`, bounded by the array the run sized.
      */
-    adapter: CorpusAdapter;
+    readonly FingerprintTable: "fingerprint-table";
     /**
-     * Options handed to the adapter (input path, country filter, limit, signal).
+     * A V8 `Set` of key strings, capped because V8 refuses a 16,777,216th entry.
      */
+    readonly CappedSet: "capped-set";
+};
+export type DedupStore = (typeof DedupStore)[keyof typeof DedupStore];
+export interface RunAdapterOptions {
+    adapter: CorpusAdapter;
     adapterOptions: AdapterOptions;
     /**
-     * Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it.
+     * Hold dedup keys in a V8 `Set` capped at this many, rather than in a fingerprint table.
+     *
+     * The cap is what the fingerprint table replaces.
+     * A run that sets this reproduces the pre-2026-09-28 behavior.
+     *
+     * Past the cap, the runner still drops a duplicate of a key it holds
+     * and writes a duplicate of a key first seen after the cap.
+     *
+     * Over `v0.7.0-de-holdout` that wrote 1,490,992 duplicate rows across three adapters.
+     * A test sets it low to reach exhaustion in a few rows.
+     *
+     * After reaching the cap, the runner still drops duplicates for keys already in the set.
+     * It writes duplicates for keys first seen after the cap.
+     */
+    dedupMaxSize?: number;
+    /**
+     * Base-2 logarithm of the fingerprint table's slot count.
+     *
+     * Defaults to {@linkcode DEFAULT_DEDUP_SLOTS_LOG2}.
+     * A test sets it low to keep the allocation small.
+     */
+    dedupSlotsLog2?: number;
+    /**
+     * Root output directory.
+     * The runner creates `<outputDir>/<adapter.id>/` under it.
      */
     outputDir: PathBuilderLike;
     /**
-     * Corpus version stamped onto every row. Locked together with the tokenizer version.
+     * Corpus version stamped onto every row, locked together with the tokenizer version.
      */
     corpusVersion: string;
     /**
-     * Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once at the end of the
-     * run. Errors thrown from this callback abort the run.
+     * The `source` id stamped on every emitted row, when it differs from the adapter's own id.
+     *
+     * A source id is a wire identifier keyed by `source_weights`, so re-using a name a built
+     * corpus already contains makes this run's rows indistinguishable from that corpus's.
+     * When absent, rows use `adapter.id`.
+     */
+    sourceName?: string;
+    /**
+     * Invoked every `progressEvery` rows yielded and once at the end.
+     * A thrown error aborts the run.
      */
     onProgress?: (snapshot: RunnerProgress) => void;
     /**
-     * Yielded-row interval at which `onProgress` fires. Defaults to 1000. The terminal tick is always emitted regardless
-     * of this value.
+     * Yielded-row interval at which `onProgress` fires.
+     *
+     * Defaults to 1000 rows per callback.
+     * The runner always emits a terminal update.
      */
     progressEvery?: number;
 }
 /**
- * Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk.
+ * Return value of `runAdapter`, the same shape as the manifest written to disk.
  */
 export interface AdapterRunManifest {
     adapter_id: string;
@@ -97,6 +120,38 @@ export interface AdapterRunManifest {
     yielded: number;
     written: number;
     deduped: number;
+    /**
+     * Input records the adapter refused or trimmed before yielding, keyed by reason.
+     *
+     * See {@link AdapterOptions.dropped} for the key shapes.
+     * An empty object means the adapter does not count its drops, so the number it dropped is unmeasured.
+     */
+    dropped: Record<string, number>;
+    /**
+     * The `yielded` count at which the dedup set stopped growing, or `null` where it never did.
+     *
+     * A run reporting a number here deduplicated its rows completely up to that point.
+     * After that point, it still drops duplicates for keys already held.
+     *
+     * It writes duplicates for keys first seen after the cap.
+     * `deduped` by itself cannot identify which, so a consumer comparing two builds'
+     * duplicate counts needs this beside it.
+     */
+    dedup_exhausted_at_yielded: number | null;
+    /**
+     * How the run held its dedup keys: `fingerprint-table` or `capped-set`.
+     *
+     * The two reach different row counts on the same input, so a consumer comparing
+     * two builds needs to know which each used.
+     * `capped-set` writes a duplicate of any key first seen after its cap.
+     */
+    dedup_store: DedupStore;
+    /**
+     * Distinct dedup keys the run held at the end.
+     *
+     * Under `fingerprint-table` this is a fingerprint count, so two keys sharing all 128 bits count once.
+     */
+    dedup_keys: number;
     bytes: number;
     sha256: string;
     jsonl_path: string;
@@ -105,26 +160,31 @@ export interface AdapterRunManifest {
     elapsed_ms: number;
 }
 /**
- * Drive a single adapter to completion.
- *
- * Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under `outputDir/<adapter.id>/`.
- * Throws if the output directory cannot be created, if a row arrives with a missing required field, or if the abort
- * signal fires.
+ * Drive a single adapter to completion, writing its jsonl and manifest under `outputDir/<adapter.id>/`.
  */
 export declare function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest>;
 /**
- * Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the registry before
- * calling if partial-failure is desired).
- *
- * Returns the manifests in registry insertion order.
+ * Drive every adapter in a registry sequentially, stopping on the first failure.
  */
 export declare function runAllAdapters(registry: AdapterRegistry, common: Omit<RunAdapterOptions, "adapter"> & {
     adapterOptionsFor?: (a: CorpusAdapter) => AdapterOptions;
 }): Promise<AdapterRunManifest[]>;
 /**
- * Promise-ify a single event emission. Used to await `drain` / `close` on the write stream. Exported for `build.ts`,
- * whose stage streams await `close` the same way; unlike a bare two-listener race, the loser listener is detached so a
- * long-lived stream does not accumulate one orphan handler per wait.
+ * Resolve once everything written to `stream` so far has reached the file.
+ *
+ * A zero-length write acts as a barrier.
+ * Its callback runs after earlier queued writes.
+ *
+ * A caller can then record a byte offset for the bytes in the file.
+ * `drain` fires only when the buffer was full.
+ *
+ * `bytesWritten` excludes queued data.
+ * Neither value answers when all earlier writes have reached the file.
+ */
+export declare function flushStream(stream: WriteStream): Promise<void>;
+/**
+ * Promise-ify a single `drain` or `close` emission, detaching the loser listener
+ * so a long-lived stream does not accumulate an orphan handler per wait.
  */
 export declare function once(emitter: WriteStream, event: "drain" | "close"): Promise<void>;
 //# sourceMappingURL=runner.d.ts.map