@mailwoman/corpus 9.4.0 → 10.1.0

package/out/types.d.ts

@@ -3,122 +3,263 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Canonical row schemas for the corpus pipeline (per #6 / Phase 1 plan).
+ *   Row, adapter and provenance types for the corpus pipeline.
+ */
+import type { BIOLabel, ComponentTag } from "@mailwoman/codex/component";
+import type { PathBuilderLike } from "path-ts";
+/**
+ * The role that an address plays for the entity that the source describes.
+ * The values are wire values.
  *
- *   The corpus pipeline produces two row shapes:
+ * Roles have different grammars.
+ * For example, mailing addresses often use post-office boxes, so mixing roles in
+ * a corpus shifts what the premise parser learns.
+ */
+export declare const AddressRole: {
+    /**
+     * The location of the addressable object, as in a national address register.
+     */
+    readonly Premise: "premise";
+    /**
+     * The seat that an entity files with a registrar, such as a siège social.
+     */
+    readonly RegisteredOffice: "registered-office";
+    /**
+     * The address a third party records for a transacting party on its own transaction record,
+     * such as the supplier on a government contract notice.
+     *
+     * The party supplied the address to the recording body rather than to a registrar.
+     * No registrar filed it as a seat, and no address register confirmed it as a premise.
+     * A sole trader's entry can hold a home address.
+     */
+    readonly Counterparty: "counterparty";
+    /**
+     * The place where post is delivered.
+     * This can be a post-office box.
+     */
+    readonly Mailing: "mailing";
+    /**
+     * The place where a licensed person or organization practices.
+     */
+    readonly Practice: "practice";
+    /**
+     * A site that an entity operates, such as a clinic, a school or a plant.
+     */
+    readonly Facility: "facility";
+    /**
+     * The address of a property's owner, recorded against the property.
+     */
+    readonly Owner: "owner";
+    /**
+     * The place where a service is delivered or an obligation is discharged.
+     */
+    readonly Service: "service";
+    /**
+     * The address for service of legal process.
+     */
+    readonly LegalNotice: "legal-notice";
+    /**
+     * A role that this vocabulary does not cover.
+     *
+     * Add a new constant when a second uncovered role appears.
+     */
+    readonly Other: "other";
+};
+/**
+ * One of the {@link AddressRole} values.
+ */
+export type AddressRole = (typeof AddressRole)[keyof typeof AddressRole];
+/**
+ * The role of a row without an `addressRole` field.
+ *
+ * Corpora written before the field existed came from premise sources.
+ */
+export declare const DEFAULT_ADDRESS_ROLE: AddressRole;
+/**
+ * Returns the row's address role, or {@link DEFAULT_ADDRESS_ROLE} when the field is absent.
+ */
+export declare function addressRoleOf(row: Pick<CanonicalRow, "addressRole">): AddressRole;
+/**
+ * How a row's `raw` text was produced.
+ * The values are wire values.
  *
- *   1. `CanonicalRow`: an adapter's raw output. Carries a free-form `raw` string, a per-component
- *        ground-truth dict, provenance, and an optional augmentation marker. Adapters emit these.
- *   2. `LabeledRow`: alignment's output. Adds a SentencePiece token list and a parallel BIO label list,
- *        suitable for direct ingestion by the neural training loop.
+ * This is separate from {@link CanonicalRow.register}, which records where the underlying record came from.
+ * A rendering of a real register record is `Rendered` or `Composed`, and it is still a real address.
+ */
+export declare const SurfaceOrigin: {
+    /**
+     * The publisher's own string, preserved unchanged.
+     *
+     * No adapter in the tree emits this today: each one builds its line from the source's fields.
+     * The value is reserved for a source that publishes the written line itself,
+     * such as a register storing `address1` as a person typed it.
+     */
+    readonly Attested: "attested";
+    /**
+     * An adapter assembled the line from one record's fields, in that locale's canonical order.
+     *
+     * The order comes from the codex layout or the adapter's own template rather than from the
+     * publisher, so the components are the source's and the written form is this repository's.
+     * Distinct from {@link SurfaceOrigin.Composed} because an adapter renders one canonical
+     * form per record while a recipe varies the form deliberately.
+     *
+     * A source id beginning `rendered-` does not take this value.
+     * `SourceOperation.Rendered` in `#recipes/sources` names a recipe operation with the same word,
+     * and those recipes record {@link SurfaceOrigin.Composed} because they vary the form.
+     *
+     * Read the row's `surface` rather than its source name to learn which of the two produced it.
+     */
+    readonly Rendered: "rendered";
+    /**
+     * A recipe assembled the text from the published fields of one real record.
+     *
+     * The recipe chooses the order, punctuation and casing.
+     * It varies them to teach a surface.
+     */
+    readonly Composed: "composed";
+    /**
+     * A synthetic row, such as a post-office box or an intersection, that corresponds to no published record.
+     */
+    readonly Invented: "invented";
+};
+/**
+ * One of the {@link SurfaceOrigin} values.
+ */
+export type SurfaceOrigin = (typeof SurfaceOrigin)[keyof typeof SurfaceOrigin];
+/**
+ * Returns the `surface` that a JSONL row declares.
  *
- *   `CorpusAdapter` is the contract every data source implements; `AdapterOptions` is the
- *   per-invocation knob set (input path, optional country filter, row cap, abort signal).
+ * @throws When the row lacks a valid `surface`.
+ * Older files must be regenerated instead of read with a default.
  */
-import type { BIOLabel, ComponentTag } from "@mailwoman/core/types";
+export declare function requireSurface(raw: Record<string, unknown>, producer: string): SurfaceOrigin;
 /**
- * Provenance + augmentation metadata that travels with every corpus row.
+ * Provenance fields present on every corpus row.
  *
- * `synth` is `undefined` for natural (un-augmented) rows; present only when a row was produced by the synthesis
- * pipeline (see `synthesize.ts`).
+ * Two related facts are recorded per source rather than per row.
+ * `createIneligibilityReader` in `build/eligibility.ts` joins `source` against the
+ * address-source register and refuses the row at build time, so a release-profile corpus
+ * holds eligible rows by construction and `TRAINING_SOURCES.json` records each refusal.
+ *
+ * A per-row eligibility field would copy a per-source decision onto every row
+ * and would disagree with the register after a review changed a license's state.
+ * Whether a row reached a checkpoint is a property of a run rather than of the corpus.
+ *
+ * And the sampler draws a different sample per epoch, so `deriveEffectiveTrainingManifest`
+ * records it per source from a config and an `audit_epoch_mixture` output.
  */
 export interface SourceProvenance {
     /**
-     * Adapter id that emitted this row, e.g. `"wof-admin"`, `"ban"`, `"openaddresses"`.
+     * The id of the adapter or recipe that emitted the row, such as `"ban"` or `"openaddresses"`.
      */
     source: string;
     /**
-     * Stable id within the adapter's source. For SQLite-backed adapters this is the row's primary key; for CSV/GeoJSON, a
-     * hash of the canonical components. Must be stable across reruns so that dedup and holdout manifests are
-     * reproducible.
+     * An id that is unique within the source and stable across reruns,
+     * so that deduplication and holdouts reproduce.
+     *
+     * SQLite-backed adapters use the primary key.
+     * File-backed adapters hash the components.
      */
     source_id: string;
     /**
-     * Corpus version string. Stamped by the runner, not the adapter. Locked together with the tokenizer version:
-     * `corpus-v0.1.0` ships with `tokenizer-v0.1.0`.
+     * The corpus version stamped by the runner.
      */
     corpus_version: string;
     /**
-     * Short license label or SPDX id for _this_ row. Defaults to the adapter's `defaultLicense`, but per-row sources
-     * (OpenAddresses) override.
+     * The license label or SPDX id for this row.
+     *
+     * It defaults to the adapter's `defaultLicense`, and sources with per-record terms override it.
      */
     license: string;
 }
 /**
- * Marker placed on rows produced by `synthesize.ts`. Allows downstream code to weight, stratify, or exclude
- * augmentations.
+ * The recipe that produced a row and the row it was derived from.
  */
-export interface SynthMarker {
+export interface RecipeMarker {
     /**
-     * Pipeline id describing what augmentation produced this row. Free-form but stable — e.g. `"case-perturb"`,
-     * `"accent-strip"`, `"abbrev-swap"`, `"compose:case-perturb+typo"`.
+     * A stable recipe id, such as `"german"` or `"compose:case-perturb+typo"`.
      */
-    method: string;
+    recipe: string;
     /**
-     * `source_id` of the un-augmented row this was derived from. Allows tracing every synthetic row back to its natural
-     * ancestor.
+     * The `source_id` of the row that this row was derived from.
      */
     base_source_id: string;
 }
 /**
- * One address row, before tokenization + BIO labeling.
- *
- * `raw` is what a parser would see in the wild — possibly multi-line, with arbitrary whitespace. `components` is the
- * ground-truth tagging: every `ComponentTag` present in the source data, mapped to its surface form _as it appears in
- * `raw`_. Alignment uses this to assign BIO labels.
+ * One address row before tokenization and labeling.
  *
- * Country is ISO 3166-1 alpha-2 (`"US"`, `"FR"`). Locale is BCP-47 (`"en-US"`, `"fr-FR"`) and is optional; adapters
- * that can't be sure leave it empty and let the runner default by country.
+ * Each value in `components` must appear in `raw`, within the alignment edit-distance
+ * threshold, or alignment quarantines the row.
  */
 export interface CanonicalRow extends SourceProvenance {
     /**
-     * Address string as it might appear in source data.
+     * The address text as it might appear in source data, possibly spanning lines.
      */
     raw: string;
     /**
-     * Component-by-tag ground truth. Surface forms must occur in `raw` (within the alignment edit distance threshold) or
-     * the row will land in the quarantine pile.
+     * The ground-truth text of each component, as it appears in `raw`.
      */
     components: Partial<Record<ComponentTag, string>>;
     /**
-     * ISO 3166-1 alpha-2 country code.
+     * The ISO 3166-1 alpha-2 country code.
      */
     country: string;
     /**
-     * Optional BCP-47 locale. Defaulted by country if absent.
+     * The BCP-47 locale.
+     * The runner derives it from the country when absent.
      */
     locale?: string;
     /**
-     * Present only on synthetic rows.
+     * The role of this address.
+     *
+     * The runner stamps the adapter's `addressRole` on rows that omit it.
+     * An adapter sets it per row only when one source has several roles.
+     * Read it with {@link addressRoleOf}.
+     */
+    addressRole?: AddressRole;
+    /**
+     * The stable id of the publication that the row's record came from, such as `"us-census-tiger"`.
+     *
+     * `null` means that the row corresponds to no published record.
+     * The runner stamps {@link CorpusAdapter.register} on rows that leave it unset.
+     * Rights records and supply counts read this field.
+     */
+    register?: string | null;
+    /**
+     * How the row's `raw` text was produced.
+     *
+     * The runner stamps {@link CorpusAdapter.surface} on rows that omit it.
+     */
+    surface?: SurfaceOrigin;
+    /**
+     * The recipe that produced the row, when a recipe did.
      */
-    synth?: SynthMarker;
+    recipe?: RecipeMarker;
 }
 /**
- * Output of `align.ts`. Carries everything `CanonicalRow` does, plus parallel `tokens` and `labels` arrays of identical
- * length (`labels[i]` is the BIO tag for `tokens[i]`) and — as of the v0.5.0 char-offset migration (#519) — parallel
- * char-span arrays addressing `raw` directly.
+ * A row after alignment, with tokens, BIO labels and character spans.
  *
- * The span triple is the v0.5.0 source of truth; `tokens`/`labels` remain emitted during the transition (and stay
- * derivable afterwards: whitespace split + span lookup). The reverse derivation — today's token labels — is the lossy
- * direction (punctuation-mute).
+ * The span arrays are the source of truth.
+ * Consumers can derive token labels from them.
  */
 export interface LabeledRow extends CanonicalRow {
     /**
-     * SentencePiece subword tokens for `raw`.
+     * Subword tokens for `raw`.
      */
     tokens: readonly string[];
     /**
-     * BIO labels, one per token. Same length as `tokens`.
+     * BIO labels, one per token.
      */
     labels: readonly BIOLabel[];
     /**
-     * Char-offset label spans over `raw` (parallel arrays, per the #519 ruling): `span_starts[i]` is the inclusive start
-     * offset (UTF-16 code units) of span `i`, `span_ends[i]` its exclusive end, `span_tags[i]` its component tag.
-     * Invariants — enforced loudly by `alignRow`, documented for every other producer: sorted ascending by start,
-     * non-overlapping. `raw` must be NFC-normalized or the offsets are ambiguous (also enforced by `alignRow`).
+     * Inclusive start offsets of the label spans over `raw`, in UTF-16 code units.
+     *
+     * The three span arrays are parallel.
+     * Spans are sorted by start and never overlap.
      *
-     * Optional during the v0.4.x → v0.5.0 transition only: alignment always emits the triple; frozen historical corpora
-     * and not-yet-migrated synthesis paths may lack it. Required once v0.5.0 lands and the token path is deleted.
+     * `raw` must be NFC-normalized.
+     * `alignRow` enforces both rules.
+     * Older corpora and some synthesis paths omit the spans.
      */
     span_starts?: readonly number[];
     /**
@@ -131,90 +272,118 @@ export interface LabeledRow extends CanonicalRow {
     span_tags?: readonly ComponentTag[];
 }
 /**
- * A row that alignment refused to label. Lands in `/data/corpus/quarantine/` for human review.
- *
- * The `reason` is human-readable; common values are `"component-not-found:<tag>"`, `"edit-distance-exceeded:<tag>"`,
- * `"raw-empty"`. Re-running alignment after a fix should re-emit the quarantined rows; the runner keys them by
- * `source_id`.
+ * A row that alignment refused to label, with a reason such as `"component-not-found:<tag>"`.
  */
 export interface QuarantinedRow {
     row: CanonicalRow;
     reason: string;
 }
 /**
- * Per-invocation knobs handed to an adapter by the runner.
- *
- * `inputPath` is interpreted by the adapter — it might be a single file path, a directory of slices, or even an HTTPS
- * URL. Each adapter documents its own expected shape in its README.
- *
- * `country` filters to a single ISO 3166-1 alpha-2 country _at the adapter level_. Adapters that hold multi-country
- * data (OSM PBF, OpenAddresses) MUST honor this; single-country adapters (BAN) may ignore it but should reject
- * mismatches.
- *
- * `limit` is a soft cap on rows emitted; useful for fixture-driven tests and smoke runs.
- *
- * `signal` allows the runner to cancel a long-running scan cleanly.
+ * Per-run options that the runner passes to an adapter.
  */
 export interface AdapterOptions {
     /**
-     * Path to the adapter's input data (file, directory, or URL — adapter-specific).
+     * The adapter's input.
+     * Each adapter interprets it as a file, directory or URL.
      */
-    inputPath: string;
+    inputPath: PathBuilderLike;
     /**
-     * Optional output directory, available to adapters that maintain side state (rare).
+     * An output directory for adapters that keep side state.
      */
     outputDir?: string;
     /**
-     * ISO 3166-1 alpha-2 country filter.
+     * An ISO 3166-1 alpha-2 country filter.
+     *
+     * Multi-country adapters must apply it.
+     * Single-country adapters should reject a mismatch.
      */
     country?: string;
     /**
-     * Soft row cap. Adapters should stop iterating once this is reached.
+     * A soft cap on emitted rows.
      */
     limit?: number;
     /**
-     * Cancellation hook. Adapters should respect this on every iteration boundary.
+     * The fraction of rows that include an explicit `country` component.
+     *
+     * A source without country tokens teaches the model that country tokens are rare.
+     * That lowers `country` recall.
+     *
+     * A value of `0` or no value adds no country and consumes no random draw,
+     * so existing outputs stay identical.
+     */
+    countryFraction?: number;
+    /**
+     * The seed for the {@linkcode AdapterOptions.countryFraction} draw.
+     * The draw has a fixed default.
+     */
+    seed?: number;
+    /**
+     * A cancellation signal that adapters should check at each iteration.
      */
     signal?: AbortSignal;
+    /**
+     * Counts of input records the adapter refused or trimmed, keyed by reason.
+     *
+     * An adapter that drops a record increments a `row:<reason>` key, and one that keeps the row
+     * but discards a component increments a `component:<tag>:<reason>` key.
+     * A `kept:<condition>` key counts records kept under a condition the reader should see,
+     * such as a country the record does not state.
+     *
+     * The runner writes the map into `MANIFEST.json` as `dropped`, so the gap between a
+     * source's record count and the rows written has a counted reason.
+     * An adapter that does not count leaves the map empty, which the manifest records
+     * as an empty object rather than as zero drops of any named kind.
+     */
+    dropped?: Map<string, number>;
 }
 /**
- * The contract every data source implements.
- *
- * Adapters are async generators: they yield `CanonicalRow`s one at a time, the runner consumes them (writing JSONL +
- * maintaining checksums + driving alignment). Streaming is mandatory — many sources are tens of millions of rows and
- * cannot be buffered.
+ * Adds one to `reason` in an adapter's {@link AdapterOptions.dropped} map, when the caller supplied one.
+ */
+export declare function countDropped(opts: Pick<AdapterOptions, "dropped">, reason: string): void;
+/**
+ * The interface that every data source implements.
  *
- * `defaultLicense` is stamped onto every emitted row's `license` field unless the adapter sets `license` explicitly
- * (e.g. OpenAddresses, which carries per-source licenses).
+ * Adapters stream rows, because many sources hold tens of millions of rows.
  */
 export interface CorpusAdapter {
     /**
-     * Stable, machine-friendly id used in paths and CLI args. E.g. `"wof-admin"`.
+     * A stable id used in paths and CLI arguments, such as `"wof-admin"`.
      */
     readonly id: string;
     /**
-     * Default SPDX-ish license label for rows from this adapter. Per-row overrides allowed.
+     * The license label stamped on rows that do not set their own.
      */
     readonly defaultLicense: string;
     /**
-     * One-sentence description shown by `npx mailwoman corpus list`.
+     * The dominant role of the source's addresses.
+     *
+     * The field has no default because only the adapter's author knows the source.
+     * The runner stamps it on rows that leave `addressRole` unset.
      */
-    readonly description: string;
+    readonly addressRole: AddressRole;
     /**
-     * Async iterable of canonical rows.
+     * The publication that this adapter reads, or `null` for an adapter that invents rows.
      *
-     * Implementations MUST:
-     *
-     * - Honor `opts.country` (filter or reject mismatches).
-     * - Honor `opts.limit` (stop after N rows).
-     * - Respect `opts.signal` on every iteration.
-     * - Set `source` to `this.id` on every emitted row.
-     * - Set `license` to `this.defaultLicense` unless overriding per-row.
+     * The field has no default because only the adapter's author knows the source.
+     * The runner stamps it on rows that leave `register` unset.
+     */
+    readonly register: string | null;
+    /**
+     * The dominant way the adapter produces `raw` text.
      *
-     * Implementations MUST NOT:
+     * The runner stamps it on rows that leave `surface` unset.
+     */
+    readonly surface: SurfaceOrigin;
+    /**
+     * The one-sentence description that `mailwoman corpus list` shows.
+     */
+    readonly description: string;
+    /**
+     * Streams canonical rows.
      *
-     * - Set `corpus_version` (the runner stamps it).
-     * - Mutate previously-yielded rows.
+     * Implementations must apply `opts.country` and `opts.limit`, check `opts.signal` at each
+     * iteration, set `source` to `this.id`, and set `license` unless a row overrides it.
+     * They must not set `corpus_version`, which the runner stamps, or mutate rows already yielded.
      */
     rows(opts: AdapterOptions): AsyncIterable<CanonicalRow>;
 }

package/out/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../lib/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAA;IAEtB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC3B;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAA;CACtB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,YAAa,SAAQ,gBAAgB;IACrD;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;;OAGG;IACH,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEjD;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,KAAK,CAAC,EAAE,WAAW,CAAA;CACnB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC/C;;OAEG;IACH,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IAEzB;;OAEG;IACH,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAA;IAE3B;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAE/B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAE7B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAA;CACnC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,YAAY,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IAEnB;;OAEG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAE/B;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAE5B;;;;;;;;;;;;;;;OAeG;IACH,IAAI,CAAC,IAAI,EAAE,cAAc,GAAG,aAAa,CAAC,YAAY,CAAC,CAAA;CACvD"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../lib/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAExE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW;IACvB;;OAEG;aACH,OAAO,EAAE,SAAS;IAClB;;OAEG;aACH,gBAAgB,EAAE,mBAAmB;IACrC;;;;;;;OAOG;aACH,YAAY,EAAE,cAAc;IAC5B;;;OAGG;aACH,OAAO,EAAE,SAAS;IAClB;;OAEG;aACH,QAAQ,EAAE,UAAU;IACpB;;OAEG;aACH,QAAQ,EAAE,UAAU;IACpB;;OAEG;aACH,KAAK,EAAE,OAAO;IACd;;OAEG;aACH,OAAO,EAAE,SAAS;IAClB;;OAEG;aACH,WAAW,EAAE,cAAc;IAC3B;;;;OAIG;aACH,KAAK,EAAE,OAAO;CACL,CAAA;AAEV;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,OAAO,WAAW,CAAC,CAAA;AAExE;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,EAAE,WAAiC,CAAA;AAEpE;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,GAAG,WAAW,CAEjF;AAED;;;;;;GAMG;AACH,eAAO,MAAM,aAAa;IACzB;;;;;;OAMG;aACH,QAAQ,EAAE,UAAU;IACpB;;;;;;;;;;;;;OAaG;aACH,QAAQ,EAAE,UAAU;IACpB;;;;;OAKG;aACH,QAAQ,EAAE,UAAU;IACpB;;OAEG;aACH,QAAQ,EAAE,UAAU;CACX,CAAA;AAEV;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAA;AAI9E;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,aAAa,CAU5F;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;IAEtB;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;CACtB;AAED;;;;;GAKG;AACH,MAAM,WAAW,YAAa,SAAQ,gBAAgB;IACrD;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;OAEG;IACH,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEjD;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,WAAW,CAAA;IAEzB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAExB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAA;IAEvB;;OAEG;IACH,MAAM,CAAC,EAAE,YAAY,CAAA;CACrB;AAED;;;;;GAKG;AACH,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC/C;;OAEG;IACH,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IAEzB;;OAEG;IACH,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAA;IAE3B;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAE/B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAE7B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,YAAY,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B;;;OAGG;IACH,SAAS,EAAE,eAAe,CAAA;IAE1B;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IAExB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAA;IAEpB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAExF;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IAEnB;;OAEG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAE/B;;;;;OAKG;IACH,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAA;IAEjC;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IAEhC;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAA;IAE/B;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAE5B;;;;;;OAMG;IACH,IAAI,CAAC,IAAI,EAAE,cAAc,GAAG,aAAa,CAAC,YAAY,CAAC,CAAA;CACvD"}

package/out/types.js

@@ -3,17 +3,141 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Canonical row schemas for the corpus pipeline (per #6 / Phase 1 plan).
+ *   Row, adapter and provenance types for the corpus pipeline.
+ */
+import { stringifyJSON } from "@mailwoman/core/json";
+/**
+ * The role that an address plays for the entity that the source describes.
+ * The values are wire values.
  *
- *   The corpus pipeline produces two row shapes:
+ * Roles have different grammars.
+ * For example, mailing addresses often use post-office boxes, so mixing roles in
+ * a corpus shifts what the premise parser learns.
+ */
+export const AddressRole = {
+    /**
+     * The location of the addressable object, as in a national address register.
+     */
+    Premise: "premise",
+    /**
+     * The seat that an entity files with a registrar, such as a siège social.
+     */
+    RegisteredOffice: "registered-office",
+    /**
+     * The address a third party records for a transacting party on its own transaction record,
+     * such as the supplier on a government contract notice.
+     *
+     * The party supplied the address to the recording body rather than to a registrar.
+     * No registrar filed it as a seat, and no address register confirmed it as a premise.
+     * A sole trader's entry can hold a home address.
+     */
+    Counterparty: "counterparty",
+    /**
+     * The place where post is delivered.
+     * This can be a post-office box.
+     */
+    Mailing: "mailing",
+    /**
+     * The place where a licensed person or organization practices.
+     */
+    Practice: "practice",
+    /**
+     * A site that an entity operates, such as a clinic, a school or a plant.
+     */
+    Facility: "facility",
+    /**
+     * The address of a property's owner, recorded against the property.
+     */
+    Owner: "owner",
+    /**
+     * The place where a service is delivered or an obligation is discharged.
+     */
+    Service: "service",
+    /**
+     * The address for service of legal process.
+     */
+    LegalNotice: "legal-notice",
+    /**
+     * A role that this vocabulary does not cover.
+     *
+     * Add a new constant when a second uncovered role appears.
+     */
+    Other: "other",
+};
+/**
+ * The role of a row without an `addressRole` field.
  *
- *   1. `CanonicalRow`: an adapter's raw output. Carries a free-form `raw` string, a per-component
- *        ground-truth dict, provenance, and an optional augmentation marker. Adapters emit these.
- *   2. `LabeledRow`: alignment's output. Adds a SentencePiece token list and a parallel BIO label list,
- *        suitable for direct ingestion by the neural training loop.
+ * Corpora written before the field existed came from premise sources.
+ */
+export const DEFAULT_ADDRESS_ROLE = AddressRole.Premise;
+/**
+ * Returns the row's address role, or {@link DEFAULT_ADDRESS_ROLE} when the field is absent.
+ */
+export function addressRoleOf(row) {
+    return row.addressRole ?? DEFAULT_ADDRESS_ROLE;
+}
+/**
+ * How a row's `raw` text was produced.
+ * The values are wire values.
+ *
+ * This is separate from {@link CanonicalRow.register}, which records where the underlying record came from.
+ * A rendering of a real register record is `Rendered` or `Composed`, and it is still a real address.
+ */
+export const SurfaceOrigin = {
+    /**
+     * The publisher's own string, preserved unchanged.
+     *
+     * No adapter in the tree emits this today: each one builds its line from the source's fields.
+     * The value is reserved for a source that publishes the written line itself,
+     * such as a register storing `address1` as a person typed it.
+     */
+    Attested: "attested",
+    /**
+     * An adapter assembled the line from one record's fields, in that locale's canonical order.
+     *
+     * The order comes from the codex layout or the adapter's own template rather than from the
+     * publisher, so the components are the source's and the written form is this repository's.
+     * Distinct from {@link SurfaceOrigin.Composed} because an adapter renders one canonical
+     * form per record while a recipe varies the form deliberately.
+     *
+     * A source id beginning `rendered-` does not take this value.
+     * `SourceOperation.Rendered` in `#recipes/sources` names a recipe operation with the same word,
+     * and those recipes record {@link SurfaceOrigin.Composed} because they vary the form.
+     *
+     * Read the row's `surface` rather than its source name to learn which of the two produced it.
+     */
+    Rendered: "rendered",
+    /**
+     * A recipe assembled the text from the published fields of one real record.
+     *
+     * The recipe chooses the order, punctuation and casing.
+     * It varies them to teach a surface.
+     */
+    Composed: "composed",
+    /**
+     * A synthetic row, such as a post-office box or an intersection, that corresponds to no published record.
+     */
+    Invented: "invented",
+};
+const SURFACE_ORIGINS = new Set(Object.values(SurfaceOrigin));
+/**
+ * Returns the `surface` that a JSONL row declares.
  *
- *   `CorpusAdapter` is the contract every data source implements; `AdapterOptions` is the
- *   per-invocation knob set (input path, optional country filter, row cap, abort signal).
+ * @throws When the row lacks a valid `surface`.
+ * Older files must be regenerated instead of read with a default.
+ */
+export function requireSurface(raw, producer) {
+    const value = raw["surface"];
+    if (typeof value === "string" && SURFACE_ORIGINS.has(value))
+        return value;
+    throw new Error(`${producer}: row ${stringifyJSON(raw["source_id"])} declares no surface. ` +
+        `Every row carries one of ${[...SURFACE_ORIGINS].join(", ")}. ` +
+        `A jsonl written before the field existed has to be regenerated rather than read with a default.`);
+}
+/**
+ * Adds one to `reason` in an adapter's {@link AdapterOptions.dropped} map, when the caller supplied one.
  */
-export {};
+export function countDropped(opts, reason) {
+    opts.dropped?.set(reason, (opts.dropped.get(reason) ?? 0) + 1);
+}
 //# sourceMappingURL=types.js.map