@mailwoman/corpus 9.2.0 → 9.3.0

package/out/synthesizers/utils.d.ts

@@ -0,0 +1,241 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Synthesis / augmentation per Phase 1 task #6.
+ *
+ *   An `Augmentation` is a pure function that takes a `CanonicalRow` and either returns a new
+ *   `CanonicalRow` (with `raw` AND `components` transformed in lockstep so alignment still
+ *   succeeds) or `null` when the augmentation doesn't apply to the row's shape.
+ *
+ *   Synthesis runs **before** alignment: augmentations transform raw + components together, and the
+ *   runner reruns alignment on each augmented row to produce its labels. This keeps the synthesis
+ *   surface small (no token/label arithmetic) at the cost of a re-run.
+ *
+ *   Every augmented row carries the `synth` marker:
+ *
+ *   - `method`: the augmentation's stable id (e.g. `"case-upper"`, `"accent-strip"`).
+ *   - `base_source_id`: the source_id of the un-augmented (or upstream-augmented) row, so ancestry is
+ *       traceable.
+ *
+ *   Phase 1 implements the locale-agnostic + most useful US/FR augmentations. Typo injection (#530)
+ *   is now implemented ({@link typoInject}) — the "seed-aware API" the deferral asked for is
+ *   resolved by seeding the PRNG from each row's `source_id`. It ships in {@link AUGMENTATIONS} but
+ *   is kept OUT of the default set ({@link defaultAugmentationsForCountry}) until its on-model
+ *   effect is measured; see the note there.
+ */
+import type { CanonicalRow, LabeledRow, QuarantinedRow } from "#types";
+import { type Tokenizer } from "#utils/tokenize";
+/**
+ * An augmentation transforms a single row. Return `null` if the augmentation doesn't apply (e.g. accent-strip on a row
+ * that has no accents; particle-strip on a US row).
+ */
+export type Augmentation = (row: CanonicalRow) => CanonicalRow | null;
+/**
+ * Upper-case raw + every component value. Returns null if already all-upper.
+ */
+export declare const caseUpper: Augmentation;
+/**
+ * Lower-case raw + every component value. Returns null if already all-lower.
+ */
+export declare const caseLower: Augmentation;
+/**
+ * Drop commas from `raw`. Components unchanged (they didn't carry commas).
+ */
+export declare const dropCommas: Augmentation;
+/**
+ * Replace single spaces with double spaces in `raw` AND in every component value. The component update is essential for
+ * alignment: `alignRow` substring-searches each component's surface form inside `raw`, so doubling the spaces in `raw`
+ * only would leave single-spaced components unfindable (this was the bug behind v0.1.1's first build attempt — 99.9% of
+ * quarantined rows traced back to this augmentation). Doubling both keeps the substring contract intact.
+ */
+export declare const doubleSpace: Augmentation;
+/**
+ * Strip Unicode combining marks (accents, diacritics) from raw + components. "Hôtel" → "Hotel"; "Île-de-France" →
+ * "Ile-de-France". Returns null if the row has no accents.
+ */
+export declare const accentStrip: Augmentation;
+/**
+ * Inject ONE realistic typo — an adjacent-QWERTY-key substitution OR an adjacent-character transposition — into a
+ * single alpha name component (street/locality/region…), teaching the model to recover from real-world misspellings
+ * ("Cupertino" → "Cupertimo"). The edit is applied to BOTH `raw` and the component so the substring contract `alignRow`
+ * depends on holds. Number / postcode / unit components are never touched (they fail {@link ALPHA_NAME}). Deterministic
+ * per row (seeded from `source_id`). Returns `null` when no eligible component exists or the edit is a no-op.
+ */
+export declare const typoInject: Augmentation;
+/**
+ * US: substitute the full state name for its alpha-2 abbreviation.
+ */
+export declare const stateExpand: Augmentation;
+/**
+ * US: substitute the alpha-2 abbreviation for the full state name.
+ */
+export declare const stateAbbreviate: Augmentation;
+/**
+ * US: expand directional abbreviations in `street`/`street_suffix` (NW → Northwest).
+ */
+export declare const directionalExpand: Augmentation;
+/**
+ * US: abbreviate directional words (Northwest → NW).
+ */
+export declare const directionalAbbreviate: Augmentation;
+/**
+ * US: swap the trailing street-suffix word in `components.street` to its preferred USPS abbreviation, preserving case.
+ * `"5th Avenue"` → `"5th Ave"`; `"5TH AVENUE"` → `"5TH AVE"`; `"main street"` → `"main st"`. Returns null when no
+ * trailing suffix is recognized, when the trailing word is already the preferred abbreviation, or when the swap would
+ * leave `raw` un- touched (alignment requires both raw and components to move in lockstep).
+ *
+ * Targets the trailing word only to avoid mangling streets like "Avenue of the Americas" where the suffix-shaped word
+ * is part of the proper name rather than a USPS suffix.
+ */
+export declare const streetSuffixAbbreviate: Augmentation;
+/**
+ * US: swap the trailing street-suffix word in `components.street` to its full canonical form, preserving case. `"5th
+ * Ave"` → `"5th Avenue"`; `"5TH AVE"` → `"5TH AVENUE"`; `"main st"` → `"main street"`. Returns null when no trailing
+ * suffix is recognized, when the trailing word is already the canonical full form, or when the swap would leave `raw`
+ * untouched.
+ *
+ * Same trailing-word-only rule as `streetSuffixAbbreviate`.
+ */
+export declare const streetSuffixExpand: Augmentation;
+/**
+ * US: swap the LEADING secondary-unit designator in `components.unit` to its approved USPS abbreviation, preserving
+ * case + the identifier. `"Apartment 4B"` → `"Apt 4B"`; `"SUITE 200"` → `"STE 200"`; `"floor 3"` → `"fl 3"`. Returns
+ * null when the unit has no recognized leading designator (a bare `"4B"` / `"#210"`), the designator is already the
+ * approved abbreviation, or the swap would leave `raw` untouched.
+ *
+ * Mirrors `streetSuffixAbbreviate`, but designators LEAD the unit (vs suffixes that trail the street). Sourced from the
+ * USPS Pub-28 C2 codex — the data-generation counterpart to the runtime `UnitDesignatorClassifier`.
+ */
+export declare const unitDesignatorAbbreviate: Augmentation;
+/**
+ * US: swap the LEADING secondary-unit designator in `components.unit` to its full canonical form, preserving case + the
+ * identifier. `"Apt 4B"` → `"Apartment 4B"`; `"STE 200"` → `"SUITE 200"`. Returns null when there's no recognized
+ * leading designator, it's already the canonical word, or the swap would leave `raw` untouched. Same leading-word-only
+ * rule as `unitDesignatorAbbreviate`.
+ */
+export declare const unitDesignatorExpand: Augmentation;
+/**
+ * US: ZIP+4 form `12345-6789` → `123456789` (dash dropped).
+ */
+export declare const zipPlus4DashDrop: Augmentation;
+/**
+ * FR: drop the article particle from a street ("Rue de la République" → "Rue République").
+ */
+export declare const particleStrip: Augmentation;
+/**
+ * Stable id → augmentation table.
+ */
+export declare const AUGMENTATIONS: Record<string, Augmentation>;
+/**
+ * Default augmentation set, by country. Phase 1: US + FR; others get the locale-agnostic set.
+ */
+export declare function defaultAugmentationsForCountry(country: string): readonly Augmentation[];
+/**
+ * Run every augmentation against a row; collect the non-null outputs. The augmentations are pure, so callers can
+ * compose them off this generator (e.g. nesting accent-strip ∘ state-abbreviate).
+ */
+export declare function synthesizeRow(row: CanonicalRow, augmentations?: readonly Augmentation[]): Generator<CanonicalRow>;
+/**
+ * One element of `arr`, drawn with the supplied unit-interval source.
+ */
+export declare function pick<T>(arr: ReadonlyArray<T>, random: () => number): T;
+/**
+ * One element of `items`, drawn with probability proportional to `weightOf(item)`.
+ *
+ * One `random()` draw per call. `inclusive` (the default) keeps an item whose cumulative weight lands exactly on the
+ * draw (`r <= 0` after subtraction); pass `inclusive: false` for the strict `r < 0` boundary — the two callers this
+ * consolidates disagreed on that float-exact edge, and each keeps its own reading so its draw stream is unchanged.
+ */
+export declare function weightedPick<T>(items: readonly T[], random: () => number, weightOf: (item: T) => number, options?: {
+    inclusive?: boolean;
+}): T;
+/**
+ * One band of a {@link tieredNumber} distribution: a tier draw below `cutoff` (the last band omits it) yields `base +
+ * floor(random() * span)`.
+ */
+export interface TieredNumberBand {
+    cutoff?: number;
+    base: number;
+    span: number;
+}
+/**
+ * A number drawn from a tiered magnitude distribution — the house/box-number shape the synthesizers share.
+ *
+ * Exactly two `random()` draws per call, in the order every caller established: one tier draw, then one value draw
+ * inside the chosen band.
+ */
+export declare function tieredNumber(random: () => number, bands: readonly TieredNumberBand[]): string;
+/**
+ * The primary locale a synthesizer renders for a country — ISO-3166-1 alpha-2, alpha-3, or the English display name,
+ * case- and whitespace-tolerant. Unknown countries render as `en-US`.
+ *
+ * The PO-box synthesizer keeps its own template-scoped `countryToLocale` on top of this one: it maps any locale WITHOUT
+ * a PO-box template back to `en-US` (so `DE` still renders the en-US box vocabulary there).
+ */
+export declare function countryToLocale(country: string): string;
+/**
+ * Options accepted by `composeAdversarialRow`.
+ */
+export interface ComposeAdversarialOptions {
+    /**
+     * Stable pattern label written into the emitted row's `synth.method` field (as `compose:<pattern>`). Free-form but
+     * should be one of a small set of canonical pattern names so downstream filtering / stratification can target
+     * individual patterns.
+     *
+     * Recommended values (Phase 1.6 §2.1):
+     *
+     * - `"place-name-venue"` — venue token shared with locality (`Buffalo Health Clinic, Buffalo NY`).
+     * - `"place-shaped-venue"` — venue contains a place-shaped substring (`New York, New York Steakhouse, Las Vegas NV`).
+     * - `"particle-honorific"` — apostrophe + St./Saint ambiguity (`P'tit St. Denis Street Café`).
+     */
+    pattern: string;
+    /**
+     * Separator inserted between the venue and the address `raw`. Default `", "`. Single space (`" "`) produces the
+     * harder unpunctuated variant; newline (`"\n"`) the multi-line variant.
+     */
+    separator?: string;
+    /**
+     * Tokenizer to apply to the venue prefix. Default `whitespaceTokenizer()`. The address half uses the same tokenizer
+     * when re-aligned — pass a consistent one if customizing.
+     */
+    tokenizer?: Tokenizer;
+}
+/**
+ * Either a successful labeled composition or a quarantined attempt.
+ */
+export type ComposeResult = {
+    kind: "labeled";
+    row: LabeledRow;
+} | {
+    kind: "quarantined";
+    row: QuarantinedRow;
+};
+/**
+ * Compose a venue string + an address row into a single adversarial `LabeledRow`.
+ *
+ * The emitted row's `raw` is `${venue}${separator}${address.raw}`. Tokens are produced by tokenizing the two halves
+ * independently and concatenating; labels are venue tokens → `B-venue` / `I-venue` followed by the address's labels
+ * (obtained by aligning the input address in isolation). This deterministic boundary is the entire point of the
+ * primitive: the embedded place-shaped tokens in the venue stay labeled as `venue`, never as the address's locality /
+ * region / etc., even when they share surface forms.
+ *
+ * The char-offset span triple (#519) is re-targeted to the composed surface by the same deterministic boundary: one
+ * `venue` span over `[0, venue.length)` (no re-search), then the address's own spans shifted by `venue.length +
+ * separator.length` — plain offset arithmetic, no token indirection. The separator chars sit outside every span
+ * (deliberately unlabeled — now expressible). The composed triple is passed through `assertSpanInvariants` so a
+ * composition bug can't ride into a corpus.
+ *
+ * The address's components are forwarded as-is (alignment ran on them and they survived); `venue` is added on top with
+ * the trimmed venue string as its surface form.
+ *
+ * Returns `{ kind: "quarantined" }` when:
+ *
+ * - The venue is empty or whitespace-only.
+ * - The venue is not NFC-normalized (char offsets over a non-NFC raw are ambiguous — the same discipline `alignRow`
+ *   enforces on adapter rows, surfaced as quarantine here because the venue is caller-supplied data).
+ * - The address row fails alignment in isolation (the underlying failure reason is propagated).
+ */
+export declare function composeAdversarialRow(venue: string, address: CanonicalRow, options: ComposeAdversarialOptions): ComposeResult;
+//# sourceMappingURL=utils.d.ts.map

package/out/synthesizers/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../lib/synthesizers/utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAeH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAA;AAEtE,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAErE;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,YAAY,KAAK,YAAY,GAAG,IAAI,CAAA;AA4BrE;;GAEG;AACH,eAAO,MAAM,SAAS,EAAE,YAYvB,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,SAAS,EAAE,YAYvB,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,EAAE,YAKxB,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,WAAW,EAAE,YAYzB,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,WAAW,EAAE,YAazB,CAAA;AAyDD;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,EAAE,YAyDxB,CAAA;AAmED;;GAEG;AACH,eAAO,MAAM,WAAW,EAAE,YAiBzB,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,EAAE,YAe7B,CAAA;AAiBD;;GAEG;AACH,eAAO,MAAM,iBAAiB,EAAE,YAuB/B,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,YA2BnC,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,EAAE,YAwBpC,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,kBAAkB,EAAE,YAuBhC,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,wBAAwB,EAAE,YAwBtC,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,EAAE,YAuBlC,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE,YAW9B,CAAA;AAMD;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,YAc3B,CAAA;AAMD;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAiBtD,CAAA;AAED;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,YAAY,EAAE,CA2BvF;AAED;;;GAGG;AACH,wBAAiB,aAAa,CAC7B,GAAG,EAAE,YAAY,EACjB,aAAa,GAAE,SAAS,YAAY,EAAgD,GAClF,SAAS,CAAC,YAAY,CAAC,CAQzB;AAED;;GAEG;AACH,wBAAgB,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,MAAM,GAAG,CAAC,CAEtE;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAC7B,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,MAAM,EAAE,MAAM,MAAM,EACpB,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC7B,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAO,GACnC,CAAC,CAYH;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,MAAM,EAAE,KAAK,EAAE,SAAS,gBAAgB,EAAE,GAAG,MAAM,CAa7F;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAwBvD;AAiCD;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACzC;;;;;;;;;;OAUG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,GAAG,EAAE,cAAc,CAAA;CAAE,CAAA;AAE/G;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,qBAAqB,CACpC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,yBAAyB,GAChC,aAAa,CAyFf"}