@mailwoman/corpus 2.0.0

package/out/src/synthesize.js

@@ -0,0 +1,472 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Synthesis / augmentation per Phase 1 task #6.
+ *
+ *   An `Augmentation` is a pure function that takes a `CanonicalRow` and either returns a new
+ *   `CanonicalRow` (with `raw` AND `components` transformed in lockstep so alignment still
+ *   succeeds) or `null` when the augmentation doesn't apply to the row's shape.
+ *
+ *   Synthesis runs **before** alignment: augmentations transform raw + components together, and the
+ *   runner reruns alignment on each augmented row to produce its labels. This keeps the synthesis
+ *   surface small (no token/label arithmetic) at the cost of a re-run.
+ *
+ *   Every augmented row carries the `synth` marker:
+ *
+ *   - `method`: the augmentation's stable id (e.g. `"case-upper"`, `"accent-strip"`).
+ *   - `base_source_id`: the source_id of the un-augmented (or upstream-augmented) row, so ancestry is
+ *       traceable.
+ *
+ *   Phase 1 implements the locale-agnostic + most useful US/FR augmentations. Typo injection and
+ *   other stochastic augmentations are intentionally deferred — they need a seed-aware API and are
+ *   most useful at training time, not corpus build time.
+ */
+import { alignRow } from "./align.js";
+import { US_STREET_SUFFIX_PREFERRED_ABBR, matchCase, matchTrailingSuffix } from "./codex/us-street-suffix.js";
+import { whitespaceTokenizer } from "./tokenize.js";
+/** Helper: build the augmented row with synth marker + chained source_id. */
+function withAugmentation(source, method, newRaw, newComponents) {
+    const baseId = source.synth?.base_source_id ?? source.source_id;
+    return {
+        ...source,
+        raw: newRaw,
+        components: newComponents,
+        source_id: `${source.source_id}+${method}`,
+        synth: { method, base_source_id: baseId },
+    };
+}
+// ===========================================================================
+// Locale-agnostic augmentations
+// ===========================================================================
+/** Upper-case raw + every component value. Returns null if already all-upper. */
+export const caseUpper = (row) => {
+    if (row.raw === row.raw.toUpperCase())
+        return null;
+    const upRaw = row.raw.toUpperCase();
+    const upComponents = {};
+    for (const [k, v] of Object.entries(row.components)) {
+        if (v)
+            upComponents[k] = v.toUpperCase();
+    }
+    return withAugmentation(row, "case-upper", upRaw, upComponents);
+};
+/** Lower-case raw + every component value. Returns null if already all-lower. */
+export const caseLower = (row) => {
+    if (row.raw === row.raw.toLowerCase())
+        return null;
+    const downRaw = row.raw.toLowerCase();
+    const downComponents = {};
+    for (const [k, v] of Object.entries(row.components)) {
+        if (v)
+            downComponents[k] = v.toLowerCase();
+    }
+    return withAugmentation(row, "case-lower", downRaw, downComponents);
+};
+/** Drop commas from `raw`. Components unchanged (they didn't carry commas). */
+export const dropCommas = (row) => {
+    if (!row.raw.includes(","))
+        return null;
+    const newRaw = row.raw.replace(/,/g, "").replace(/\s+/g, " ").trim();
+    return withAugmentation(row, "drop-commas", newRaw, { ...row.components });
+};
+/**
+ * Replace single spaces with double spaces in `raw` AND in every component value. The component
+ * update is load-bearing for alignment: `alignRow` substring-searches each component's surface form
+ * inside `raw`, so doubling the spaces in `raw` only would leave single-spaced components
+ * unfindable (this was the bug behind v0.1.1's first build attempt — 99.9% of quarantined rows
+ * traced back to this augmentation). Doubling both keeps the substring contract intact.
+ */
+export const doubleSpace = (row) => {
+    if (!/ /.test(row.raw))
+        return null;
+    const newRaw = row.raw.replace(/ /g, "  ");
+    const newComponents = {};
+    for (const [k, v] of Object.entries(row.components)) {
+        if (v)
+            newComponents[k] = v.replace(/ /g, "  ");
+    }
+    return withAugmentation(row, "double-space", newRaw, newComponents);
+};
+/**
+ * Strip Unicode combining marks (accents, diacritics) from raw + components. "Hôtel" → "Hotel";
+ * "Île-de-France" → "Ile-de-France". Returns null if the row has no accents.
+ */
+export const accentStrip = (row) => {
+    const stripped = stripAccents(row.raw);
+    if (stripped === row.raw)
+        return null;
+    const newComponents = {};
+    for (const [k, v] of Object.entries(row.components)) {
+        if (v)
+            newComponents[k] = stripAccents(v);
+    }
+    return withAugmentation(row, "accent-strip", stripped, newComponents);
+};
+function stripAccents(s) {
+    return s.normalize("NFD").replace(/\p{M}/gu, "");
+}
+// ===========================================================================
+// US-specific augmentations
+// ===========================================================================
+/** US state full ↔ alpha-2 mapping. Two-way: `STATE_TO_ABBR["Oregon"] = "OR"`. */
+const STATE_NAME_TO_ABBR = {
+    Alabama: "AL",
+    Alaska: "AK",
+    Arizona: "AZ",
+    Arkansas: "AR",
+    California: "CA",
+    Colorado: "CO",
+    Connecticut: "CT",
+    Delaware: "DE",
+    Florida: "FL",
+    Georgia: "GA",
+    Hawaii: "HI",
+    Idaho: "ID",
+    Illinois: "IL",
+    Indiana: "IN",
+    Iowa: "IA",
+    Kansas: "KS",
+    Kentucky: "KY",
+    Louisiana: "LA",
+    Maine: "ME",
+    Maryland: "MD",
+    Massachusetts: "MA",
+    Michigan: "MI",
+    Minnesota: "MN",
+    Mississippi: "MS",
+    Missouri: "MO",
+    Montana: "MT",
+    Nebraska: "NE",
+    Nevada: "NV",
+    "New Hampshire": "NH",
+    "New Jersey": "NJ",
+    "New Mexico": "NM",
+    "New York": "NY",
+    "North Carolina": "NC",
+    "North Dakota": "ND",
+    Ohio: "OH",
+    Oklahoma: "OK",
+    Oregon: "OR",
+    Pennsylvania: "PA",
+    "Rhode Island": "RI",
+    "South Carolina": "SC",
+    "South Dakota": "SD",
+    Tennessee: "TN",
+    Texas: "TX",
+    Utah: "UT",
+    Vermont: "VT",
+    Virginia: "VA",
+    Washington: "WA",
+    "West Virginia": "WV",
+    Wisconsin: "WI",
+    Wyoming: "WY",
+    "District of Columbia": "DC",
+};
+const STATE_ABBR_TO_NAME = Object.fromEntries(Object.entries(STATE_NAME_TO_ABBR).map(([k, v]) => [v, k]));
+/** US: substitute the full state name for its alpha-2 abbreviation. */
+export const stateExpand = (row) => {
+    if (row.country !== "US")
+        return null;
+    const region = row.components.region;
+    if (!region)
+        return null;
+    const full = STATE_ABBR_TO_NAME[region];
+    if (!full)
+        return null;
+    // Replace the bounded "OR" surface form with "Oregon" in raw. Use word boundaries so we
+    // don't match inside "Stop" or similar.
+    const re = new RegExp(`\\b${region}\\b`, "g");
+    if (!re.test(row.raw))
+        return null;
+    const newRaw = row.raw.replace(new RegExp(`\\b${region}\\b`, "g"), full);
+    const newComponents = { ...row.components, region: full };
+    return withAugmentation(row, "state-expand", newRaw, newComponents);
+};
+/** US: substitute the alpha-2 abbreviation for the full state name. */
+export const stateAbbreviate = (row) => {
+    if (row.country !== "US")
+        return null;
+    const region = row.components.region;
+    if (!region)
+        return null;
+    const abbr = STATE_NAME_TO_ABBR[region];
+    if (!abbr)
+        return null;
+    const re = new RegExp(`\\b${region}\\b`, "g");
+    if (!re.test(row.raw))
+        return null;
+    const newRaw = row.raw.replace(new RegExp(`\\b${region}\\b`, "g"), abbr);
+    const newComponents = { ...row.components, region: abbr };
+    return withAugmentation(row, "state-abbreviate", newRaw, newComponents);
+};
+const DIRECTIONAL_FULL_TO_ABBR = {
+    North: "N",
+    South: "S",
+    East: "E",
+    West: "W",
+    Northeast: "NE",
+    Northwest: "NW",
+    Southeast: "SE",
+    Southwest: "SW",
+};
+const DIRECTIONAL_ABBR_TO_FULL = Object.fromEntries(Object.entries(DIRECTIONAL_FULL_TO_ABBR).map(([k, v]) => [v, k]));
+/** US: expand directional abbreviations in `street`/`street_suffix` (NW → Northwest). */
+export const directionalExpand = (row) => {
+    if (row.country !== "US")
+        return null;
+    const tagsToCheck = ["street", "street_suffix", "street_prefix"];
+    let changed = false;
+    let newRaw = row.raw;
+    const newComponents = { ...row.components };
+    for (const tag of tagsToCheck) {
+        const v = newComponents[tag];
+        if (!v)
+            continue;
+        const replaced = v.replace(/\b(N|S|E|W|NE|NW|SE|SW)\b/g, (m) => DIRECTIONAL_ABBR_TO_FULL[m] ?? m);
+        if (replaced !== v) {
+            newComponents[tag] = replaced;
+            newRaw = newRaw.replace(new RegExp(`\\b${escapeRegex(v)}\\b`, "g"), replaced);
+            changed = true;
+        }
+    }
+    if (!changed)
+        return null;
+    return withAugmentation(row, "directional-expand", newRaw, newComponents);
+};
+/** US: abbreviate directional words (Northwest → NW). */
+export const directionalAbbreviate = (row) => {
+    if (row.country !== "US")
+        return null;
+    const tagsToCheck = ["street", "street_suffix", "street_prefix"];
+    let changed = false;
+    let newRaw = row.raw;
+    const newComponents = { ...row.components };
+    for (const tag of tagsToCheck) {
+        const v = newComponents[tag];
+        if (!v)
+            continue;
+        const replaced = v.replace(/\b(North|South|East|West|Northeast|Northwest|Southeast|Southwest)\b/g, (m) => DIRECTIONAL_FULL_TO_ABBR[m] ?? m);
+        if (replaced !== v) {
+            newComponents[tag] = replaced;
+            newRaw = newRaw.replace(new RegExp(`\\b${escapeRegex(v)}\\b`, "g"), replaced);
+            changed = true;
+        }
+    }
+    if (!changed)
+        return null;
+    return withAugmentation(row, "directional-abbreviate", newRaw, newComponents);
+};
+/**
+ * US: swap the trailing street-suffix word in `components.street` to its preferred USPS
+ * abbreviation, preserving case. `"5th Avenue"` → `"5th Ave"`; `"5TH AVENUE"` → `"5TH AVE"`; `"main
+ * street"` → `"main st"`. Returns null when no trailing suffix is recognized, when the trailing
+ * word is already the preferred abbreviation, or when the swap would leave `raw` un- touched
+ * (alignment requires both raw and components to move in lockstep).
+ *
+ * Targets the trailing word only to avoid mangling streets like "Avenue of the Americas" where the
+ * suffix-shaped word is part of the proper name rather than a USPS suffix.
+ */
+export const streetSuffixAbbreviate = (row) => {
+    if (row.country !== "US")
+        return null;
+    const street = row.components.street;
+    if (!street)
+        return null;
+    const match = matchTrailingSuffix(street);
+    if (!match)
+        return null;
+    const preferred = US_STREET_SUFFIX_PREFERRED_ABBR[match.canonical];
+    const target = matchCase(preferred, match.matched);
+    if (target === match.matched)
+        return null;
+    const newStreet = `${street.slice(0, street.lastIndexOf(match.matched))}${target}`;
+    if (newStreet === street)
+        return null;
+    const newComponents = { ...row.components, street: newStreet };
+    const newRaw = row.raw.replace(new RegExp(`\\b${escapeRegex(street)}\\b`, "g"), newStreet);
+    if (newRaw === row.raw)
+        return null;
+    return withAugmentation(row, "us-street-suffix-abbreviate", newRaw, newComponents);
+};
+/**
+ * US: swap the trailing street-suffix word in `components.street` to its full canonical form,
+ * preserving case. `"5th Ave"` → `"5th Avenue"`; `"5TH AVE"` → `"5TH AVENUE"`; `"main st"` → `"main
+ * street"`. Returns null when no trailing suffix is recognized, when the trailing word is already
+ * the canonical full form, or when the swap would leave `raw` untouched.
+ *
+ * Same trailing-word-only rule as `streetSuffixAbbreviate`.
+ */
+export const streetSuffixExpand = (row) => {
+    if (row.country !== "US")
+        return null;
+    const street = row.components.street;
+    if (!street)
+        return null;
+    const match = matchTrailingSuffix(street);
+    if (!match)
+        return null;
+    const target = matchCase(match.canonical, match.matched);
+    if (target === match.matched)
+        return null;
+    const newStreet = `${street.slice(0, street.lastIndexOf(match.matched))}${target}`;
+    if (newStreet === street)
+        return null;
+    const newComponents = { ...row.components, street: newStreet };
+    const newRaw = row.raw.replace(new RegExp(`\\b${escapeRegex(street)}\\b`, "g"), newStreet);
+    if (newRaw === row.raw)
+        return null;
+    return withAugmentation(row, "us-street-suffix-expand", newRaw, newComponents);
+};
+/** US: ZIP+4 form `12345-6789` → `123456789` (dash dropped). */
+export const zipPlus4DashDrop = (row) => {
+    if (row.country !== "US")
+        return null;
+    const postcode = row.components.postcode;
+    if (!postcode || !/^\d{5}-\d{4}$/.test(postcode))
+        return null;
+    const noDash = postcode.replace("-", "");
+    const newRaw = row.raw.replace(postcode, noDash);
+    if (newRaw === row.raw)
+        return null;
+    return withAugmentation(row, "zip-plus4-dash-drop", newRaw, { ...row.components, postcode: noDash });
+};
+// ===========================================================================
+// FR-specific augmentations
+// ===========================================================================
+/** FR: drop the article particle from a street ("Rue de la République" → "Rue République"). */
+export const particleStrip = (row) => {
+    if (row.country !== "FR")
+        return null;
+    const particle = row.components.street_prefix_particle;
+    if (!particle)
+        return null;
+    const newComponents = { ...row.components };
+    delete newComponents.street_prefix_particle;
+    // Drop the particle from raw, then collapse any double spaces.
+    const re = new RegExp(`\\s+${escapeRegex(particle)}\\s+`, "g");
+    if (!re.test(row.raw))
+        return null;
+    const newRaw = row.raw.replace(re, " ").replace(/\s+/g, " ").trim();
+    return withAugmentation(row, "particle-strip", newRaw, newComponents);
+};
+// ===========================================================================
+// Registry + default policies
+// ===========================================================================
+/** Stable id → augmentation table. */
+export const AUGMENTATIONS = {
+    "case-upper": caseUpper,
+    "case-lower": caseLower,
+    "drop-commas": dropCommas,
+    "double-space": doubleSpace,
+    "accent-strip": accentStrip,
+    "state-expand": stateExpand,
+    "state-abbreviate": stateAbbreviate,
+    "directional-expand": directionalExpand,
+    "directional-abbreviate": directionalAbbreviate,
+    "us-street-suffix-abbreviate": streetSuffixAbbreviate,
+    "us-street-suffix-expand": streetSuffixExpand,
+    "zip-plus4-dash-drop": zipPlus4DashDrop,
+    "particle-strip": particleStrip,
+};
+/** Default augmentation set, by country. Phase 1: US + FR; others get the locale-agnostic set. */
+export function defaultAugmentationsForCountry(country) {
+    const universal = [caseUpper, caseLower, dropCommas, doubleSpace];
+    switch (country) {
+        case "US":
+            return [
+                ...universal,
+                stateExpand,
+                stateAbbreviate,
+                directionalExpand,
+                directionalAbbreviate,
+                streetSuffixAbbreviate,
+                streetSuffixExpand,
+                zipPlus4DashDrop,
+            ];
+        case "FR":
+            return [...universal, accentStrip, particleStrip];
+        default:
+            return universal;
+    }
+}
+/**
+ * Run every augmentation against a row; collect the non-null outputs. The augmentations are pure,
+ * so callers can compose them off this generator (e.g. nesting accent-strip ∘ state-abbreviate).
+ */
+export function* synthesizeRow(row, augmentations = defaultAugmentationsForCountry(row.country)) {
+    for (const aug of augmentations) {
+        const out = aug(row);
+        if (out)
+            yield out;
+    }
+}
+function escapeRegex(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/**
+ * Compose a venue string + an address row into a single adversarial `LabeledRow`.
+ *
+ * The emitted row's `raw` is `${venue}${separator}${address.raw}`. Tokens are produced by
+ * tokenizing the two halves independently and concatenating; labels are venue tokens → `B-venue` /
+ * `I-venue` followed by the address's labels (obtained by aligning the input address in isolation).
+ * This deterministic boundary is the entire point of the primitive: the embedded place-shaped
+ * tokens in the venue stay labeled as `venue`, never as the address's locality / region / etc.,
+ * even when they share surface forms.
+ *
+ * The address's components are forwarded as-is (alignment ran on them and they survived); `venue`
+ * is added on top with the trimmed venue string as its surface form.
+ *
+ * Returns `{ kind: "quarantined" }` when:
+ *
+ * - The venue is empty or whitespace-only.
+ * - The address row fails alignment in isolation (the underlying failure reason is propagated).
+ */
+export function composeAdversarialRow(venue, address, options) {
+    const separator = options.separator ?? ", ";
+    const tokenizer = options.tokenizer ?? whitespaceTokenizer();
+    const venueTrimmed = venue.trim();
+    if (!venueTrimmed) {
+        return { kind: "quarantined", row: { row: address, reason: "venue-empty" } };
+    }
+    const addressAligned = alignRow(address, { tokenizer });
+    if (addressAligned.kind !== "labeled") {
+        // Surface the address's quarantine reason but tag it with the compose attempt for
+        // debugging. The original CanonicalRow stays on the QuarantinedRow so callers can
+        // inspect the address payload.
+        return {
+            kind: "quarantined",
+            row: { row: address, reason: `compose-address-${addressAligned.row.reason}` },
+        };
+    }
+    const venueTokens = tokenizer.tokenize(venueTrimmed);
+    if (venueTokens.length === 0) {
+        return { kind: "quarantined", row: { row: address, reason: "venue-no-tokens" } };
+    }
+    const venueLabels = venueTokens.map((_, i) => (i === 0 ? "B-venue" : "I-venue"));
+    const tokens = [...venueTokens.map((t) => t.text), ...addressAligned.row.tokens];
+    const labels = [...venueLabels, ...addressAligned.row.labels];
+    const composedRaw = `${venueTrimmed}${separator}${address.raw}`;
+    const composedComponents = {
+        venue: venueTrimmed,
+        ...address.components,
+    };
+    const baseSourceId = address.synth?.base_source_id ?? address.source_id;
+    const method = `compose:${options.pattern}`;
+    const composed = {
+        raw: composedRaw,
+        components: composedComponents,
+        country: address.country,
+        locale: address.locale,
+        source: address.source,
+        source_id: `${address.source_id}+${method}`,
+        corpus_version: address.corpus_version,
+        license: address.license,
+        synth: { method, base_source_id: baseSourceId },
+        tokens,
+        labels,
+    };
+    return { kind: "labeled", row: composed };
+}
+//# sourceMappingURL=synthesize.js.map

package/out/src/synthesize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"synthesize.js","sourceRoot":"","sources":["../../src/synthesize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,+BAA+B,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAA;AAC7G,OAAO,EAAE,mBAAmB,EAAkB,MAAM,eAAe,CAAA;AAWnE,6EAA6E;AAC7E,SAAS,gBAAgB,CACxB,MAAoB,EACpB,MAAc,EACd,MAAc,EACd,aAA4B;IAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,cAAc,IAAI,MAAM,CAAC,SAAS,CAAA;IAC/D,OAAO;QACN,GAAG,MAAM;QACT,GAAG,EAAE,MAAM;QACX,UAAU,EAAE,aAAa;QACzB,SAAS,EAAE,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,EAAE;QAC1C,KAAK,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE;KACzC,CAAA;AACF,CAAC;AAED,8EAA8E;AAC9E,gCAAgC;AAChC,8EAA8E;AAE9E,iFAAiF;AACjF,MAAM,CAAC,MAAM,SAAS,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC9C,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE;QAAE,OAAO,IAAI,CAAA;IAClD,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAA;IACnC,MAAM,YAAY,GAAkB,EAAE,CAAA;IACtC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC;YAAE,YAAY,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;IACzD,CAAC;IACD,OAAO,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,CAAC,CAAA;AAChE,CAAC,CAAA;AAED,iFAAiF;AACjF,MAAM,CAAC,MAAM,SAAS,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC9C,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE;QAAE,OAAO,IAAI,CAAA;IAClD,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAA;IACrC,MAAM,cAAc,GAAkB,EAAE,CAAA;IACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC;YAAE,cAAc,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;IAC3D,CAAC;IACD,OAAO,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;AACpE,CAAC,CAAA;AAED,+EAA+E;AAC/E,MAAM,CAAC,MAAM,UAAU,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACvC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACpE,OAAO,gBAAgB,CAAC,GAAG,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;AAC3E,CAAC,CAAA;AAED;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,WAAW,GAAiB,CAAC,GAAG,EAAE,EAAE;IAChD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACnC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAC1C,MAAM,aAAa,GAAkB,EAAE,CAAA;IACvC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC;YAAE,aAAa,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAChE,CAAC;IACD,OAAO,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACpE,CAAC,CAAA;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAiB,CAAC,GAAG,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACtC,IAAI,QAAQ,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,aAAa,GAAkB,EAAE,CAAA;IACvC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC;YAAE,aAAa,CAAC,CAAiB,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;IAC1D,CAAC;IACD,OAAO,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAA;AACtE,CAAC,CAAA;AAED,SAAS,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;AACjD,CAAC;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,kFAAkF;AAClF,MAAM,kBAAkB,GAA2B;IAClD,OAAO,EAAE,IAAI;IACb,MAAM,EAAE,IAAI;IACZ,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,UAAU,EAAE,IAAI;IAChB,QAAQ,EAAE,IAAI;IACd,WAAW,EAAE,IAAI;IACjB,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,IAAI;IACb,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,IAAI;IACX,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,IAAI;IACV,MAAM,EAAE,IAAI;IACZ,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,KAAK,EAAE,IAAI;IACX,QAAQ,EAAE,IAAI;IACd,aAAa,EAAE,IAAI;IACnB,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,WAAW,EAAE,IAAI;IACjB,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,MAAM,EAAE,IAAI;IACZ,eAAe,EAAE,IAAI;IACrB,YAAY,EAAE,IAAI;IAClB,YAAY,EAAE,IAAI;IAClB,UAAU,EAAE,IAAI;IAChB,gBAAgB,EAAE,IAAI;IACtB,cAAc,EAAE,IAAI;IACpB,IAAI,EAAE,IAAI;IACV,QAAQ,EAAE,IAAI;IACd,MAAM,EAAE,IAAI;IACZ,YAAY,EAAE,IAAI;IAClB,cAAc,EAAE,IAAI;IACpB,gBAAgB,EAAE,IAAI;IACtB,cAAc,EAAE,IAAI;IACpB,SAAS,EAAE,IAAI;IACf,KAAK,EAAE,IAAI;IACX,IAAI,EAAE,IAAI;IACV,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE,IAAI;IACrB,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,IAAI;IACb,sBAAsB,EAAE,IAAI;CAC5B,CAAA;AAED,MAAM,kBAAkB,GAA2B,MAAM,CAAC,WAAW,CACpE,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAC1D,CAAA;AAED,uEAAuE;AACvE,MAAM,CAAC,MAAM,WAAW,GAAiB,CAAC,GAAG,EAAE,EAAE;IAChD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IACpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAA;IACvC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,wFAAwF;IACxF,wCAAwC;IACxC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,CAAA;IAC7C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;IACxE,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;IACxE,OAAO,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACpE,CAAC,CAAA;AAED,uEAAuE;AACvE,MAAM,CAAC,MAAM,eAAe,GAAiB,CAAC,GAAG,EAAE,EAAE;IACpD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IACpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAA;IACvC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,CAAA;IAC7C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;IACxE,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;IACxE,OAAO,gBAAgB,CAAC,GAAG,EAAE,kBAAkB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACxE,CAAC,CAAA;AAED,MAAM,wBAAwB,GAA2B;IACxD,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;CACf,CAAA;AACD,MAAM,wBAAwB,GAA2B,MAAM,CAAC,WAAW,CAC1E,MAAM,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAChE,CAAA;AAED,yFAAyF;AACzF,MAAM,CAAC,MAAM,iBAAiB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACtD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,WAAW,GAAmB,CAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAA;IAChF,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG,CAAA;IACpB,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAC1D,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,CAAC,CAAC;YAAE,SAAQ;QAChB,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,4BAA4B,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QACjG,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACpB,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAA;YAC7E,OAAO,GAAG,IAAI,CAAA;QACf,CAAC;IACF,CAAC;IACD,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IACzB,OAAO,gBAAgB,CAAC,GAAG,EAAE,oBAAoB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AAC1E,CAAC,CAAA;AAED,yDAAyD;AACzD,MAAM,CAAC,MAAM,qBAAqB,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC1D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,WAAW,GAAmB,CAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAA;IAChF,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG,CAAA;IACpB,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAC1D,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,CAAC,CAAC;YAAE,SAAQ;QAChB,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CACzB,sEAAsE,EACtE,CAAC,CAAC,EAAE,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,CACvC,CAAA;QACD,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACpB,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAA;YAC7E,OAAO,GAAG,IAAI,CAAA;QACf,CAAC;IACF,CAAC;IACD,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IACzB,OAAO,gBAAgB,CAAC,GAAG,EAAE,wBAAwB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AAC9E,CAAC,CAAA;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC3D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IACpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,SAAS,GAAG,+BAA+B,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IAClD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,EAAE,CAAA;IAClF,IAAI,SAAS,KAAK,MAAM;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;IAC7E,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;IAC1F,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IACnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,6BAA6B,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACnF,CAAC,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACvD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IACpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IACxD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,EAAE,CAAA;IAClF,IAAI,SAAS,KAAK,MAAM;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;IAC7E,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;IAC1F,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IACnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,yBAAyB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AAC/E,CAAC,CAAA;AAED,gEAAgE;AAChE,MAAM,CAAC,MAAM,gBAAgB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACrD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAA;IACxC,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAA;IAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;IACxC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAChD,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IACnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;AACrG,CAAC,CAAA;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,+FAA+F;AAC/F,MAAM,CAAC,MAAM,aAAa,GAAiB,CAAC,GAAG,EAAE,EAAE;IAClD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,sBAAsB,CAAA;IACtD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAC1B,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAC1D,OAAO,aAAa,CAAC,sBAAsB,CAAA;IAC3C,+DAA+D;IAC/D,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,OAAO,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC9D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACnE,OAAO,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACtE,CAAC,CAAA;AAED,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E,sCAAsC;AACtC,MAAM,CAAC,MAAM,aAAa,GAAiC;IAC1D,YAAY,EAAE,SAAS;IACvB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,UAAU;IACzB,cAAc,EAAE,WAAW;IAC3B,cAAc,EAAE,WAAW;IAC3B,cAAc,EAAE,WAAW;IAC3B,kBAAkB,EAAE,eAAe;IACnC,oBAAoB,EAAE,iBAAiB;IACvC,wBAAwB,EAAE,qBAAqB;IAC/C,6BAA6B,EAAE,sBAAsB;IACrD,yBAAyB,EAAE,kBAAkB;IAC7C,qBAAqB,EAAE,gBAAgB;IACvC,gBAAgB,EAAE,aAAa;CAC/B,CAAA;AAED,kGAAkG;AAClG,MAAM,UAAU,8BAA8B,CAAC,OAAe;IAC7D,MAAM,SAAS,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAA;IACjE,QAAQ,OAAO,EAAE,CAAC;QACjB,KAAK,IAAI;YACR,OAAO;gBACN,GAAG,SAAS;gBACZ,WAAW;gBACX,eAAe;gBACf,iBAAiB;gBACjB,qBAAqB;gBACrB,sBAAsB;gBACtB,kBAAkB;gBAClB,gBAAgB;aAChB,CAAA;QACF,KAAK,IAAI;YACR,OAAO,CAAC,GAAG,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAA;QAClD;YACC,OAAO,SAAS,CAAA;IAClB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,SAAS,CAAC,CAAC,aAAa,CAC7B,GAAiB,EACjB,gBAAyC,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC;IAEpF,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAA;QACpB,IAAI,GAAG;YAAE,MAAM,GAAG,CAAA;IACnB,CAAC;AACF,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC7B,OAAO,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAA;AAChD,CAAC;AAiED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,qBAAqB,CACpC,KAAa,EACb,OAAqB,EACrB,OAAkC;IAElC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAA;IAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IAE5D,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IACjC,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,CAAA;IAC7E,CAAC;IAED,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,IAAI,cAAc,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACvC,kFAAkF;QAClF,kFAAkF;QAClF,+BAA+B;QAC/B,OAAO;YACN,IAAI,EAAE,aAAa;YACnB,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE;SAC7E,CAAA;IACF,CAAC;IAED,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;IACpD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,EAAE,CAAA;IACjF,CAAC;IAED,MAAM,WAAW,GAAe,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAA;IAE5F,MAAM,MAAM,GAAa,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAC1F,MAAM,MAAM,GAAe,CAAC,GAAG,WAAW,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAEzE,MAAM,WAAW,GAAG,GAAG,YAAY,GAAG,SAAS,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;IAC/D,MAAM,kBAAkB,GAAG;QAC1B,KAAK,EAAE,YAAY;QACnB,GAAG,OAAO,CAAC,UAAU;KACrB,CAAA;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,EAAE,cAAc,IAAI,OAAO,CAAC,SAAS,CAAA;IACvE,MAAM,MAAM,GAAG,WAAW,OAAO,CAAC,OAAO,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAe;QAC5B,GAAG,EAAE,WAAW;QAChB,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE;QAC3C,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,KAAK,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE;QAC/C,MAAM;QACN,MAAM;KACN,CAAA;IAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAA;AAC1C,CAAC"}

package/out/src/tokenize.d.ts

@@ -0,0 +1,47 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Tokenizer interface for alignment.
+ *
+ *   Two implementations live in the corpus package:
+ *
+ *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
+ *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
+ *        default for in-container alignment tests and as a fallback when no SentencePiece model is
+ *        available.
+ *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
+ *        trained on the corpus. Same interface, different splits. Locked against the corpus version
+ *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
+ *
+ *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
+ *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
+ *   spans, independent of how the tokenizer chose its splits.
+ */
+/** A token with its character span in the source string. */
+export interface TokenSpan {
+    /** The token text, possibly normalized (case unchanged here; tokenizers may differ). */
+    text: string;
+    /** Inclusive start offset (UTF-16 code-unit index) in the source string. */
+    start: number;
+    /** Exclusive end offset in the source string. `text === source.slice(start, end)`. */
+    end: number;
+}
+/** A tokenizer that maps a string to a sequence of `TokenSpan`s. */
+export interface Tokenizer {
+    tokenize(text: string): readonly TokenSpan[];
+}
+/**
+ * Whitespace + punctuation tokenizer (pure JS).
+ *
+ * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}`
+ * marks, plus `'`, `-`, `_`). Everything else — whitespace, punctuation, symbols — is treated as a
+ * separator and **not** emitted as a token. The resulting spans cover the original string only on
+ * token regions; in-between regions belong to no token.
+ *
+ * This is intentionally lossy at the edges (alignment can still label every meaningful span). A
+ * future SentencePiece tokenizer will preserve all bytes via byte-fallback.
+ */
+export declare function whitespaceTokenizer(): Tokenizer;
+//# sourceMappingURL=tokenize.d.ts.map

package/out/src/tokenize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,4DAA4D;AAC5D,MAAM,WAAW,SAAS;IACzB,wFAAwF;IACxF,IAAI,EAAE,MAAM,CAAA;IAEZ,4EAA4E;IAC5E,KAAK,EAAE,MAAM,CAAA;IAEb,sFAAsF;IACtF,GAAG,EAAE,MAAM,CAAA;CACX;AAED,oEAAoE;AACpE,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,SAAS,EAAE,CAAA;CAC5C;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,IAAI,SAAS,CAe/C"}

package/out/src/tokenize.js

@@ -0,0 +1,49 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Tokenizer interface for alignment.
+ *
+ *   Two implementations live in the corpus package:
+ *
+ *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
+ *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
+ *        default for in-container alignment tests and as a fallback when no SentencePiece model is
+ *        available.
+ *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
+ *        trained on the corpus. Same interface, different splits. Locked against the corpus version
+ *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
+ *
+ *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
+ *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
+ *   spans, independent of how the tokenizer chose its splits.
+ */
+/**
+ * Whitespace + punctuation tokenizer (pure JS).
+ *
+ * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}`
+ * marks, plus `'`, `-`, `_`). Everything else — whitespace, punctuation, symbols — is treated as a
+ * separator and **not** emitted as a token. The resulting spans cover the original string only on
+ * token regions; in-between regions belong to no token.
+ *
+ * This is intentionally lossy at the edges (alignment can still label every meaningful span). A
+ * future SentencePiece tokenizer will preserve all bytes via byte-fallback.
+ */
+export function whitespaceTokenizer() {
+    // Maximal runs of letters/digits/marks plus the joiners common to addresses
+    // (apostrophe, hyphen, underscore). Comma/space/period etc. are not in the set.
+    const tokenRe = /[\p{L}\p{N}\p{M}'_-]+/gu;
+    return {
+        tokenize(text) {
+            const out = [];
+            tokenRe.lastIndex = 0;
+            let m;
+            while ((m = tokenRe.exec(text))) {
+                out.push({ text: m[0], start: m.index, end: m.index + m[0].length });
+            }
+            return out;
+        },
+    };
+}
+//# sourceMappingURL=tokenize.js.map

package/out/src/tokenize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../../src/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAmBH;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB;IAClC,4EAA4E;IAC5E,gFAAgF;IAChF,MAAM,OAAO,GAAG,yBAAyB,CAAA;IACzC,OAAO;QACN,QAAQ,CAAC,IAAY;YACpB,MAAM,GAAG,GAAgB,EAAE,CAAA;YAC3B,OAAO,CAAC,SAAS,GAAG,CAAC,CAAA;YACrB,IAAI,CAAyB,CAAA;YAC7B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YACrE,CAAC;YACD,OAAO,GAAG,CAAA;QACX,CAAC;KACD,CAAA;AACF,CAAC"}