@mailwoman/corpus 10.0.0 → 10.1.0

package/out/tools/sub/venue/table.js

@@ -1,108 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   @file The sub-venue lexicon's record schema and the shipped vocabulary it is seeded from — the
- *   emitted artifact's shape plus the designator, modifier and Wikidata-concept tables every build
- *   starts out holding.
- *
- *   The shape follows `@mailwoman/poi-taxonomy`'s `taxonomy.json` idiom exactly — typed records plus a
- *   FLAT phrase array keyed back to a record id, which is what makes a longest-match phrase index cheap
- *   to build over it. {@link SubVenueSurface} is this table's `SynonymEntry`.
- *
- *   The seeds DUPLICATE `neural/venue-structure.ts` knowingly: `@mailwoman/corpus` does not depend on
- *   `@mailwoman/neural` (the dependency runs the other way for the training path, and pulling
- *   onnxruntime into a corpus build to read three string arrays would be absurd), so the shipped
- *   vocabulary is re-declared here. That is a drift surface and it is stated rather than hidden —
- *   `sub-venue-lexicon.test.ts` pins the seed's contents literally, so a change in either place fails a
- *   test rather than passing silently.
- */
-/**
- * This table's own data version. Bump when the source vintages or the build semantics change.
- *
- * `0.2.0` — wave 2: per-region attestation, the phrase-attribution fix (finding 3 above), derived head nouns, the
- * Overture source, and the `promotions[]` receipts section.
- */
-export const SUBVENUE_LEXICON_VERSION = "0.2.0";
-/**
- * Which side of the containment relation a designator names. Mirrors `@mailwoman/osm/sdk`'s `SubVenueTier`; re-declared
- * for the same dependency-direction reason as the seed.
- */
-export const LexiconTier = {
-    SubVenue: "subvenue",
-    Venue: "venue",
-};
-/**
- * The vocabulary that already ships in `neural/venue-structure.ts`, re-declared. See the module docstring for why this
- * duplication exists.
- *
- * `tier` is added here (the shipped list has no such field): the seven WOF placetypes plus `terminal`/`gate` are all
- * venue-INTERIOR, except `campus` and `building`, which name a whole venue as often as a part of one. They are marked
- * `subvenue` anyway, because that is the role the span proposer uses them in — `Building 43, Googleplex` is a unit
- * inside a venue.
- */
-export const SHIPPED_DESIGNATOR_SEED = [
-    { id: "arcade", modifierEligible: true, provenance: ["wof:placetype"] },
-    { id: "building", modifierEligible: false, provenance: ["wof:placetype"] },
-    { id: "campus", modifierEligible: true, provenance: ["wof:placetype"] },
-    { id: "concourse", modifierEligible: true, provenance: ["wof:placetype"] },
-    { id: "enclosure", modifierEligible: false, provenance: ["wof:placetype"] },
-    { id: "gate", modifierEligible: false, provenance: ["osm:aeroway=gate"] },
-    { id: "installation", modifierEligible: false, provenance: ["wof:placetype"] },
-    { id: "terminal", modifierEligible: true, provenance: ["osm:aeroway=terminal"] },
-    { id: "wing", modifierEligible: true, provenance: ["wof:placetype"] },
-];
-/**
- * The shipped positional modifiers, re-declared from `neural/venue-structure.ts`'s `VENUE_STRUCTURE_MODIFIERS`.
- */
-export const SHIPPED_MODIFIER_SEED = [
-    "central",
-    "east",
-    "front",
-    "inner",
-    "lower",
-    "main",
-    "north",
-    "outer",
-    "rear",
-    "south",
-    "upper",
-    "west",
-];
-/**
- * Designators the lexicon ADDS beyond what ships, each with the source that attests it.
- *
- * `platform`, `station` and `airport` come from the OSM extractor's rule table and are the rail/aviation venue-side
- * vocabulary the corpus line needs. `hall` and `satellite` come from Wikidata concepts and from
- * `wof-osm-placetype-map.mdx`'s own "plausible additions" note, which lists `hall` explicitly. `pier` joins them in
- * wave 2 on 282 Overture attestations in the `pier` category plus 162 in the GB extract — the corpus task names `Pier
- * C` as a target shape, so the record has to exist before a slice can generate it.
- *
- * None is `modifierEligible`: that claim needs a confound board per term AND per locale, and `sub-venue-promotions.ts`
- * is where those live. A promotion marks a SURFACE usable; it does not widen the modifier grammar.
- */
-export const PROPOSED_DESIGNATORS = [
-    { id: "airport", tier: LexiconTier.Venue, provenance: ["osm:aeroway=aerodrome"] },
-    { id: "hall", tier: LexiconTier.SubVenue, provenance: ["wikidata:Q240854"] },
-    { id: "pier", tier: LexiconTier.SubVenue, provenance: ["overture:pier"] },
-    { id: "platform", tier: LexiconTier.SubVenue, provenance: ["osm:public_transport=platform", "osm:railway=platform"] },
-    { id: "satellite", tier: LexiconTier.SubVenue, provenance: ["wikidata:Q15990706"] },
-    { id: "station", tier: LexiconTier.Venue, provenance: ["osm:railway=station"] },
-];
-/**
- * `designatorID` → Wikidata QID, mirroring `fetch/wikidata-subvenue.ts`'s `SUBVENUE_CONCEPTS`. Re-declared here so the
- * builder stays a pure function over PARSED input rather than reaching into a fetch module for a constant; the test
- * pins the two against each other.
- */
-export const CONCEPT_QIDS = {
-    terminal: "Q849706",
-    gate: "Q247739",
-    concourse: "Q862212",
-    campus: "Q209465",
-    building: "Q41176",
-    arcade: "Q186637",
-    hall: "Q240854",
-    satellite: "Q15990706",
-};
-//# sourceMappingURL=table.js.map

package/out/tools/sub/venue/table.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"table.js","sourceRoot":"","sources":["../../../../lib/tools/sub/venue/table.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,OAAO,CAAA;AAE/C;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IAC1B,QAAQ,EAAE,UAAU;IACpB,KAAK,EAAE,OAAO;CACL,CAAA;AAgKV;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAI/B;IACJ,EAAE,EAAE,EAAE,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,eAAe,CAAC,EAAE;IACvE,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,eAAe,CAAC,EAAE;IAC1E,EAAE,EAAE,EAAE,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,eAAe,CAAC,EAAE;IACvE,EAAE,EAAE,EAAE,WAAW,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,eAAe,CAAC,EAAE;IAC1E,EAAE,EAAE,EAAE,WAAW,EAAE,gBAAgB,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,eAAe,CAAC,EAAE;IAC3E,EAAE,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,kBAAkB,CAAC,EAAE;IACzE,EAAE,EAAE,EAAE,cAAc,EAAE,gBAAgB,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,eAAe,CAAC,EAAE;IAC9E,EAAE,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,sBAAsB,CAAC,EAAE;IAChF,EAAE,EAAE,EAAE,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,eAAe,CAAC,EAAE;CACrE,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAsB;IACvD,SAAS;IACT,MAAM;IACN,OAAO;IACP,OAAO;IACP,OAAO;IACP,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;CACN,CAAA;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAI5B;IACJ,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,uBAAuB,CAAC,EAAE;IACjF,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,kBAAkB,CAAC,EAAE;IAC5E,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,eAAe,CAAC,EAAE;IACzE,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,+BAA+B,EAAE,sBAAsB,CAAC,EAAE;IACrH,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,WAAW,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,oBAAoB,CAAC,EAAE;IACnF,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,qBAAqB,CAAC,EAAE;CAC/E,CAAA;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,YAAY,GAAqC;IAC7D,QAAQ,EAAE,SAAS;IACnB,IAAI,EAAE,SAAS;IACf,SAAS,EAAE,SAAS;IACpB,MAAM,EAAE,SAAS;IACjB,QAAQ,EAAE,QAAQ;IAClB,MAAM,EAAE,SAAS;IACjB,IAAI,EAAE,SAAS;IACf,SAAS,EAAE,WAAW;CACtB,CAAA"}

package/out/utils/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../lib/utils/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,cAAc,CAAA;AAC5B,cAAc,QAAQ,CAAA;AACtB,cAAc,eAAe,CAAA;AAC7B,cAAc,gBAAgB,CAAA;AAC9B,cAAc,gBAAgB,CAAA;AAC9B,cAAc,cAAc,CAAA;AAC5B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,iBAAiB,CAAA"}

package/out/utils/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/utils/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,cAAc,CAAA;AAC5B,cAAc,QAAQ,CAAA;AACtB,cAAc,eAAe,CAAA;AAC7B,cAAc,gBAAgB,CAAA;AAC9B,cAAc,gBAAgB,CAAA;AAC9B,cAAc,cAAc,CAAA;AAC5B,cAAc,iBAAiB,CAAA;AAC/B,cAAc,iBAAiB,CAAA"}

package/out/utils/parquet.d.ts

@@ -1,178 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Final output sliceer for the corpus pipeline.
- *
- *   Phase 1 (#9) shipped JSONL slices + a Python (PyArrow) converter as the path to binary Parquet —
- *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
- *   writer. The build pipeline no longer touches Python at all in its hot path; the only remaining
- *   Python is the one-shot `train_tokenizer.py` SentencePiece step.
- *
- *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`; parquet-wasm supports SNAPPY,
- *   which is the standard ML-corpus default (and PyArrow's default). Documented in
- *   `DECISIONS.md`.
- *
- *   Layout under `<outputDir>`:
- *
- *   ```
- *   corpus-v<version>/
- *   MANIFEST.json
- *   train/
- *     part-0000.parquet
- *     part-0001.parquet
- *     ...
- *   val/
- *     part-0000.parquet
- *   test/
- *     part-0000.parquet
- * ```
- *
- *   Each slice caps at `rowsPerSlice` (default 1_000_000); within a slice, DuckDB writes row
- *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
- *   slice's path, row count, byte size, and SHA-256 (computed by re-reading the slice once after
- *   close — cheap relative to writing it).
- */
-import { type PathBuilderLike } from "path-ts";
-import type { LabeledRow } from "#types";
-import type { SplitName } from "#utils/split";
-/**
- * Row groups are written at this cadence within a slice.
- */
-export declare const ROW_GROUP_SIZE = 50000;
-/**
- * Escape `value` for a single-quoted SQL string literal; the caller supplies the quotes.
- */
-export declare function escapeSQLString(value: string): string;
-/**
- * An open DuckDB connection, re-exported so consumers of {@link connectDuckDB} can name the type without their own
- * static dependency on the optional peer.
- */
-export type { DuckDBConnection } from "@duckdb/node-api";
-/**
- * Open an in-memory DuckDB connection. `@duckdb/node-api` is an optional peer — lazy import (the pipeline convention),
- * so the heavy native module loads only on the paths that read or write Parquet through DuckDB.
- */
-export declare function connectDuckDB(): Promise<import("@duckdb/node-api").DuckDBConnection>;
-/**
- * Snappy is the codec selected for corpus slices.
- */
-export declare const SLICE_COMPRESSION: "SNAPPY";
-export interface ParquetFieldDefinition {
-    type: "UTF8" | "INT32";
-    compression: typeof SLICE_COMPRESSION;
-    repeated?: boolean;
-    optional?: boolean;
-}
-export type ParquetSchemaDefinition<T> = Record<Extract<keyof T, string>, ParquetFieldDefinition>;
-/**
- * A single Parquet row shape. The index signature allows callers to carry source fields before projection.
- *
- * Optional fields are represented as null in the Arrow table and read back as null.
- */
-export interface ParquetRow {
-    raw: string;
-    tokens: readonly string[];
-    labels: readonly string[];
-    span_starts: readonly number[];
-    span_ends: readonly number[];
-    span_tags: readonly string[];
-    country: string;
-    locale?: string | null;
-    source: string;
-    source_id: string;
-    corpus_version: string;
-    license: string;
-    synth_method?: string | null;
-    synth_base_id?: string | null;
-    [key: string]: unknown;
-}
-/**
- * Column names emitted into every slice. Matches `ParquetRow`.
- */
-export declare const PARQUET_COLUMNS: readonly ["raw", "tokens", "labels", "span_starts", "span_ends", "span_tags", "country", "locale", "source", "source_id", "corpus_version", "license", "synth_method", "synth_base_id"];
-/**
- * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
- * tokens/labels arrays. Compression is per-column SNAPPY.
- */
-export declare const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow>;
-export declare function writeParquetRows(rows: readonly ParquetRow[], path: string): Promise<void>;
-/**
- * Stream rows from a local Parquet file in DuckDB-managed chunks.
- *
- * DuckDB opens the path itself and exposes its DataChunks through `fetchChunk()`. Rows are converted and yielded one
- * chunk at a time; the complete Parquet file and complete result set are never copied into JavaScript memory.
- */
-export declare function streamParquetRows<T>(path: string, columns?: readonly string[], options?: {
-    limit?: number;
-}): AsyncGenerator<T>;
-/**
- * Per-slice metadata captured in `MANIFEST.json`.
- */
-export interface SliceDescriptor {
-    split: SplitName;
-    path: string;
-    format: "parquet";
-    compression: typeof SLICE_COMPRESSION;
-    rows: number;
-    bytes: number;
-    sha256: string;
-    first_source_id: string;
-    last_source_id: string;
-    /**
-     * The slice's corpus source slug, when the writer knows it. `audit.ts` prefers this over inferring the source from
-     * `first_source_id`'s prefix; `writeSlices` itself writes multi-source slices and leaves it unset.
-     */
-    source?: string;
-}
-export interface SliceManifest {
-    corpus_version: string;
-    schema: readonly string[];
-    rows_per_slice: number;
-    row_group_size: number;
-    slices: SliceDescriptor[];
-    counts: Record<SplitName, number>;
-    total_rows: number;
-}
-export interface WriteSlicesOptions {
-    /**
-     * Root output directory; corpus version dir is created beneath.
-     */
-    outputDir: PathBuilderLike;
-    /**
-     * Corpus version stamped onto rows + into the output directory name.
-     */
-    corpusVersion: string;
-    /**
-     * Max rows per `.parquet` slice. Default 1_000_000 per the Phase 1 plan.
-     */
-    rowsPerSlice?: number;
-}
-/**
- * Pre-partitioned labeled-row streams, one per split. Callers (`buildCorpus`) decide each row's split inline at align
- * time via `splitForRow` and route rows to the matching stream, eliminating the prior `Map<source_id, SplitName>` O(n)
- * lookup table.
- *
- * Splits with no rows can be omitted (or passed as an empty iterable); `writeSlices` skips them.
- */
-export type PerSplitRows = Partial<Record<SplitName, AsyncIterable<LabeledRow>>>;
-/**
- * Project a labeled row to the Parquet schema.
- *
- * The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
- * from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the slice (the "builders
- * before parquet = silent loss" hazard). Loud failure, naming the row, instead.
- */
-export declare function rowToParquet(row: LabeledRow): ParquetRow;
-/**
- * Stream labeled rows into `.parquet` slices, one set of slices per split. Splits are processed sequentially so that
- * only one slice is open at a time. Rows are staged to newline-delimited JSON with backpressure, then DuckDB writes the
- * Parquet file from disk.
- *
- * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
- * because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
- * for the new wire-up.
- */
-export declare function writeSlices(perSplit: PerSplitRows, opts: WriteSlicesOptions): Promise<SliceManifest>;
-//# sourceMappingURL=parquet.d.ts.map

package/out/utils/parquet.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../lib/utils/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAUH,OAAO,EAAQ,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAE7C;;GAEG;AACH,eAAO,MAAM,cAAc,QAAS,CAAA;AAEpC;;GAEG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;;GAGG;AACH,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAExD;;;GAGG;AACH,wBAAsB,aAAa,IAAI,OAAO,CAAC,OAAO,kBAAkB,EAAE,gBAAgB,CAAC,CAK1F;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,EAAG,QAAiB,CAAA;AAElD,MAAM,WAAW,sBAAsB;IAEtC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAA;IACtB,WAAW,EAAE,OAAO,iBAAiB,CAAA;IACrC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,MAAM,uBAAuB,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,EAAE,sBAAsB,CAAC,CAAA;AAEjG;;;;GAIG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;IAC9B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,YAC3B,KAAK,EACL,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,WAAW,EACX,WAAW,EACX,SAAS,EACT,QAAQ,EACR,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,cAAc,EACd,eAAe,CACN,CAAA;AAMV;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,uBAAuB,CAAC,UAAU,CAkBlE,CAAA;AAmFD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAa/F;AAYD;;;;;GAKG;AACH,wBAAuB,iBAAiB,CAAC,CAAC,EACzC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,EAC3B,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAC1B,cAAc,CAAC,CAAC,CAAC,CAgCnB;AAyCD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,SAAS,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,CAAA;IACjB,WAAW,EAAE,OAAO,iBAAiB,CAAA;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,eAAe,EAAE,MAAM,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;IACtB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,aAAa;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,eAAe,EAAE,CAAA;IACzB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,EAAE,eAAe,CAAA;IAE1B;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAEhF;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,UAAU,CAqCxD;AAED;;;;;;;;GAQG;AACH,wBAAsB,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,aAAa,CAAC,CAoH1G"}

package/out/utils/parquet.js

@@ -1,371 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Final output sliceer for the corpus pipeline.
- *
- *   Phase 1 (#9) shipped JSONL slices + a Python (PyArrow) converter as the path to binary Parquet —
- *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
- *   writer. The build pipeline no longer touches Python at all in its hot path; the only remaining
- *   Python is the one-shot `train_tokenizer.py` SentencePiece step.
- *
- *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`; parquet-wasm supports SNAPPY,
- *   which is the standard ML-corpus default (and PyArrow's default). Documented in
- *   `DECISIONS.md`.
- *
- *   Layout under `<outputDir>`:
- *
- *   ```
- *   corpus-v<version>/
- *   MANIFEST.json
- *   train/
- *     part-0000.parquet
- *     part-0001.parquet
- *     ...
- *   val/
- *     part-0000.parquet
- *   test/
- *     part-0000.parquet
- * ```
- *
- *   Each slice caps at `rowsPerSlice` (default 1_000_000); within a slice, DuckDB writes row
- *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
- *   slice's path, row count, byte size, and SHA-256 (computed by re-reading the slice once after
- *   close — cheap relative to writing it).
- */
-import { tryStat } from "@mailwoman/core/fs/readers";
-import { openWriteStream } from "@mailwoman/core/fs/streams";
-import { temporaryDirectory } from "@mailwoman/core/fs/temporary";
-import { writeLocalBuffer, writeLocalJSONFile, makeDirectories } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { once } from "@mailwoman/core/utils/events";
-import { Field, Int32, List, Table as ArrowTable, tableToIPC, Utf8, vectorFromArray } from "apache-arrow";
-import { Compression, Table as WasmTable, WriterPropertiesBuilder, writeParquet } from "parquet-wasm";
-import { join } from "path-ts";
-/**
- * Row groups are written at this cadence within a slice.
- */
-export const ROW_GROUP_SIZE = 50_000;
-/**
- * Escape `value` for a single-quoted SQL string literal; the caller supplies the quotes.
- */
-export function escapeSQLString(value) {
-    return value.replaceAll("'", "''");
-}
-/**
- * Open an in-memory DuckDB connection. `@duckdb/node-api` is an optional peer — lazy import (the pipeline convention),
- * so the heavy native module loads only on the paths that read or write Parquet through DuckDB.
- */
-export async function connectDuckDB() {
-    const { DuckDBInstance } = await import("@duckdb/node-api");
-    const instance = await DuckDBInstance.create();
-    return await instance.connect();
-}
-/**
- * Snappy is the codec selected for corpus slices.
- */
-export const SLICE_COMPRESSION = "SNAPPY";
-/**
- * Column names emitted into every slice. Matches `ParquetRow`.
- */
-export const PARQUET_COLUMNS = [
-    "raw",
-    "tokens",
-    "labels",
-    "span_starts",
-    "span_ends",
-    "span_tags",
-    "country",
-    "locale",
-    "source",
-    "source_id",
-    "corpus_version",
-    "license",
-    "synth_method",
-    "synth_base_id",
-];
-/* oxlint-disable unicorn/text-encoding-identifier-case -- `"UTF8"` below is a ParquetType enum member,
-   not a text-encoding identifier. Lowercasing it does not type-check against ParquetSchemaDefinition,
-   and the rule has no way to tell the two apart. */
-/**
- * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
- * tokens/labels arrays. Compression is per-column SNAPPY.
- */
-export const LABELED_ROW_SCHEMA = {
-    raw: { type: "UTF8", compression: SLICE_COMPRESSION },
-    tokens: { type: "UTF8", repeated: true, compression: SLICE_COMPRESSION },
-    labels: { type: "UTF8", repeated: true, compression: SLICE_COMPRESSION },
-    // v0.5.0 char-offset label spans (#519): parallel arrays over `raw` (UTF-16 code units,
-    // [start, end) exclusive-end, sorted, non-overlapping). INT32 — raw is a short address string,
-    // and INT32 round-trips as `number` where parquetjs INT64 would surface bigint.
-    span_starts: { type: "INT32", repeated: true, compression: SLICE_COMPRESSION },
-    span_ends: { type: "INT32", repeated: true, compression: SLICE_COMPRESSION },
-    span_tags: { type: "UTF8", repeated: true, compression: SLICE_COMPRESSION },
-    country: { type: "UTF8", compression: SLICE_COMPRESSION },
-    locale: { type: "UTF8", compression: SLICE_COMPRESSION, optional: true },
-    source: { type: "UTF8", compression: SLICE_COMPRESSION },
-    source_id: { type: "UTF8", compression: SLICE_COMPRESSION },
-    corpus_version: { type: "UTF8", compression: SLICE_COMPRESSION },
-    license: { type: "UTF8", compression: SLICE_COMPRESSION },
-    synth_method: { type: "UTF8", compression: SLICE_COMPRESSION, optional: true },
-    synth_base_id: { type: "UTF8", compression: SLICE_COMPRESSION, optional: true },
-};
-const stringListType = new List(new Field("item", new Utf8(), true));
-const int32ListType = new List(new Field("item", new Int32(), true));
-const PARQUET_COLUMN_TYPES = {
-    raw: "VARCHAR",
-    tokens: "VARCHAR[]",
-    labels: "VARCHAR[]",
-    span_starts: "INTEGER[]",
-    span_ends: "INTEGER[]",
-    span_tags: "VARCHAR[]",
-    country: "VARCHAR",
-    locale: "VARCHAR",
-    source: "VARCHAR",
-    source_id: "VARCHAR",
-    corpus_version: "VARCHAR",
-    license: "VARCHAR",
-    synth_method: "VARCHAR",
-    synth_base_id: "VARCHAR",
-};
-function parquetTable(rows) {
-    return new ArrowTable({
-        raw: vectorFromArray(rows.map((row) => row.raw), new Utf8()),
-        tokens: vectorFromArray(rows.map((row) => row.tokens), stringListType),
-        labels: vectorFromArray(rows.map((row) => row.labels), stringListType),
-        span_starts: vectorFromArray(rows.map((row) => row.span_starts), int32ListType),
-        span_ends: vectorFromArray(rows.map((row) => row.span_ends), int32ListType),
-        span_tags: vectorFromArray(rows.map((row) => row.span_tags), stringListType),
-        country: vectorFromArray(rows.map((row) => row.country), new Utf8()),
-        locale: vectorFromArray(rows.map((row) => row.locale ?? null), new Utf8()),
-        source: vectorFromArray(rows.map((row) => row.source), new Utf8()),
-        source_id: vectorFromArray(rows.map((row) => row.source_id), new Utf8()),
-        corpus_version: vectorFromArray(rows.map((row) => row.corpus_version), new Utf8()),
-        license: vectorFromArray(rows.map((row) => row.license), new Utf8()),
-        synth_method: vectorFromArray(rows.map((row) => row.synth_method ?? null), new Utf8()),
-        synth_base_id: vectorFromArray(rows.map((row) => row.synth_base_id ?? null), new Utf8()),
-    });
-}
-export async function writeParquetRows(rows, path) {
-    const arrow = parquetTable(rows);
-    const wasmTable = WasmTable.fromIPCStream(tableToIPC(arrow, "stream"));
-    const properties = new WriterPropertiesBuilder()
-        .setCompression(Compression.SNAPPY)
-        .setCreatedBy("mailwoman")
-        .setMaxRowGroupSize(ROW_GROUP_SIZE)
-        .build();
-    // parquet-wasm serializes key-value metadata through a hash map, whose order is not stable between writes.
-    // Slice identity and provenance live in MANIFEST.json, so omitting file metadata preserves deterministic bytes.
-    await writeLocalBuffer(writeParquet(wasmTable, properties), path);
-}
-function normalizeDuckDBValue(value) {
-    if (value && typeof value === "object" && "items" in value && Array.isArray(value.items)) {
-        return value.items.map(normalizeDuckDBValue);
-    }
-    if (Array.isArray(value))
-        return value.map(normalizeDuckDBValue);
-    return value;
-}
-/**
- * Stream rows from a local Parquet file in DuckDB-managed chunks.
- *
- * DuckDB opens the path itself and exposes its DataChunks through `fetchChunk()`. Rows are converted and yielded one
- * chunk at a time; the complete Parquet file and complete result set are never copied into JavaScript memory.
- */
-export async function* streamParquetRows(path, columns, options) {
-    const db = await connectDuckDB();
-    const projection = columns?.length ? columns.map(escapeSQLIdentifier).join(", ") : "*";
-    const limit = options?.limit;
-    const limitClause = limit === undefined ? "" : ` LIMIT ${validateLimit(limit)}`;
-    const sql = `SELECT ${projection} FROM read_parquet('${escapeSQLString(path)}')${limitClause}`;
-    try {
-        const stream = await db.stream(sql);
-        const columnNames = stream.columnNames();
-        for (let chunk = await stream.fetchChunk(); chunk && chunk.rowCount > 0; chunk = await stream.fetchChunk()) {
-            const rows = chunk.getRowObjects(columnNames);
-            for (const row of rows) {
-                yield Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeDuckDBValue(value)]));
-            }
-        }
-    }
-    catch (error) {
-        if (columns && /Binder Error|Referenced column|does not exist/.test(String(error))) {
-            const missing = String(error).match(/(?:Referenced column|field named) ["']([^"']+)["']/)?.[1] ?? columns.join(", ");
-            throw new Error(`Parquet projection requested column absent from the file schema: ${missing}`, {
-                cause: error,
-            });
-        }
-        throw error;
-    }
-    finally {
-        db.closeSync();
-    }
-}
-function escapeSQLIdentifier(value) {
-    return `"${value.replaceAll('"', '""')}"`;
-}
-function validateLimit(value) {
-    if (!Number.isSafeInteger(value) || value < 0)
-        throw new Error(`Parquet row limit must be a non-negative integer: ${value}`);
-    return value;
-}
-async function writeStagedParquet(stagePath, outputPath) {
-    const db = await connectDuckDB();
-    const columns = [...PARQUET_COLUMNS];
-    const columnsLiteral = "{" + columns.map((column) => `'${column}': '${PARQUET_COLUMN_TYPES[column]}'`).join(", ") + "}";
-    const selectList = columns.map(escapeSQLIdentifier).join(", ");
-    try {
-        await db.run("SET preserve_insertion_order=true");
-        await db.run(`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath)}', ` +
-            `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
-            `TO '${escapeSQLString(outputPath)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${ROW_GROUP_SIZE})`);
-    }
-    finally {
-        db.closeSync();
-    }
-}
-async function writeStagedRow(stage, row) {
-    if (!stage.write(JSON.stringify(row) + "\n")) {
-        await once(stage, "drain");
-    }
-}
-/**
- * Project a labeled row to the Parquet schema.
- *
- * The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
- * from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the slice (the "builders
- * before parquet = silent loss" hazard). Loud failure, naming the row, instead.
- */
-export function rowToParquet(row) {
-    const { span_starts, span_ends, span_tags } = row;
-    if (span_starts === undefined || span_ends === undefined || span_tags === undefined) {
-        throw new Error(`rowToParquet: row is missing the char-offset span triple (#519) — ` +
-            `span_starts=${span_starts !== undefined} span_ends=${span_ends !== undefined} span_tags=${span_tags !== undefined} ` +
-            `(source=${row.source}, source_id=${row.source_id}). ` +
-            `Every parquet-bound row must carry span_starts/span_ends/span_tags; ` +
-            `producers that emit tokens/labels only have not migrated to the v0.5.0 format.`);
-    }
-    if (span_starts.length !== span_ends.length || span_starts.length !== span_tags.length) {
-        throw new Error(`rowToParquet: span triple arrays are not parallel — ` +
-            `starts=${span_starts.length} ends=${span_ends.length} tags=${span_tags.length} ` +
-            `(source=${row.source}, source_id=${row.source_id})`);
-    }
-    return {
-        raw: row.raw,
-        tokens: row.tokens,
-        labels: row.labels,
-        span_starts,
-        span_ends,
-        span_tags,
-        country: row.country,
-        locale: row.locale ?? null,
-        source: row.source,
-        source_id: row.source_id,
-        corpus_version: row.corpus_version,
-        license: row.license,
-        synth_method: row.synth?.method ?? null,
-        synth_base_id: row.synth?.base_source_id ?? null,
-    };
-}
-/**
- * Stream labeled rows into `.parquet` slices, one set of slices per split. Splits are processed sequentially so that
- * only one slice is open at a time. Rows are staged to newline-delimited JSON with backpressure, then DuckDB writes the
- * Parquet file from disk.
- *
- * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
- * because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
- * for the new wire-up.
- */
-export async function writeSlices(perSplit, opts) {
-    const rowsPerSlice = opts.rowsPerSlice ?? 1_000_000;
-    const corpusDir = join(opts.outputDir, `corpus-v${opts.corpusVersion}`);
-    await makeDirectories(corpusDir);
-    const slices = [];
-    const counts = { train: 0, val: 0, test: 0 };
-    let totalRows = 0;
-    for (const split of ["train", "val", "test"]) {
-        const rows = perSplit[split];
-        if (!rows)
-            continue;
-        await using staging = await temporaryDirectory(`mailwoman-parquet-${split}-`);
-        let sliceIndex = 0;
-        let path = "";
-        let stagePath = "";
-        let sliceRows = 0;
-        let stage = null;
-        let firstSourceID = "";
-        let lastSourceID = "";
-        const openSlice = async () => {
-            const splitDir = join(corpusDir, split);
-            await makeDirectories(splitDir);
-            path = join(splitDir, `part-${String(sliceIndex).padStart(4, "0")}.parquet`);
-            stagePath = staging.resolve(`part-${String(sliceIndex).padStart(4, "0")}.ndjson`);
-            stage = staging.use(openWriteStream(stagePath));
-            sliceRows = 0;
-            firstSourceID = "";
-            lastSourceID = "";
-        };
-        const closeSlice = async () => {
-            const activeStage = stage;
-            if (!activeStage)
-                return;
-            await new Promise((resolve, reject) => {
-                activeStage.end((error) => (error ? reject(error) : resolve()));
-            });
-            if (sliceRows) {
-                await writeStagedParquet(stagePath, path);
-                const fileStat = await tryStat(path);
-                const sha256 = await sha256File(path);
-                slices.push({
-                    split,
-                    path,
-                    format: "parquet",
-                    compression: SLICE_COMPRESSION,
-                    rows: sliceRows,
-                    bytes: fileStat?.size ?? 0,
-                    sha256,
-                    first_source_id: firstSourceID,
-                    last_source_id: lastSourceID,
-                });
-            }
-            stage = null;
-            sliceRows = 0;
-        };
-        for await (const row of rows) {
-            if (!path) {
-                await openSlice();
-            }
-            const pq = rowToParquet(row);
-            if (!stage)
-                throw new Error("Parquet slice writer was not opened");
-            await writeStagedRow(stage, pq);
-            sliceRows++;
-            if (sliceRows === 1) {
-                firstSourceID = row.source_id;
-            }
-            lastSourceID = row.source_id;
-            counts[split]++;
-            totalRows++;
-            if (sliceRows >= rowsPerSlice) {
-                await closeSlice();
-                sliceIndex++;
-                path = "";
-            }
-        }
-        await closeSlice();
-    }
-    slices.sort((a, b) => (a.split === b.split ? a.path.localeCompare(b.path) : a.split.localeCompare(b.split)));
-    const manifest = {
-        corpus_version: opts.corpusVersion,
-        schema: PARQUET_COLUMNS,
-        rows_per_slice: rowsPerSlice,
-        row_group_size: ROW_GROUP_SIZE,
-        slices,
-        counts,
-        total_rows: totalRows,
-    };
-    await writeLocalJSONFile(manifest, corpusDir, "MANIFEST.json");
-    return manifest;
-}
-//# sourceMappingURL=parquet.js.map