@mailwoman/corpus 2.0.0

package/out/src/parquet-wrapper/writer.d.ts

@@ -0,0 +1,41 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed Parquet writer. Two static constructors mirror the base class:
+ *
+ *   - `openStream`: wrap an existing writable stream.
+ *   - `openFile`: open a path on disk, ensuring the parent directory exists first.
+ *
+ *   Implements `AsyncDisposable` so `await using writer = await ParquetWriter.openFile(...)` flushes
+ *   and closes cleanly. `close()` internally serializes against any in-flight flush so back-to-back
+ *   dispose calls don't race.
+ *
+ *   Salvaged 2026-05-17 from `isp-nexus/universe@6eeb7bd99643a6d62a8b8abbd50968a1e492b90b`
+ *   `sdk/parquet/writer.ts` (originally copyright OpenISP, Inc.; both projects are AGPL-3.0). One
+ *   trim relative to the original: dropped the `@isp.nexus/core/polyfills/promises/withResolvers`
+ *   import — Node 22 (mailwoman's runtime) has `Promise.withResolvers` natively.
+ */
+import { ParquetWriter as BaseParquetWriter } from "@dsnp/parquetjs";
+import type { WriterOptions } from "@dsnp/parquetjs/dist/lib/declare.js";
+import { type WriteStreamMinimal } from "@dsnp/parquetjs/dist/lib/util.js";
+import { type ParquetRecordLike, ParquetSchema, type ParquetSchemaDefinition, ParquetSchemaDefinitionCache } from "./schema.js";
+/** A typed Parquet writer, wrapping the base Parquet writer. */
+export declare class ParquetWriter<T extends ParquetRecordLike> extends BaseParquetWriter implements AsyncDisposable {
+    #private;
+    schema: ParquetSchema<T>;
+    protected static readonly SchemaDefinitionCache: ParquetSchemaDefinitionCache;
+    static openStream<T extends ParquetRecordLike>(schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>, outputStream: WriteStreamMinimal, opts?: WriterOptions): Promise<ParquetWriter<T>>;
+    /** Convenience method to create a new buffered parquet writer that writes to the specified file. */
+    static openFile<T extends ParquetRecordLike>(schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>, sourcePath: string | Buffer | URL, opts?: WriterOptions): Promise<ParquetWriter<T>>;
+    /** Set a metadata key-value pair on the writer. */
+    setMetadata(key: string, value: string): void;
+    /** Append a row to the buffer. If the buffer is full, the data will be written to disk. */
+    appendRow(row: T): Promise<void>;
+    /** Flush all buffered data to disk, close the file, and release resources. */
+    close(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=writer.d.ts.map

package/out/src/parquet-wrapper/writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qCAAqC,CAAA;AACxE,OAAO,EAAU,KAAK,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAIlF,OAAO,EACN,KAAK,iBAAiB,EACtB,aAAa,EACb,KAAK,uBAAuB,EAC5B,4BAA4B,EAC5B,MAAM,aAAa,CAAA;AAEpB,gEAAgE;AAChE,qBAAa,aAAa,CAAC,CAAC,SAAS,iBAAiB,CAAE,SAAQ,iBAAkB,YAAW,eAAe;;IACnG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAA;IAChC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,+BAAqC;WAG9D,UAAU,CAAC,CAAC,SAAS,iBAAiB,EAC3D,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,EACzD,YAAY,EAAE,kBAAkB,EAChC,IAAI,GAAE,aAAkB,GACtB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAW5B,oGAAoG;WAC9E,QAAQ,CAAC,CAAC,SAAS,iBAAiB,EACzD,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,EACzD,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,GAAG,EACjC,IAAI,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAU5B,mDAAmD;IACnC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAI7D,2FAA2F;IACrE,SAAS,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAItD,8EAA8E;IACxD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAa/B,CAAC,MAAM,CAAC,YAAY,CAAC;IAIrB,OAAO;CAGpB"}

package/out/src/parquet-wrapper/writer.js

@@ -0,0 +1,71 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Typed Parquet writer. Two static constructors mirror the base class:
+ *
+ *   - `openStream`: wrap an existing writable stream.
+ *   - `openFile`: open a path on disk, ensuring the parent directory exists first.
+ *
+ *   Implements `AsyncDisposable` so `await using writer = await ParquetWriter.openFile(...)` flushes
+ *   and closes cleanly. `close()` internally serializes against any in-flight flush so back-to-back
+ *   dispose calls don't race.
+ *
+ *   Salvaged 2026-05-17 from `isp-nexus/universe@6eeb7bd99643a6d62a8b8abbd50968a1e492b90b`
+ *   `sdk/parquet/writer.ts` (originally copyright OpenISP, Inc.; both projects are AGPL-3.0). One
+ *   trim relative to the original: dropped the `@isp.nexus/core/polyfills/promises/withResolvers`
+ *   import — Node 22 (mailwoman's runtime) has `Promise.withResolvers` natively.
+ */
+import { ParquetWriter as BaseParquetWriter } from "@dsnp/parquetjs";
+import { osopen } from "@dsnp/parquetjs/dist/lib/util.js";
+import { ParquetEnvelopeWriter } from "@dsnp/parquetjs/dist/lib/writer.js";
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { ParquetSchema, ParquetSchemaDefinitionCache, } from "./schema.js";
+/** A typed Parquet writer, wrapping the base Parquet writer. */
+export class ParquetWriter extends BaseParquetWriter {
+    static SchemaDefinitionCache = new ParquetSchemaDefinitionCache();
+    #flushing = Promise.resolve();
+    static async openStream(schemaLike, outputStream, opts = {}) {
+        const schema = schemaLike instanceof ParquetSchema
+            ? schemaLike
+            : ParquetWriter.SchemaDefinitionCache.findOrCreateSchema(schemaLike);
+        const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
+        return new ParquetWriter(schema, envelopeWriter, opts);
+    }
+    /** Convenience method to create a new buffered parquet writer that writes to the specified file. */
+    static async openFile(schemaLike, sourcePath, opts) {
+        if (typeof sourcePath === "string") {
+            await fs.mkdir(path.dirname(sourcePath), { recursive: true });
+        }
+        const outputStream = await osopen(sourcePath, opts);
+        return ParquetWriter.openStream(schemaLike, outputStream, opts);
+    }
+    // @note This fixes invalid Markdown in the base class JSDoc.
+    /** Set a metadata key-value pair on the writer. */
+    setMetadata(key, value) {
+        return super.setMetadata(key, value);
+    }
+    /** Append a row to the buffer. If the buffer is full, the data will be written to disk. */
+    async appendRow(row) {
+        return super.appendRow(row);
+    }
+    /** Flush all buffered data to disk, close the file, and release resources. */
+    async close() {
+        await this.#flushing;
+        if (this.closed)
+            return;
+        const { promise, resolve, reject } = Promise.withResolvers();
+        super.close().then(resolve, reject);
+        this.#flushing = promise;
+        return this.#flushing;
+    }
+    async [Symbol.asyncDispose]() {
+        return this.close();
+    }
+    async dispose() {
+        return this[Symbol.asyncDispose]();
+    }
+}
+//# sourceMappingURL=writer.js.map

package/out/src/parquet-wrapper/writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,MAAM,EAA2B,MAAM,kCAAkC,CAAA;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAC1E,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AACjC,OAAO,EAEN,aAAa,EAEb,4BAA4B,GAC5B,MAAM,aAAa,CAAA;AAEpB,gEAAgE;AAChE,MAAM,OAAO,aAA2C,SAAQ,iBAAiB;IAEtE,MAAM,CAAU,qBAAqB,GAAG,IAAI,4BAA4B,EAAE,CAAA;IACpF,SAAS,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAA;IAE5C,MAAM,CAAU,KAAK,CAAC,UAAU,CAC/B,UAAyD,EACzD,YAAgC,EAChC,OAAsB,EAAE;QAExB,MAAM,MAAM,GACX,UAAU,YAAY,aAAa;YAClC,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,aAAa,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAA;QAEtE,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;QAEzF,OAAO,IAAI,aAAa,CAAI,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;IAC1D,CAAC;IAED,oGAAoG;IACpG,MAAM,CAAU,KAAK,CAAC,QAAQ,CAC7B,UAAyD,EACzD,UAAiC,EACjC,IAAoB;QAEpB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;QACnD,OAAO,aAAa,CAAC,UAAU,CAAI,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;IACnE,CAAC;IAED,6DAA6D;IAC7D,mDAAmD;IACnC,WAAW,CAAC,GAAW,EAAE,KAAa;QACrD,OAAO,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACrC,CAAC;IAED,2FAA2F;IAC3E,KAAK,CAAC,SAAS,CAAC,GAAM;QACrC,OAAO,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,8EAA8E;IAC9D,KAAK,CAAC,KAAK;QAC1B,MAAM,IAAI,CAAC,SAAS,CAAA;QACpB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAM;QAEvB,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAA;QAElE,KAAK,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAEnC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAA;QAExB,OAAO,IAAI,CAAC,SAAS,CAAA;IACtB,CAAC;IAEM,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;IACnC,CAAC"}

package/out/src/parquet.d.ts

@@ -0,0 +1,122 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Final output sharder for the corpus pipeline.
+ *
+ *   Phase 1 (#9) shipped JSONL shards + a Python (PyArrow) converter as the path to binary Parquet —
+ *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
+ *   writer based on the salvaged `@dsnp/parquetjs` wrapper from isp-nexus (now in
+ *   `./parquet-wrapper/`). The build pipeline no longer touches Python at all in its hot path; the
+ *   only remaining Python is the one-shot `train_tokenizer.py` SentencePiece step.
+ *
+ *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`, but `@dsnp/parquetjs` 1.7.0 only
+ *   supports UNCOMPRESSED / GZIP / SNAPPY / BROTLI (see `node_modules/@dsnp/parquetjs/dist/lib/
+ *   compression.js`). SNAPPY is the standard ML-corpus default (PyArrow's default too) and is the
+ *   closest substitute on speed; revisit if @dsnp/parquetjs gains zstd support. Documented in
+ *   `DECISIONS.md`.
+ *
+ *   Layout under `<outputDir>`:
+ *
+ *   ```
+ *   corpus-v<version>/
+ *   MANIFEST.json
+ *   train/
+ *     part-0000.parquet
+ *     part-0001.parquet
+ *     ...
+ *   val/
+ *     part-0000.parquet
+ *   test/
+ *     part-0000.parquet
+ * ```
+ *
+ *   Each shard caps at `rowsPerShard` (default 1_000_000); within a shard, parquetjs flushes row
+ *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
+ *   shard's path, row count, byte size, and SHA-256 (computed by re-reading the shard once after
+ *   close — cheap relative to writing it).
+ */
+import { type ParquetSchemaDefinition } from "./parquet-wrapper/index.js";
+import type { SplitName } from "./split.js";
+import type { LabeledRow } from "./types.js";
+/** Row groups flush at this many rows (parquetjs internal cadence within a shard). */
+export declare const ROW_GROUP_SIZE = 50000;
+/** Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0. */
+export declare const SHARD_COMPRESSION: "SNAPPY";
+/**
+ * A single Parquet-style row shape. The `[key: string]: unknown` index signature is required for
+ * compatibility with `ParquetRecordLike` in the wrapper — parquetjs accepts any string key on
+ * rows.
+ */
+export interface ParquetRow {
+    raw: string;
+    tokens: readonly string[];
+    labels: readonly string[];
+    country: string;
+    locale: string | null;
+    source: string;
+    source_id: string;
+    corpus_version: string;
+    license: string;
+    synth_method: string | null;
+    synth_base_id: string | null;
+    [key: string]: unknown;
+}
+/** Column names emitted into every shard. Matches `ParquetRow`. */
+export declare const PARQUET_COLUMNS: readonly ["raw", "tokens", "labels", "country", "locale", "source", "source_id", "corpus_version", "license", "synth_method", "synth_base_id"];
+/**
+ * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8
+ * columns capture tokens/labels arrays. Compression is per-column SNAPPY.
+ */
+export declare const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow>;
+/** Per-shard metadata captured in `MANIFEST.json`. */
+export interface ShardDescriptor {
+    split: SplitName;
+    path: string;
+    format: "parquet";
+    compression: typeof SHARD_COMPRESSION;
+    rows: number;
+    bytes: number;
+    sha256: string;
+    first_source_id: string;
+    last_source_id: string;
+}
+export interface ShardManifest {
+    corpus_version: string;
+    schema: readonly string[];
+    rows_per_shard: number;
+    row_group_size: number;
+    shards: ShardDescriptor[];
+    counts: Record<SplitName, number>;
+    total_rows: number;
+}
+export interface WriteShardsOptions {
+    /** Root output directory; corpus version dir is created beneath. */
+    outputDir: string;
+    /** Corpus version stamped onto rows + into the output directory name. */
+    corpusVersion: string;
+    /** Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan. */
+    rowsPerShard?: number;
+}
+/**
+ * Pre-partitioned labeled-row streams, one per split. Callers (`buildCorpus`) decide each row's
+ * split inline at align time via `splitForRow` and route rows to the matching stream, eliminating
+ * the prior `Map<source_id, SplitName>` O(n) lookup table.
+ *
+ * Splits with no rows can be omitted (or passed as an empty iterable); `writeShards` skips them.
+ */
+export type PerSplitRows = Partial<Record<SplitName, AsyncIterable<LabeledRow>>>;
+/** Project a labeled row to the Parquet schema. */
+export declare function rowToParquet(row: LabeledRow): ParquetRow;
+/**
+ * Stream labeled rows into `.parquet` shards, one set of shards per split. Splits are processed
+ * sequentially so that only one shard writer is open at a time — memory cost is bounded by the
+ * parquetjs row-group buffer (~`ROW_GROUP_SIZE × row_size`), not by the labeled-row count.
+ *
+ * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior
+ * `splitFor(sourceId)` callback is gone because pre-partitioning at the caller eliminates the O(n)
+ * `Map<source_id, SplitName>` it required. See `buildCorpus` for the new wire-up.
+ */
+export declare function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptions): Promise<ShardManifest>;
+//# sourceMappingURL=parquet.d.ts.map

package/out/src/parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAMH,OAAO,EAAiB,KAAK,uBAAuB,EAAE,MAAM,4BAA4B,CAAA;AACxF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE5C,sFAAsF;AACtF,eAAO,MAAM,cAAc,QAAS,CAAA;AAEpC,mFAAmF;AACnF,eAAO,MAAM,iBAAiB,EAAG,QAAiB,CAAA;AAElD;;;;GAIG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED,mEAAmE;AACnE,eAAO,MAAM,eAAe,gJAYlB,CAAA;AAEV;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,uBAAuB,CAAC,UAAU,CAYlE,CAAA;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,SAAS,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,CAAA;IACjB,WAAW,EAAE,OAAO,iBAAiB,CAAA;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,eAAe,EAAE,MAAM,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,eAAe,EAAE,CAAA;IACzB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,kBAAkB;IAClC,oEAAoE;IACpE,SAAS,EAAE,MAAM,CAAA;IAEjB,yEAAyE;IACzE,aAAa,EAAE,MAAM,CAAA;IAErB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAEhF,mDAAmD;AACnD,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,UAAU,CAcxD;AAwBD;;;;;;;;GAQG;AACH,wBAAsB,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,aAAa,CAAC,CAwF1G"}

package/out/src/parquet.js

@@ -0,0 +1,220 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Final output sharder for the corpus pipeline.
+ *
+ *   Phase 1 (#9) shipped JSONL shards + a Python (PyArrow) converter as the path to binary Parquet —
+ *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
+ *   writer based on the salvaged `@dsnp/parquetjs` wrapper from isp-nexus (now in
+ *   `./parquet-wrapper/`). The build pipeline no longer touches Python at all in its hot path; the
+ *   only remaining Python is the one-shot `train_tokenizer.py` SentencePiece step.
+ *
+ *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`, but `@dsnp/parquetjs` 1.7.0 only
+ *   supports UNCOMPRESSED / GZIP / SNAPPY / BROTLI (see `node_modules/@dsnp/parquetjs/dist/lib/
+ *   compression.js`). SNAPPY is the standard ML-corpus default (PyArrow's default too) and is the
+ *   closest substitute on speed; revisit if @dsnp/parquetjs gains zstd support. Documented in
+ *   `DECISIONS.md`.
+ *
+ *   Layout under `<outputDir>`:
+ *
+ *   ```
+ *   corpus-v<version>/
+ *   MANIFEST.json
+ *   train/
+ *     part-0000.parquet
+ *     part-0001.parquet
+ *     ...
+ *   val/
+ *     part-0000.parquet
+ *   test/
+ *     part-0000.parquet
+ * ```
+ *
+ *   Each shard caps at `rowsPerShard` (default 1_000_000); within a shard, parquetjs flushes row
+ *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
+ *   shard's path, row count, byte size, and SHA-256 (computed by re-reading the shard once after
+ *   close — cheap relative to writing it).
+ */
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { mkdir, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { ParquetWriter } from "./parquet-wrapper/index.js";
+/** Row groups flush at this many rows (parquetjs internal cadence within a shard). */
+export const ROW_GROUP_SIZE = 50_000;
+/** Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0. */
+export const SHARD_COMPRESSION = "SNAPPY";
+/** Column names emitted into every shard. Matches `ParquetRow`. */
+export const PARQUET_COLUMNS = [
+    "raw",
+    "tokens",
+    "labels",
+    "country",
+    "locale",
+    "source",
+    "source_id",
+    "corpus_version",
+    "license",
+    "synth_method",
+    "synth_base_id",
+];
+/**
+ * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8
+ * columns capture tokens/labels arrays. Compression is per-column SNAPPY.
+ */
+export const LABELED_ROW_SCHEMA = {
+    raw: { type: "UTF8", compression: SHARD_COMPRESSION },
+    tokens: { type: "UTF8", repeated: true, compression: SHARD_COMPRESSION },
+    labels: { type: "UTF8", repeated: true, compression: SHARD_COMPRESSION },
+    country: { type: "UTF8", compression: SHARD_COMPRESSION },
+    locale: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
+    source: { type: "UTF8", compression: SHARD_COMPRESSION },
+    source_id: { type: "UTF8", compression: SHARD_COMPRESSION },
+    corpus_version: { type: "UTF8", compression: SHARD_COMPRESSION },
+    license: { type: "UTF8", compression: SHARD_COMPRESSION },
+    synth_method: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
+    synth_base_id: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
+};
+/** Project a labeled row to the Parquet schema. */
+export function rowToParquet(row) {
+    return {
+        raw: row.raw,
+        tokens: row.tokens,
+        labels: row.labels,
+        country: row.country,
+        locale: row.locale ?? null,
+        source: row.source,
+        source_id: row.source_id,
+        corpus_version: row.corpus_version,
+        license: row.license,
+        synth_method: row.synth?.method ?? null,
+        synth_base_id: row.synth?.base_source_id ?? null,
+    };
+}
+/**
+ * Project a `ParquetRow` for `appendRow`. parquetjs treats `null` as "skip" for `optional` columns;
+ * passing it explicitly is fine, but cleaner to omit so the on-disk Definition Levels match what
+ * PyArrow / DuckDB / etc. produce for the same logical row.
+ */
+function appendShape(row) {
+    const out = {
+        raw: row.raw,
+        tokens: row.tokens,
+        labels: row.labels,
+        country: row.country,
+        source: row.source,
+        source_id: row.source_id,
+        corpus_version: row.corpus_version,
+        license: row.license,
+    };
+    if (row.locale !== null)
+        out.locale = row.locale;
+    if (row.synth_method !== null)
+        out.synth_method = row.synth_method;
+    if (row.synth_base_id !== null)
+        out.synth_base_id = row.synth_base_id;
+    return out;
+}
+/**
+ * Stream labeled rows into `.parquet` shards, one set of shards per split. Splits are processed
+ * sequentially so that only one shard writer is open at a time — memory cost is bounded by the
+ * parquetjs row-group buffer (~`ROW_GROUP_SIZE × row_size`), not by the labeled-row count.
+ *
+ * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior
+ * `splitFor(sourceId)` callback is gone because pre-partitioning at the caller eliminates the O(n)
+ * `Map<source_id, SplitName>` it required. See `buildCorpus` for the new wire-up.
+ */
+export async function writeShards(perSplit, opts) {
+    const rowsPerShard = opts.rowsPerShard ?? 1_000_000;
+    const corpusDir = join(opts.outputDir, `corpus-v${opts.corpusVersion}`);
+    await mkdir(corpusDir, { recursive: true });
+    const shards = [];
+    const counts = { train: 0, val: 0, test: 0 };
+    let totalRows = 0;
+    for (const split of ["train", "val", "test"]) {
+        const rows = perSplit[split];
+        if (!rows)
+            continue;
+        let shardIndex = 0;
+        let writer = null;
+        let path = "";
+        let shardRows = 0;
+        let firstSourceId = "";
+        let lastSourceId = "";
+        const openShard = async () => {
+            const splitDir = join(corpusDir, split);
+            await mkdir(splitDir, { recursive: true });
+            path = join(splitDir, `part-${String(shardIndex).padStart(4, "0")}.parquet`);
+            writer = await ParquetWriter.openFile(LABELED_ROW_SCHEMA, path, {
+                rowGroupSize: ROW_GROUP_SIZE,
+            });
+            writer.setMetadata("mailwoman.corpus_version", opts.corpusVersion);
+            writer.setMetadata("mailwoman.split", split);
+            writer.setMetadata("mailwoman.shard_index", String(shardIndex));
+            shardRows = 0;
+            firstSourceId = "";
+            lastSourceId = "";
+        };
+        const closeShard = async () => {
+            if (!writer)
+                return;
+            await writer.close();
+            if (shardRows > 0) {
+                const fileStat = await stat(path);
+                const sha256 = await hashFile(path);
+                shards.push({
+                    split,
+                    path,
+                    format: "parquet",
+                    compression: SHARD_COMPRESSION,
+                    rows: shardRows,
+                    bytes: fileStat.size,
+                    sha256,
+                    first_source_id: firstSourceId,
+                    last_source_id: lastSourceId,
+                });
+            }
+            writer = null;
+        };
+        for await (const row of rows) {
+            if (!writer)
+                await openShard();
+            const pq = rowToParquet(row);
+            await writer.appendRow(appendShape(pq));
+            if (shardRows === 0)
+                firstSourceId = row.source_id;
+            lastSourceId = row.source_id;
+            shardRows++;
+            counts[split]++;
+            totalRows++;
+            if (shardRows >= rowsPerShard) {
+                await closeShard();
+                shardIndex++;
+            }
+        }
+        await closeShard();
+    }
+    shards.sort((a, b) => (a.split === b.split ? a.path.localeCompare(b.path) : a.split.localeCompare(b.split)));
+    const manifest = {
+        corpus_version: opts.corpusVersion,
+        schema: PARQUET_COLUMNS,
+        rows_per_shard: rowsPerShard,
+        row_group_size: ROW_GROUP_SIZE,
+        shards,
+        counts,
+        total_rows: totalRows,
+    };
+    await writeFile(join(corpusDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
+    return manifest;
+}
+/** Single-pass SHA-256 over the file at `path`. Cheap relative to Parquet write throughput. */
+async function hashFile(path) {
+    const hash = createHash("sha256");
+    const stream = createReadStream(path);
+    for await (const chunk of stream)
+        hash.update(chunk);
+    return hash.digest("hex");
+}
+//# sourceMappingURL=parquet.js.map

package/out/src/parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,aAAa,EAAgC,MAAM,4BAA4B,CAAA;AAIxF,sFAAsF;AACtF,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAA;AAEpC,mFAAmF;AACnF,MAAM,CAAC,MAAM,iBAAiB,GAAG,QAAiB,CAAA;AAsBlD,mEAAmE;AACnE,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAwC;IACtE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACrD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxD,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3D,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAChE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,YAAY,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC9E,aAAa,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;CAC/E,CAAA;AA6CD,mDAAmD;AACnD,MAAM,UAAU,YAAY,CAAC,GAAe;IAC3C,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;QAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI;QACvC,aAAa,EAAE,GAAG,CAAC,KAAK,EAAE,cAAc,IAAI,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,GAAe;IACnC,MAAM,GAAG,GAA4B;QACpC,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;KACpB,CAAA;IACD,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI;QAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;IAChD,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI;QAAE,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAA;IAClE,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI;QAAE,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAA;IACrE,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAsB,EAAE,IAAwB;IACjF,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;IACvE,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,MAAM,MAAM,GAAsB,EAAE,CAAA;IACpC,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAC5B,IAAI,CAAC,IAAI;YAAE,SAAQ;QAEnB,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,MAAM,GAAqC,IAAI,CAAA;QACnD,IAAI,IAAI,GAAG,EAAE,CAAA;QACb,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,aAAa,GAAG,EAAE,CAAA;QACtB,IAAI,YAAY,GAAG,EAAE,CAAA;QAErB,MAAM,SAAS,GAAG,KAAK,IAAmB,EAAE;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YACvC,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC1C,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAA;YAC5E,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAa,kBAAkB,EAAE,IAAI,EAAE;gBAC3E,YAAY,EAAE,cAAc;aAC5B,CAAC,CAAA;YACF,MAAM,CAAC,WAAW,CAAC,0BAA0B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;YAClE,MAAM,CAAC,WAAW,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAA;YAC5C,MAAM,CAAC,WAAW,CAAC,uBAAuB,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;YAC/D,SAAS,GAAG,CAAC,CAAA;YACb,aAAa,GAAG,EAAE,CAAA;YAClB,YAAY,GAAG,EAAE,CAAA;QAClB,CAAC,CAAA;QAED,MAAM,UAAU,GAAG,KAAK,IAAmB,EAAE;YAC5C,IAAI,CAAC,MAAM;gBAAE,OAAM;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACpB,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAA;gBACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAA;gBACnC,MAAM,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,IAAI;oBACJ,MAAM,EAAE,SAAS;oBACjB,WAAW,EAAE,iBAAiB;oBAC9B,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,QAAQ,CAAC,IAAI;oBACpB,MAAM;oBACN,eAAe,EAAE,aAAa;oBAC9B,cAAc,EAAE,YAAY;iBAC5B,CAAC,CAAA;YACH,CAAC;YACD,MAAM,GAAG,IAAI,CAAA;QACd,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM;gBAAE,MAAM,SAAS,EAAE,CAAA;YAC9B,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAC5B,MAAM,MAAO,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAA0B,CAAC,CAAA;YACjE,IAAI,SAAS,KAAK,CAAC;gBAAE,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;YAClD,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;YAC5B,SAAS,EAAE,CAAA;YACX,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;YACf,SAAS,EAAE,CAAA;YAEX,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;gBAC/B,MAAM,UAAU,EAAE,CAAA;gBAClB,UAAU,EAAE,CAAA;YACb,CAAC;QACF,CAAC;QAED,MAAM,UAAU,EAAE,CAAA;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAE5G,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,YAAY;QAC5B,cAAc,EAAE,cAAc;QAC9B,MAAM;QACN,MAAM;QACN,UAAU,EAAE,SAAS;KACrB,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACnG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,+FAA+F;AAC/F,KAAK,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IACjC,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IACrC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM;QAAE,IAAI,CAAC,MAAM,CAAC,KAAe,CAAC,CAAA;IAC9D,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC1B,CAAC"}

package/out/src/runner.d.ts

@@ -0,0 +1,100 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Adapter runner — drives a `CorpusAdapter` to completion and writes intermediate JSONL + a
+ *   per-shard manifest.
+ *
+ *   Output layout under `outputDir`:
+ *
+ *   ```
+ *   <outputDir>/<adapter.id>/
+ *   canonical.jsonl       # one row per line, in emission order
+ *   MANIFEST.json         # adapter id, version, row count, sha256, license, started_at, ended_at
+ * ```
+ *
+ *   The runner is responsible for everything an adapter is **not** responsible for:
+ *
+ *   - Stamping `corpus_version` on every row (adapters must NOT set it).
+ *   - Applying `canonicalDedupKey` and skipping duplicates.
+ *   - Streaming sha256 over JSONL bytes so the manifest checksum doesn't require a re-read.
+ *   - Honoring backpressure on the output write stream.
+ *   - Counting + emitting periodic progress to an optional callback.
+ *   - Honoring `signal` (delegates to adapter's iteration boundary).
+ *
+ *   The runner does NOT perform alignment, tokenization, synthesis, or sharding into Parquet. Those
+ *   steps run later, consuming the JSONL shards this writes.
+ */
+import { type AdapterRegistry } from "./adapter.js";
+import type { AdapterOptions, CorpusAdapter } from "./types.js";
+/** Snapshot of the runner's state, emitted on every progress tick. */
+export interface RunnerProgress {
+    /** Adapter being driven. */
+    adapterId: string;
+    /** Total rows the adapter has yielded (before dedup). */
+    yielded: number;
+    /** Rows actually written to JSONL (after dedup). */
+    written: number;
+    /** Bytes written to JSONL so far. */
+    bytes: number;
+    /** Wall-clock milliseconds since the run started. */
+    elapsed_ms: number;
+}
+/** Per-invocation options for `runAdapter`. */
+export interface RunAdapterOptions {
+    /** Adapter to drive. */
+    adapter: CorpusAdapter;
+    /** Options handed to the adapter (input path, country filter, limit, signal). */
+    adapterOptions: AdapterOptions;
+    /** Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it. */
+    outputDir: string;
+    /** Corpus version stamped onto every row. Locked together with the tokenizer version. */
+    corpusVersion: string;
+    /**
+     * Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once
+     * at the end of the run. Errors thrown from this callback abort the run.
+     */
+    onProgress?: (snapshot: RunnerProgress) => void;
+    /**
+     * Yielded-row interval at which `onProgress` fires. Defaults to 1000. The terminal tick is always
+     * emitted regardless of this value.
+     */
+    progressEvery?: number;
+}
+/** Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk. */
+export interface AdapterRunManifest {
+    adapter_id: string;
+    corpus_version: string;
+    default_license: string;
+    description: string;
+    yielded: number;
+    written: number;
+    deduped: number;
+    bytes: number;
+    sha256: string;
+    jsonl_path: string;
+    started_at: string;
+    ended_at: string;
+    elapsed_ms: number;
+}
+/**
+ * Drive a single adapter to completion.
+ *
+ * Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under
+ * `outputDir/<adapter.id>/`. Throws if the output directory cannot be created, if a row arrives
+ * with a missing required field, or if the abort signal fires.
+ */
+export declare function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest>;
+/**
+ * Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the
+ * registry before calling if partial-failure is desired).
+ *
+ * Returns the manifests in registry insertion order.
+ */
+export declare function runAllAdapters(registry: AdapterRegistry, common: Omit<RunAdapterOptions, "adapter"> & {
+    adapterOptionsFor?: (a: CorpusAdapter) => AdapterOptions;
+}): Promise<AdapterRunManifest[]>;
+/** Convenience: ensure the parent directory of `filePath` exists. */
+export declare function ensureParentDir(filePath: string): Promise<void>;
+//# sourceMappingURL=runner.d.ts.map

package/out/src/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAKH,OAAO,EAAsC,KAAK,eAAe,EAAwB,MAAM,cAAc,CAAA;AAC7G,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAE,MAAM,YAAY,CAAA;AAE7E,sEAAsE;AACtE,MAAM,WAAW,cAAc;IAC9B,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAA;IAEjB,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAA;IAEf,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAA;IAEf,qCAAqC;IACrC,KAAK,EAAE,MAAM,CAAA;IAEb,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IACjC,wBAAwB;IACxB,OAAO,EAAE,aAAa,CAAA;IAEtB,iFAAiF;IACjF,cAAc,EAAE,cAAc,CAAA;IAE9B,sFAAsF;IACtF,SAAS,EAAE,MAAM,CAAA;IAEjB,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,IAAI,CAAA;IAE/C;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,+EAA+E;AAC/E,MAAM,WAAW,kBAAkB;IAClC,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAkGrF;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,CAAC,GAAG;IAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,cAAc,CAAA;CAAE,GACvG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAa/B;AAwCD,qEAAqE;AACrE,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE"}