@mailwoman/corpus 9.2.0 → 9.4.0

package/src/parquet-wrapper/reader.ts

@@ -1,97 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Typed wrapper around `@dsnp/parquetjs`'s `ParquetReader` that narrows the row-iterator generic to
- *   a user-supplied record type and adds `AsyncDisposable` support so `await using` cleans up the
- *   envelope reader without an explicit `close()`.
- */
-
-import { ParquetReader as BaseParquetReader } from "@dsnp/parquetjs"
-import type { BufferReaderOptions } from "@dsnp/parquetjs/dist/lib/bufferReader.js"
-import { ParquetEnvelopeReader } from "@dsnp/parquetjs/dist/lib/reader.js"
-
-import type { ParquetSchema, ParquetRecordLike } from "./schema.ts"
-
-/**
- * A typed Parquet reader, wrapping the base Parquet reader.
- */
-export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReader implements AsyncDisposable {
-	declare schema: ParquetSchema<T>
-
-	static override async openFile<T extends ParquetRecordLike>(
-		filePath: string | URL,
-		options?: BufferReaderOptions
-	): Promise<ParquetReader<T>> {
-		const envelopeReader = await ParquetEnvelopeReader.openFile(filePath.toString(), options)
-
-		return ParquetReader.openEnvelopeReader<T>(envelopeReader, options)
-	}
-
-	static override async openBuffer<T extends ParquetRecordLike>(buffer: Buffer, options?: BufferReaderOptions) {
-		const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer, options)
-
-		return this.openEnvelopeReader<T>(envelopeReader, options)
-	}
-
-	static override async openEnvelopeReader<T extends ParquetRecordLike>(
-		envelopeReader: ParquetEnvelopeReader,
-		opts?: BufferReaderOptions
-	) {
-		if (opts?.metadata) {
-			return new ParquetReader<T>(opts.metadata, envelopeReader, opts)
-		}
-
-		try {
-			await envelopeReader.readHeader()
-
-			const metadata = await envelopeReader.readFooter()
-
-			return new ParquetReader<T>(metadata, envelopeReader, opts)
-		} catch (error) {
-			await envelopeReader.close()
-			throw error
-		}
-	}
-
-	public override [Symbol.asyncIterator](): AsyncGenerator<T, void, unknown> {
-		return super[Symbol.asyncIterator]() as AsyncGenerator<T, void, unknown>
-	}
-
-	/**
-	 * Iterate a subset of the columns, narrowed to the keys asked for.
-	 *
-	 * Parquet is columnar, so a projection is read avoided rather than read-then-discarded: on a 14-column corpus shard,
-	 * two columns cost 93 ms against 253 ms for the whole row. The default iterator reads every column, which is what a
-	 * caller wanting the whole record should use.
-	 */
-	public async *project<K extends keyof T>(...columns: K[]): AsyncGenerator<Pick<T, K>, void, unknown> {
-		const missing = columns.filter((column) => !Object.hasOwn(this.schema.schema, column))
-
-		if (missing.length) {
-			throw new Error(
-				`Parquet projection requested column${missing.length === 1 ? "" : "s"} absent from the file schema: ` +
-					missing.map(String).join(", ")
-			)
-		}
-
-		const cursor = this.getCursor(columns.map((column) => [column as string]))
-
-		for (;;) {
-			const row = (await cursor.next()) as Pick<T, K> | null
-
-			if (!row) return
-
-			yield row
-		}
-	}
-
-	public async [Symbol.asyncDispose]() {
-		return this.close()
-	}
-
-	public async dispose() {
-		return this[Symbol.asyncDispose]()
-	}
-}

package/src/parquet-wrapper/schema.ts

@@ -1,88 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Typed wrapper around `@dsnp/parquetjs`'s schema definition. Adds:
- *
- *   - `ParquetSchema<T>`: a generic class narrowing the base schema's `schema` property to a
- *       field-by-field typed dict.
- *   - `ParquetSchemaDefinitionCache`: an LRU lookup so hot paths that compute the same schema
- *       repeatedly pay the cost once. Implements `Disposable` so `using` works.
- *   - `createBloomFilters<T>`: helper that takes a schema and a list of columns and returns the
- *       `@dsnp/parquetjs`-shaped bloom-filter spec array.
- *
- *   `Symbol.dispose` is sync (the original was async, but `Disposable`'s contract is sync — async
- *   cleanup belongs on `AsyncDisposable`).
- */
-
-import { ParquetSchema as BaseParquetSchema } from "@dsnp/parquetjs"
-import type { createSBBFParams as BloomFilterCreation } from "@dsnp/parquetjs/dist/lib/bloomFilterIO/bloomFilterWriter.js"
-import type { FieldDefinition } from "@dsnp/parquetjs/dist/lib/declare.js"
-import { LRUCache } from "lru-cache"
-
-/**
- * A Parquet record-like object, i.e. a record with string keys and JSON-serializable values.
- */
-/**
- * Shape a row type must satisfy. Declare yours as a `type` alias — TypeScript gives an alias an implicit index
- * signature and an `interface` none, so an otherwise-identical interface does not satisfy this.
- */
-export type ParquetRecordLike = Record<string, unknown | undefined>
-
-/**
- * Typed Parquet schema definition.
- */
-export type ParquetSchemaDefinition<T = ParquetRecordLike> = Record<Extract<keyof T, string>, FieldDefinition>
-
-/**
- * Typed Parquet schema.
- */
-export class ParquetSchema<T> extends BaseParquetSchema {
-	declare schema: ParquetSchemaDefinition<T>
-}
-
-/**
- * Given a Parquet schema and a list of columns, create a list of Bloom filters for those columns.
- */
-export function createBloomFilters<T>(
-	parquetSchemaDef: ParquetSchemaDefinition<T>,
-	columns: Extract<keyof T, string>[]
-) {
-	const bloomFilters: BloomFilterCreation[] = []
-
-	for (const column of columns) {
-		if (!parquetSchemaDef[column]) {
-			throw new Error(`Bloom filter column ${column} not found in Parquet schema`)
-		}
-
-		bloomFilters.push({ column })
-	}
-
-	return bloomFilters
-}
-
-export class ParquetSchemaDefinitionCache
-	extends LRUCache<ParquetSchemaDefinition<ParquetRecordLike>, ParquetSchema<ParquetRecordLike>>
-	implements Disposable
-{
-	constructor(max = 1000) {
-		super({ max })
-	}
-
-	public findOrCreateSchema<T extends ParquetRecordLike>(schemaDef: ParquetSchemaDefinition<T>): ParquetSchema<T> {
-		const key = schemaDef as ParquetSchemaDefinition<ParquetRecordLike>
-		let schema = this.get(key) as ParquetSchema<T> | undefined
-
-		if (!schema) {
-			schema = new ParquetSchema<T>(schemaDef)
-			this.set(key, schema as ParquetSchema<ParquetRecordLike>)
-		}
-
-		return schema
-	}
-
-	public [Symbol.dispose]() {
-		this.clear()
-	}
-}

package/src/parquet-wrapper/writer.ts

@@ -1,110 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Typed Parquet writer. Two static constructors mirror the base class:
- *
- *   - `openStream`: wrap an existing writable stream.
- *   - `openFile`: open a path on disk, ensuring the parent directory exists first.
- *
- *   Implements `AsyncDisposable` so `await using writer = await ParquetWriter.openFile(...)` flushes
- *   and closes cleanly. `close()` internally serializes against any in-flight flush so back-to-back
- *   dispose calls don't race.
- */
-
-import * as fs from "node:fs/promises"
-import * as path from "node:path"
-
-import { ParquetWriter as BaseParquetWriter } from "@dsnp/parquetjs"
-import type { WriterOptions } from "@dsnp/parquetjs/dist/lib/declare.js"
-import { osopen, type WriteStreamMinimal } from "@dsnp/parquetjs/dist/lib/util.js"
-import { ParquetEnvelopeWriter } from "@dsnp/parquetjs/dist/lib/writer.js"
-
-import {
-	type ParquetRecordLike,
-	ParquetSchema,
-	type ParquetSchemaDefinition,
-	ParquetSchemaDefinitionCache,
-} from "./schema.ts"
-
-/**
- * A typed Parquet writer, wrapping the base Parquet writer.
- */
-export class ParquetWriter<T extends ParquetRecordLike> extends BaseParquetWriter implements AsyncDisposable {
-	declare schema: ParquetSchema<T>
-	protected static readonly SchemaDefinitionCache = new ParquetSchemaDefinitionCache()
-	#flushing: Promise<void> = Promise.resolve()
-
-	static override async openStream<T extends ParquetRecordLike>(
-		schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>,
-		outputStream: WriteStreamMinimal,
-		opts: WriterOptions = {}
-	): Promise<ParquetWriter<T>> {
-		const schema =
-			schemaLike instanceof ParquetSchema
-				? schemaLike
-				: ParquetWriter.SchemaDefinitionCache.findOrCreateSchema(schemaLike)
-
-		const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts)
-
-		return new ParquetWriter<T>(schema, envelopeWriter, opts)
-	}
-
-	/**
-	 * Convenience method to create a new buffered parquet writer that writes to the specified file.
-	 */
-	static override async openFile<T extends ParquetRecordLike>(
-		schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>,
-		sourcePath: string | Buffer | URL,
-		opts?: WriterOptions
-	): Promise<ParquetWriter<T>> {
-		if (typeof sourcePath === "string") {
-			await fs.mkdir(path.dirname(sourcePath), { recursive: true })
-		}
-
-		const outputStream = await osopen(sourcePath, opts)
-
-		return ParquetWriter.openStream<T>(schemaLike, outputStream, opts)
-	}
-
-	// @note This fixes invalid Markdown in the base class JSDoc.
-	/**
-	 * Set a metadata key-value pair on the writer.
-	 */
-	public override setMetadata(key: string, value: string): void {
-		return super.setMetadata(key, value)
-	}
-
-	/**
-	 * Append a row to the buffer. If the buffer is full, the data will be written to disk.
-	 */
-	public override async appendRow(row: T): Promise<void> {
-		return super.appendRow(row)
-	}
-
-	/**
-	 * Flush all buffered data to disk, close the file, and release resources.
-	 */
-	public override async close(): Promise<void> {
-		await this.#flushing
-
-		if (this.closed) return
-
-		const { promise, resolve, reject } = Promise.withResolvers<void>()
-
-		super.close().then(resolve, reject)
-
-		this.#flushing = promise
-
-		return this.#flushing
-	}
-
-	public async [Symbol.asyncDispose]() {
-		return this.close()
-	}
-
-	public async dispose() {
-		return this[Symbol.asyncDispose]()
-	}
-}

package/src/runner.ts

@@ -1,317 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Adapter runner — drives a `CorpusAdapter` to completion and writes intermediate JSONL + a
- *   per-shard manifest.
- *
- *   Output layout under `outputDir`:
- *
- *   ```
- *   <outputDir>/<adapter.id>/
- *   canonical.jsonl       # one row per line, in emission order
- *   MANIFEST.json         # adapter id, version, row count, sha256, license, started_at, ended_at
- * ```
- *
- *   The runner is responsible for everything an adapter is **not** responsible for:
- *
- *   - Stamping `corpus_version` on every row (adapters must NOT set it).
- *   - Applying `canonicalDedupKey` and skipping duplicates.
- *   - Streaming sha256 over JSONL bytes so the manifest checksum doesn't require a re-read.
- *   - Honoring backpressure on the output write stream.
- *   - Counting + emitting periodic progress to an optional callback.
- *   - Honoring `signal` (delegates to adapter's iteration boundary).
- *
- *   The runner does NOT perform alignment, tokenization, synthesis, or sharding into Parquet. Those
- *   steps run later, consuming the JSONL shards this writes.
- */
-
-import { createWriteStream, type WriteStream } from "node:fs"
-import { mkdir, writeFile } from "node:fs/promises"
-import { dirname, join } from "node:path"
-
-import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "#adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Snapshot of the runner's state, emitted on every progress tick.
- */
-export interface RunnerProgress {
-	/**
-	 * Adapter being driven.
-	 */
-	adapterID: string
-
-	/**
-	 * Total rows the adapter has yielded (before dedup).
-	 */
-	yielded: number
-
-	/**
-	 * Rows actually written to JSONL (after dedup).
-	 */
-	written: number
-
-	/**
-	 * Bytes written to JSONL so far.
-	 */
-	bytes: number
-
-	/**
-	 * Wall-clock milliseconds since the run started.
-	 */
-	elapsed_ms: number
-}
-
-/**
- * Per-invocation options for `runAdapter`.
- */
-export interface RunAdapterOptions {
-	/**
-	 * Adapter to drive.
-	 */
-	adapter: CorpusAdapter
-
-	/**
-	 * Options handed to the adapter (input path, country filter, limit, signal).
-	 */
-	adapterOptions: AdapterOptions
-
-	/**
-	 * Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it.
-	 */
-	outputDir: string
-
-	/**
-	 * Corpus version stamped onto every row. Locked together with the tokenizer version.
-	 */
-	corpusVersion: string
-
-	/**
-	 * Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once at the end of the
-	 * run. Errors thrown from this callback abort the run.
-	 */
-	onProgress?: (snapshot: RunnerProgress) => void
-
-	/**
-	 * Yielded-row interval at which `onProgress` fires. Defaults to 1000. The terminal tick is always emitted regardless
-	 * of this value.
-	 */
-	progressEvery?: number
-}
-
-/**
- * Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk.
- */
-export interface AdapterRunManifest {
-	adapter_id: string
-	corpus_version: string
-	default_license: string
-	description: string
-	yielded: number
-	written: number
-	deduped: number
-	bytes: number
-	sha256: string
-	jsonl_path: string
-	started_at: string
-	ended_at: string
-	elapsed_ms: number
-}
-
-/**
- * Drive a single adapter to completion.
- *
- * Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under `outputDir/<adapter.id>/`.
- * Throws if the output directory cannot be created, if a row arrives with a missing required field, or if the abort
- * signal fires.
- */
-export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest> {
-	const { adapter, adapterOptions, outputDir, corpusVersion } = opts
-	const progressEvery = opts.progressEvery ?? 1000
-
-	const adapterDir = join(outputDir, adapter.id)
-	await mkdir(adapterDir, { recursive: true })
-
-	const jsonlPath = join(adapterDir, "canonical.jsonl")
-	const manifestPath = join(adapterDir, "MANIFEST.json")
-
-	const startedAt = new Date()
-	const t0 = performance.now()
-
-	const stream = createWriteStream(jsonlPath, { encoding: "utf8" })
-	const hasher: StreamingHasher = streamingSha256()
-	const seen = new Set<string>()
-	const DEDUP_MAX_SIZE = 10_000_000
-	let dedupExhausted = false
-
-	let yielded = 0
-	let written = 0
-	let bytes = 0
-
-	const emitProgress = (): void => {
-		opts.onProgress?.({
-			adapterID: adapter.id,
-			yielded,
-			written,
-			bytes,
-			elapsed_ms: performance.now() - t0,
-		})
-	}
-
-	try {
-		for await (const row of adapter.rows(adapterOptions)) {
-			if (adapterOptions.signal?.aborted) {
-				throw new DOMException("Adapter run aborted by signal", "AbortError")
-			}
-
-			yielded++
-			assertEmittedRow(adapter, row)
-
-			const stamped: CanonicalRow = { ...row, corpus_version: corpusVersion }
-			const key = canonicalDedupKey(stamped)
-
-			if (!dedupExhausted) {
-				if (seen.has(key)) {
-					if (yielded % progressEvery === 0) {
-						emitProgress()
-					}
-
-					continue
-				}
-
-				if (seen.size >= DEDUP_MAX_SIZE) {
-					dedupExhausted = true
-
-					process.stderr.write(
-						`  runner: dedup set full at ${DEDUP_MAX_SIZE.toLocaleString()} — skipping dedup for remaining rows\n`
-					)
-				} else {
-					seen.add(key)
-				}
-			}
-
-			const line = `${JSON.stringify(stamped)}\n`
-			hasher.update(line)
-			bytes += Buffer.byteLength(line, "utf8")
-
-			written++
-
-			if (!stream.write(line)) {
-				await once(stream, "drain")
-			}
-
-			if (yielded % progressEvery === 0) {
-				emitProgress()
-			}
-		}
-	} finally {
-		stream.end()
-		await once(stream, "close")
-	}
-
-	const endedAt = new Date()
-	const elapsed_ms = performance.now() - t0
-	emitProgress()
-
-	const manifest: AdapterRunManifest = {
-		adapter_id: adapter.id,
-		corpus_version: corpusVersion,
-		default_license: adapter.defaultLicense,
-		description: adapter.description,
-		yielded,
-		written,
-		deduped: yielded - written,
-		bytes,
-		sha256: hasher.digest(),
-		jsonl_path: jsonlPath,
-		started_at: startedAt.toISOString(),
-		ended_at: endedAt.toISOString(),
-		elapsed_ms,
-	}
-
-	await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
-
-	return manifest
-}
-
-/**
- * Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the registry before
- * calling if partial-failure is desired).
- *
- * Returns the manifests in registry insertion order.
- */
-export async function runAllAdapters(
-	registry: AdapterRegistry,
-	common: Omit<RunAdapterOptions, "adapter"> & { adapterOptionsFor?: (a: CorpusAdapter) => AdapterOptions }
-): Promise<AdapterRunManifest[]> {
-	const out: AdapterRunManifest[] = []
-
-	for (const adapter of registry.list()) {
-		const adapterOptions = common.adapterOptionsFor?.(adapter) ?? common.adapterOptions
-
-		out.push(
-			await runAdapter({
-				...common,
-				adapter,
-				adapterOptions,
-			})
-		)
-	}
-
-	return out
-}
-
-/**
- * Validate an emitted row. Cheap; runs once per row. Catches adapter bugs early so the JSONL doesn't end up
- * half-malformed.
- */
-function assertEmittedRow(adapter: CorpusAdapter, row: CanonicalRow): void {
-	if (row.source !== adapter.id) {
-		throw new Error(`adapter ${adapter.id}: row.source must equal adapter.id (got ${JSON.stringify(row.source)})`)
-	}
-
-	if (!row.source_id) {
-		throw new Error(`adapter ${adapter.id}: row.source_id is empty`)
-	}
-
-	if (!row.raw) {
-		throw new Error(`adapter ${adapter.id}: row.raw is empty for source_id=${row.source_id}`)
-	}
-
-	if (!row.country) {
-		throw new Error(`adapter ${adapter.id}: row.country is empty for source_id=${row.source_id}`)
-	}
-
-	if (!row.license) {
-		throw new Error(`adapter ${adapter.id}: row.license is empty for source_id=${row.source_id}`)
-	}
-}
-
-/**
- * Promise-ify a single event emission. Used to await `drain` / `close` on the write stream.
- */
-function once(emitter: WriteStream, event: "drain" | "close"): Promise<void> {
-	return new Promise((resolve, reject) => {
-		const onEvent = (): void => {
-			emitter.off("error", onError)
-			resolve()
-		}
-
-		const onError = (err: Error): void => {
-			emitter.off(event, onEvent)
-			reject(err)
-		}
-
-		emitter.once(event, onEvent)
-		emitter.once("error", onError)
-	})
-}
-
-/**
- * Convenience: ensure the parent directory of `filePath` exists.
- */
-export async function ensureParentDir(filePath: string): Promise<void> {
-	await mkdir(dirname(filePath), { recursive: true })
-}