@mailwoman/corpus 9.1.0 → 9.2.0

package/src/utils/align.ts

@@ -34,9 +34,10 @@
  */
 
 import type { BIOLabel, ComponentTag } from "@mailwoman/core/types"
-import type { CanonicalRow, LabeledRow, QuarantinedRow } from "@mailwoman/corpus/types"
 import { distance as levenshteinDistance } from "fastest-levenshtein"
 
+import type { CanonicalRow, LabeledRow, QuarantinedRow } from "#types"
+
 import { whitespaceTokenizer, type TokenSpan, type Tokenizer } from "./tokenize.ts"
 
 /**

package/src/utils/golden.ts

@@ -63,7 +63,7 @@ export interface GoldenReport {
 export function parseGoldenLine(line: string): GoldenEntry {
 	// The throw IS the result: `validateGoldenFile` catches it and records the message against the line
 	// number, so a tolerant parse would report a corrupt row as valid.
-	// oxlint-disable-next-line no-restricted-properties
+	// oxlint-disable-next-line no-restricted-properties -- throw-on-corrupt is recorded against this JSONL line by the caller
 	const obj = JSON.parse(line) as Partial<GoldenEntry> & Record<string, unknown>
 
 	if (typeof obj.raw !== "string" || !obj.raw.length) {

package/src/utils/license.ts

@@ -36,7 +36,7 @@ export function compileLicenseExcludes(spec: string): RegExp[] {
 	return spec
 		.split(",")
 		.map((s) => s.trim())
-		.filter(Boolean)
+		.filter((entry) => entry !== "")
 		.map((s) => new RegExp("^" + s.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"))
 }
 

package/src/utils/parquet.ts

@@ -41,8 +41,9 @@ import { mkdir, stat, writeFile } from "node:fs/promises"
 import { join } from "node:path"
 
 import { sha256File } from "@mailwoman/core/utils"
-import { ParquetWriter, type ParquetSchemaDefinition } from "@mailwoman/corpus/parquet-wrapper"
-import type { LabeledRow } from "@mailwoman/corpus/types"
+
+import { ParquetWriter, type ParquetSchemaDefinition } from "#parquet-wrapper"
+import type { LabeledRow } from "#types"
 
 import type { SplitName } from "./split.ts"
 
@@ -59,6 +60,10 @@ export const SHARD_COMPRESSION = "SNAPPY" as const
 /**
  * A single Parquet-style row shape. The `[key: string]: unknown` index signature is required for compatibility with
  * `ParquetRecordLike` in the wrapper — parquetjs accepts any string key on rows.
+ *
+ * The three columns the schema marks `optional: true` are optional here too: `appendRow` receives the row with them
+ * OMITTED rather than null (see {@link appendShape}), and a reader gets back whichever of null/absent parquetjs
+ * surfaces.
  */
 export interface ParquetRow {
 	raw: string
@@ -68,13 +73,13 @@ export interface ParquetRow {
 	span_ends: readonly number[]
 	span_tags: readonly string[]
 	country: string
-	locale: string | null
+	locale?: string | null
 	source: string
 	source_id: string
 	corpus_version: string
 	license: string
-	synth_method: string | null
-	synth_base_id: string | null
+	synth_method?: string | null
+	synth_base_id?: string | null
 	[key: string]: unknown
 }
 
@@ -139,6 +144,11 @@ export interface ShardDescriptor {
 	sha256: string
 	first_source_id: string
 	last_source_id: string
+	/**
+	 * The shard's corpus source slug, when the writer knows it. `audit.ts` prefers this over inferring the source from
+	 * `first_source_id`'s prefix; `writeShards` itself writes multi-source shards and leaves it unset.
+	 */
+	source?: string
 }
 
 export interface ShardManifest {
@@ -227,9 +237,13 @@ export function rowToParquet(row: LabeledRow): ParquetRow {
  * Project a `ParquetRow` for `appendRow`. parquetjs treats `null` as "skip" for `optional` columns; passing it
  * explicitly is fine, but cleaner to omit so the on-disk Definition Levels match what PyArrow / DuckDB / etc. produce
  * for the same logical row.
+ *
+ * EXPORTED because it was not, and a second producer hand-rolled its own copy that predates the v0.5.0 span triple
+ * (#519). `LABELED_ROW_SCHEMA` declares those three columns `repeated` and NOT optional, so the copy wrote empty arrays
+ * under a schema promising values. Every new column has to arrive here once, not once per writer.
  */
-function appendShape(row: ParquetRow): Record<string, unknown> {
-	const out: Record<string, unknown> = {
+export function appendShape(row: ParquetRow): ParquetRow {
+	const out: ParquetRow = {
 		raw: row.raw,
 		tokens: row.tokens,
 		labels: row.labels,
@@ -335,7 +349,7 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 			}
 
 			const pq = rowToParquet(row)
-			await writer!.appendRow(appendShape(pq) as unknown as ParquetRow)
+			await writer!.appendRow(appendShape(pq))
 
 			if (shardRows === 0) {
 				firstSourceID = row.source_id

package/src/utils/split.ts

@@ -31,9 +31,10 @@ import { mkdir, unlink, writeFile } from "node:fs/promises"
 import { join } from "node:path"
 
 import { childEnv } from "@mailwoman/core/scripting/utils"
-import type { CanonicalRow, LabeledRow } from "@mailwoman/corpus/types"
 import { JSONSpliterator } from "spliterator"
 
+import type { CanonicalRow, LabeledRow } from "#types"
+
 export type SplitName = "train" | "val" | "test"
 
 export interface SplitOptions {
@@ -216,7 +217,7 @@ export async function writeSplitManifestsFromLabeledFiles(opts: {
 	for (const split of ["train", "val", "test"] as const) {
 		const labeledPath = opts.labeledPaths[split]
 		const outPath = join(opts.outputDir, `${split}.txt`)
-		await streamSortedSourceIds(labeledPath, outPath)
+		await streamSortedSourceIDs(labeledPath, outPath)
 	}
 
 	const total = opts.counts.train + opts.counts.val + opts.counts.test
@@ -236,7 +237,7 @@ export async function writeSplitManifestsFromLabeledFiles(opts: {
  * Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input → empty output file (not
  * absent). Uses `sort(1)` for disk-spilling external sort so peak memory stays O(1) regardless of labeled-row count.
  */
-async function streamSortedSourceIds(labeledJsonlPath: string, outPath: string): Promise<void> {
+async function streamSortedSourceIDs(labeledJsonlPath: string, outPath: string): Promise<void> {
 	const unsortedPath = `${outPath}.unsorted`
 	const out = createWriteStream(unsortedPath, { encoding: "utf8" })