@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/recipes/po/box/cedex/sources.ts

@@ -0,0 +1,189 @@
+/**
+ * @copyright Sister Software
+ */
+
+import { isAuPostcode, isAuStateAbbreviation } from "@mailwoman/codex/au"
+import { isNZPostcode } from "@mailwoman/codex/nz"
+import { dataRootPath } from "@mailwoman/core/data-root"
+import { pathExists } from "@mailwoman/core/fs/readers"
+import { readZipEntry } from "@mailwoman/core/fs/zip"
+import type { PathBuilderLike } from "path-ts"
+import { TextSpliterator, TSVSpliterator } from "spliterator"
+
+import type { AUTuple, FRTuple, NZTuple, USTuple } from "#recipes/po/box/cedex/types"
+import { readCSVRecords, readZippedCSVRecords } from "#recipes/scaffold"
+
+/**
+ * Non-Vermont OpenAddresses files used for training rows.
+ */
+export const US_TRAIN_SOURCES = [
+	{ zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
+	{ zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
+	{ zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
+	{ zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
+	{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
+	{ zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
+	{ zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
+]
+
+/**
+ * Vermont OpenAddresses file reserved for golden rows.
+ */
+export const US_EVAL_SOURCE = {
+	zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
+	csv: "us/vt/statewide.csv",
+	region: "VT",
+}
+
+/**
+ * Countrywide French OpenAddresses source.
+ */
+export const FR_SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }
+/**
+ * GeoNames Canada feature dump.
+ */
+export const GEONAMES_CA = dataRootPath("geonames", "CA.zip")
+/**
+ * GeoNames Australian postal-code dump.
+ */
+export const GEONAMES_POSTAL_AU = { zip: dataRootPath("geonames-postal", "AU.zip"), txt: "AU.txt" }
+/**
+ * GeoNames New Zealand postal-code dump.
+ */
+export const GEONAMES_POSTAL_NZ = { zip: dataRootPath("geonames-postal", "NZ.zip"), txt: "NZ.txt" }
+
+const MAX_LOCALITY_LENGTH = 40
+const FR_STRIDE = 211
+const FR_STRIDE_OFFSET = 3
+const CA_LOCALITY_MIN_POPULATION = 1000
+
+export const localityHash = (name: string): number => {
+	let hash = 5381
+
+	for (const character of name.toLowerCase()) {
+		hash = ((hash << 5) + hash + character.charCodeAt(0)) >>> 0
+	}
+
+	return hash
+}
+
+export const isHoldoutLocality = (name: string): boolean => localityHash(name) % 10 === 0
+
+const cleanLocality = (locality: string) =>
+	locality && locality.length <= MAX_LOCALITY_LENGTH && !/\d|,/.test(locality) && !/cedex/i.test(locality)
+
+export async function readUsTuples(source: { zip: PathBuilderLike; csv: string; region: string }): Promise<USTuple[]> {
+	const tuples: USTuple[] = []
+	const seen = new Set<string>()
+
+	for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
+		const locality = row.city ?? ""
+
+		if (!cleanLocality(locality)) continue
+		const street = row.street ?? ""
+		const pairKey = `${locality}|${street}`.toLowerCase()
+
+		if (seen.has(pairKey)) continue
+		seen.add(pairKey)
+
+		tuples.push({
+			house_number: row.number ?? "",
+			street,
+			locality,
+			region: source.region,
+			postcode: row.postcode ?? "",
+		})
+	}
+
+	return tuples
+}
+
+export async function readFrTuples(limit: number): Promise<FRTuple[]> {
+	if (!(await pathExists(FR_SOURCE.zip))) {
+		console.error(`  WARN: ${FR_SOURCE.zip} is not cached — skipping ${FR_SOURCE.csv}`)
+
+		return []
+	}
+
+	const encoder = new TextEncoder()
+
+	const strided = TextSpliterator.fromAsync(readZipEntry(FR_SOURCE.zip, FR_SOURCE.csv), { skipEmpty: false })
+		.filter((_line, index) => index === 0 || (index + 1) % FR_STRIDE === FR_STRIDE_OFFSET)
+		.take(limit + 1)
+		.map((line) => encoder.encode(line + "\n"))
+
+	const tuples: FRTuple[] = []
+	const seen = new Set<string>()
+
+	for await (const row of readCSVRecords(strided)) {
+		const locality = row.city ?? "",
+			postcode = row.postcode ?? "",
+			street = row.street ?? "",
+			house_number = row.number ?? ""
+
+		if (!cleanLocality(locality) || !/^\d{5}$/.test(postcode) || !street || !house_number) continue
+		const key = `${locality}|${street}`.toLowerCase()
+
+		if (seen.has(key)) continue
+		seen.add(key)
+		tuples.push({ house_number, street, locality, postcode })
+	}
+
+	return tuples
+}
+
+export async function readCaLocalities(admin1: string): Promise<string[]> {
+	if (!(await pathExists(GEONAMES_CA))) {
+		console.error(`  WARN: ${GEONAMES_CA} is not cached — skipping CA localities (admin1=${admin1})`)
+
+		return []
+	}
+
+	const localities = new Set<string>()
+
+	for await (const columns of TSVSpliterator.fromAsync(readZipEntry(GEONAMES_CA, "CA.txt"), { header: false })) {
+		if (
+			columns[6] === "P" &&
+			columns[8] === "CA" &&
+			columns[10] === admin1 &&
+			Number(columns[14]) > CA_LOCALITY_MIN_POPULATION &&
+			cleanLocality(columns[1] ?? "")
+		) {
+			localities.add(columns[1]!)
+		}
+	}
+
+	return [...localities]
+}
+
+export async function readPostalTuples(
+	source: { zip: PathBuilderLike; txt: string },
+	opts: { withState: boolean }
+): Promise<Array<AUTuple | NZTuple>> {
+	if (!(await pathExists(source.zip))) {
+		console.error(`  WARN: ${source.zip} is not cached — skipping ${source.txt}`)
+
+		return []
+	}
+
+	const tuples: Array<AUTuple | NZTuple> = []
+	const seen = new Set<string>()
+	const validPostcode = opts.withState ? isAuPostcode : isNZPostcode
+
+	for await (const columns of TSVSpliterator.fromAsync(readZipEntry(source.zip, source.txt), { header: false })) {
+		const postcode = (columns[1] ?? "").trim(),
+			locality = (columns[2] ?? "").trim(),
+			region = (columns[4] ?? "").trim()
+
+		if (!cleanLocality(locality) || !validPostcode(postcode) || (opts.withState && !isAuStateAbbreviation(region)))
+			continue
+
+		const key = `${locality}|${postcode}`.toLowerCase()
+
+		if (seen.has(key)) continue
+		seen.add(key)
+		tuples.push(opts.withState ? { locality, region, postcode } : { locality, postcode })
+	}
+
+	return tuples
+}

package/lib/recipes/po/box/cedex/types.ts

@@ -0,0 +1,35 @@
+/**
+ * @copyright Sister Software
+ */
+
+export interface USTuple {
+	house_number: string
+	street: string
+	locality: string
+	region: string
+	postcode: string
+}
+
+export interface FRTuple {
+	house_number: string
+	street: string
+	locality: string
+	postcode: string
+}
+
+export interface AUTuple {
+	locality: string
+	region: string
+	postcode: string
+}
+
+export interface NZTuple {
+	locality: string
+	postcode: string
+}
+
+export interface Rendered {
+	fmt: string
+	raw: string
+	components: Record<string, string>
+}

package/lib/recipes/po/box/cedex/us.ts

@@ -0,0 +1,69 @@
+/**
+ * @copyright Sister Software
+ */
+
+import { makePoBoxPhrase } from "#recipes/po/box/cedex/phrases"
+import type { Rendered, USTuple } from "#recipes/po/box/cedex/types"
+import { US_LEADERS_COMMON, US_LEADERS_RARE, US_PMB_LEADERS, VENUES_EN } from "#recipes/po/box/cedex/vocabulary"
+import { pick } from "#synthesizers/utils"
+
+const US_FULL_CUTOFF = 0.4,
+	US_NO_POSTCODE_CUTOFF = 0.55,
+	US_BARE_CUTOFF = 0.75,
+	US_VENUE_CUTOFF = 0.9
+
+const PMB_AFTER_STREET_CUTOFF = 0.5,
+	PMB_COMMA_CUTOFF = 0.85
+
+export function renderPoBoxUs(random: () => number, tuple: USTuple): Rendered {
+	const phrase = makePoBoxPhrase(random, US_LEADERS_COMMON, US_LEADERS_RARE)
+	const { locality, region, postcode } = tuple
+	const base = { po_box: phrase, locality, region }
+	const draw = random()
+
+	if (draw < US_FULL_CUTOFF && postcode)
+		return { fmt: "full", raw: `${phrase}, ${locality}, ${region} ${postcode}`, components: { ...base, postcode } }
+
+	if (draw < US_NO_POSTCODE_CUTOFF)
+		return { fmt: "no-postcode", raw: `${phrase}, ${locality}, ${region}`, components: base }
+
+	if (draw < US_BARE_CUTOFF) return { fmt: "bare", raw: phrase, components: { po_box: phrase } }
+
+	if (draw < US_VENUE_CUTOFF) {
+		const venue = pick(VENUES_EN, random)
+
+		return {
+			fmt: "venue",
+			raw: postcode
+				? `${venue}, ${phrase}, ${locality}, ${region} ${postcode}`
+				: `${venue}, ${phrase}, ${locality}, ${region}`,
+			components: { venue, ...base, ...(postcode ? { postcode } : {}) },
+		}
+	}
+
+	const upper = (value: string) => value.toUpperCase()
+
+	return {
+		fmt: "label-nocomma",
+		raw: postcode
+			? `${upper(phrase)} ${upper(locality)} ${region} ${postcode}`
+			: `${upper(phrase)} ${upper(locality)} ${region}`,
+		components: { po_box: upper(phrase), locality: upper(locality), region, ...(postcode ? { postcode } : {}) },
+	}
+}
+
+export function renderPmbUs(random: () => number, tuple: USTuple): Rendered {
+	const phrase = makePoBoxPhrase(random, US_PMB_LEADERS)
+	const { house_number, street, locality, region, postcode } = tuple
+	const road = `${house_number} ${street}`
+	const components = { house_number, street, po_box: phrase, locality, region, postcode }
+	const draw = random()
+
+	if (draw < PMB_AFTER_STREET_CUTOFF)
+		return { fmt: "pmb-after-street", raw: `${road} ${phrase}, ${locality}, ${region} ${postcode}`, components }
+
+	if (draw < PMB_COMMA_CUTOFF)
+		return { fmt: "pmb-comma", raw: `${road}, ${phrase}, ${locality}, ${region} ${postcode}`, components }
+
+	return { fmt: "pmb-bare", raw: `${road} ${phrase}`, components: { house_number, street, po_box: phrase } }
+}

package/lib/recipes/po/box/cedex/vocabulary.ts

@@ -0,0 +1,106 @@
+/**
+ * @copyright Sister Software
+ */
+
+import { FSA_LETTER_TO_PROVINCE } from "@mailwoman/codex/ca"
+import { US_PO_BOX_DESIGNATORS } from "@mailwoman/codex/us"
+
+import { PO_BOX_LOCALE_TEMPLATES, type LocaleTemplate } from "#synthesizers/po-box"
+
+const templates: Record<string, LocaleTemplate> = Object.fromEntries(
+	PO_BOX_LOCALE_TEMPLATES.map((template) => [template.locale, template])
+)
+
+/**
+ * Common US PO-box leaders from the en-US corpus template.
+ */
+export const US_LEADERS_COMMON = templates["en-US"]!.leaders
+
+/**
+ * Less-common USPS designators derived from the codex list.
+ */
+export const US_LEADERS_RARE = US_PO_BOX_DESIGNATORS.filter(
+	(designator) => !["POST OFFICE BOX", "PO BOX", "P O BOX", "BOX"].includes(designator)
+).map((designator) =>
+	designator
+		.toLowerCase()
+		.split(" ")
+		.map((word) => `${word[0]!.toUpperCase()}${word.slice(1)}`)
+		.join(" ")
+)
+
+/**
+ * US private-mailbox leaders excluding the conflicting hash sign.
+ */
+export const US_PMB_LEADERS = templates["en-US"]!.pmb!.filter((leader) => leader !== "#")
+/**
+ * French PO-box leaders from the fr-FR corpus template.
+ */
+export const FR_LEADERS = templates["fr-FR"]!.leaders
+/**
+ * French-language Canadian PO-box leaders from the fr-CA template.
+ */
+export const CA_FR_LEADERS = templates["fr-CA"]!.leaders
+/**
+ * English-language Canadian PO-box leaders from the en-CA template.
+ */
+export const CA_EN_LEADERS = templates["en-CA"]!.leaders
+/**
+ * Current Australian delivery-service leaders.
+ */
+export const AU_LEADERS_CURRENT = ["PO Box", "P.O. Box", "Post Office Box", "GPO Box", "Locked Bag", "Private Bag"]
+/**
+ * Recognized legacy Australian delivery-service leaders.
+ */
+export const AU_LEADERS_LEGACY = ["RMB", "RSD", "CMB"]
+/**
+ * Common New Zealand delivery-service leaders.
+ */
+export const NZ_LEADERS_COMMON = ["PO Box", "Private Bag"]
+/**
+ * Rare New Zealand delivery-service leader.
+ */
+export const NZ_LEADERS_RARE = ["CMB"]
+
+/**
+ * Quebec first-letter choices from the Canadian FSA prior.
+ */
+export const QC_FSA_LETTERS = Object.entries(FSA_LETTER_TO_PROVINCE)
+	.filter(([, province]) => province === "QC")
+	.map(([letter]) => letter)
+
+/**
+ * Ontario first-letter choices from the Canadian FSA prior.
+ */
+export const ON_FSA_LETTERS = Object.entries(FSA_LETTER_TO_PROVINCE)
+	.filter(([, province]) => province === "ON")
+	.map(([letter]) => letter)
+
+/**
+ * Valid Canadian postal-code interior letters.
+ */
+export const CA_INTERIOR_LETTERS = "ABCEGHJKLMNPRSTVWXYZ"
+
+/**
+ * Weighted country and address-form mix for generated rows.
+ */
+export const CLASS_MIX: ReadonlyArray<[string, number]> = [
+	["po-box-us", 0.27],
+	["po-box-us-military", 0.05],
+	["pmb-us", 0.07],
+	["bp-fr", 0.1],
+	["cedex-fr", 0.17],
+	["cp-ca-fr", 0.12],
+	["po-box-ca-en", 0.04],
+	["po-box-au", 0.12],
+	["po-box-nz", 0.06],
+]
+
+/**
+ * English recipient prefixes used in venue layouts.
+ */
+export const VENUES_EN = ["John Doe", "Jane Smith", "Acme Inc", "Wayne Enterprises", "Maria Garcia", "Riverside Clinic"]
+/**
+ * French recipient prefixes used in venue layouts.
+ */
+export const VENUES_FR = ["Société Dupont", "Cabinet Martin", "Hôpital Central", "Mairie Annexe", "Imprimerie Moderne"]

package/lib/recipes/po/box/cedex.ts

@@ -0,0 +1,9 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * The recipe's implementation is partitioned by data source and postal system in `cedex/`.
+ */
+
+export { poBoxCedexRecipe } from "#recipes/po/box/cedex/recipe"

package/lib/recipes/po/box/index.ts

@@ -0,0 +1,128 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `po-box` slice recipe — synthetic PO box rows: tuples → {@link synthesizePoBoxRow} → aligned
+ *   LabeledRow, plus optional self-contained US military/diplomatic rows (#517) at
+ *   `--military-ratio`. Region is required EXCEPT region-less locales (NZ). Ported from
+ *   scripts/build-po-box-slice.mjs.
+ */
+
+import { makeLcg } from "@mailwoman/core/utils"
+
+import {
+	alignAndWrite,
+	readTuples,
+	sliceSourceID,
+	SYNTHETIC_TUPLE_LICENSE as LICENSE,
+	type CorpusRecipe,
+} from "#recipes/scaffold"
+import { synthesizeMilitaryPoBoxRow, synthesizePoBoxRow, type PoBoxBaseTuple } from "#synthesizers/po-box"
+
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const poBoxRecipe: CorpusRecipe = {
+	name: "po-box",
+	description: "PO box rows: tuples → synthesizePoBoxRow (+ optional US military/diplomatic rows)",
+	mode: "tuples",
+	options: [
+		{ flag: "--pmb-ratio <p>", description: "P(private-mailbox layout). Default 0.15" },
+		{ flag: "--military-ratio <p>", description: "P(emit one US military/diplomatic row per input, #517). Default 0" },
+	],
+	async run(opts, write) {
+		if (!opts.input) throw new Error("po-box recipe requires --input <tuples.jsonl>")
+		const random = makeLcg(opts.seed)
+		const pmbRatio = opts.pmbRatio ?? 0.15
+		const militaryRatio = opts.militaryRatio ?? 0
+		// `--source-name` so a slice built for one class carries its own source label and its own dose. A
+		// military-only slice (`--variants 0 --military-ratio 1`) is otherwise indistinguishable from the
+		// leader-template rows in the mixture, and the two are dosed for different reasons (#517).
+		const source = opts.sourceName ?? "synth-po-box"
+		let read = 0
+		let emitted = 0
+		let skipped = 0
+
+		for await (const tuple of readTuples(opts.input)) {
+			read++
+			// Region required EXCEPT region-less locales (NZ: "Private Bag 12, Auckland 1010", #517).
+			const regionOptional = ["NZ", "NZL", "NEW ZEALAND"].includes(String(tuple.country || "").toUpperCase())
+
+			if (!tuple.locality || !tuple.postcode || !tuple.country || (!tuple.region && !regionOptional)) {
+				skipped++
+
+				continue
+			}
+
+			for (let v = 0; v < opts.variants; v++) {
+				const synth = synthesizePoBoxRow(tuple as PoBoxBaseTuple, { random, pmbRatio })
+
+				if (!synth) continue
+
+				const ok = alignAndWrite(
+					write,
+					{
+						raw: synth.raw,
+						components: synth.components,
+						country: tuple.country,
+						locale: synth.locale,
+						source,
+						source_id: sliceSourceID(source, {
+							locality: tuple.locality,
+							region: tuple.region,
+							postcode: tuple.postcode,
+							country: tuple.country,
+							v: String(v),
+						}),
+						corpus_version: "0.4.0",
+						license: LICENSE,
+					},
+					synth.template
+				)
+
+				if (ok) {
+					emitted++
+				} else {
+					skipped++
+				}
+			}
+
+			// US military/diplomatic rows (#517): self-contained, one per input line at --military-ratio.
+			// Default 0 → byte-stable (random() not called when off). US-only.
+			if (militaryRatio > 0 && random() < militaryRatio) {
+				const mil = synthesizeMilitaryPoBoxRow({ random })
+
+				const ok = alignAndWrite(
+					write,
+					{
+						raw: mil.raw,
+						components: mil.components,
+						country: "US",
+						locale: mil.locale,
+						source,
+						source_id: sliceSourceID(source, {
+							po_box: mil.components.po_box,
+							locality: mil.components.locality,
+							region: mil.components.region,
+							postcode: mil.components.postcode,
+							v: `mil${emitted}`,
+						}),
+						corpus_version: "0.4.0",
+						license: LICENSE,
+					},
+					mil.template
+				)
+
+				if (ok) {
+					emitted++
+				} else {
+					skipped++
+				}
+			}
+		}
+
+		return { read, emitted, skipped }
+	},
+}

package/lib/recipes/scaffold.ts

@@ -79,7 +79,6 @@ export interface SliceTuple {
  * A recipe must seed the same one its `.mjs` did, the same way it did (usually `seed`, but some derive a per-stream
  * seed), or `--seed N` stops being byte-reproducible.
  */
-export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@mailwoman/core/utils"
 
 /**
  * One CSV record, keyed by its lower-cased header name, every value trimmed. A column the header does not declare reads
@@ -88,45 +87,32 @@ export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@m
 export type CSVRecord = Record<string, string | undefined>
 
 /**
- * Quote handling spans the ROW split, not only the column split: a newline inside a quoted field belongs to a single
- * record, so the record boundaries can only be found by a scanner that already knows where the quotes are. Find the
- * lines first and unquote afterwards and such a record splits in two — the first half short by however many columns
- * followed the newline, which is how a `street` value comes to be read as `city`.
+ * Line breaks inside a value become single spaces, and every value is trimmed.
  *
- * `header: false` keeps the first record in the stream, so this module owns the lower-casing rather than
- * `normalizeKeys` — see {@link toHeader}.
- */
-const CSV_INIT = { header: false, enableQuoteHandling: true } as const
-
-/**
- * Header names are lower-cased on the way in. `normalizeKeys` will not do it: it leaves an ALL CAPS header alone, and
- * OpenAddresses ships `LON,LAT,NUMBER,STREET` while other extracts ship the same names lower-case. A recipe names its
- * columns in lower case either way.
- */
-function toHeader(cells: readonly string[]): string[] {
-	return cells.map((name) => name.trim().toLowerCase())
-}
-
-/**
- * Line breaks inside a value become single spaces. A quote-aware parse is the first thing here able to return a value
- * CONTAINING one — `us/ia/statewide.csv` has 12, all unit designators like `"#2\n#2"` — and every consumer synthesizes
- * one-line address text from these cells with no guard, because until that parse landed no value could carry one.
- * Collapsing keeps the record (the address is fine; the source's line break is not part of it) without emitting a
- * training row with a newline inside it.
+ * A quote-aware parse is the first thing able to return a value CONTAINING a line break — `us/ia/statewide.csv` has 12,
+ * all unit designators like `"#2\n#2"` — and every consumer synthesizes one-line address text from these cells with no
+ * guard, because until that parse landed no value could carry one. Collapsing keeps the record (the address is fine;
+ * the source's line break is not part of it) without emitting a training row with a newline inside it.
  *
  * Only `\r` and `\n`, deliberately — NOT `\s`. Runs of spaces and tabs pass through exactly as the source wrote them
  * (OA's IA extract writes `NORTH`, three spaces, `MAIN STREET`), because those could always appear and every slice
  * built to date contains them. Widening this to `\s+` silently rewrites values on rows with no line break at all.
- * `scaffold.test.ts` pins both halves, against both readers.
+ * `scaffold.test.ts` pins both halves.
+ *
+ * This is what a CSV reader here still owns. Quote handling, object rows, and lower-case keys are `CSVSpliterator`'s
+ * defaults, so a recipe reads a source with `CSVSpliterator.fromAsync(source).map(withoutLineBreaks)` and needs nothing
+ * else.
+ *
+ * @category CSV
  */
-function toRecord(header: readonly string[], cells: readonly string[]): CSVRecord {
-	const record: CSVRecord = {}
+export function withoutLineBreaks(record: CSVRecord): CSVRecord {
+	const out: CSVRecord = {}
 
-	for (let i = 0; i < header.length; i++) {
-		record[header[i]!] = (cells[i] ?? "").replaceAll(/[\r\n]+/g, " ").trim()
+	for (const [key, value] of Object.entries(record)) {
+		out[key] = (value ?? "").replaceAll(/[\r\n]+/g, " ").trim()
 	}
 
-	return record
+	return out
 }
 
 /**
@@ -142,17 +128,7 @@ function toRecord(header: readonly string[], cells: readonly string[]): CSVRecor
  * @category CSV
  */
 export function readCSVRecords(source: AsyncDataResource | AsyncChunkIterator): AsyncSequence<CSVRecord> {
-	let header: string[] | null = null
-
-	return CSVSpliterator.fromAsync<string[]>(source, CSV_INIT)
-		.filter((cells) => {
-			if (header) return true
-
-			header = toHeader(cells)
-
-			return false
-		})
-		.map((cells) => toRecord(header!, cells))
+	return CSVSpliterator.fromAsync<CSVRecord>(source).map(withoutLineBreaks)
 }
 
 /**