@mailwoman/corpus 9.2.0 → 9.4.0

package/lib/tools/corpus-stats.ts

@@ -0,0 +1,136 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
+ *
+ *   Reads one or more Parquet slices, builds per-(token, label) and per-(bigram, label-bigram)
+ *   histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-slice.ts`
+ *   as the baseline against which a new slice is compared.
+ *
+ *   Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
+ *   linter invocations. Re-run this script whenever the corpus changes substantially (a new
+ *   mainline slice added, a source-pool re-weighted, etc.).
+ *
+ *   Output schema:
+ *
+ *   ```ts
+ *   interface CorpusStats {
+ *     row_count: number
+ *     slice_paths: string[]
+ *     tokens: { [token: string]: { [label: string]: number } }
+ *     bigrams: { [token_bigram: string]: { [label_bigram: string]: number } }
+ *     // token_bigram = "tok1tok2" (US sep), label_bigram = "lab1lab2"
+ *     // For memory: only keep bigrams with count >= MIN_BIGRAM_COUNT (2).
+ *   }
+ * ```
+ *
+ *   Usage: node scripts/build-corpus-stats.ts\
+ *   --slices <glob-pattern-or-dir>\
+ *   --output <stats.json>
+ *
+ *   For a quick local-corpus baseline (limited but useful for linter testing): node
+ *   scripts/build-corpus-stats.ts\
+ *   --slices $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
+ *   --output /tmp/corpus-stats-local.json
+ */
+
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { readDirectory, statPath } from "@mailwoman/core/fs/readers"
+import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
+import { join } from "path-ts"
+
+import { accumulateCooccurrences, createCooccurrenceStats, streamTokenLabelRows } from "#utils/slice-stats"
+
+const MIN_BIGRAM_COUNT = 2
+
+export interface CorpusStatsOptions {
+	slicesArg: string
+	outputPath: string
+	limitPerSlice?: number
+}
+
+async function discoverSlices(slicesArg: string): Promise<string[]> {
+	const stat = await statPath(slicesArg)
+
+	if (stat.isDirectory()) {
+		return (await readDirectory(slicesArg)).filter((f) => f.endsWith(".parquet")).map((f) => join(slicesArg, f))
+	}
+
+	if (stat.isFile() && slicesArg.endsWith(".parquet")) return [slicesArg]
+
+	// Otherwise treat as a literal path list (one per line if it's stdin-friendly).
+	return [slicesArg]
+}
+
+export async function buildCorpusStats(args: CorpusStatsOptions): Promise<void> {
+	const slicePaths = await discoverSlices(args.slicesArg)
+
+	console.error(`Discovered ${slicePaths.length} parquet slice(s)`)
+
+	const stats = createCooccurrenceStats()
+	const { tokens: tokenStats, bigrams: bigramStats } = stats
+	let totalRows = 0
+
+	for (const path of slicePaths) {
+		console.error(`Reading ${path}...`)
+
+		const before = totalRows
+
+		for await (const { tokens, labels } of streamTokenLabelRows(path, args.limitPerSlice)) {
+			totalRows++
+
+			if (tokens.length !== labels.length) continue // skip malformed
+
+			accumulateCooccurrences(stats, tokens, labels)
+		}
+
+		console.error(
+			`  ${totalRows - before} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`
+		)
+	}
+
+	// Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane. Token stats
+	// stay complete — they're cheap and we need accuracy at the long tail for label-vacuum
+	// detection.
+	let prunedBigrams = 0
+
+	for (const [k, labelMap] of bigramStats) {
+		let total = 0
+
+		for (const v of labelMap.values()) {
+			total += v
+		}
+
+		if (total < MIN_BIGRAM_COUNT) {
+			bigramStats.delete(k)
+
+			prunedBigrams++
+		}
+	}
+
+	console.error(`Pruned ${prunedBigrams} singleton bigrams; ${bigramStats.size} remain`)
+
+	const out = {
+		row_count: totalRows,
+		slice_paths: slicePaths,
+		tokens: {} as Record<string, Record<string, number>>,
+		bigrams: {} as Record<string, Record<string, number>>,
+	}
+
+	for (const [tk, labelMap] of tokenStats) {
+		out.tokens[tk] = Object.fromEntries(labelMap)
+	}
+
+	for (const [k, labelMap] of bigramStats) {
+		out.bigrams[k] = Object.fromEntries(labelMap)
+	}
+
+	await writeLocalJSONFile(out, args.outputPath)
+	const formattedSize = ByteFormatter.formatIEC(Buffer.byteLength(JSON.stringify(out)))
+
+	console.error(
+		`Wrote ${args.outputPath} (${formattedSize}) — ${totalRows} rows, ${tokenStats.size} tokens, ${bigramStats.size} bigrams`
+	)
+}

package/lib/tools/fetch/acra-sg.ts

@@ -0,0 +1,174 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch Singapore's ACRA corporate-entity register from data.gov.sg: 27 monthly CSVs (one per first
+ *   letter of the entity name plus "others"), 53 columns, the registered address FIELDED — `Block`,
+ *   `Street Name`, `Level No`, `Unit No`, `Building Name`, `Postal Code` — which is the NPPES shape,
+ *   so the corpus adapter renders the string and the spans fall out by construction. A Singapore
+ *   postcode names one building, so the postcode column is also the join to the OneMap/Overture rows.
+ *
+ *   License: Singapore Open Data Licence version 1.0 (attribution; commercial use, modification and
+ *   adaptation permitted; no share-alike). The prescribed attribution sentence goes in the manifest.
+ *
+ *   data.gov.sg serves a dataset in two calls: `initiate-download` prepares a signed URL and
+ *   `poll-download` answers it once the export is ready.
+ *
+ *   Invoke via `mailwoman corpus fetch acra-sg --out-root <path>`.
+ */
+
+import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
+import { pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { sleep } from "@mailwoman/core/utils/sleep"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
+import { loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download"
+
+const SLUG = "acra-sg"
+const COLLECTION_URL = "https://api-production.data.gov.sg/v2/public/api/collections/2/metadata"
+const DATASET_API = "https://api-production.data.gov.sg/v2/public/api/datasets"
+const DOWNLOAD_API = "https://api-open.data.gov.sg/v1/public/api/datasets"
+const LICENSE = "Singapore Open Data Licence version 1.0 — https://data.gov.sg/open-data-licence"
+
+const ATTRIBUTION =
+	"Contains information from ACRA Information on Corporate Entities accessed on <date> from data.gov.sg which is made available under the terms of the Singapore Open Data Licence version 1.0 https://data.gov.sg/open-data-licence"
+
+const POLL_INTERVAL_MS = 3000
+const POLL_ATTEMPTS = 40
+
+export type FetchACRASGOptions = BaseFetchOptions
+
+interface CollectionMetadata {
+	data?: { collectionMetadata?: { childDatasets?: string[] } }
+}
+
+interface DatasetMetadata {
+	data?: { name?: string; lastUpdatedAt?: string }
+}
+
+interface PollResponse {
+	data?: { status?: string; url?: string }
+}
+
+async function readJSON<T>(url: string): Promise<T> {
+	const res = await fetch(url, { headers: { accept: "application/json" } })
+
+	if (!res.ok) throw new Error(`acra-sg: ${url} answered HTTP ${res.status}`)
+
+	return (await res.json()) as T
+}
+
+/**
+ * Ask the portal to prepare the CSV export, then poll until it names the signed URL.
+ */
+async function signedURLFor(datasetID: string): Promise<string | undefined> {
+	await readJSON(`${DOWNLOAD_API}/${datasetID}/initiate-download`)
+
+	for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) {
+		await sleep(POLL_INTERVAL_MS)
+		const poll = await readJSON<PollResponse>(`${DOWNLOAD_API}/${datasetID}/poll-download`)
+
+		if (poll.data?.status === "DOWNLOAD_SUCCESS" && poll.data.url) return poll.data.url
+	}
+
+	return undefined
+}
+
+function filenameFor(name: string, datasetID: string): string {
+	const letter = /\('([^']+)'\)/.exec(name)?.[1]
+
+	return `acra-entities-${(letter ?? datasetID).toLowerCase().replaceAll(/[^a-z0-9]+/g, "-")}.csv`
+}
+
+export async function fetchACRASG(options: FetchACRASGOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	await makeDirectories(destDir)
+	const manifestPath = join(destDir, "MANIFEST.json")
+
+	const collection = await readJSON<CollectionMetadata>(COLLECTION_URL)
+	const datasetIDs = collection.data?.collectionMetadata?.childDatasets ?? []
+	report?.(`=== ${SLUG}: ${datasetIDs.length} datasets in collection 2`)
+
+	const previous = await loadCollectionFiles(manifestPath)
+	const files = new Map<string, SourceManifest>()
+	let fetched = 0
+	let skipped = 0
+	const failedCodes: string[] = []
+
+	for (const datasetID of datasetIDs) {
+		const metadata = await readJSON<DatasetMetadata>(`${DATASET_API}/${datasetID}/metadata`)
+		const name = metadata.data?.name ?? datasetID
+		const filename = filenameFor(name, datasetID)
+		const dest = join(destDir, filename)
+		const before = previous.get(filename)
+
+		if (before && (await pathExists(dest))) {
+			files.set(filename, before)
+
+			skipped++
+
+			continue
+		}
+
+		report?.(`--- ${name} (${datasetID})`)
+		let bytes: number
+
+		try {
+			bytes = await withRetries(
+				async () => {
+					const url = await signedURLFor(datasetID)
+
+					if (!url) throw new Error(`the export never reached DOWNLOAD_SUCCESS for ${datasetID}`)
+					const res = await fetch(url, { signal: AbortSignal.timeout(1_800_000) })
+
+					if (!res.ok) throw new Error(`HTTP ${res.status} on the signed URL for ${datasetID}`)
+
+					return streamBodyToFile(res, dest)
+				},
+				{ report, label: name }
+			)
+		} catch (error) {
+			report?.(`  ✗ ${(error as Error).message}`)
+			failedCodes.push(datasetID)
+
+			continue
+		}
+
+		if (bytes < BYTES_PER_KIB) {
+			report?.(`  ✗ ${bytes} bytes — an error page, not the register`)
+			failedCodes.push(datasetID)
+
+			continue
+		}
+
+		const sha = await sha256File(dest)
+
+		files.set(filename, {
+			source_url: `https://data.gov.sg/datasets/${datasetID}/view`,
+			downloaded_at: new Date().toISOString(),
+			filename,
+			sha256: sha,
+			bytes,
+		})
+
+		fetched++
+		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
+
+		const manifest: SourceCollectionManifest = {
+			source: SLUG,
+			source_url: "https://data.gov.sg/collections/2/view",
+			license: LICENSE,
+			attribution: ATTRIBUTION,
+			downloaded_at: new Date().toISOString(),
+			files: [...files.values()],
+		}
+
+		await writeManifest(manifestPath, manifest)
+	}
+
+	return { fetched, skipped, failed: failedCodes.length, failedCodes }
+}

package/lib/tools/fetch/ban.ts

@@ -0,0 +1,289 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the full French BAN (Base Adresse Nationale) — all metropolitan départements (01-95, 2A,
+ *   2B) plus 5 overseas DOM/TOM (971-976 excl. 975).
+ *
+ *   Source: https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/
+ *   Licence: Licence Ouverte 2.0 (attribution required — Tier B).
+ *
+ *   Files already present with matching sha256 are skipped (re-runnable). Downloads `.csv.gz`,
+ *   decompresses to `.csv`, deletes the `.gz` artifact. One shared `MANIFEST.json` at
+ *   `<outRoot>/ban/MANIFEST.json` covers all codes.
+ *
+ *   Invoke via `mailwoman corpus fetch ban --out-root <path>`. Built-in `fetch` with gzip/brotli
+ *   decompression replaces curl; native `node:zlib` gunzip replaces the `gunzip` subprocess; no
+ *   Python.
+ */
+
+import { gunzip } from "@mailwoman/core/fs/compression"
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { isFile, readLocalBuffer, tryStat } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePath, writeLocalFile } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { sleep } from "@mailwoman/core/utils/sleep"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { downloadToFile, loadManifestEntries, writeManifest } from "#tools/fetch/download"
+
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+
+const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv"
+
+/**
+ * All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
+ * change.
+ */
+const DEPT_CODES = [
+	"01",
+	"02",
+	"03",
+	"04",
+	"05",
+	"06",
+	"07",
+	"08",
+	"09",
+	"10",
+	"11",
+	"12",
+	"13",
+	"14",
+	"15",
+	"16",
+	"17",
+	"18",
+	"19",
+	"21",
+	"22",
+	"23",
+	"24",
+	"25",
+	"26",
+	"27",
+	"28",
+	"29",
+	"2A",
+	"2B",
+	"30",
+	"31",
+	"32",
+	"33",
+	"34",
+	"35",
+	"36",
+	"37",
+	"38",
+	"39",
+	"40",
+	"41",
+	"42",
+	"43",
+	"44",
+	"45",
+	"46",
+	"47",
+	"48",
+	"49",
+	"50",
+	"51",
+	"52",
+	"53",
+	"54",
+	"55",
+	"56",
+	"57",
+	"58",
+	"59",
+	"60",
+	"61",
+	"62",
+	"63",
+	"64",
+	"65",
+	"66",
+	"67",
+	"68",
+	"69",
+	"70",
+	"71",
+	"72",
+	"73",
+	"74",
+	"75",
+	"76",
+	"77",
+	"78",
+	"79",
+	"80",
+	"81",
+	"82",
+	"83",
+	"84",
+	"85",
+	"86",
+	"87",
+	"88",
+	"89",
+	"90",
+	"91",
+	"92",
+	"93",
+	"94",
+	"95",
+	"971",
+	"972",
+	"973",
+	"974",
+	"976",
+]
+
+export interface BanManifestEntry {
+	dept_code: string
+	filename: string
+	source_url: string
+	downloaded_at: string
+	sha256: string
+	bytes: number
+}
+
+export type FetchBanOptions = BaseFetchOptions
+
+export async function fetchBan(options: FetchBanOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const banDir = join(options.outRoot, "ban")
+	const manifestPath = join(banDir, "MANIFEST.json")
+	await makeDirectories(banDir)
+
+	// Load existing entries (code -> entry): skip detection + preservation of untouched codes.
+	const entries = await loadManifestEntries<BanManifestEntry>(manifestPath, (entry) => entry.dept_code)
+
+	let fetched = 0
+	let skipped = 0
+	let failed = 0
+	const failedCodes: string[] = []
+
+	for (const code of DEPT_CODES) {
+		const filename = `adresses-${code}.csv`
+		const gzFile = join(banDir, `${filename}.gz`)
+		const csvFile = join(banDir, filename)
+		const url = `${BASE_URL}/adresses-${code}.csv.gz`
+
+		report?.(`=== dept ${code}`)
+
+		// If the CSV already exists, compare its sha256 against the manifest.
+		if (await isFile(csvFile)) {
+			const existingSha = await sha256File(csvFile)
+			const recordedSha = entries.get(code)?.sha256
+
+			if (recordedSha && existingSha === recordedSha) {
+				report?.(`  → already present + sha matches — skipping`)
+
+				skipped++
+
+				continue
+			}
+
+			report?.(`  → present but sha mismatch or no manifest entry — re-fetching`)
+			await removePath(csvFile)
+		}
+
+		// Download the gzipped CSV.
+		try {
+			await downloadToFile({
+				url,
+				dest: gzFile,
+				timeoutMs: 600_000,
+				headers: { "Accept-Encoding": "gzip, br" },
+				report,
+			})
+		} catch (error) {
+			report?.(`  ✗ download failed: ${url} (${(error as Error).message})`)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		// Guard against truncated 404/error pages.
+		const gzSize = (await tryStat(gzFile))?.size ?? 0
+
+		if (gzSize < BYTES_PER_KIB) {
+			report?.(`  ✗ response too small (${gzSize} bytes) — probable 404 / error page`)
+			await removePath(gzFile)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		// Decompress in-place; delete the .gz.
+		try {
+			await writeLocalFile(await gunzip(await readLocalBuffer(gzFile)), csvFile)
+		} catch (error) {
+			report?.(`  ✗ decompress failed: ${(error as Error).message}`)
+			await removePath(gzFile)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		await removePath(gzFile)
+
+		if (!(await isFile(csvFile))) {
+			report?.(`  ✗ decompressed file not found at ${csvFile}`)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		const bytes = (await tryStat(csvFile))?.size ?? 0
+		const sha = await sha256File(csvFile)
+
+		entries.set(code, {
+			dept_code: code,
+			filename,
+			source_url: url,
+			downloaded_at: new Date().toISOString(),
+			sha256: sha,
+			bytes,
+		})
+
+		report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha}`)
+
+		fetched++
+
+		// Be a polite citizen — short pause between requests.
+		await sleep(200)
+	}
+
+	// Write the consolidated MANIFEST.json (entries sorted by dept_code, codepoint order).
+	const sorted = [...entries.values()].toSorted((a, b) =>
+		a.dept_code < b.dept_code ? -1 : a.dept_code > b.dept_code ? 1 : 0
+	)
+
+	await writeManifest(manifestPath, sorted)
+	report?.(`Wrote ${manifestPath} with ${sorted.length} entries.`)
+
+	report?.(`=== summary ===`)
+	report?.(`fetched:  ${fetched}`)
+	report?.(`skipped:  ${skipped} (already present + sha matched)`)
+	report?.(`failed:   ${failed}`)
+
+	if (failedCodes.length) {
+		report?.(`failed codes: ${failedCodes.join(" ")}`)
+	}
+
+	return { fetched, skipped, failed, failedCodes }
+}