@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/ro/tools/fetch/retea-scolara.ts

@@ -0,0 +1,187 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Downloads Romania's school network workbook for the `ro-retea-scolara` adapter.
+ *
+ * The Ministry of Education publishes a new workbook each school year under one data.gov.ro dataset,
+ * and each upload has a new resource URL. The fetcher therefore asks the portal's CKAN API for the
+ * dataset's resources and downloads the newest XLSX, rather than holding a URL that the next upload
+ * retires. The manifest records the resource URL, the CKAN `last_modified` and the license id the
+ * portal states, so a corpus row traces to one upload.
+ */
+
+import { APIClient } from "@mailwoman/core/api"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { tryStat } from "@mailwoman/core/fs/readers"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { RO_RETEA_SCOLARA_ADAPTER_ID, RO_RETEA_SCOLARA_LICENSE } from "#ro/adapters/retea-scolara/adapter"
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The data.gov.ro dataset that holds every school year's workbook.
+ */
+export const RO_RETEA_SCOLARA_DATASET_ID = "69392a50-5750-4a54-90c1-d461de44da6d"
+
+/**
+ * The CKAN call that lists the dataset's resources and its license.
+ */
+export const RO_RETEA_SCOLARA_PACKAGE_URL = `https://data.gov.ro/api/3/action/package_show?id=${RO_RETEA_SCOLARA_DATASET_ID}`
+
+/**
+ * The file name the adapter reads, whatever the upload's own name.
+ */
+export const RO_RETEA_SCOLARA_FILENAME = "retea-scolara.xlsx"
+
+const SLUG = RO_RETEA_SCOLARA_ADAPTER_ID
+
+/**
+ * One CKAN resource, as `package_show` lists it.
+ */
+export interface CKANResource {
+	url: string
+	format?: string | null
+	name?: string | null
+	last_modified?: string | null
+	created?: string | null
+}
+
+interface CKANPackage {
+	license_id?: string | null
+	resources?: CKANResource[]
+}
+
+export interface ReteaScolaraManifest extends SourceManifest {
+	resource_name: string | null
+	last_modified: string | null
+	license_id: string | null
+}
+
+/**
+ * The newest XLSX resource, by its `last_modified` or else its `created` date.
+ *
+ * @throws When the dataset lists no XLSX resource.
+ */
+export function newestXLSXResource(resources: readonly CKANResource[]): CKANResource {
+	const dated = resources
+		.filter((resource) => (resource.format ?? "").toUpperCase() === "XLSX")
+		.map((resource) => ({ resource, at: resource.last_modified ?? resource.created ?? "" }))
+		.toSorted((a, b) => b.at.localeCompare(a.at))
+
+	if (!dated.length) throw new Error(`${SLUG}: dataset ${RO_RETEA_SCOLARA_DATASET_ID} lists no XLSX resource`)
+
+	return dated[0]!.resource
+}
+
+export interface DownloadReteaScolaraOptions {
+	outputDir: PathBuilderLike
+	force?: boolean
+	retries?: number
+	retryDelayMs?: number
+	signal?: AbortSignal
+	report?: (line: string) => void
+}
+
+export async function downloadReteaScolara(
+	client: Pick<APIClient, "fetch">,
+	options: DownloadReteaScolaraOptions
+): Promise<FetchSummary> {
+	const { report } = options
+	const destDir = PathBuilder.from(options.outputDir)
+	const workbookPath = destDir(RO_RETEA_SCOLARA_FILENAME)
+	const manifestPath = destDir("MANIFEST.json")
+
+	await makeDirectories(destDir)
+	report?.(`=== ${SLUG}`)
+
+	const response = await client.fetch<{ result: CKANPackage }>({
+		url: RO_RETEA_SCOLARA_PACKAGE_URL,
+		signal: options.signal,
+	})
+
+	const pkg = response.data.result
+	const resource = newestXLSXResource(pkg.resources ?? [])
+	const lastModified = resource.last_modified ?? resource.created ?? null
+
+	if (pkg.license_id !== RO_RETEA_SCOLARA_LICENSE) {
+		report?.(`  the portal now states license ${pkg.license_id ?? "none"}, not ${RO_RETEA_SCOLARA_LICENSE}`)
+	}
+
+	const recorded = await readManifest<ReteaScolaraManifest>(manifestPath)
+	const stat = await tryStat(workbookPath)
+
+	if (
+		!options.force &&
+		recorded &&
+		stat &&
+		recorded.source_url === resource.url &&
+		recorded.last_modified === lastModified &&
+		recorded.bytes === stat.size
+	) {
+		report?.(`  present, and the portal lists the same upload`)
+
+		return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+	}
+
+	const { bytes } = await downloadToFile({
+		url: resource.url,
+		dest: workbookPath,
+		retries: options.retries,
+		retryDelayMs: options.retryDelayMs,
+		report,
+	})
+
+	const sha256 = await sha256File(workbookPath)
+
+	report?.(`  ✓ ${resource.name ?? resource.url} ${ByteFormatter.formatIEC(bytes)} sha256=${sha256}`)
+
+	const manifest: ReteaScolaraManifest = {
+		source_url: resource.url,
+		filename: RO_RETEA_SCOLARA_FILENAME,
+		bytes,
+		sha256,
+		downloaded_at: new Date().toISOString(),
+		resource_name: resource.name ?? null,
+		last_modified: lastModified,
+		license_id: pkg.license_id ?? null,
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}
+
+/**
+ * The workbook path the adapter reads under a fetch root.
+ */
+export function reteaScolaraInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike {
+	return outRoot(SLUG, RO_RETEA_SCOLARA_FILENAME)
+}
+
+export interface FetchReteaScolaraOptions extends BaseFetchOptions {
+	force?: boolean
+	retries?: number
+	signal?: AbortSignal
+}
+
+export async function fetchReteaScolara(
+	options: FetchReteaScolaraOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	await using client = new APIClient({ displayName: SLUG, retry: true })
+
+	// Awaited rather than returned, so the client is disposed after the download finishes.
+	return await downloadReteaScolara(client, {
+		outputDir: options.outRoot(SLUG),
+		force: options.force,
+		retries: options.retries,
+		retryDelayMs: options.retryDelayMs,
+		signal: options.signal,
+		report,
+	})
+}

package/lib/runner.ts

@@ -3,106 +3,142 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Adapter runner — drives a `CorpusAdapter` to completion and writes intermediate JSONL + a
- *   per-slice manifest.
- *
- *   Output layout under `outputDir`:
- *
- *   ```
- *   <outputDir>/<adapter.id>/
- *   canonical.jsonl       # one row per line, in emission order
- *   MANIFEST.json         # adapter id, version, row count, sha256, license, started_at, ended_at
- * ```
- *
- *   The runner is responsible for everything an adapter is **not** responsible for:
- *
- *   - Stamping `corpus_version` on every row (adapters must NOT set it).
- *   - Applying `canonicalDedupKey` and skipping duplicates.
- *   - Streaming sha256 over JSONL bytes so the manifest checksum doesn't require a re-read.
- *   - Honoring backpressure on the output write stream.
- *   - Counting + emitting periodic progress to an optional callback.
- *   - Honoring `signal` (delegates to adapter's iteration boundary).
- *
- *   The runner does NOT perform alignment, tokenization, synthesis, or slicing into Parquet. Those
- *   steps run later, consuming the JSONL slices this writes.
+ *   Adapter runner — drives a `CorpusAdapter` to completion and writes `canonical.jsonl` plus a
+ *   `MANIFEST.json` per adapter. It does not align, tokenize, synthesize, or write Parquet.
  */
 
+import { isAlpha2CodeShape } from "@mailwoman/codex/country"
 import { openWriteStream, type WriteStream } from "@mailwoman/core/fs/streams"
 import { writeLocalJSONFile, makeDirectories } from "@mailwoman/core/fs/writers"
-import { join, type PathBuilderLike } from "path-ts"
+import { stringifyJSON } from "@mailwoman/core/json"
+import { type PathBuilderLike, resolvePathBuilder } from "path-ts"
 
-import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "#adapters/utils"
+import { canonicalDedupKey } from "#adapters/dedup-key"
+import type { AdapterRegistry } from "#adapters/registry"
+import { streamingSha256, type StreamingHasher } from "#adapters/sha256-stream"
+import { FingerprintSet } from "#fingerprints"
 import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
 
 /**
  * Snapshot of the runner's state, emitted on every progress tick.
  */
 export interface RunnerProgress {
-	/**
-	 * Adapter being driven.
-	 */
 	adapterID: string
 
 	/**
-	 * Total rows the adapter has yielded (before dedup).
+	 * Total rows the adapter has yielded, before dedup.
 	 */
 	yielded: number
 
-	/**
-	 * Rows actually written to JSONL (after dedup).
-	 */
 	written: number
 
-	/**
-	 * Bytes written to JSONL so far.
-	 */
 	bytes: number
 
-	/**
-	 * Wall-clock milliseconds since the run started.
-	 */
 	elapsed_ms: number
 }
 
 /**
- * Per-invocation options for `runAdapter`.
+ * Base-2 logarithm of the fingerprint table's slot count.
+ *
+ * 2^27 slots hold 93,952,409 keys before the load limit.
+ * The largest measured adapter has 57,570,829 distinct keys.
+ * The table occupies 2.0 GiB outside the V8 heap.
  */
-export interface RunAdapterOptions {
+export const DEFAULT_DEDUP_SLOTS_LOG2 = 27
+
+/**
+ * How many distinct dedup keys a `Set`-backed run holds before it stops adding new ones.
+ *
+ * A V8 `Set` refuses a 16,777,216th entry outright, so this cap was a real bound rather than a preference.
+ * It applies only to a run that opts out of the fingerprint table
+ * through {@linkcode RunAdapterOptions.dedupMaxSize}.
+ */
+export const DEFAULT_DEDUP_MAX_SIZE = 10_000_000
+
+/**
+ * Where a run held its dedup keys.
+ */
+export const DedupStore = {
 	/**
-	 * Adapter to drive.
+	 * 128-bit fingerprints in a flat `Uint32Array`, bounded by the array the run sized.
 	 */
-	adapter: CorpusAdapter
+	FingerprintTable: "fingerprint-table",
 
 	/**
-	 * Options handed to the adapter (input path, country filter, limit, signal).
+	 * A V8 `Set` of key strings, capped because V8 refuses a 16,777,216th entry.
 	 */
+	CappedSet: "capped-set",
+} as const
+
+export type DedupStore = (typeof DedupStore)[keyof typeof DedupStore]
+
+export interface RunAdapterOptions {
+	adapter: CorpusAdapter
+
 	adapterOptions: AdapterOptions
 
 	/**
-	 * Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it.
+	 * Hold dedup keys in a V8 `Set` capped at this many, rather than in a fingerprint table.
+	 *
+	 * The cap is what the fingerprint table replaces.
+	 * A run that sets this reproduces the pre-2026-09-28 behavior.
+	 *
+	 * Past the cap, the runner still drops a duplicate of a key it holds
+	 * and writes a duplicate of a key first seen after the cap.
+	 *
+	 * Over `v0.7.0-de-holdout` that wrote 1,490,992 duplicate rows across three adapters.
+	 * A test sets it low to reach exhaustion in a few rows.
+	 *
+	 * After reaching the cap, the runner still drops duplicates for keys already in the set.
+	 * It writes duplicates for keys first seen after the cap.
+	 */
+	dedupMaxSize?: number
+
+	/**
+	 * Base-2 logarithm of the fingerprint table's slot count.
+	 *
+	 * Defaults to {@linkcode DEFAULT_DEDUP_SLOTS_LOG2}.
+	 * A test sets it low to keep the allocation small.
+	 */
+	dedupSlotsLog2?: number
+
+	/**
+	 * Root output directory.
+	 * The runner creates `<outputDir>/<adapter.id>/` under it.
 	 */
 	outputDir: PathBuilderLike
 
 	/**
-	 * Corpus version stamped onto every row. Locked together with the tokenizer version.
+	 * Corpus version stamped onto every row, locked together with the tokenizer version.
 	 */
 	corpusVersion: string
 
 	/**
-	 * Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once at the end of the
-	 * run. Errors thrown from this callback abort the run.
+	 * The `source` id stamped on every emitted row, when it differs from the adapter's own id.
+	 *
+	 * A source id is a wire identifier keyed by `source_weights`, so re-using a name a built
+	 * corpus already contains makes this run's rows indistinguishable from that corpus's.
+	 * When absent, rows use `adapter.id`.
+	 */
+	sourceName?: string
+
+	/**
+	 * Invoked every `progressEvery` rows yielded and once at the end.
+	 * A thrown error aborts the run.
 	 */
 	onProgress?: (snapshot: RunnerProgress) => void
 
 	/**
-	 * Yielded-row interval at which `onProgress` fires. Defaults to 1000. The terminal tick is always emitted regardless
-	 * of this value.
+	 * Yielded-row interval at which `onProgress` fires.
+	 *
+	 * Defaults to 1000 rows per callback.
+	 * The runner always emits a terminal update.
 	 */
 	progressEvery?: number
 }
 
 /**
- * Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk.
+ * Return value of `runAdapter`, the same shape as the manifest written to disk.
  */
 export interface AdapterRunManifest {
 	adapter_id: string
@@ -112,6 +148,38 @@ export interface AdapterRunManifest {
 	yielded: number
 	written: number
 	deduped: number
+	/**
+	 * Input records the adapter refused or trimmed before yielding, keyed by reason.
+	 *
+	 * See {@link AdapterOptions.dropped} for the key shapes.
+	 * An empty object means the adapter does not count its drops, so the number it dropped is unmeasured.
+	 */
+	dropped: Record<string, number>
+	/**
+	 * The `yielded` count at which the dedup set stopped growing, or `null` where it never did.
+	 *
+	 * A run reporting a number here deduplicated its rows completely up to that point.
+	 * After that point, it still drops duplicates for keys already held.
+	 *
+	 * It writes duplicates for keys first seen after the cap.
+	 * `deduped` by itself cannot identify which, so a consumer comparing two builds'
+	 * duplicate counts needs this beside it.
+	 */
+	dedup_exhausted_at_yielded: number | null
+	/**
+	 * How the run held its dedup keys: `fingerprint-table` or `capped-set`.
+	 *
+	 * The two reach different row counts on the same input, so a consumer comparing
+	 * two builds needs to know which each used.
+	 * `capped-set` writes a duplicate of any key first seen after its cap.
+	 */
+	dedup_store: DedupStore
+	/**
+	 * Distinct dedup keys the run held at the end.
+	 *
+	 * Under `fingerprint-table` this is a fingerprint count, so two keys sharing all 128 bits count once.
+	 */
+	dedup_keys: number
 	bytes: number
 	sha256: string
 	jsonl_path: string
@@ -121,30 +189,38 @@ export interface AdapterRunManifest {
 }
 
 /**
- * Drive a single adapter to completion.
- *
- * Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under `outputDir/<adapter.id>/`.
- * Throws if the output directory cannot be created, if a row arrives with a missing required field, or if the abort
- * signal fires.
+ * Drive a single adapter to completion, writing its jsonl and manifest under `outputDir/<adapter.id>/`.
  */
 export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest> {
-	const { adapter, adapterOptions, outputDir, corpusVersion } = opts
+	const { adapter, outputDir, corpusVersion } = opts
+	// A caller that passes its own map reads the counts back from it.
+	// Otherwise the runner supplies one, so every manifest carries the adapter's drop reasons.
+	const dropped = opts.adapterOptions.dropped ?? new Map<string, number>()
+	const adapterOptions: AdapterOptions = { ...opts.adapterOptions, dropped }
 	const progressEvery = opts.progressEvery ?? 1000
 
-	const adapterDir = join(outputDir, adapter.id)
+	const adapterDir = resolvePathBuilder(outputDir, adapter.id)
 	await makeDirectories(adapterDir)
 
-	const jsonlPath = join(adapterDir, "canonical.jsonl")
-	const manifestPath = join(adapterDir, "MANIFEST.json")
+	const jsonlPath = adapterDir("canonical.jsonl")
+	const manifestPath = adapterDir("MANIFEST.json")
 
 	const startedAt = new Date()
 	const t0 = performance.now()
 
 	const stream = openWriteStream(jsonlPath, { encoding: "utf8" })
 	const hasher: StreamingHasher = streamingSha256()
-	const seen = new Set<string>()
-	const DEDUP_MAX_SIZE = 10_000_000
+
+	// The `Set` path serves callers that cannot accept a fingerprint collision dropping a legitimate row.
+	// A test also uses it to reach the capped behavior in three rows.
+	// Every other run takes the table.
+	// Its entry count is bounded by the array it sized rather than by V8's `Set` limit.
+	const capped = opts.dedupMaxSize !== undefined
+	const seen = capped ? new Set<string>() : null
+	const fingerprints = capped ? null : new FingerprintSet(opts.dedupSlotsLog2 ?? DEFAULT_DEDUP_SLOTS_LOG2)
+	const dedupMaxSize = opts.dedupMaxSize ?? DEFAULT_DEDUP_MAX_SIZE
 	let dedupExhausted = false
+	let dedupExhaustedAtYielded: number | null = null
 
 	let yielded = 0
 	let written = 0
@@ -169,30 +245,61 @@ export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunMan
 			yielded++
 			assertEmittedRow(adapter, row)
 
-			const stamped: CanonicalRow = { ...row, corpus_version: corpusVersion }
+			const stamped: CanonicalRow = {
+				...row,
+				// The rename is the runner's, after `assertEmittedRow` has held the adapter to emitting its own id.
+				source: opts.sourceName ?? row.source,
+				corpus_version: corpusVersion,
+				addressRole: row.addressRole ?? adapter.addressRole,
+				// `register` is nullable and null is a statement rather than an absence,
+				// so only an undefined field takes the adapter's declaration.
+				register: row.register === undefined ? adapter.register : row.register,
+				surface: row.surface ?? adapter.surface,
+			}
+
 			const key = canonicalDedupKey(stamped)
 
-			if (!dedupExhausted) {
-				if (seen.has(key)) {
+			if (fingerprints) {
+				// The table holds every key the run has seen, so a duplicate is refused wherever it arrives.
+				if (!fingerprints.add(key)) {
 					if (yielded % progressEvery === 0) {
 						emitProgress()
 					}
 
 					continue
 				}
+			} else if (seen) {
+				// The membership test runs whether or not the set is full.
+				// The cap stops the set from growing.
+				// The set still rejects every duplicate of a key it holds.
+				// Without the membership test at the cap, duplicates of the first
+				// `dedupMaxSize` keys passed through.
+				// This ordering keeps those duplicates out.
+				if (seen.has(key)) {
+					if (yielded % progressEvery === 0) {
+						emitProgress()
+					}
 
-				if (seen.size >= DEDUP_MAX_SIZE) {
-					dedupExhausted = true
+					continue
+				}
 
-					process.stderr.write(
-						`  runner: dedup set full at ${DEDUP_MAX_SIZE.toLocaleString()} — skipping dedup for remaining rows\n`
-					)
-				} else {
-					seen.add(key)
+				if (!dedupExhausted) {
+					if (seen.size >= dedupMaxSize) {
+						dedupExhausted = true
+						dedupExhaustedAtYielded = yielded
+
+						process.stderr.write(
+							`  runner: dedup set full at ${dedupMaxSize.toLocaleString()} keys after ${yielded.toLocaleString()} ` +
+								`yielded rows — a later row duplicating a key already held is still dropped, and a duplicate of ` +
+								`a key first seen from here on is written\n`
+						)
+					} else {
+						seen.add(key)
+					}
 				}
 			}
 
-			const line = `${JSON.stringify(stamped)}\n`
+			const line = `${stringifyJSON(stamped)}\n`
 			hasher.update(line)
 			bytes += Buffer.byteLength(line, "utf8")
 
@@ -211,6 +318,10 @@ export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunMan
 		await once(stream, "close")
 	}
 
+	// Every adapter honors `signal` by returning instead, so without this a truncated
+	// `canonical.jsonl` would be recorded as a finished run.
+	adapterOptions.signal?.throwIfAborted()
+
 	const endedAt = new Date()
 	const elapsed_ms = performance.now() - t0
 	emitProgress()
@@ -223,9 +334,13 @@ export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunMan
 		yielded,
 		written,
 		deduped: yielded - written,
+		dropped: Object.fromEntries([...dropped].toSorted(([a], [b]) => a.localeCompare(b))),
+		dedup_exhausted_at_yielded: dedupExhaustedAtYielded,
+		dedup_store: fingerprints ? DedupStore.FingerprintTable : DedupStore.CappedSet,
+		dedup_keys: fingerprints?.size ?? seen?.size ?? 0,
 		bytes,
 		sha256: hasher.digest(),
-		jsonl_path: jsonlPath,
+		jsonl_path: jsonlPath.toString(),
 		started_at: startedAt.toISOString(),
 		ended_at: endedAt.toISOString(),
 		elapsed_ms,
@@ -237,10 +352,7 @@ export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunMan
 }
 
 /**
- * Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the registry before
- * calling if partial-failure is desired).
- *
- * Returns the manifests in registry insertion order.
+ * Drive every adapter in a registry sequentially, stopping on the first failure.
  */
 export async function runAllAdapters(
 	registry: AdapterRegistry,
@@ -263,13 +375,9 @@ export async function runAllAdapters(
 	return out
 }
 
-/**
- * Validate an emitted row. Cheap; runs once per row. Catches adapter bugs early so the JSONL doesn't end up
- * half-malformed.
- */
 function assertEmittedRow(adapter: CorpusAdapter, row: CanonicalRow): void {
 	if (row.source !== adapter.id) {
-		throw new Error(`adapter ${adapter.id}: row.source must equal adapter.id (got ${JSON.stringify(row.source)})`)
+		throw new Error(`adapter ${adapter.id}: row.source must equal adapter.id (got ${stringifyJSON(row.source)})`)
 	}
 
 	if (!row.source_id) {
@@ -284,15 +392,43 @@ function assertEmittedRow(adapter: CorpusAdapter, row: CanonicalRow): void {
 		throw new Error(`adapter ${adapter.id}: row.country is empty for source_id=${row.source_id}`)
 	}
 
+	// `country_weights` and every country filter use this value.
+	// Validate its ISO 3166-1 alpha-2 shape.
+	// `ZZ` and `XK` are legitimate codes even though ISO membership does not list them.
+	if (!isAlpha2CodeShape(row.country)) {
+		throw new Error(
+			`adapter ${adapter.id}: row.country ${stringifyJSON(row.country)} is not two upper-case letters ` +
+				`for source_id=${row.source_id}. ISO 3166-1 alpha-2 is the shape country_weights and every ` +
+				`country filter key on, so a code outside it trains on nothing and matches no filter.`
+		)
+	}
+
 	if (!row.license) {
 		throw new Error(`adapter ${adapter.id}: row.license is empty for source_id=${row.source_id}`)
 	}
 }
 
 /**
- * Promise-ify a single event emission. Used to await `drain` / `close` on the write stream. Exported for `build.ts`,
- * whose stage streams await `close` the same way; unlike a bare two-listener race, the loser listener is detached so a
- * long-lived stream does not accumulate one orphan handler per wait.
+ * Resolve once everything written to `stream` so far has reached the file.
+ *
+ * A zero-length write acts as a barrier.
+ * Its callback runs after earlier queued writes.
+ *
+ * A caller can then record a byte offset for the bytes in the file.
+ * `drain` fires only when the buffer was full.
+ *
+ * `bytesWritten` excludes queued data.
+ * Neither value answers when all earlier writes have reached the file.
+ */
+export function flushStream(stream: WriteStream): Promise<void> {
+	return new Promise((resolve, reject) => {
+		stream.write("", (error) => (error ? reject(error) : resolve()))
+	})
+}
+
+/**
+ * Promise-ify a single `drain` or `close` emission, detaching the loser listener
+ * so a long-lived stream does not accumulate an orphan handler per wait.
  */
 export function once(emitter: WriteStream, event: "drain" | "close"): Promise<void> {
 	return new Promise((resolve, reject) => {