@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/it/tools/fetch/anac.ts

@@ -0,0 +1,391 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download ANAC's OCDS contract releases for the `it-anac` adapter.
+ *
+ * The Open Contracting Partnership's Data Registry serves publication 117 at
+ * `https://data.open-contracting.org/en/publication/117/download?name=<edition>.jsonl.gz`, where an
+ * edition is a year, `undated`, or `full` for every year in one file. Each file is JSON Lines, one
+ * OCDS release per line. That is the shape `#it/adapters/anac/adapter` reads.
+ *
+ * The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the
+ * `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`, so
+ * this module sits beside `#fi/tools/fetch/ryhti` rather than in an `sdk/` root the workspace does
+ * not declare.
+ *
+ * Three host behaviors decide what this module does:
+ *
+ * 1. **The host honors `Range`.** It answers `accept-ranges: bytes`, so an interrupted transfer
+ *    resumes through {@linkcode resumableDownload} rather than restarting. That matters most for
+ *    the `full` edition.
+ * 2. **An edition's length and modification time are both served by `HEAD`.** The re-run check
+ *    compares the pair against the manifest, so one small request per edition replaces the body.
+ * 3. **This module moves bytes and decodes none of them.** The compressed body is streamed to disk
+ *    and then decompressed stream to stream, so no chunk boundary is interpreted as a character
+ *    boundary. ANAC's data holds accented Italian place names, and a `chunk.toString("utf8")` per
+ *    gzip chunk would write U+FFFD at every split multi-byte character.
+ *
+ * ANAC updates the publication monthly, on the second of the month. The current year's
+ * edition is re-fetched on a `HEAD` disagreement rather than treated as final.
+ *
+ * The registry publishes this release under CC BY 4.0. The address-source register elected it.
+ * The manifest records the license and the attribution, and the adapter records the license on every
+ * row.
+ */
+
+import { APIClient } from "@mailwoman/core/api"
+import { gunzipChunks } from "@mailwoman/core/fs/compression"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { tryStat } from "@mailwoman/core/fs/readers/stat"
+import { openReadStream, openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { IT_ANAC_ADAPTER_ID, IT_ANAC_ATTRIBUTION, IT_ANAC_LICENSE } from "#it/adapters/anac/adapter"
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
+import { loadCollectionFiles, resumableDownload, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The publication the registry serves this release under.
+ *
+ * Italy has four publications on the registry and id 117 is ANAC's.
+ * Id 87 is OpenTender's Italian publication, so a reader that filters by country
+ * would take the wrong publication.
+ */
+export const IT_ANAC_PUBLICATION_ID = 117
+
+/**
+ * The download endpoint for one edition.
+ */
+export function anacEditionURL(edition: string): string {
+	return `https://data.open-contracting.org/en/publication/${IT_ANAC_PUBLICATION_ID}/download?name=${edition}.jsonl.gz`
+}
+
+/**
+ * The directory the editions are written under, the adapter's `inputPath`.
+ */
+const SLUG = IT_ANAC_ADAPTER_ID
+
+/**
+ * The edition fetched when a caller names none.
+ *
+ * `full` is every year in one file.
+ * A corpus build reads that edition.
+ * A caller measuring one year names that year instead.
+ */
+export const IT_ANAC_DEFAULT_EDITIONS: readonly string[] = ["full"]
+
+/**
+ * What the manifest records per edition.
+ *
+ * {@linkcode SourceManifest}'s five fields describe the decompressed `.jsonl`,
+ * which is what the adapter reads and what the digest covers.
+ * The two added fields describe the transfer and are what the re-run check compares
+ * before it downloads the body again.
+ */
+export interface ANACEditionManifest extends SourceManifest {
+	/**
+	 * The `last-modified` the host served the compressed edition under, or `null` where it served none.
+	 */
+	last_modified: string | null
+	/**
+	 * The `content-length` the host stated for the compressed edition, or `null` where it stated none.
+	 */
+	compressed_bytes: number | null
+}
+
+/**
+ * What the host states about one edition without sending it.
+ */
+export interface ANACEditionHead {
+	lastModified: string | null
+	contentLength: number | null
+	contentType: string | null
+}
+
+/**
+ * Read what the host states about one edition.
+ *
+ * Through `APIClient`, which is the interface for a repeated request with a small response.
+ * The bodies are not taken this way: `APIClient` buffers a response, and the `full` edition is 184 MB.
+ *
+ * A header the host does not send is reported as `null` rather than raised on,
+ * and the caller then downloads rather than reading the absence as a match.
+ */
+export async function readANACEditionHead(client: Pick<APIClient, "fetch">, edition: string): Promise<ANACEditionHead> {
+	const response = await client.fetch<unknown>({ method: "head", url: anacEditionURL(edition) })
+	const headers = response.headers as Record<string, string> | undefined
+	const length = headers?.["content-length"]
+
+	return {
+		lastModified: headers?.["last-modified"] ?? null,
+		contentLength: length && /^\d+$/u.test(length) ? Number(length) : null,
+		contentType: headers?.["content-type"] ?? null,
+	}
+}
+
+export interface DownloadANACOptions {
+	/**
+	 * Where the editions and the manifest are written.
+	 * The directory itself is the adapter's `inputPath`.
+	 */
+	outputDir: PathBuilderLike
+	/**
+	 * Which editions to fetch: a year, `undated`, or `full`.
+	 * @defaultValue {@linkcode IT_ANAC_DEFAULT_EDITIONS}
+	 */
+	editions?: readonly string[]
+	/**
+	 * Keep the compressed copy beside the `.jsonl`.
+	 *
+	 * The adapter reads the `.jsonl`, so the default removes the archive once it is written.
+	 */
+	keepCompressed?: boolean
+	/**
+	 * Re-read the `.jsonl`'s sha256 on a re-run instead of comparing its byte count.
+	 */
+	verifyDigest?: boolean
+	/**
+	 * Download even where the `HEAD` agrees with the manifest.
+	 */
+	force?: boolean
+	/**
+	 * Pause between range attempts, in milliseconds.
+	 */
+	retryDelayMs?: number
+	signal?: AbortSignal
+	report?: (line: string) => void
+}
+
+export type FetchITANACOptions = BaseFetchOptions & Omit<DownloadANACOptions, "outputDir" | "report">
+
+/**
+ * The directory `#it/adapters/anac/adapter` reads under a fetch root.
+ *
+ * The adapter takes the directory rather than one file, because a build reading
+ * several years reads every edition in it.
+ */
+export function itANACInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike {
+	return outRoot(SLUG)
+}
+
+/**
+ * Whether an edition on disk is still the one the manifest describes and the host still serves.
+ *
+ * The `HEAD` decides first, because ANAC republishes the publication monthly
+ * and the host states a new length and modification time when it does.
+ * The file's own length is then checked, so a truncated local copy is re-fetched even
+ * where the host has not changed.
+ *
+ * Exported so a test can exercise the decision without a transfer: the alternative is
+ * letting {@linkcode downloadITANAC} reach the host to find out which branch it took.
+ */
+export async function isANACEditionCurrent(
+	recorded: ANACEditionManifest | null,
+	head: ANACEditionHead,
+	path: PathBuilderLike,
+	verifyDigest: boolean
+): Promise<boolean> {
+	if (!recorded) return false
+
+	// A host that states neither cannot be compared against, so the edition is downloaded.
+	if (!head.lastModified && head.contentLength === null) return false
+
+	if (head.lastModified && head.lastModified !== recorded.last_modified) return false
+
+	if (head.contentLength !== null && head.contentLength !== recorded.compressed_bytes) return false
+
+	const stat = await tryStat(path)
+
+	if (!stat || stat.size !== recorded.bytes) return false
+
+	if (!verifyDigest) return true
+
+	return (await sha256File(path)) === recorded.sha256
+}
+
+/**
+ * Download each requested edition into `options.outputDir` and leave them beside one manifest.
+ *
+ * Re-runnable per edition: an edition whose `HEAD` matches the manifest's recorded
+ * modification time and compressed length, and whose `.jsonl` is still on disk at
+ * the recorded length, makes no request for the body.
+ */
+export async function downloadITANAC(
+	client: Pick<APIClient, "fetch">,
+	options: DownloadANACOptions
+): Promise<FetchSummary> {
+	const { report } = options
+	const destDir = PathBuilder.from(options.outputDir)
+	const editions = options.editions?.length ? options.editions : IT_ANAC_DEFAULT_EDITIONS
+
+	await makeDirectories(destDir)
+
+	const manifestPath = destDir("MANIFEST.json")
+	const recorded = (await loadCollectionFiles(manifestPath)) as Map<string, ANACEditionManifest>
+	const written = new Map<string, ANACEditionManifest>(recorded)
+	const summary: FetchSummary = { fetched: 0, skipped: 0, failed: 0, failedCodes: [] }
+
+	report?.(`=== ${SLUG}`)
+
+	for (const edition of editions) {
+		const filename = `${edition}.jsonl`
+		const editionPath = destDir(filename)
+		const compressedPath = destDir(`${filename}.gz`)
+
+		let head: ANACEditionHead
+
+		try {
+			head = await readANACEditionHead(client, edition)
+		} catch (error) {
+			report?.(`  ✗ ${edition}: ${error instanceof Error ? error.message : String(error)}`)
+
+			summary.failed++
+			summary.failedCodes.push(edition)
+
+			continue
+		}
+
+		report?.(
+			`  ${edition}: ${head.contentLength === null ? "no content-length" : `${ByteFormatter.formatIEC(head.contentLength)} compressed`}` +
+				`, last-modified ${head.lastModified ?? "absent"}`
+		)
+
+		const current =
+			!options.force &&
+			(await isANACEditionCurrent(recorded.get(filename) ?? null, head, editionPath, options.verifyDigest ?? false))
+
+		if (current) {
+			report?.(`  ✓ ${edition} already current — no download.`)
+
+			summary.skipped++
+
+			continue
+		}
+
+		let compressedBytes: number
+
+		try {
+			compressedBytes = await resumableDownload({
+				url: anacEditionURL(edition),
+				dest: compressedPath,
+				headers: { accept: "application/gzip, application/x-gzip, */*" },
+				retryDelayMs: options.retryDelayMs,
+				report,
+			})
+		} catch (error) {
+			report?.(`  ✗ ${edition} download failed: ${error instanceof Error ? error.message : String(error)}`)
+			await removePathIfPresent(compressedPath)
+
+			summary.failed++
+			summary.failedCodes.push(edition)
+
+			continue
+		}
+
+		// The delivered count rather than the stated one.
+		// ANAC republishes monthly, so a `HEAD` taken before the transfer can describe
+		// a different edition than the body that arrived.
+		if (head.contentLength !== null && head.contentLength !== compressedBytes) {
+			report?.(
+				`  ! ${edition}: the HEAD stated ${head.contentLength} compressed bytes and ${compressedBytes} arrived, ` +
+					`so the edition was republished between the two requests`
+			)
+		}
+
+		try {
+			await decompressToJSONL(compressedPath, editionPath)
+		} catch (error) {
+			report?.(`  ✗ ${edition} decompress failed: ${error instanceof Error ? error.message : String(error)}`)
+			await removePathIfPresent(compressedPath)
+
+			summary.failed++
+			summary.failedCodes.push(edition)
+
+			continue
+		}
+
+		const stat = await tryStat(editionPath)
+
+		if (!stat) {
+			report?.(`  ✗ ${edition}: the decompressed edition is not at ${editionPath.toString()}`)
+
+			summary.failed++
+			summary.failedCodes.push(edition)
+
+			continue
+		}
+
+		if (!options.keepCompressed) {
+			await removePathIfPresent(compressedPath)
+		}
+
+		written.set(filename, {
+			source_url: anacEditionURL(edition),
+			downloaded_at: new Date().toISOString(),
+			filename,
+			sha256: await sha256File(editionPath),
+			bytes: stat.size,
+			last_modified: head.lastModified,
+			compressed_bytes: compressedBytes,
+		})
+
+		report?.(`  ✓ ${edition}: ${ByteFormatter.formatIEC(stat.size)} decompressed`)
+
+		summary.fetched++
+	}
+
+	const manifest: SourceCollectionManifest = {
+		source: SLUG,
+		source_url: `https://data.open-contracting.org/en/publication/${IT_ANAC_PUBLICATION_ID}`,
+		license: IT_ANAC_LICENSE,
+		attribution: IT_ANAC_ATTRIBUTION,
+		downloaded_at: new Date().toISOString(),
+		files: [...written.values()].toSorted((left, right) => left.filename.localeCompare(right.filename)),
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  MANIFEST written to ${manifestPath.toString()}`)
+
+	return summary
+}
+
+/**
+ * Download ANAC's OCDS editions into `<outRoot>/it-anac/`.
+ *
+ * The registry entry point.
+ * `#it/adapters/anac/adapter` is pointed at {@linkcode itANACInputPath},
+ * the directory itself, so it reads every edition inside it.
+ */
+export async function fetchITANAC(options: FetchITANACOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	await using client = new APIClient({ displayName: SLUG, retry: true })
+
+	// Awaited rather than returned: `await using` disposes the client when this scope exits,
+	// and a disposed `APIClient` refuses every later request.
+	return await downloadITANAC(client, {
+		outputDir: options.outRoot(SLUG),
+		editions: options.editions,
+		keepCompressed: options.keepCompressed,
+		verifyDigest: options.verifyDigest,
+		force: options.force,
+		retryDelayMs: options.retryDelayMs,
+		signal: options.signal,
+		report,
+	})
+}
+
+/**
+ * Decompress one gzip file to another path, stream to stream.
+ *
+ * Exported because a test exercises it directly: an accented Italian place name
+ * split across two gzip chunks must arrive whole.
+ */
+export async function decompressToJSONL(source: PathBuilderLike, destination: PathBuilderLike): Promise<void> {
+	const decompressed = gunzipChunks(openReadStream(source))
+
+	await pipeline(Readable.fromWeb(decompressed as Parameters<typeof Readable.fromWeb>[0]), openWriteStream(destination))
+}

package/lib/jp/adapters/wof-admin/adapter.ts

@@ -5,28 +5,28 @@
  *
  *   `wof-admin-jp`: Japanese admin-hierarchy adapter.
  *
- *   PROTOTYPE — not yet wired into the Stage 3 training corpus. Demonstrates the JP address shape and
+ *   prototype — not yet wired into the Stage 3 training corpus. Demonstrates the JP address shape and
  *   synthesizes BIO-labeled training rows from the global WOF SQLite.
  *
  *   JP addresses differ from US/EU in three ways:
  *
  *   1. **Reversed ordering** — region → county → locality → block → house_number "東京都世田谷区南町1-2-3" not
  *        "1-2-3 Minamicho, Setagaya, Tokyo"
- *   2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighbourhood"
+ *   2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighborhood"
  *        placetype (丁目) is the closest analog to a street but is actually a grid cell.
- *   3. **Postcode-first convention** — addresses are often prefixed with `〒NNN-NNNN`.
+ *   3. **Postcode-first convention** — addresses are often prefixed with `〒NNN-nnnn`.
  *
  *   Schema mapping to ComponentTags (subset of Stage 3 + JP-specific Phase 6 tags):
  *
  *   | JP concept | WOF placetype | ComponentTag (Phase 6) |
  *   |-----------------|--------------------|-----------------------| | 都道府県 (prefecture) | region |
  *   region (or prefecture) | | 市区町村 (city/ward) | county/locality | locality (or municipality) | |
- *   丁目 (chome) | neighbourhood | block (Phase 6 tag) | | 番地 (banchi) | (synth) | sub_block | | 号
+ *   丁目 (chome) | neighborhood | block (Phase 6 tag) | | 番地 (banchi) | (synth) | sub_block | | 号
  *   (gō) | (synth) | house_number | | 〒 (postcode prefix) | — | postcode |
  *
  *   This adapter currently emits only the admin chain (region → locality → block). House numbers and
  *   sub-blocks require a different data source (JP postcode database or real-world address dumps
- *   from MLIT/JapanPost).
+ *   from mlit/JapanPost).
  */
 
 import { isPresent } from "@mailwoman/core/objects"
@@ -34,7 +34,8 @@ import { allRows, getRow } from "@mailwoman/core/utils"
 import type { WOFDatabase } from "@mailwoman/resolver-wof-sqlite/schema"
 import { DatabaseClient } from "@mailwoman/sqlite/client"
 
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
 
 /**
  * Ancestry hops walked upward from a WOF record before giving up — deeper than any real JP admin chain.
@@ -42,8 +43,9 @@ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
 const MAX_ANCESTRY_DEPTH = 6
 
 /**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
  */
 export const WOF_ADMIN_JP_ADAPTER_ID = "wof-admin-jp"
 
@@ -58,16 +60,15 @@ interface PlaceRow {
 /**
  * Walk parent chain up to 6 levels, over a place table already resident in memory.
  *
- * `resolve` is the override for a parent outside the preloaded set (a chain that leaves the country); it is a point
- * query, so keeping the common case out of it is the whole point. The GeoJSON adapters' in-memory analogue is
- * `buildAncestryIndex` (`#utils`); the two stay separate because this one's rows come from SQLite and a chain can leave
- * the preloaded country set.
+ * `resolve` is the override for a parent outside the preloaded set (a chain that leaves the country);
+ * it is a point query, so keeping the common case out of it is the whole point.
+ * The GeoJSON adapters' in-memory analogue is `buildAncestorNameIndex` (`#utils`), and the two stay
+ * separate because this one's rows come from SQLite and a chain can leave the preloaded country set.
+ *
+ * That analogue stores the nearest ancestor's name for each tag.
+ * This function receives one `PlaceRow` per level and reads the chain directly.
  */
-function chainOf(
-	byID: Map<number, PlaceRow>,
-	resolve: (id: number) => PlaceRow | undefined,
-	startID: number
-): PlaceRow[] {
+function chainOf(byID: Map<number, PlaceRow>, resolve: (id: number) => PlaceRow | null, startID: number): PlaceRow[] {
 	const out: PlaceRow[] = []
 	let id = startID
 
@@ -94,7 +95,7 @@ function pickName(row: PlaceRow, jpnNames: Map<number, string>): string {
  *
  * Format: `〒<postcode>? <region><locality><neighbourhood>?`
  *
- * No house numbers yet — needs MLIT data.
+ * No house numbers yet — needs mlit data.
  */
 export function synthesizeJpAddress(
 	chain: PlaceRow[],
@@ -130,13 +131,18 @@ export function synthesizeJpAddress(
 }
 
 /**
- * Build the JP adapter. Reads from the unified global WOF SQLite, walks admin chains starting from neighbourhoods, and
- * yields canonical rows.
+ * Build the JP adapter.
+ *
+ * Reads from the unified global WOF SQLite and walks admin chains from neighborhoods.
+ * Yields canonical rows.
  */
 export function createWOFAdminJpAdapter(): CorpusAdapter {
 	return {
 		id: WOF_ADMIN_JP_ADAPTER_ID,
 		defaultLicense: "CC-BY-4.0",
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.WhosOnFirst,
+		surface: SurfaceOrigin.Rendered,
 		description: "Japanese admin hierarchy from WOF (synthesized addresses without house numbers).",
 
 		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
@@ -155,8 +161,8 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
 				}
 			}
 
-			// One read of the JP place table instead of a fresh `prepare` + up to six point queries per
-			// seed; there are tens of thousands of neighbourhood seeds.
+			// One read of the JP place table instead of a fresh `prepare` + up to six point queries per seed.
+			// There are tens of thousands of neighborhood seeds.
 			const byID = new Map<number, PlaceRow>()
 
 			for (const row of allRows<PlaceRow>(
@@ -167,7 +173,7 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
 
 			const outsideStmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
 
-			const resolveOutside = (id: number): PlaceRow | undefined => {
+			const resolveOutside = (id: number): PlaceRow | null => {
 				const row = getRow<PlaceRow>(outsideStmt, id)
 
 				if (row) {

package/lib/jp/tools/fetch/houjin.ts

@@ -6,13 +6,16 @@
  *   Fetch the National Tax Agency's corporate-number register (法人番号公表サイト 全件データ): every
  *   corporation assigned a number, about 5,000,000 rows, with the head-office address as one string
  *   (国内所在地) plus the prefecture and municipality split out and the postcode. The nationwide CSV
- *   in Unicode is one zip of about 255 MB. This is the NOISY source for Japanese addresses; the
- *   Overture rows the JP corpus is built from are the LABEL half.
+ *   in Unicode is one zip of about 255 MB.
+ *
+ *   The agency issues the corporate number, so the identifier is an `identity` assertion. The head-office
+ *   string beside it is an `observation` the corporation filed, on `registered-office` rows. The Overture
+ *   rows the JP corpus is built from assert `address` on `premise` rows. That distinction matters when the two are mixed.
  *
  *   The agency states the three published fields may be used freely by anyone ("どなたでも自由にご利用
  *   いただくことができます"), with no attribution condition on the download page.
  *
- *   The file sits behind a form: the page hands out a per-session token, and a POST with that token,
+ *   The file sits behind a form. The page hands out a per-session token. A post with that token,
  *   `event=download` and the file number of the nationwide Unicode CSV answers the zip. The file
  *   number is read off the page rather than pinned, because the agency re-issues the files monthly.
  *
@@ -21,16 +24,9 @@
 
 import { makeDirectories } from "@mailwoman/core/fs/writers"
 import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
 
-import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest } from "#tools/fetch/download/index"
-import {
-	attachmentFilename,
-	cookieHeader,
-	streamBodyToFile,
-	withRetries,
-	writeManifest,
-} from "#tools/fetch/download/index"
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest } from "#tools/fetch/download"
+import { attachmentFilename, cookieHeader, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download"
 
 const SLUG = "houjin-jp"
 const PAGE_URL = "https://www.houjin-bangou.nta.go.jp/download/zenken/"
@@ -43,23 +39,23 @@ const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch"
 export type FetchHoujinJPOptions = BaseFetchOptions
 
 /**
- * The nationwide row of the "CSV形式・Unicode" table carries the file number in its `doDownload(N)` handler.
+ * The nationwide row of the "CSV形式・Unicode" table stores the file number in its `doDownload(N)` handler.
  */
-function nationwideUnicodeFileNumber(html: string): string | undefined {
+function nationwideUnicodeFileNumber(html: string): string | null {
 	const start = html.indexOf('id="csv-unicode"')
 
-	if (start === -1) return undefined
+	if (start === -1) return null
 	const section = html.slice(start, html.indexOf('id="xml-unicode"', start))
 	const row = /全国[\s\S]{0,600}?doDownload\((\d+)\)/.exec(section)
 
-	return row?.[1]
+	return row?.[1] ?? null
 }
 
 export async function fetchHoujinJP(
 	options: FetchHoujinJPOptions,
 	report?: (line: string) => void
 ): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
+	const destDir = options.outRoot(SLUG)
 	await makeDirectories(destDir)
 
 	const page = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } })
@@ -85,7 +81,9 @@ export async function fetchHoujinJP(
 	try {
 		bytes = await withRetries(
 			async () => {
-				// A fresh page per attempt: the token is bound to the session cookie, and both may have lapsed.
+				// A fresh page per attempt.
+				// The token is bound to the session cookie.
+				// Either may have lapsed.
 				const attemptPage = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } })
 
 				const attemptToken = new RegExp(`${TOKEN_FIELD.replaceAll(".", "\\.")}" value="([^"]+)"`).exec(
@@ -112,7 +110,7 @@ export async function fetchHoujinJP(
 
 				filename = attachmentFilename(res, filename)
 
-				return streamBodyToFile(res, join(destDir, filename))
+				return streamBodyToFile(res, destDir(filename))
 			},
 			{ report, label: filename }
 		)
@@ -122,7 +120,7 @@ export async function fetchHoujinJP(
 		return { fetched: 0, skipped: 0, failed: 1, failedCodes: ["zenkoku-csv-unicode"] }
 	}
 
-	const dest = join(destDir, filename)
+	const dest = destDir(filename)
 	const sha = await sha256File(dest)
 	report?.(`  ✓ ${filename} ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
 
@@ -135,7 +133,7 @@ export async function fetchHoujinJP(
 		files: [{ source_url: FORM_URL, downloaded_at: new Date().toISOString(), filename, sha256: sha, bytes }],
 	}
 
-	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+	await writeManifest(destDir("MANIFEST.json"), manifest)
 
 	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
 }