@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/us/tools/fetch/state/sources.ts

@@ -0,0 +1,146 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the state-level open-data sources (NY/TX/DE/or notaries, IA contractors, WA health
+ *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
+ *
+ *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools`. Its
+ *   upstream is an xlsx workbook that requires an openpyxl-driven sheet-concatenation pre-step
+ *   before the adapter can consume it.
+ *
+ *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `manifest.json`
+ *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
+ *
+ *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
+ *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
+ */
+
+import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, writeManifest } from "#tools/fetch/download"
+
+/**
+ * Bytes per KiB — the divisor for human-readable sizes.
+ *
+ * It is the floor below which a "download" is an error page rather than data.
+ */
+
+export type FetchStateSourcesOptions = BaseFetchOptions
+
+interface Source {
+	slug: string
+	filename: string
+	url: string
+}
+
+const SOURCES: readonly Source[] = [
+	{
+		slug: "state-ny-notaries",
+		filename: "NY_Commissioned_Notaries.csv",
+		url: "https://data.ny.gov/api/views/rwbv-mz6z/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-tx-notaries",
+		filename: "TX_Notary_Public_Commissions.csv",
+		url: "https://data.texas.gov/api/views/gmd3-bnrd/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-de-notaries",
+		filename: "DE_Notaries_Commissioned.csv",
+		url: "https://data.delaware.gov/api/views/q8dr-mj6p/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-or-notaries",
+		filename: "OR_Active_Notaries.csv",
+		url: "https://data.oregon.gov/api/views/j2pk-zk6z/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-ia-contractors",
+		filename: "IA_Active_Construction_Contractor_Registrations.csv",
+		url: "https://data.iowa.gov/api/views/dpf3-iz94/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-wa-health-providers",
+		filename: "WA_Health_Care_Provider_Credential_Data.csv",
+		url: "https://data.wa.gov/api/views/qxh8-f4bd/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-hi-lobbyists",
+		filename: "HI_Lobbyist_Registration_Statements.csv",
+		url: "https://data.hawaii.gov/api/views/cm7c-skav/rows.csv?accessType=DOWNLOAD",
+	},
+]
+
+export async function fetchStateSources(
+	options: FetchStateSourcesOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	await makeDirectories(options.outRoot)
+
+	let fetched = 0
+	let failed = 0
+	const failedCodes: string[] = []
+
+	for (const { slug, filename, url } of SOURCES) {
+		const destDir = options.outRoot(slug)
+		await makeDirectories(destDir)
+		const dest = destDir(filename)
+
+		report?.(`=== ${slug} / ${filename}`)
+
+		let bytes: number
+
+		try {
+			;({ bytes } = await downloadToFile({
+				url,
+				dest,
+				timeoutMs: 600_000,
+				headers: { "Accept-Encoding": "gzip, br" },
+				report,
+			}))
+		} catch (error) {
+			report?.(`  ✗ download failed for ${url}: ${(error as Error).message}`)
+
+			failed++
+			failedCodes.push(slug)
+
+			continue
+		}
+
+		if (bytes < BYTES_PER_KIB) {
+			report?.(`  ✗ response too small (${bytes} bytes) — probable 404 / error page`)
+
+			failed++
+			failedCodes.push(slug)
+
+			continue
+		}
+
+		const sha = await sha256File(dest)
+
+		const manifest: SourceManifest = {
+			source_url: url,
+			downloaded_at: new Date().toISOString(),
+			filename,
+			sha256: sha,
+			bytes,
+		}
+
+		await writeManifest(destDir("MANIFEST.json"), manifest)
+
+		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
+
+		fetched++
+	}
+
+	report?.(`=== summary ===`)
+	report?.(`fetched: ${fetched}`)
+	report?.(`failed:  ${failed}`)
+
+	return { fetched, skipped: 0, failed, failedCodes }
+}

package/lib/us/tools/fetch/tiger-full.ts

@@ -0,0 +1,256 @@
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Fetch the full TIGER 2024 addrfeat dataset for all US counties from
+ * `https://www2.census.gov/geo/tiger/TIGER2024/addrfeat/`, where files use the pattern
+ * `tl_2024_<statefips><countyfips>_addrfeat.zip`.
+ *
+ * Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
+ * `manifest.json` recording filename, sha256 and bytes so re-runs skip already-verified files.
+ * Extraction and ogr2ogr ingestion happen later, in the `tiger` adapter.
+ */
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { isoSeconds } from "@mailwoman/core/utils"
+import { sleep } from "@mailwoman/core/utils/sleep"
+import type { PathBuilder } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { readManifest, streamDownload, writeManifest } from "#tools/fetch/download"
+
+const HTTP_OK = 200
+const HTTP_REDIRECT = 300
+
+const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT"
+
+export interface FetchTigerFullOptions extends BaseFetchOptions {
+	skipStateFips?: string
+	rateSleep?: number
+	maxParallel?: number
+	dryRun?: boolean
+}
+
+interface CountyEntry {
+	filename: string
+	sha256: string
+	bytes: number
+}
+
+async function readCountyManifest(manifestPath: PathBuilder): Promise<Map<string, CountyEntry>> {
+	const map = new Map<string, CountyEntry>()
+	const parsed = await readManifest<{ counties?: CountyEntry[] }>(manifestPath)
+
+	for (const c of parsed?.counties ?? []) {
+		if (c.filename) {
+			map.set(c.filename, { filename: c.filename, sha256: c.sha256, bytes: c.bytes })
+		}
+	}
+
+	return map
+}
+
+async function fileMatchesSha(path: PathBuilder, expectedSha: string, expectedBytes: number): Promise<boolean> {
+	if (!(await pathExists(path))) return false
+
+	if ((await statPath(path)).size !== expectedBytes) return false
+
+	return (await sha256File(path)) === expectedSha
+}
+
+type CountyResult =
+	| { ok: true; filename: string; sha256: string; bytes: number }
+	| { ok: false; filename: string; reason: string }
+
+async function downloadCounty(url: string, dest: PathBuilder): Promise<CountyResult> {
+	const filename = dest.basename()
+	const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 })
+
+	if (status < HTTP_OK || status >= HTTP_REDIRECT) {
+		return { ok: false, filename, reason: `HTTP ${status}` }
+	}
+
+	const bytes = (await statPath(dest)).size
+
+	if (bytes < BYTES_PER_KIB) {
+		await removePathIfPresent(dest)
+
+		return { ok: false, filename, reason: `too small (${bytes} bytes)` }
+	}
+
+	const sha256 = await sha256File(dest)
+
+	return { ok: true, filename, sha256, bytes }
+}
+
+export async function fetchTigerFull(
+	options: FetchTigerFullOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length)
+	const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000)
+	const maxParallel = options.maxParallel ?? 4
+	const dryRun = options.dryRun ?? false
+
+	const addrfeatDir = options.outRoot("tiger", "addrfeat")
+	await makeDirectories(addrfeatDir)
+
+	// MARK: Step 1 — discover the county file list
+
+	report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`)
+
+	const listingRes = await new APIClient({
+		displayName: "tiger-listing",
+		retry: true,
+		axios: { headers: { "Accept-Encoding": "gzip, br" } },
+	})
+		.fetch<string>({ url: `${TIGER_BASE_URL}/`, responseType: "text", timeout: 60_000 })
+		.then(pluckResponseData)
+
+	const html = listingRes
+	const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted()
+	const totalCounties = allZips.length
+	report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`)
+
+	// `tl_2024_SSCCC_addrfeat.zip`: SS is the 2-digit state FIPS at chars 8-9.
+	const stateFiles = new Map<string, string[]>()
+
+	for (const fname of allZips) {
+		const stateFips = fname.slice(8, 10)
+		const list = stateFiles.get(stateFips) ?? []
+		list.push(fname)
+		stateFiles.set(stateFips, list)
+	}
+
+	report?.(`  Spans ${stateFiles.size} state/territory FIPS codes.`)
+
+	// MARK: Step 2 — download the county ZIPs
+
+	let totalFetched = 0
+	let totalSkipped = 0
+	let totalSkippedState = 0
+	let totalFailed = 0
+	let totalBytesFetched = 0
+	const failedCodes: string[] = []
+
+	const sortedStates = [...stateFiles.keys()].toSorted()
+
+	for (const stateFips of sortedStates) {
+		const countyFiles = stateFiles.get(stateFips) ?? []
+
+		if (skipStateFips.includes(stateFips)) {
+			report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`)
+			totalSkippedState += countyFiles.length
+
+			continue
+		}
+
+		const stateDir = addrfeatDir(`state-${stateFips}`)
+		await makeDirectories(stateDir)
+		const manifestPath = stateDir("MANIFEST.json")
+
+		const manifest = await readCountyManifest(manifestPath)
+
+		report?.(`--- State ${stateFips} — ${countyFiles.length} counties`)
+
+		const pending: Array<{ url: string; dest: PathBuilder }> = []
+
+		for (const fname of countyFiles) {
+			const dest = stateDir(fname)
+			const url = `${TIGER_BASE_URL}/${fname}`
+			const known = manifest.get(fname)
+
+			if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
+				report?.(`  skip (verified) ${fname}`)
+
+				totalSkipped++
+
+				continue
+			}
+
+			if (dryRun) {
+				report?.(`  would fetch: ${url}`)
+
+				totalFetched++
+
+				continue
+			}
+
+			pending.push({ url, dest })
+		}
+
+		if (dryRun) continue
+
+		if (!pending.length) continue
+
+		const results: CountyResult[] = Array.from({ length: pending.length })
+		let cursor = 0
+
+		const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
+			while (true) {
+				const i = cursor++
+
+				if (i >= pending.length) return
+				const item = pending[i]!
+				await sleep(rateSleepMs)
+				results[i] = await downloadCounty(item.url, item.dest)
+			}
+		})
+
+		await Promise.all(workers)
+
+		for (const result of results) {
+			if (result.ok) {
+				report?.(
+					`  ok ${result.filename}  ${ByteFormatter.formatIEC(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`
+				)
+
+				manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes })
+
+				totalFetched++
+				totalBytesFetched += result.bytes
+			} else {
+				report?.(`  FAIL ${result.filename} -- ${result.reason}`)
+
+				totalFailed++
+				failedCodes.push(result.filename)
+			}
+		}
+
+		const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename))
+
+		const manifestDoc = {
+			state_fips: stateFips,
+			updated_at: isoSeconds(),
+			tiger_base_url: TIGER_BASE_URL,
+			counties,
+		}
+
+		await writeManifest(manifestPath, manifestDoc)
+	}
+
+	// MARK: Summary
+
+	report?.(`=== Summary ===`)
+	report?.(`  Total counties in index   : ${totalCounties}`)
+	report?.(`  State(s) fully skipped    : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`)
+	report?.(`  Counties already present  : ${totalSkipped}`)
+	report?.(`  Counties fetched this run : ${totalFetched}`)
+	report?.(`  Counties failed           : ${totalFailed}`)
+
+	if (totalBytesFetched > 0) {
+		report?.(`  Bytes fetched this run    : ${ByteFormatter.formatIEC(totalBytesFetched)} (${totalBytesFetched})`)
+	}
+
+	if (totalFailed > 0) {
+		report?.(`WARNING: ${totalFailed} download(s) failed. Re-run to retry.`)
+	}
+
+	return { fetched: totalFetched, skipped: totalSkipped + totalSkippedState, failed: totalFailed, failedCodes }
+}

package/lib/utils/align.ts

@@ -19,21 +19,20 @@
  *        character range. First token in a component span → `B-<tag>`; subsequent tokens →
  *        `I-<tag>`; no overlap → `O`.
  *   5. Emit the located char spans verbatim as `span_starts[]` / `span_ends[]` / `span_tags[]` (the
- *        v0.5.0 char-offset format, #519). The token quantization in step 4 is the part the v0.5.0
- *        rebuild deletes; during the transition both representations ride on every labeled row.
+ *        v0.5.0 char-offset format).
  *
  *   Structural invariants the function preserves (the span ones loudly — a violation throws rather
- *   than quarantines, because it indicates a bug here, not bad source data):
+ *   than quarantines, because it indicates a bug here rather than bad source data):
  *
  *   - `tokens.length === labels.length` always.
  *   - Each component contributes at most one contiguous BIO run (no `B-tag … O … I-tag` gaps). This is
  *       enforced by greedy first-match span assignment + ordered token iteration.
  *   - The span triple is sorted ascending by start and non-overlapping.
- *   - `raw` is NFC-normalized (asserted per row; a non-NFC raw makes char offsets ambiguous downstream
+ *   - `raw` is NFC-normalized (asserted per row. a non-NFC raw makes char offsets ambiguous downstream
  *       — NFD `é` occupies two code units where NFC `é` occupies one — and silently so).
  */
 
-import type { BIOLabel, ComponentTag } from "@mailwoman/core/types"
+import type { BIOLabel, ComponentTag } from "@mailwoman/codex/component"
 import { distance as levenshteinDistance } from "fastest-levenshtein"
 
 import type { CanonicalRow, LabeledRow, QuarantinedRow } from "#types"
@@ -44,22 +43,29 @@ import { whitespaceTokenizer, type TokenSpan, type Tokenizer } from "#utils/toke
  */
 export interface AlignOptions {
 	/**
-	 * Tokenizer to use. Defaults to `whitespaceTokenizer()`.
+	 * Tokenizer to use.
+	 *
+	 * Defaults to `whitespaceTokenizer()`.
 	 */
 	tokenizer?: Tokenizer
 
 	/**
-	 * Max Levenshtein edit distance to accept when a verbatim substring match fails. Set `0` to require verbatim matches
-	 * only. Default `2`.
+	 * Max Levenshtein edit distance to accept when a verbatim substring match fails.
+	 *
+	 * Set `0` to require verbatim matches only.
+	 * Default `2`.
 	 *
-	 * Distance is computed against same-length windows in `raw`, so the threshold scales naturally with the component
-	 * value length.
+	 * Distance is computed against same-length windows in `raw`, so the threshold
+	 * scales naturally with the component value length.
 	 */
 	maxEditDistance?: number
 
 	/**
-	 * Case-insensitive comparison for substring search. Default `true`. The retained span in `raw` is the original case;
-	 * only matching is case-insensitive.
+	 * Case-insensitive comparison for substring search.
+	 *
+	 * Default `true`.
+	 * The retained span in `raw` is the original case.
+	 * Only matching is case-insensitive.
 	 */
 	caseInsensitive?: boolean
 }
@@ -70,8 +76,7 @@ export interface AlignOptions {
 export type AlignmentResult = { kind: "labeled"; row: LabeledRow } | { kind: "quarantined"; row: QuarantinedRow }
 
 /**
- * One located char-offset label span over a row's `raw` ([start, end) in UTF-16 code units). The element type behind
- * the parallel `span_starts[]`/`span_ends[]`/`span_tags[]` triple on `LabeledRow` (#519).
+ * One located char-offset label span over a row's `raw` ([start, end) in UTF-16 code units). The element type behind the parallel `span_starts[]`/`span_ends[]`/`span_tags[]` triple on `LabeledRow`.
  */
 export interface ComponentSpan {
 	tag: ComponentTag
@@ -91,10 +96,6 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 		return { kind: "quarantined", row: { row, reason: "raw-empty" } }
 	}
 
-	// #519 NFC handling — relaxed from a hard throw to normalization (2026-06-12, DeepSeek-validated):
-	// one non-NFC row (e.g. a non-Latin name variant like "দক্ষিণ কোরিয়া") must not crash a multi-hour
-	// build. Normalize `raw` AND every component value to NFC, compute spans over the NFC raw, and
-	// store the NFC raw — preserving the #519 single-normalization-form principle while keeping the row.
 	const raw = row.raw.normalize("NFC")
 	const components = { ...row.components }
 
@@ -111,10 +112,9 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 
 	const haystack = caseInsensitive ? raw.toLowerCase() : raw
 
-	// Longest value first: a short component must not claim a word that a longer, more specific
-	// component owns ("Alaska Regional Dr, Alaska" — region "Alaska" stealing the street's first
-	// word quarantined the street; pilot2's residual class). Emit order is unaffected — spans are
-	// re-sorted by start below.
+	// Longest value first: a short component must not claim a word that a longer,
+	// more specific component owns.
+	// Emit order is unaffected, since spans are re-sorted by start below.
 	const entries = (Object.entries(components) as Array<[ComponentTag, string | undefined]>).toSorted(
 		(a, b) => (b[1]?.length ?? 0) - (a[1]?.length ?? 0)
 	)
@@ -132,12 +132,8 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 			}
 		}
 
-		// Defensive bounds quarantine (2026-06-12): locateSpan's fuzzy/boundary logic can over-run the
-		// raw by a code unit on some non-Latin / combining-mark strings (e.g. Bengali name variants),
-		// yielding a span past the end. An out-of-bounds offset can never be a valid char span —
-		// quarantine the row rather than crash assertSpanInvariants and take down a multi-hour build.
-		// (If this class proves large in the quarantine report, locateSpan's boundary logic needs a
-		// combining-mark fix to KEEP these non-Latin rows.)
+		// An out-of-bounds offset can never be a valid char span, so quarantine the row
+		// rather than crash assertSpanInvariants and take down a multi-hour build.
 		if (span.start < 0 || span.end > raw.length || span.start >= span.end) {
 			return {
 				kind: "quarantined",
@@ -160,8 +156,6 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 		components,
 		tokens: tokens.map((t) => t.text),
 		labels,
-		// The v0.5.0 char-offset triple (#519): the located spans, emitted verbatim. The token
-		// quantization above is what the rebuild deletes; both ride during the transition.
 		span_starts: componentSpans.map((s) => s.start),
 		span_ends: componentSpans.map((s) => s.end),
 		span_tags: componentSpans.map((s) => s.tag),
@@ -171,13 +165,13 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 }
 
 /**
- * Enforce the #519 span-triple invariants — in-bounds, sorted ascending by start, non-overlapping — loudly.
+ * Enforce the span-triple invariants: in-bounds, sorted ascending by start, non-overlapping.
  *
- * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible and the caller sorts, so
- * a violation here is a bug in this file, not bad source data: throw (naming the row) rather than quarantine, so the
- * corruption can't ride into a corpus. Exported for every OTHER span producer (`composeAdversarialRow`'s offset
- * arithmetic, future synthesis paths) — any code that emits the triple without going through `alignRow` must pass its
- * output through this.
+ * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible
+ * and the caller sorts, so a violation here is a bug in this file rather than bad source data.
+ * Throw rather than quarantine, so the corruption cannot ride into a corpus.
+ *
+ * Any code that emits the triple without going through `alignRow` must pass its output through this.
  */
 export function assertSpanInvariants(
 	spans: readonly ComponentSpan[],
@@ -213,11 +207,13 @@ export function assertSpanInvariants(
 }
 
 /**
- * Locate `needle` in `haystack` (both already normalized for case if requested), preferring verbatim substring match.
- * Falls back to a fuzzy window scan when verbatim fails and `maxEditDistance > 0`. Already-claimed spans are skipped so
- * two components don't grab overlapping ranges.
+ * Locate `needle` in `haystack` (both already normalized for case if requested),
+ * preferring verbatim substring match.
+ *
+ * Falls back to a fuzzy window scan when verbatim fails and `maxEditDistance > 0`.
+ * Already-claimed spans are skipped so two components don't grab overlapping ranges.
  *
- * Returns the span in the original `raw` (not the lower-cased `haystack`).
+ * @returns The span in the original `raw` (not the lower-cased `haystack`).
  */
 function locateSpan(args: {
 	haystack: string
@@ -230,12 +226,11 @@ function locateSpan(args: {
 
 	if (!needle.length) return undefined
 
-	// Pass 1: verbatim substring. Word-boundary-aligned matches are PREFERRED over intra-word ones
-	// — leftmost-substring alone let a short value claim the inside of an earlier word (region "AK"
-	// matched inside "Umak"/"Lake", scrambling every later span; caught by the v0.5.0 pilot build).
+	// Pass 1: verbatim substring.
+	// Word-boundary-aligned matches are preferred over intra-word ones, so a short
+	// value cannot claim the inside of an earlier word.
 	// Intra-word matches stay allowed as the fallback because they are essential for affix
-	// supervision (street_suffix "straße" inside "Hauptstraße" has no boundary-aligned occurrence —
-	// sub-word spans are the point of the char-offset format).
+	// supervision (street_suffix "straße" inside "Hauptstraße" has no boundary-aligned occurrence).
 	let intraWord: { start: number; end: number } | undefined
 	let from = 0
 
@@ -257,15 +252,14 @@ function locateSpan(args: {
 
 	if (maxEditDistance <= 0) return undefined
 
-	// Pass 2: fuzzy sliding-window. Walk over candidate windows of length `needle.length`
-	// across haystack, compute Levenshtein, pick the leftmost window under the threshold.
+	// Pass 2: fuzzy sliding-window.
 	const len = needle.length
 
 	for (let i = 0; i + len <= haystack.length; i++) {
 		if (overlapsClaimed(i, i + len, claimed)) continue
 		const window = haystack.slice(i, i + len)
 
-		if (window === needle) return { start: i, end: i + len } // covered by pass 1, but cheap
+		if (window === needle) return { start: i, end: i + len }
 		const d = levenshteinDistance(window, needle)
 
 		if (d <= maxEditDistance) return { start: i, end: i + len }
@@ -295,16 +289,12 @@ function overlapsClaimed(start: number, end: number, claimed: Array<[number, num
 }
 
 /**
- * Assign BIO labels to tokens given the component spans. Components MUST be sorted by start offset. For each token,
- * find the first component span that contains the token's start offset; if the token is the first one inside that span
- * emit `B-<tag>`, else `I-<tag>`.
+ * Assign BIO labels to tokens from component spans sorted by start offset.
  */
 function labelTokens(tokens: readonly TokenSpan[], spans: readonly ComponentSpan[]): readonly BIOLabel[] {
 	const out: BIOLabel[] = []
 	const seenSpan = new Set<number>()
 
-	// index into `spans`
-
 	for (const token of tokens) {
 		let assigned: BIOLabel = "O"