lazycodex-ai 5.1.48 → 5.1.54

package/packages/shared-skills/skills/ulw-research/scripts/fidelity-values.mjs

@@ -7,12 +7,16 @@
 // No model, no network.
 
 import { existsSync, readFileSync } from "node:fs"
-import { join, resolve, sep } from "node:path"
+import { resolve, sep } from "node:path"
 
-import { sha256Of } from "./cli-support.mjs"
 import { sameValue, unitKey } from "./fidelity-binding.mjs"
+import { readCsv } from "./fidelity-csv.mjs"
 import { claimChecks } from "./fidelity-claims.mjs"
+import { sourceChecks } from "./fidelity-sources.mjs"
+import { convertedHalfFits, followsOriginal, isDerivedRow, originalProblem, renderLocator, staleRender, textAfter } from "./fidelity-originals.mjs"
+import { snapshotCause, unreadableSnapshots } from "./fidelity-snapshot-readability.mjs"
 import { STRUCTURAL, clip, traceChecks } from "./fidelity-trace.mjs"
+import { fillEntriesByNode, valueUnitProblem } from "./fidelity-value-unit.mjs"
 import { parseHtml, proseText } from "./html-lite.mjs"
 import { parseNumeral } from "./numerals.mjs"
 import { div, fromDecimal, sub } from "./rational.mjs"
@@ -45,11 +49,14 @@ function quoteProblem(row, shown, selector) {
 	return bare(shown) === bare(want) ? null : ["quote_not_verbatim", `"${clip(shown)}" is not the ${selector} selection of ${row.fact_id}: "${clip(want)}"`]
 }
 
-function factProblem(row, shown, attrs = {}) {
+function factProblem(row, shown, attrs = {}, rows = new Map()) {
+	const original = originalProblem(row, shown, rows, attrs["data-fact-mode"] === "value")
+	if (original) return original
 	if (attrs["data-unit"] !== undefined && unitKey(attrs["data-unit"]) !== unitKey(row.unit)) return ["unit_mismatch", `data-unit "${attrs["data-unit"]}" but ${row.fact_id} is in "${row.unit}"`]
 	if (attrs.value !== undefined && !sameValue(attrs.value, row.value)) return ["value_mismatch", `value="${attrs.value}" but ${row.fact_id} is ${row.value}`]
+	if (convertedHalfFits(row, shown)) return null
 	const text = normalizeText(shown)
-	const forms = [row.raw, row.render, row.value, row.unit && `${row.value} ${row.unit}`].filter(Boolean).map(normalizeText)
+	const forms = [...(["converted", "derived"].includes(row.kind) ? [] : [row.raw, row.render]), row.value, row.unit && `${row.value} ${row.unit}`].filter(Boolean).map(normalizeText)
 	if (forms.some((form) => text === form || (text.endsWith(` ${form}`) && !/\d/.test(text.slice(0, -form.length))))) return null
 	const parsed = parseNumeral(text)
 	if (!parsed) return unitKey(text) === unitKey(row.unit) ? null : ["value_mismatch", `"${clip(shown)}" shows no value of ${row.fact_id} (${row.value})`]
@@ -73,16 +80,6 @@ export function exportRows(ledger) {
 	return (ledger?.rows ?? []).filter((row) => row.status === "active" && row.kind !== "quote" && row.value !== null).map((row) => ({ fact_id: row.fact_id, label: row.label, plot_value: row.value, unit: row.unit ?? "" }))
 }
 
-function readCsv(path) {
-	if (!existsSync(path)) return null
-	const lines = readFileSync(path, "utf8").split(/\r?\n/).filter((line) => line.trim() !== "")
-	const cells = (line) => line.split(",").map((cell) => cell.trim().replace(/^"(.*)"$/, "$1"))
-	const head = cells(lines[0] ?? "")
-	const [id, value, unit] = ["fact_id", "plot_value", "unit"].map((name) => head.indexOf(name))
-	if (id < 0 || value < 0 || unit < 0 || lines.length < 2) return null
-	return lines.slice(1).map((line) => cells(line)).map((c) => ({ fact_id: c[id], plot_value: c[value], unit: c[unit] ?? "" }))
-}
-
 const walk = (node) => node.children.filter((child) => child.tag !== "#text").flatMap((child) => [child, ...walk(child)])
 
 /** SVG text nodes of a chart: data labels, or ticks with their declared grid. */
@@ -140,46 +137,57 @@ function chartProblem(ctx, fig, labels, ledger) {
  * @param {object} ctx buildContext(...) from gates-static.mjs
  * @param {{ rows: object[] }} ledger
  * @param {{ fact_id: string, rendered: string, selector?: string }[] | null} fillMap Markdown fill map, or null
- * @param {string | null} sourcesDir directory of <S-id>/{text.txt, meta.json} snapshots
+ * @param {string | null} sourcesDir directory of <S-id>/ snapshots (source-view.mjs reads them)
  * @param {{ recompute?: (ledger: object, row: object) => { ok: boolean }, asOf?: string | null }} options
  */
 export function valueChecks(ctx, ledger, fillMap, sourcesDir, options = {}) {
 	const out = []
-	const flag = (code, node, message) => out.push(ctx.defect(code, node, message))
+	const flag = (code, node, message, extra = {}) => out.push({ ...ctx.defect(code, node, message), ...extra })
 	const rows = new Map((ledger?.rows ?? []).map((row) => [row.fact_id, row]))
 	const cited = new Map()
+	const fills = (fillMap ?? []).map((entry) => entry.rendered)
 	const check = (id, node, problemOf) => {
 		const row = rows.get(id)
 		if (!row) return flag("unresolved_placeholder", node, `${id} names no ledger row`)
 		if (!cited.has(id)) cited.set(id, node)
 		const problem = problemOf(row)
-		if (problem) flag(problem[0], node, problem[1])
+		if (problem) out.push({ ...ctx.defect(problem[0], node, problem[1]), ...(problem[2] ? { part: problem[2] } : {}) })
 	}
 	for (const node of ctx.all.filter((cur) => cur.attrs["data-fact"] !== undefined)) {
 		const selector = node.attrs["data-quote-sel"] ?? "full"
-		check(node.attrs["data-fact"], node, (row) => (row.kind === "quote" ? quoteProblem(row, withoutFooter(node), selector) : factProblem(row, proseText(node), node.attrs)))
+		const glued = (row) => isDerivedRow(row) && node.attrs["data-fact-mode"] !== "value" && followsOriginal(row, rows, fills)(textAfter(node)) ? ["value_mismatch", `another original follows ${row.fact_id}: "${clip(textAfter(node))}"`, "original"] : null
+		check(node.attrs["data-fact"], node, (row) => (row.kind === "quote" ? quoteProblem(row, withoutFooter(node), selector) : (factProblem(row, proseText(node), node.attrs, rows) ?? glued(row))))
 	}
 	const docText = bare(proseText(ctx.body))
 	for (const entry of fillMap ?? []) {
 		check(entry.fact_id, ctx.body, (row) => {
-			const problem = row.kind === "quote" ? quoteProblem(row, entry.rendered, entry.selector ?? "full") : factProblem(row, entry.rendered)
-			if (problem || docText.includes(bare(entry.rendered))) return problem
-			return [row.kind === "quote" ? "quote_not_verbatim" : "value_mismatch", `filled text "${clip(entry.rendered)}" of ${row.fact_id} is no longer in the report`]
+			const problem = row.kind === "quote" ? quoteProblem(row, entry.rendered, entry.selector ?? "full") : factProblem(row, entry.rendered, { "data-fact-mode": entry.mode }, rows)
+			const stale = isDerivedRow(row) ? staleRender(row, entry.rendered, proseText(ctx.body), followsOriginal(row, rows, fills)) : docText.includes(bare(entry.rendered)) ? null : "gone"
+			if (problem || stale === null) return problem
+			return [row.kind === "quote" ? "quote_not_verbatim" : "value_mismatch", `filled text "${clip(entry.rendered)}" of ${row.fact_id} is no longer in the report${stale === "original" ? " (its original was edited)" : ""}`, ...(stale === "original" ? ["original"] : [])]
 		})
 	}
 	rowChecks(ctx, ledger, rows, cited, options, flag)
+	// options.fillEntries: the edition's fill-map entries whose element is in the checked HTML (value mode, fill's unit_bound).
+	const facts = ctx.all.filter((cur) => cur.attrs["data-fact"] !== undefined)
+	const entries = fillEntriesByNode(facts, options.fillEntries)
+	for (const node of facts) {
+		const row = rows.get(node.attrs["data-fact"])
+		const problem = row && row.kind !== "quote" && !factProblem(row, proseText(node), node.attrs, rows) && valueUnitProblem(node, row, entries.get(node))
+		if (problem) flag(problem[0], node, problem[1])
+	}
 	for (const tr of ctx.all.filter((node) => node.tag === "tr" && node.attrs["data-row-kind"] !== "comparison")) {
 		const groups = new Set(ctx.util.descendants(tr).map((node) => rows.get(node.attrs["data-fact"])?.observation_id).filter(Boolean))
 		if (groups.size > 1) flag("join_integrity_fail", tr, `one table row joins observations ${[...groups].join(", ")}`)
 	}
-	const meta = sourceChecks(ctx, rows, cited, sourcesDir, flag)
+	const meta = sourceChecks(ctx, rows, cited, sourcesDir, flag, unreadableSnapshots(ctx, rows, cited, sourcesDir, out))
 	claimChecks(ctx, rows, (row) => provenance(sourcesDir, meta, row), flag)
 	const charts = chartFigures(ctx)
 	for (const { node, scope } of ctx.elements) {
 		const kind = ctx.kindOf(scope)
 		if ((node.tag === "q" || node.tag === "blockquote") && node.attrs["data-fact"] === undefined && kind !== "sources" && kind !== "toc") flag("quote_unbound", node, `<${node.tag}> with no Q row: "${clip(proseText(node))}"`)
 	}
-	traceChecks(ctx, charts, fillMap, flag, fillSiteFits(rows))
+	traceChecks(ctx, charts, fillMap, flag, fillSiteFits(rows), renderLocator(rows, fillMap))
 	for (const [fig, labels] of charts) {
 		const problem = chartProblem(ctx, fig, labels, ledger)
 		if (problem) flag("chart_value_untraceable", fig, problem)
@@ -190,16 +198,23 @@ export function valueChecks(ctx, ledger, fillMap, sourcesDir, options = {}) {
 function rowChecks(ctx, ledger, rows, cited, options, flag) {
 	// An as-of of "latest ..." or "... per snapshot" names no fixed revision, so quote_stale_version is skipped.
 	const asOf = options.asOf && !/^latest\b|per snapshot/i.test(options.asOf) ? options.asOf : null
+	// A sentence-final period ("release v1.2.") is punctuation, not part of the revision token.
+	const target = asOf?.replace(/\.(?=\s|$)/gu, "")
 	for (const [id, node] of cited) {
 		const row = rows.get(id)
 		if (row.fuzzy === true) flag("quote_fuzzy_match", node, `${id} was registered from a similarity match on an OCR-noisy snapshot`)
 		const revision = typeof row.revision === "string" ? row.revision : (row.revision?.id ?? row.revision?.tag ?? null)
-		if (asOf && revision && !asOf.includes(revision)) flag("quote_stale_version", node, `${id} is from revision ${revision}; the brief targets "${asOf}"`)
+		const revisionPattern = revision && new RegExp(`(?<![\\w.-])${revision.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![\\w.-])`, "u")
+		if (asOf && revisionPattern && !revisionPattern.test(target)) flag("quote_stale_version", node, `${id} is from revision ${revision}; the brief targets "${asOf}"`)
 		if ((row.kind === "derived" || row.kind === "converted") && options.recompute?.(ledger, row)?.ok === false) flag("value_mismatch", node, `${id} does not recompute from its operands`)
 		const window = row.snapshot_window
 		if (window && typeof window === "object" && window.start && window.end) {
 			const [start, end] = [Date.parse(window.start), Date.parse(window.end)]
-			const outside = (row.operands ?? []).find((op) => !(Date.parse(rows.get(op)?.observed_at) >= start && Date.parse(rows.get(op)?.observed_at) < end))
+			const [absent, present] = [(row.operands ?? []).filter((op) => !rows.has(op)), (row.operands ?? []).filter((op) => rows.has(op))]
+			for (const op of absent) flag("operand_missing", node, `operand ${op} of ${id} is not in the fact ledger`)
+			const missing = present.filter((op) => !rows.get(op).observed_at)
+			for (const op of missing) flag("operand_timestamp_missing", node, `operand ${op} of ${id} has no observed_at timestamp`)
+			const outside = present.find((op) => !missing.includes(op) && !(Date.parse(rows.get(op).observed_at) >= start && Date.parse(rows.get(op).observed_at) < end))
 			if (outside) flag("snapshot_mismatch", node, `operand ${outside} of ${id} was observed outside ${window.start} to ${window.end}`)
 		}
 	}
@@ -213,17 +228,3 @@ function provenance(sourcesDir, meta, row) {
 	const path = resolve(dir, artifact)
 	return path.startsWith(`${dir}${sep}`) && existsSync(path) ? meta.get(row.source_id)?.kind : undefined
 }
-
-function sourceChecks(ctx, rows, cited, sourcesDir, flag) {
-	const ids = new Set([...cited.keys()].map((id) => rows.get(id)?.source_id).filter(Boolean))
-	for (const match of proseText(ctx.body).matchAll(/\[S(\d+)\]/g)) ids.add(`S${match[1]}`)
-	const meta = new Map()
-	for (const id of sourcesDir ? ids : []) {
-		const [metaPath, textPath] = [join(sourcesDir, id, "meta.json"), join(sourcesDir, id, "text.txt")]
-		if (!existsSync(metaPath) || !existsSync(textPath)) continue
-		meta.set(id, JSON.parse(readFileSync(metaPath, "utf8")))
-		const actual = sha256Of(readFileSync(textPath))
-		if (meta.get(id).sha256 !== actual) flag("snapshot_hash_mismatch", ctx.body, `${id}/text.txt changed after its snapshot was pinned`)
-	}
-	return meta
-}

package/packages/shared-skills/skills/ulw-research/scripts/fidelity-values.test.ts

@@ -1,36 +1,15 @@
+/// <reference types="bun-types" />
 import { describe, expect, test } from "bun:test"
-import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { cpSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"
 import { tmpdir } from "node:os"
 import { join } from "node:path"
 
-import { exportRows, valueChecks } from "./fidelity-values.mjs"
-import { buildContext } from "./gates-static.mjs"
-import { parseHtml } from "./html-lite.mjs"
+import { exportRows } from "./fidelity-values.mjs"
 import { similarity } from "./text-similarity.mjs"
+import { LEDGER, SOURCES, ledgerWith, run, type Defect, type Row } from "./tests/fidelity-values-support"
 
-type Defect = { code: string; severity: string; selector: string; line: number; message: string; hint: string }
-type Row = Record<string, unknown> & { fact_id: string }
-type Ledger = { schemaVersion: number; rows: Row[]; events: unknown[] }
-type Options = { ledger?: Ledger; fillMap?: unknown[] | null; sourcesDir?: string | null; baseDir?: string; recompute?: (ledger: Ledger, row: Row) => { ok: boolean }; asOf?: string }
-
-const FIX = join(import.meta.dir, "tests", "fixtures", "fidelity")
-const SOURCES = join(FIX, "sources")
-const LEDGER: Ledger = JSON.parse(readFileSync(join(FIX, "ledger-verify.json"), "utf8"))
 const FULL = "we estimate completion no earlier than 2027"
 
-function ledgerWith(id: string, patch: Record<string, unknown>): Ledger {
-	const copy = structuredClone(LEDGER)
-	copy.rows = copy.rows.map((row) => (row.fact_id === id ? { ...row, ...patch } : row))
-	return copy
-}
-
-function run(body: string, options: Options = {}): Defect[] {
-	const html = /<h2/.test(body) ? body : `<h2>Findings</h2>\n${body}`
-	const ctx = buildContext(parseHtml(html), { baseDir: options.baseDir ?? FIX })
-	const sourcesDir = options.sourcesDir === undefined ? SOURCES : options.sourcesDir
-	return valueChecks(ctx, options.ledger ?? LEDGER, options.fillMap ?? null, sourcesDir, options) as Defect[]
-}
-
 const codes = (defects: Defect[]) => defects.map((defect) => defect.code)
 const fact = (id: string, text: string, unit = "") => `<data class="fact" data-fact="${id}"${unit ? ` data-unit="${unit}"` : ""}>${text}</data>`
 const quote = (sel: string, text: string) => `<p>They wrote <q data-fact="Q1" data-quote-sel="${sel}">${text}</q> [S4].</p>`
@@ -53,8 +32,8 @@ describe("untraceable numbers", () => {
 		expect(codes(run("<h2>Appendix</h2><p>Revenue grew 12% [S4].</p>"))).toEqual(["untraceable_number"])
 	})
 
-	test("#given the same sentence under Sources #when checked #then it is out of scope", () => {
-		expect(run("<h2>Sources</h2><p>Revenue grew 12% [S4].</p>")).toEqual([])
+	test("#given the same sentence under Sources #when checked #then prose still needs a ledger row", () => {
+		expect(codes(run("<h2>Sources</h2><p>Revenue grew 12% [S4].</p>"))).toEqual(["untraceable_number"])
 	})
 
 	test("#given a date in prose #when checked #then dates are in scope", () => {
@@ -218,14 +197,14 @@ describe("verification claims are structural (data-claim-kind)", () => {
 		`<p>The bridge, which ran across the river, spans ${fact("F1", "up to 320 m", "m")} [S4].</p>`,
 		`<p>The span is ${fact("F1", "up to 320 m", "m")}, as confirmed with the annual inspection report [S4].</p>`,
 		`<p>Clinical tests confirmed the span of ${fact("F1", "up to 320 m", "m")} [S4].</p>`,
-		`<p>확인된 길이는 ${fact("F1", "최대 320 m", "m")} [S4].</p>`,
+		`<p>\uD655\uC778\uB41C \uAE38\uC774\uB294 ${fact("F1", "\uCD5C\uB300 320 m", "m")} [S4].</p>`,
 	]
 	for (const body of quiet) {
 		test(`#given findings prose ${JSON.stringify(body.replace(/<[^>]+>/g, ""))} #when checked #then no warning and no major`, () => expect(run(body)).toEqual([]))
 	}
 
 	test("#given an UNMARKED Korean run-and-check sentence #when checked #then verification_claim_unmarked minor only", () => {
-		expect(severities(run(`<p>우리가 실행해 확인한 길이는 ${fact("F1", "최대 320 m", "m")} [S4].</p>`))).toEqual(["verification_claim_unmarked:minor"])
+		expect(severities(run(`<p>\uC6B0\uB9AC\uAC00 \uC2E4\uD589\uD574 \uD655\uC778\uD55C \uAE38\uC774\uB294 ${fact("F1", "\uCD5C\uB300 320 m", "m")} [S4].</p>`))).toEqual(["verification_claim_unmarked:minor"])
 	})
 
 	test("#given the same run-and-check sentence MARKED verified on a command row #when checked #then no warning", () => {

package/packages/shared-skills/skills/ulw-research/scripts/gates-fidelity.mjs

@@ -20,11 +20,13 @@ import { parseHtml } from "./html-lite.mjs"
 
 /** Every code this gate can raise. */
 export const FIDELITY_GATE_CODES = Object.freeze([
+	"unit_unbound",
 	"fidelity_interview_incomplete", "unresolved_placeholder", "value_mismatch", "unit_mismatch", "quote_not_verbatim",
 	"quote_unbound", "claim_status_invalid", "snapshot_hash_mismatch", "chart_value_untraceable", "untraceable_number",
 	"unsupported_verification", "conflicting_readings", "hedge_dropped", "direction_flipped", "causality_added",
-	"qualifier_dropped", "unbound_sentence", "snapshot_mismatch", "join_integrity_fail", "quote_stale_version",
+	"qualifier_dropped", "unbound_sentence", "snapshot_mismatch", "operand_timestamp_missing", "operand_missing", "join_integrity_fail", "quote_stale_version",
 	"quote_fuzzy_match", "name_outside_quote", "scope_widened_warn", "claim_kind_mismatch", "verification_claim_unmarked",
+	"snapshot_not_raw", "relocated", "quote_converted_only",
 ])
 const RANK = { blocker: 0, major: 1, minor: 2 }
 
@@ -40,8 +42,8 @@ export function deliveryBlockReason(defect) {
 }
 
 function withIntegrity(defect) {
-	const { code, severity, selector, line, message, hint } = defect
-	return { code, severity, integrity: DEFECT_CODES[code].integrity, selector, line, message, hint }
+	const { code, severity, selector, line, message, hint, part, fact_id } = defect
+	return { code, severity, integrity: DEFECT_CODES[code].integrity, selector, line, message, hint, ...(part ? { part } : {}), ...(fact_id ? { fact_id } : {}) }
 }
 
 function compareDefects(a, b) {
@@ -93,17 +95,17 @@ function incomplete(parsed) {
 
 /**
  * @param {string} html the filled report
- * @param {{ ledger: { rows: object[] }, sourcesDir?: string | null, claims?: object | null, fillMap?: object[] | null,
+ * @param {{ ledger: { rows: object[] }, sourcesDir?: string | null, claims?: object | null, fillMap?: object[] | null, fillEntries?: object[] | null,
  *   brief: string | object, baseDir?: string, recompute?: (ledger: object, row: object) => { ok: boolean } }} options
  *   `brief` is the brief.md text or parseFidelityBlock output; `baseDir` resolves data-chart-source paths.
  * @returns {{ defects: object[], gatesRun: string[] }}
  */
-export function checkFidelity(html, { ledger, sourcesDir = null, claims = null, fillMap = null, brief, baseDir, recompute } = {}) {
+export function checkFidelity(html, { ledger, sourcesDir = null, claims = null, fillMap = null, fillEntries = null, brief, baseDir, recompute } = {}) {
 	const parsed = brief && typeof brief === "object" ? brief : parseFidelityBlock(String(brief ?? ""))
 	if (!isLocked(parsed)) return { defects: [incomplete(parsed)], gatesRun: ["fidelity_interview_incomplete"] }
 	const ctx = buildContext(parseHtml(html), { baseDir })
 	const defects = [
-		...valueChecks(ctx, ledger, fillMap, sourcesDir, { recompute, asOf: parsed.fields["as of"] }),
+		...valueChecks(ctx, ledger, fillMap, sourcesDir, { recompute, asOf: parsed.fields["as of"], fillEntries }),
 		...bindingChecks(ctx, claims, ledger),
 	]
 	const pairs = fidelityPairs(ctx, ledger, sourcesDir)

package/packages/shared-skills/skills/ulw-research/scripts/gates-static-headings.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, test } from "bun:test"
+
+import { buildContext, checkStatic } from "./gates-static.mjs"
+import { parseHtml } from "./html-lite.mjs"
+
+describe("HTML exact Closing headings", () => {
+	test.each([
+		["Closing", "closing", false],
+		["Closing the gap: GPU prices", null, true],
+	] as const)("#given %s as an HTML heading #when checked #then closing scope matches the whole heading", (heading: string, kind: string | null, missing: boolean) => {
+		const root = parseHtml(`<h2>${heading}</h2><p>Prices fell 8%.</p>`)
+		expect(buildContext(root, {}).sections.map((section: { kind: string | null }) => section.kind)).toEqual([kind])
+		const { defects } = checkStatic(root, {})
+		expect(defects.some((defect) => "code" in defect && defect.code === "missing_closing_section")).toBe(missing)
+		expect(defects.some((defect) => "code" in defect && defect.code === "unsourced_number")).toBe(missing)
+	})
+})

package/packages/shared-skills/skills/ulw-research/scripts/gates-static.mjs

@@ -14,6 +14,7 @@ import { figureGates } from "./gates-figures.mjs"
 import { structureGates } from "./gates-structure.mjs"
 import { textGates } from "./gates-text.mjs"
 import { PROSE_TAGS, proseText, selectorPath } from "./html-lite.mjs"
+import { sourcesClaims } from "./sources-citation.mjs"
 
 const CITATION_MARKER = /\[S\d+\]|\[Source\s*\d+\]|\[\uCD9C\uCC98[^\]]*\]|\[\^\d+\]/i
 const ANCHOR_HREF = /^#(?:s|src|source|ref|fn)/i
@@ -65,6 +66,11 @@ function hasCitation(node) {
 	return CITATION_MARKER.test(proseText(node)) || descendants(node).some(isCitationAnchor)
 }
 
+function proseScope(scope, owner) {
+	if (scope.section?.kind !== "sources" || !owner || sourcesClaims(owner).length === 0) return scope
+	return { ...scope, section: { ...scope.section, kind: null } }
+}
+
 function toRegExp(pattern) {
 	const re = pattern instanceof RegExp ? pattern : new RegExp(String(pattern), "i")
 	return new RegExp(re.source, re.flags.replace(/[gy]/g, ""))
@@ -181,7 +187,10 @@ export function checkStatic(doc, options = {}) {
 			continue
 		}
 		gatesRun.push(...gate.codes)
-		defects.push(...gate.run(ctx))
+		const context = gate.codes.includes("unsourced_number")
+			? { ...ctx, elements: ctx.elements.map(({ node, scope }) => ({ node, scope: proseScope(scope, node) })) }
+			: ctx
+		defects.push(...gate.run(context))
 	}
 	return { defects, gatesRun, gatesSkipped }
 }

package/packages/shared-skills/skills/ulw-research/scripts/html-lite.mjs

@@ -23,7 +23,7 @@ const PROSE_SKIP = new Set(["code", "pre", "kbd", "samp", "script", "style", "sv
 const SAFE_ID = /^[A-Za-z_][\w-]*$/
 
 export const PROSE_TAGS = new Set(["p", "li", ...HEADINGS, "td", "th", "figcaption", "blockquote", "dd", "dt"])
-const PROSE_BREAKS = new Set([...P_CLOSERS, ...PROSE_TAGS, "br", "tr"])
+export const PROSE_BREAKS = new Set([...P_CLOSERS, ...PROSE_TAGS, "br", "tr"])
 
 function makeNode(tag, attrs, parent, line) {
 	return { tag, attrs, children: [], parent, line, text: "", raw: null }
@@ -130,6 +130,8 @@ export function parseHtml(source) {
 	const flushText = (to) => {
 		if (to > textStart) {
 			const node = makeNode("#text", {}, top(), lineAt(starts, textStart))
+			node.source_start = textStart
+			node.source_end = to
 			node.text = decodeEntities(src.slice(textStart, to))
 			top().children.push(node)
 		}
@@ -157,7 +159,7 @@ export function parseHtml(source) {
 			const tag = readTag(src, lt + 2)
 			if (tag.name !== "body" && tag.name !== "html") {
 				for (let k = stack.length - 1; k > 0; k--) {
-					if (stack[k].tag === tag.name) { stack.length = k; break }
+					if (stack[k].tag === tag.name) { stack[k].source_end = lt; stack.length = k; break }
 				}
 			}
 			at = textStart = tag.end
@@ -169,6 +171,8 @@ export function parseHtml(source) {
 			if (P_CLOSERS.has(tag.name)) closeWithin("p", P_SCOPE)
 			if (tag.name === "li") closeWithin("li", LI_SCOPE)
 			const node = makeNode(tag.name, tag.attrs, top(), lineAt(starts, lt))
+			node.source_start = tag.end
+			node.source_end = src.length
 			top().children.push(node)
 			if (tag.name === "body") sawBody = true
 			if (tag.attrs.id) root.ids.set(tag.attrs.id, (root.ids.get(tag.attrs.id) ?? 0) + 1)

package/packages/shared-skills/skills/ulw-research/scripts/html-markup.mjs

@@ -61,10 +61,10 @@ function scriptEnd(src, from) {
 function closeOf(src, from, name) {
 	if (name === "script") return scriptEnd(src, from)
 	if (RAW.has(name)) {
-		const close = new RegExp(`</${name}(?=[\\s/>])[^>]*>`, "gi")
+		const close = new RegExp(`</${name}(?=[\\s/>])`, "gi")
 		close.lastIndex = from
 		const match = close.exec(src)
-		return match ? match.index + match[0].length : src.length
+		return match ? readTag(src, match.index + 2).end : src.length
 	}
 	let depth = 0
 	let stop = src.length

package/packages/shared-skills/skills/ulw-research/scripts/html-text-map.mjs

@@ -0,0 +1,140 @@
+// html-text-map.mjs - the reader-visible text of a page, with a source map back into the markup.
+//
+// extractHtml(html) returns { root, text, blocks, starts, ends }: `text` is one line per block
+// (prose blocks and table cells; whitespace collapsed, entities decoded) and text unit k came from
+// html.slice(starts[k], ends[k]). Only text nodes are read: attribute values, CSS, scripts, and
+// anything a reader never sees (head, script, style, svg, noscript, template, code-like elements,
+// aria-hidden="true" and their descendants, and what the page's CSS hides: css-hiding.mjs)
+// contribute nothing, so a locator over `text` can never land in markup. A synthetic separator
+// (collapsed break, line feed) maps to an empty range at the previous character's end.
+// Plain Node ESM, sibling imports only.
+
+import { cssVisibility } from "./css-hiding.mjs"
+import { decodeOne } from "./entities.mjs"
+import { parseHtml, PROSE_BREAKS, PROSE_TAGS } from "./html-lite.mjs"
+
+const SKIP = new Set(["head", "script", "style", "svg", "code", "pre", "kbd", "samp", "template", "noscript"])
+const INLINE = new Set([
+	"a", "abbr", "b", "bdi", "bdo", "cite", "data", "del", "dfn", "em", "font", "i", "ins", "label", "mark", "q", "s",
+	"small", "span", "strong", "sub", "sup", "time", "u", "var",
+])
+const SPACE = /\s/
+
+/** null when a reader never sees the element or anything inside it, else whether its own text shows (css: cssVisibility). */
+function shownState(node, css, parentShown) {
+	if (SKIP.has(node.tag) || String(node.attrs?.["aria-hidden"] ?? "").trim().toLowerCase() === "true") return null
+	return css(node, parentShown)
+}
+
+function newLine() {
+	return { node: null, text: "", starts: [], ends: [], space: null }
+}
+
+function put(line, unit, start, end) {
+	if (SPACE.test(unit)) {
+		if (line.text !== "" && line.space === null) line.space = [start, end]
+		return
+	}
+	if (line.space !== null) {
+		line.text += " "
+		line.starts.push(line.space[0])
+		line.ends.push(line.space[1])
+		line.space = null
+	}
+	line.text += unit
+	line.starts.push(start)
+	line.ends.push(end)
+}
+
+/** A collapsible break with no source character of its own. */
+function softBreak(line) {
+	if (line.text !== "" && line.space === null) {
+		const at = line.ends[line.ends.length - 1]
+		line.space = [at, at]
+	}
+}
+
+/** Append a text node's source range, decoding entities exactly as decodeEntities does. */
+function putText(line, src, from, to) {
+	for (let at = from; at < to; ) {
+		if (src[at] === "&") {
+			const semi = src.indexOf(";", at + 1)
+			const decoded = semi !== -1 && semi < to && semi - at <= 12 ? decodeOne(src.slice(at + 1, semi)) : null
+			if (decoded !== null) {
+				for (let k = 0; k < decoded.length; k++) put(line, decoded[k], at, semi + 1)
+				at = semi + 1
+				continue
+			}
+		}
+		put(line, src[at], at, at + 1)
+		at++
+	}
+}
+
+function putProse(line, src, node, css, shown) {
+	for (const child of node.children) {
+		if (child.tag === "#text") {
+			if (shown) putText(line, src, child.source_start, child.source_end)
+			continue
+		}
+		const state = shownState(child, css, shown)
+		if (state === null) continue
+		const breaks = PROSE_BREAKS.has(child.tag)
+		if (breaks) softBreak(line)
+		putProse(line, src, child, css, state)
+		if (breaks) softBreak(line)
+	}
+}
+
+/** See the file comment. blocks: [{ node, start, end }] with text.slice(start, end) one block line. */
+export function extractHtml(html) {
+	const src = String(html)
+	const root = parseHtml(src)
+	const css = cssVisibility(root)
+	const lines = []
+	let line = newLine()
+	const flush = (node) => {
+		if (line.text !== "") lines.push({ ...line, node })
+		line = newLine()
+	}
+	const walk = (node, shown) => {
+		for (const child of node.children) {
+			if (child.tag === "#text") {
+				if (shown) putText(line, src, child.source_start, child.source_end)
+				continue
+			}
+			const state = shownState(child, css, shown)
+			if (state === null) continue
+			if (child.tag === "br") flush(node)
+			else if (INLINE.has(child.tag)) walk(child, state)
+			else if (PROSE_TAGS.has(child.tag)) {
+				flush(node)
+				putProse(line, src, child, css, state)
+				flush(child)
+			} else {
+				flush(node)
+				walk(child, state)
+				flush(child)
+			}
+		}
+	}
+	walk(root, true)
+	flush(root)
+	const blocks = []
+	const starts = []
+	const ends = []
+	let text = ""
+	for (const entry of lines) {
+		if (text !== "") {
+			const at = ends[ends.length - 1]
+			text += "\n"
+			starts.push(at)
+			ends.push(at)
+		}
+		blocks.push({ node: entry.node, start: text.length, end: text.length + entry.text.length })
+		text += entry.text
+		starts.push(...entry.starts)
+		ends.push(...entry.ends)
+	}
+	return { root, text, blocks, starts, ends }
+}