oh-my-opencode 5.1.47 → 5.1.51

package/dist/skills/ulw-research/scripts/fidelity-value-unit.mjs

@@ -0,0 +1,202 @@
+// Value-mode units live outside the fact element. The documented form is the value then its unit,
+// with or without a space, a Korean particle allowed after it ({{F1:span:value}} m, {{F3:rate:value}}%,
+// {{F21:rev:value}}\uC5B5 \uC6D0\uC73C\uB85C); a currency sign right before the value; the :unit element
+// right after it; or the same visual column's table header. A currency the element's own text carries (fill renders a
+// prefix currency value from its raw text, $<data>$12</data>) binds too: every bound sign, outside or inside, must agree. Unknown words are advisory, never guessed units,
+// unless fill recorded a bound unit for the value: then losing it is a unit edit.
+import { cellsOf, rowsOf } from "./fact-locate.mjs"
+import { unitKey } from "./fidelity-binding.mjs"
+import { parseHtml, proseText } from "./html-lite.mjs"
+import { NUMBER_SOURCE } from "./number-grammar.mjs"
+import { QUALIFIERS_AFTER } from "./numerals.mjs"
+import { dimensionOf, normalizeUnit, spacedAliasUnit } from "./units.mjs"
+
+// Korean particles a unit may carry (eun/neun, i/ga, eul/reul, ui, e, eseo, ege, euro/ro, wa/gwa, do, man,
+// kkaji, buteo, boda, ssik, in, ida/yeotda/ieotda, imyeo, igo), longest first.
+const PARTICLES = ["\uC774\uC5C8\uB2E4", "\uC600\uB2E4", "\uC774\uB2E4", "\uC774\uBA70", "\uC774\uACE0", "\uC73C\uB85C", "\uC5D0\uC11C", "\uC5D0\uAC8C", "\uAE4C\uC9C0", "\uBD80\uD130", "\uBCF4\uB2E4", "\uC740", "\uB294", "\uC774", "\uAC00", "\uC744", "\uB97C", "\uC758", "\uC5D0", "\uB85C", "\uC640", "\uACFC", "\uB3C4", "\uB9CC", "\uC529", "\uC778"]
+/** The displayed value, read with the one grouped-number grammar; text around it is sign and unit. */
+const NUMBER = new RegExp(NUMBER_SOURCE, "u")
+const SIGNS = ["US$", "$", "\u20AC", "\u20A9", "\u00A3", "\u00A5", "USD", "EUR", "GBP", "JPY", "KRW"]
+
+const withoutSuffix = (word, suffixes) => {
+	const suffix = suffixes.find((cur) => word.endsWith(cur) && word.length > cur.length)
+	return suffix ? word.slice(0, -suffix.length) : word
+}
+const withoutParticle = (word) => withoutSuffix(word, PARTICLES)
+/** Words written after a value that bound or soften it (the numeral qualifiers, jeongdo, garyang, jjeum, gwaban). */
+const QUALIFIER_WORDS = [...QUALIFIERS_AFTER, "\uC815\uB3C4", "\uAC00\uB7C9", "\uCBE4", "\uACFC\uBC18"]
+/** A unit, with a particle and/or a qualifier glued on ("m\uC774\uC0C1\uC758"); a whole qualifier word is never a unit (chogwa is not cho + gwa). */
+const stemsOf = (word) => {
+	const bare = withoutParticle(word)
+	return QUALIFIER_WORDS.includes(word) ? [] : [word, bare, withoutSuffix(word, QUALIFIER_WORDS), withoutSuffix(bare, QUALIFIER_WORDS)].filter((stem) => !QUALIFIER_WORDS.includes(stem))
+}
+const stemOf = (word) => stemsOf(word).find((stem) => normalizeUnit(stem)) ?? null
+const unitOf = (word) => normalizeUnit(stemOf(word))
+/** Unit aliases that are also ordinary English words ("won awards", "second place"): read only as the row's own unit. */
+const ORDINARY_WORDS = new Set(["won", "second", "pound", "pounds"])
+/** An ordinary word, or an all-caps acronym that only matches a unit case-insensitively ("SEC filings", "WON"). */
+const ordinary = (word) => ORDINARY_WORDS.has(word.toLowerCase()) || (/^[A-Z]{3,}$/.test(word) && normalizeUnit(word) !== word)
+/** Inline elements the text after a value is read through; any other element, or another fact, ends it. */
+const INLINE = new Set(["a", "abbr", "b", "bdi", "cite", "code", "data", "dfn", "em", "i", "ins", "kbd", "mark", "samp", "small", "span", "strong", "sub", "sup", "time", "u", "var"])
+/** "3 bun-ui 1" is the fraction one third, not 3 minutes. */
+const FRACTION = /^\uBD84\uC758\s?\d/u
+const scaleOf = (unit) => /-(e\d+)$/.exec(unit ?? "")?.[1] ?? null
+
+/** English words stay apart: letter-only words join only as a spaced alias ("percentage points"), so "m in" is m, then in. */
+const LETTERS = /^[A-Za-z]+$/u
+/** The unit the words spell together: a spaced alias, or (with a symbol or Hangul in them) the words joined ("% p", "\uD37C\uC13C\uD2B8 \uD3EC\uC778\uD2B8"). */
+function unitOfWords(words) {
+	if (words.length === 1) return unitOf(words[0])
+	const spaced = stemsOf(words.at(-1)).map((last) => spacedAliasUnit([...words.slice(0, -1), last].join(" "))).find(Boolean)
+	return spaced ?? (words.every((word) => LETTERS.test(word)) ? null : unitOf(words.join("")))
+}
+
+/** The unit written right after the value: the longest known unit its first words spell ("% \uD3EC\uC778\uD2B8" is %p, not %), else the next word. */
+function unitAfter(text, row) {
+	const lead = text.replace(/^[\s\u00A0]+/u, "")
+	if (FRACTION.test(lead)) return null
+	const bracketed = /^\(([^()]{1,40})\)/u.exec(lead)
+	if (bracketed) return unitAfter(bracketed[1], row)
+	const clause = lead.split(/[,.;:!?()[\]"]/u)[0]
+	const words = clause.split(/[\s\u00A0]+/u).filter(Boolean).slice(0, 3)
+	if (words.length === 0) return null
+	for (let n = words.length; n > 1; n--) {
+		const unit = unitOfWords(words.slice(0, n))
+		if (unit) return { token: unit }
+	}
+	const first = unitOf(words[0])
+	if (!first) return null
+	// An ordinary word ("pounds", "won awards") counts only as the row's unit or beside a sign or header of its kind (boundUnits).
+	if (unitKey(first) !== unitKey(row.unit) && ordinary(stemOf(words[0]))) return { token: first, ordinary: true }
+	const currency = words[1] ? unitOf(words[1]) : null
+	if (dimensionOf(first) === "scale" && dimensionOf(currency)?.startsWith("currency:")) return { token: `${currency} ${words[0]}`, dimension: dimensionOf(currency), scale: first }
+	return { token: first }
+}
+
+function signBefore(text) {
+	const trimmed = text.replace(/[\s\u00A0]+$/u, "")
+	// A letter sign is a whole word: "USD 12", never "XUSD 12".
+	const sign = SIGNS.find((s) => trimmed.endsWith(s) && !(/^[A-Z]/u.test(s) && /[A-Za-z]$/u.test(trimmed.slice(0, -s.length))))
+	return sign ? { token: normalizeUnit(sign), sign: true } : null
+}
+
+/** A currency written after the number inside the element ("12\u20AC", "12\uC6D0"); any other unit there is part of the value. */
+function currencyAfter(text, row) {
+	const found = unitAfter(text, row)
+	const dimension = found && !found.ordinary ? (found.dimension ?? dimensionOf(found.token)) : null
+	return dimension?.startsWith("currency:") ? { token: found.scale ? dimension.slice("currency:".length) : found.token, sign: true } : null
+}
+
+function agrees(found, row) {
+	if (found.scale) return dimensionOf(row.unit) === found.dimension && scaleOf(row.unit) === found.scale
+	if (unitKey(found.token) === unitKey(row.unit)) return true
+	if (dimensionOf(found.token) === "scale") return scaleOf(row.unit) === found.token
+	if (dimensionOf(found.token)?.startsWith("currency:") && found.sign) return dimensionOf(row.unit) === dimensionOf(found.token)
+	return unitKey(found.token) === unitKey(row.unit)
+}
+
+const closest = (node, test) => {
+	for (let cur = node.parent; cur; cur = cur.parent) if (test(cur)) return cur
+	return null
+}
+/** All text under a node, code included: a unit may be set in code ("320 <code>ms</code>"). */
+const plainText = (node) => (node.tag === "#text" ? node.text : node.children.map(plainText).join(""))
+
+/** A column-header row: a thead row or a row made only of th cells. A th first in a body row is a row header. */
+const isHeaderRow = (row) => row.parent?.tag === "thead" || (cellsOf(row).length > 0 && cellsOf(row).every((cur) => cur.tag === "th"))
+
+/** The header of the cell's visual column; nothing binds in a table with a rowspan or without a header row. */
+function headerUnit(node) {
+	const cell = closest(node, (cur) => cur.tag === "td")
+	const table = cell && closest(cell, (cur) => cur.tag === "table")
+	if (!table || cell.attrs.colspan || cell.attrs.rowspan) return null
+	const rows = rowsOf(table)
+	if (rows.some((row) => row.children.some((cur) => cur.attrs?.rowspan && cur.attrs.rowspan !== "1"))) return null
+	const header = rows.find(isHeaderRow)
+	if (!header || header === cell.parent) return null
+	const head = cellsOf(header)[cellsOf(cell.parent).indexOf(cell)]
+	if (!head) return null
+	const text = proseText(head)
+	const label = (/\(([^()]+)\)\s*$/u.exec(text)?.[1] ?? text).trim()
+	const unit = unitOf(label) ?? unitOf(label.split(/\s+/u).at(-1) ?? "")
+	return unit ? { token: unit } : null
+}
+
+/** The documented split form: the next element is this fact's own :unit element. */
+function unitElementAfter(siblings, at, row) {
+	const next = siblings.slice(at + 1).find((cur) => !(cur.tag === "#text" && !cur.text.trim()))
+	return next?.attrs?.["data-fact"] === row.fact_id && unitKey(proseText(next)) === unitKey(row.unit)
+}
+
+/** The text after the element within its block, read through inline wrappers and line breaks ("320 <b>m</b>", "<b>320</b> m", "320<br>m"). */
+function textAfter(node) {
+	let text = ""
+	for (let cur = node; text.length < 80; ) {
+		const next = cur.parent?.children[cur.parent.children.indexOf(cur) + 1]
+		if (next === undefined) {
+			if (!INLINE.has(cur.parent?.tag) || cur.parent.attrs["data-fact"] !== undefined) break
+			cur = cur.parent
+		} else if (next.tag === "#text" || next.tag === "br" || (INLINE.has(next.tag) && next.attrs["data-fact"] === undefined)) {
+			text += next.tag === "br" ? "\n" : plainText(next)
+			cur = next
+		} else break
+	}
+	return text
+}
+
+const INSIDE = "written inside with"
+/**
+ * Every unit bound to a value-mode element: a currency sign before it, the unit after it (or its :unit element), its
+ * column header, and a currency sign or word inside it (prefix or suffix of the number).
+ */
+function boundUnits(node, row) {
+	const siblings = node.parent.children
+	const at = siblings.indexOf(node)
+	const prev = siblings[at - 1]
+	const shown = proseText(node)
+	const number = NUMBER.exec(shown)
+	const before = signBefore(prev?.tag === "#text" ? prev.text : "")
+	// The element's own text is the row's render: a scale word in it ("3,234\uC5B5", "12 million") is part of the value.
+	const inside = number ? [signBefore(shown.slice(0, number.index)), currencyAfter(shown.slice(number.index + number[0].length), row)] : []
+	const read = unitAfter(textAfter(node), row) ?? (unitElementAfter(siblings, at, row) ? { token: row.unit } : null)
+	const header = headerUnit(node)
+	const kind = (unit) => dimensionOf(unit.token)?.split(":")[0]
+	const after = read?.ordinary && ![before, ...inside, header].some((unit) => unit && kind(unit) === kind(read)) ? null : read
+	return [["preceded by", before], ["followed by", after], ["headed with", header], ...inside.map((unit) => [INSIDE, unit])].filter(([, unit]) => unit)
+}
+
+/** Each data-fact element's own fill-map entry: fill writes one element per entry, so ids pair up in document order. */
+export function fillEntriesByNode(nodes, entries) {
+	const queues = new Map()
+	for (const entry of entries ?? []) if (entry.replaced === undefined && entry.translates === undefined) queues.set(entry.fact_id, [...(queues.get(entry.fact_id) ?? []), entry])
+	return new Map(nodes.filter((node) => node.attrs["data-replaced"] === undefined).flatMap((node) => {
+		const entry = queues.get(node.attrs["data-fact"])?.shift()
+		return entry ? [[node, entry]] : []
+	}))
+}
+
+/** Fill's record of the unit bound to each value-mode pointer (unit_bound), read off the filled edition (HTML, or the Markdown projection). */
+export function withBoundUnits(html, entries, rows) {
+	const all = (node) => [node, ...node.children.flatMap(all)]
+	const filled = fillEntriesByNode(all(parseHtml(html)).filter((node) => node.attrs?.["data-fact"] !== undefined), entries)
+	const units = new Map([...filled].flatMap(([node, entry]) => {
+		const row = rows.get(entry.fact_id)
+		// Only a unit outside the element: the element's own text is the row's render, so it can be no record of a unit beside it.
+		const unit = entry.mode === "value" && row && unitKey(row.unit) ? boundUnits(node, row).find(([where]) => where !== INSIDE)?.[1].token : null
+		return unit ? [[entry, unit]] : []
+	}))
+	return entries.map((entry) => (units.has(entry) ? { ...entry, unit_bound: units.get(entry) } : entry))
+}
+
+/**
+ * Value mode comes from fill: the element's data-fact-mode, or its fill-map entry when the edition has one
+ * (so deleting the attribute changes nothing). `entry.unit_bound` is the unit fill saw bound to the value.
+ */
+export function valueUnitProblem(node, row, entry = null) {
+	if ((entry?.mode !== "value" && node.attrs["data-fact-mode"] !== "value") || !unitKey(row.unit)) return null
+	const found = boundUnits(node, row)
+	if (entry?.unit_bound && found.every(([where]) => where === INSIDE)) return ["unit_mismatch", `value-mode ${row.fact_id} had the unit "${entry.unit_bound}" beside it when filled, but no recognizable unit is bound beside it now; its row is in "${row.unit}"`]
+	if (found.length === 0) return ["unit_unbound", `${row.fact_id} has unit "${row.unit}" but no recognizable unit right after its value, before it as a currency sign, or in its column header`]
+	const wrong = found.find(([, unit]) => !agrees(unit, row))
+	return wrong ? ["unit_mismatch", `value-mode ${row.fact_id} is ${wrong[0]} "${wrong[1].token}", but its row is in "${row.unit}"`] : null
+}

package/dist/skills/ulw-research/scripts/fidelity-values.mjs

@@ -7,12 +7,16 @@
 // No model, no network.
 
 import { existsSync, readFileSync } from "node:fs"
-import { join, resolve, sep } from "node:path"
+import { resolve, sep } from "node:path"
 
-import { sha256Of } from "./cli-support.mjs"
 import { sameValue, unitKey } from "./fidelity-binding.mjs"
+import { readCsv } from "./fidelity-csv.mjs"
 import { claimChecks } from "./fidelity-claims.mjs"
+import { sourceChecks } from "./fidelity-sources.mjs"
+import { convertedHalfFits, followsOriginal, isDerivedRow, originalProblem, renderLocator, staleRender, textAfter } from "./fidelity-originals.mjs"
+import { snapshotCause, unreadableSnapshots } from "./fidelity-snapshot-readability.mjs"
 import { STRUCTURAL, clip, traceChecks } from "./fidelity-trace.mjs"
+import { fillEntriesByNode, valueUnitProblem } from "./fidelity-value-unit.mjs"
 import { parseHtml, proseText } from "./html-lite.mjs"
 import { parseNumeral } from "./numerals.mjs"
 import { div, fromDecimal, sub } from "./rational.mjs"
@@ -45,11 +49,14 @@ function quoteProblem(row, shown, selector) {
 	return bare(shown) === bare(want) ? null : ["quote_not_verbatim", `"${clip(shown)}" is not the ${selector} selection of ${row.fact_id}: "${clip(want)}"`]
 }
 
-function factProblem(row, shown, attrs = {}) {
+function factProblem(row, shown, attrs = {}, rows = new Map()) {
+	const original = originalProblem(row, shown, rows, attrs["data-fact-mode"] === "value")
+	if (original) return original
 	if (attrs["data-unit"] !== undefined && unitKey(attrs["data-unit"]) !== unitKey(row.unit)) return ["unit_mismatch", `data-unit "${attrs["data-unit"]}" but ${row.fact_id} is in "${row.unit}"`]
 	if (attrs.value !== undefined && !sameValue(attrs.value, row.value)) return ["value_mismatch", `value="${attrs.value}" but ${row.fact_id} is ${row.value}`]
+	if (convertedHalfFits(row, shown)) return null
 	const text = normalizeText(shown)
-	const forms = [row.raw, row.render, row.value, row.unit && `${row.value} ${row.unit}`].filter(Boolean).map(normalizeText)
+	const forms = [...(["converted", "derived"].includes(row.kind) ? [] : [row.raw, row.render]), row.value, row.unit && `${row.value} ${row.unit}`].filter(Boolean).map(normalizeText)
 	if (forms.some((form) => text === form || (text.endsWith(` ${form}`) && !/\d/.test(text.slice(0, -form.length))))) return null
 	const parsed = parseNumeral(text)
 	if (!parsed) return unitKey(text) === unitKey(row.unit) ? null : ["value_mismatch", `"${clip(shown)}" shows no value of ${row.fact_id} (${row.value})`]
@@ -73,16 +80,6 @@ export function exportRows(ledger) {
 	return (ledger?.rows ?? []).filter((row) => row.status === "active" && row.kind !== "quote" && row.value !== null).map((row) => ({ fact_id: row.fact_id, label: row.label, plot_value: row.value, unit: row.unit ?? "" }))
 }
 
-function readCsv(path) {
-	if (!existsSync(path)) return null
-	const lines = readFileSync(path, "utf8").split(/\r?\n/).filter((line) => line.trim() !== "")
-	const cells = (line) => line.split(",").map((cell) => cell.trim().replace(/^"(.*)"$/, "$1"))
-	const head = cells(lines[0] ?? "")
-	const [id, value, unit] = ["fact_id", "plot_value", "unit"].map((name) => head.indexOf(name))
-	if (id < 0 || value < 0 || unit < 0 || lines.length < 2) return null
-	return lines.slice(1).map((line) => cells(line)).map((c) => ({ fact_id: c[id], plot_value: c[value], unit: c[unit] ?? "" }))
-}
-
 const walk = (node) => node.children.filter((child) => child.tag !== "#text").flatMap((child) => [child, ...walk(child)])
 
 /** SVG text nodes of a chart: data labels, or ticks with their declared grid. */
@@ -140,46 +137,57 @@ function chartProblem(ctx, fig, labels, ledger) {
  * @param {object} ctx buildContext(...) from gates-static.mjs
  * @param {{ rows: object[] }} ledger
  * @param {{ fact_id: string, rendered: string, selector?: string }[] | null} fillMap Markdown fill map, or null
- * @param {string | null} sourcesDir directory of <S-id>/{text.txt, meta.json} snapshots
+ * @param {string | null} sourcesDir directory of <S-id>/ snapshots (source-view.mjs reads them)
  * @param {{ recompute?: (ledger: object, row: object) => { ok: boolean }, asOf?: string | null }} options
  */
 export function valueChecks(ctx, ledger, fillMap, sourcesDir, options = {}) {
 	const out = []
-	const flag = (code, node, message) => out.push(ctx.defect(code, node, message))
+	const flag = (code, node, message, extra = {}) => out.push({ ...ctx.defect(code, node, message), ...extra })
 	const rows = new Map((ledger?.rows ?? []).map((row) => [row.fact_id, row]))
 	const cited = new Map()
+	const fills = (fillMap ?? []).map((entry) => entry.rendered)
 	const check = (id, node, problemOf) => {
 		const row = rows.get(id)
 		if (!row) return flag("unresolved_placeholder", node, `${id} names no ledger row`)
 		if (!cited.has(id)) cited.set(id, node)
 		const problem = problemOf(row)
-		if (problem) flag(problem[0], node, problem[1])
+		if (problem) out.push({ ...ctx.defect(problem[0], node, problem[1]), ...(problem[2] ? { part: problem[2] } : {}) })
 	}
 	for (const node of ctx.all.filter((cur) => cur.attrs["data-fact"] !== undefined)) {
 		const selector = node.attrs["data-quote-sel"] ?? "full"
-		check(node.attrs["data-fact"], node, (row) => (row.kind === "quote" ? quoteProblem(row, withoutFooter(node), selector) : factProblem(row, proseText(node), node.attrs)))
+		const glued = (row) => isDerivedRow(row) && node.attrs["data-fact-mode"] !== "value" && followsOriginal(row, rows, fills)(textAfter(node)) ? ["value_mismatch", `another original follows ${row.fact_id}: "${clip(textAfter(node))}"`, "original"] : null
+		check(node.attrs["data-fact"], node, (row) => (row.kind === "quote" ? quoteProblem(row, withoutFooter(node), selector) : (factProblem(row, proseText(node), node.attrs, rows) ?? glued(row))))
 	}
 	const docText = bare(proseText(ctx.body))
 	for (const entry of fillMap ?? []) {
 		check(entry.fact_id, ctx.body, (row) => {
-			const problem = row.kind === "quote" ? quoteProblem(row, entry.rendered, entry.selector ?? "full") : factProblem(row, entry.rendered)
-			if (problem || docText.includes(bare(entry.rendered))) return problem
-			return [row.kind === "quote" ? "quote_not_verbatim" : "value_mismatch", `filled text "${clip(entry.rendered)}" of ${row.fact_id} is no longer in the report`]
+			const problem = row.kind === "quote" ? quoteProblem(row, entry.rendered, entry.selector ?? "full") : factProblem(row, entry.rendered, { "data-fact-mode": entry.mode }, rows)
+			const stale = isDerivedRow(row) ? staleRender(row, entry.rendered, proseText(ctx.body), followsOriginal(row, rows, fills)) : docText.includes(bare(entry.rendered)) ? null : "gone"
+			if (problem || stale === null) return problem
+			return [row.kind === "quote" ? "quote_not_verbatim" : "value_mismatch", `filled text "${clip(entry.rendered)}" of ${row.fact_id} is no longer in the report${stale === "original" ? " (its original was edited)" : ""}`, ...(stale === "original" ? ["original"] : [])]
 		})
 	}
 	rowChecks(ctx, ledger, rows, cited, options, flag)
+	// options.fillEntries: the edition's fill-map entries whose element is in the checked HTML (value mode, fill's unit_bound).
+	const facts = ctx.all.filter((cur) => cur.attrs["data-fact"] !== undefined)
+	const entries = fillEntriesByNode(facts, options.fillEntries)
+	for (const node of facts) {
+		const row = rows.get(node.attrs["data-fact"])
+		const problem = row && row.kind !== "quote" && !factProblem(row, proseText(node), node.attrs, rows) && valueUnitProblem(node, row, entries.get(node))
+		if (problem) flag(problem[0], node, problem[1])
+	}
 	for (const tr of ctx.all.filter((node) => node.tag === "tr" && node.attrs["data-row-kind"] !== "comparison")) {
 		const groups = new Set(ctx.util.descendants(tr).map((node) => rows.get(node.attrs["data-fact"])?.observation_id).filter(Boolean))
 		if (groups.size > 1) flag("join_integrity_fail", tr, `one table row joins observations ${[...groups].join(", ")}`)
 	}
-	const meta = sourceChecks(ctx, rows, cited, sourcesDir, flag)
+	const meta = sourceChecks(ctx, rows, cited, sourcesDir, flag, unreadableSnapshots(ctx, rows, cited, sourcesDir, out))
 	claimChecks(ctx, rows, (row) => provenance(sourcesDir, meta, row), flag)
 	const charts = chartFigures(ctx)
 	for (const { node, scope } of ctx.elements) {
 		const kind = ctx.kindOf(scope)
 		if ((node.tag === "q" || node.tag === "blockquote") && node.attrs["data-fact"] === undefined && kind !== "sources" && kind !== "toc") flag("quote_unbound", node, `<${node.tag}> with no Q row: "${clip(proseText(node))}"`)
 	}
-	traceChecks(ctx, charts, fillMap, flag, fillSiteFits(rows))
+	traceChecks(ctx, charts, fillMap, flag, fillSiteFits(rows), renderLocator(rows, fillMap))
 	for (const [fig, labels] of charts) {
 		const problem = chartProblem(ctx, fig, labels, ledger)
 		if (problem) flag("chart_value_untraceable", fig, problem)
@@ -190,16 +198,23 @@ export function valueChecks(ctx, ledger, fillMap, sourcesDir, options = {}) {
 function rowChecks(ctx, ledger, rows, cited, options, flag) {
 	// An as-of of "latest ..." or "... per snapshot" names no fixed revision, so quote_stale_version is skipped.
 	const asOf = options.asOf && !/^latest\b|per snapshot/i.test(options.asOf) ? options.asOf : null
+	// A sentence-final period ("release v1.2.") is punctuation, not part of the revision token.
+	const target = asOf?.replace(/\.(?=\s|$)/gu, "")
 	for (const [id, node] of cited) {
 		const row = rows.get(id)
 		if (row.fuzzy === true) flag("quote_fuzzy_match", node, `${id} was registered from a similarity match on an OCR-noisy snapshot`)
 		const revision = typeof row.revision === "string" ? row.revision : (row.revision?.id ?? row.revision?.tag ?? null)
-		if (asOf && revision && !asOf.includes(revision)) flag("quote_stale_version", node, `${id} is from revision ${revision}; the brief targets "${asOf}"`)
+		const revisionPattern = revision && new RegExp(`(?<![\\w.-])${revision.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![\\w.-])`, "u")
+		if (asOf && revisionPattern && !revisionPattern.test(target)) flag("quote_stale_version", node, `${id} is from revision ${revision}; the brief targets "${asOf}"`)
 		if ((row.kind === "derived" || row.kind === "converted") && options.recompute?.(ledger, row)?.ok === false) flag("value_mismatch", node, `${id} does not recompute from its operands`)
 		const window = row.snapshot_window
 		if (window && typeof window === "object" && window.start && window.end) {
 			const [start, end] = [Date.parse(window.start), Date.parse(window.end)]
-			const outside = (row.operands ?? []).find((op) => !(Date.parse(rows.get(op)?.observed_at) >= start && Date.parse(rows.get(op)?.observed_at) < end))
+			const [absent, present] = [(row.operands ?? []).filter((op) => !rows.has(op)), (row.operands ?? []).filter((op) => rows.has(op))]
+			for (const op of absent) flag("operand_missing", node, `operand ${op} of ${id} is not in the fact ledger`)
+			const missing = present.filter((op) => !rows.get(op).observed_at)
+			for (const op of missing) flag("operand_timestamp_missing", node, `operand ${op} of ${id} has no observed_at timestamp`)
+			const outside = present.find((op) => !missing.includes(op) && !(Date.parse(rows.get(op).observed_at) >= start && Date.parse(rows.get(op).observed_at) < end))
 			if (outside) flag("snapshot_mismatch", node, `operand ${outside} of ${id} was observed outside ${window.start} to ${window.end}`)
 		}
 	}
@@ -213,17 +228,3 @@ function provenance(sourcesDir, meta, row) {
 	const path = resolve(dir, artifact)
 	return path.startsWith(`${dir}${sep}`) && existsSync(path) ? meta.get(row.source_id)?.kind : undefined
 }
-
-function sourceChecks(ctx, rows, cited, sourcesDir, flag) {
-	const ids = new Set([...cited.keys()].map((id) => rows.get(id)?.source_id).filter(Boolean))
-	for (const match of proseText(ctx.body).matchAll(/\[S(\d+)\]/g)) ids.add(`S${match[1]}`)
-	const meta = new Map()
-	for (const id of sourcesDir ? ids : []) {
-		const [metaPath, textPath] = [join(sourcesDir, id, "meta.json"), join(sourcesDir, id, "text.txt")]
-		if (!existsSync(metaPath) || !existsSync(textPath)) continue
-		meta.set(id, JSON.parse(readFileSync(metaPath, "utf8")))
-		const actual = sha256Of(readFileSync(textPath))
-		if (meta.get(id).sha256 !== actual) flag("snapshot_hash_mismatch", ctx.body, `${id}/text.txt changed after its snapshot was pinned`)
-	}
-	return meta
-}

package/dist/skills/ulw-research/scripts/fidelity-values.test.ts

@@ -1,36 +1,15 @@
+/// <reference types="bun-types" />
 import { describe, expect, test } from "bun:test"
-import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { cpSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"
 import { tmpdir } from "node:os"
 import { join } from "node:path"
 
-import { exportRows, valueChecks } from "./fidelity-values.mjs"
-import { buildContext } from "./gates-static.mjs"
-import { parseHtml } from "./html-lite.mjs"
+import { exportRows } from "./fidelity-values.mjs"
 import { similarity } from "./text-similarity.mjs"
+import { LEDGER, SOURCES, ledgerWith, run, type Defect, type Row } from "./tests/fidelity-values-support"
 
-type Defect = { code: string; severity: string; selector: string; line: number; message: string; hint: string }
-type Row = Record<string, unknown> & { fact_id: string }
-type Ledger = { schemaVersion: number; rows: Row[]; events: unknown[] }
-type Options = { ledger?: Ledger; fillMap?: unknown[] | null; sourcesDir?: string | null; baseDir?: string; recompute?: (ledger: Ledger, row: Row) => { ok: boolean }; asOf?: string }
-
-const FIX = join(import.meta.dir, "tests", "fixtures", "fidelity")
-const SOURCES = join(FIX, "sources")
-const LEDGER: Ledger = JSON.parse(readFileSync(join(FIX, "ledger-verify.json"), "utf8"))
 const FULL = "we estimate completion no earlier than 2027"
 
-function ledgerWith(id: string, patch: Record<string, unknown>): Ledger {
-	const copy = structuredClone(LEDGER)
-	copy.rows = copy.rows.map((row) => (row.fact_id === id ? { ...row, ...patch } : row))
-	return copy
-}
-
-function run(body: string, options: Options = {}): Defect[] {
-	const html = /<h2/.test(body) ? body : `<h2>Findings</h2>\n${body}`
-	const ctx = buildContext(parseHtml(html), { baseDir: options.baseDir ?? FIX })
-	const sourcesDir = options.sourcesDir === undefined ? SOURCES : options.sourcesDir
-	return valueChecks(ctx, options.ledger ?? LEDGER, options.fillMap ?? null, sourcesDir, options) as Defect[]
-}
-
 const codes = (defects: Defect[]) => defects.map((defect) => defect.code)
 const fact = (id: string, text: string, unit = "") => `<data class="fact" data-fact="${id}"${unit ? ` data-unit="${unit}"` : ""}>${text}</data>`
 const quote = (sel: string, text: string) => `<p>They wrote <q data-fact="Q1" data-quote-sel="${sel}">${text}</q> [S4].</p>`
@@ -53,8 +32,8 @@ describe("untraceable numbers", () => {
 		expect(codes(run("<h2>Appendix</h2><p>Revenue grew 12% [S4].</p>"))).toEqual(["untraceable_number"])
 	})
 
-	test("#given the same sentence under Sources #when checked #then it is out of scope", () => {
-		expect(run("<h2>Sources</h2><p>Revenue grew 12% [S4].</p>")).toEqual([])
+	test("#given the same sentence under Sources #when checked #then prose still needs a ledger row", () => {
+		expect(codes(run("<h2>Sources</h2><p>Revenue grew 12% [S4].</p>"))).toEqual(["untraceable_number"])
 	})
 
 	test("#given a date in prose #when checked #then dates are in scope", () => {
@@ -218,14 +197,14 @@ describe("verification claims are structural (data-claim-kind)", () => {
 		`<p>The bridge, which ran across the river, spans ${fact("F1", "up to 320 m", "m")} [S4].</p>`,
 		`<p>The span is ${fact("F1", "up to 320 m", "m")}, as confirmed with the annual inspection report [S4].</p>`,
 		`<p>Clinical tests confirmed the span of ${fact("F1", "up to 320 m", "m")} [S4].</p>`,
-		`<p>확인된 길이는 ${fact("F1", "최대 320 m", "m")} [S4].</p>`,
+		`<p>\uD655\uC778\uB41C \uAE38\uC774\uB294 ${fact("F1", "\uCD5C\uB300 320 m", "m")} [S4].</p>`,
 	]
 	for (const body of quiet) {
 		test(`#given findings prose ${JSON.stringify(body.replace(/<[^>]+>/g, ""))} #when checked #then no warning and no major`, () => expect(run(body)).toEqual([]))
 	}
 
 	test("#given an UNMARKED Korean run-and-check sentence #when checked #then verification_claim_unmarked minor only", () => {
-		expect(severities(run(`<p>우리가 실행해 확인한 길이는 ${fact("F1", "최대 320 m", "m")} [S4].</p>`))).toEqual(["verification_claim_unmarked:minor"])
+		expect(severities(run(`<p>\uC6B0\uB9AC\uAC00 \uC2E4\uD589\uD574 \uD655\uC778\uD55C \uAE38\uC774\uB294 ${fact("F1", "\uCD5C\uB300 320 m", "m")} [S4].</p>`))).toEqual(["verification_claim_unmarked:minor"])
 	})
 
 	test("#given the same run-and-check sentence MARKED verified on a command row #when checked #then no warning", () => {

package/dist/skills/ulw-research/scripts/gates-fidelity.mjs

@@ -20,11 +20,13 @@ import { parseHtml } from "./html-lite.mjs"
 
 /** Every code this gate can raise. */
 export const FIDELITY_GATE_CODES = Object.freeze([
+	"unit_unbound",
 	"fidelity_interview_incomplete", "unresolved_placeholder", "value_mismatch", "unit_mismatch", "quote_not_verbatim",
 	"quote_unbound", "claim_status_invalid", "snapshot_hash_mismatch", "chart_value_untraceable", "untraceable_number",
 	"unsupported_verification", "conflicting_readings", "hedge_dropped", "direction_flipped", "causality_added",
-	"qualifier_dropped", "unbound_sentence", "snapshot_mismatch", "join_integrity_fail", "quote_stale_version",
+	"qualifier_dropped", "unbound_sentence", "snapshot_mismatch", "operand_timestamp_missing", "operand_missing", "join_integrity_fail", "quote_stale_version",
 	"quote_fuzzy_match", "name_outside_quote", "scope_widened_warn", "claim_kind_mismatch", "verification_claim_unmarked",
+	"snapshot_not_raw", "relocated", "quote_converted_only",
 ])
 const RANK = { blocker: 0, major: 1, minor: 2 }
 
@@ -40,8 +42,8 @@ export function deliveryBlockReason(defect) {
 }
 
 function withIntegrity(defect) {
-	const { code, severity, selector, line, message, hint } = defect
-	return { code, severity, integrity: DEFECT_CODES[code].integrity, selector, line, message, hint }
+	const { code, severity, selector, line, message, hint, part, fact_id } = defect
+	return { code, severity, integrity: DEFECT_CODES[code].integrity, selector, line, message, hint, ...(part ? { part } : {}), ...(fact_id ? { fact_id } : {}) }
 }
 
 function compareDefects(a, b) {
@@ -93,17 +95,17 @@ function incomplete(parsed) {
 
 /**
  * @param {string} html the filled report
- * @param {{ ledger: { rows: object[] }, sourcesDir?: string | null, claims?: object | null, fillMap?: object[] | null,
+ * @param {{ ledger: { rows: object[] }, sourcesDir?: string | null, claims?: object | null, fillMap?: object[] | null, fillEntries?: object[] | null,
  *   brief: string | object, baseDir?: string, recompute?: (ledger: object, row: object) => { ok: boolean } }} options
  *   `brief` is the brief.md text or parseFidelityBlock output; `baseDir` resolves data-chart-source paths.
  * @returns {{ defects: object[], gatesRun: string[] }}
  */
-export function checkFidelity(html, { ledger, sourcesDir = null, claims = null, fillMap = null, brief, baseDir, recompute } = {}) {
+export function checkFidelity(html, { ledger, sourcesDir = null, claims = null, fillMap = null, fillEntries = null, brief, baseDir, recompute } = {}) {
 	const parsed = brief && typeof brief === "object" ? brief : parseFidelityBlock(String(brief ?? ""))
 	if (!isLocked(parsed)) return { defects: [incomplete(parsed)], gatesRun: ["fidelity_interview_incomplete"] }
 	const ctx = buildContext(parseHtml(html), { baseDir })
 	const defects = [
-		...valueChecks(ctx, ledger, fillMap, sourcesDir, { recompute, asOf: parsed.fields["as of"] }),
+		...valueChecks(ctx, ledger, fillMap, sourcesDir, { recompute, asOf: parsed.fields["as of"], fillEntries }),
 		...bindingChecks(ctx, claims, ledger),
 	]
 	const pairs = fidelityPairs(ctx, ledger, sourcesDir)

package/dist/skills/ulw-research/scripts/gates-static-headings.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, test } from "bun:test"
+
+import { buildContext, checkStatic } from "./gates-static.mjs"
+import { parseHtml } from "./html-lite.mjs"
+
+describe("HTML exact Closing headings", () => {
+	test.each([
+		["Closing", "closing", false],
+		["Closing the gap: GPU prices", null, true],
+	] as const)("#given %s as an HTML heading #when checked #then closing scope matches the whole heading", (heading: string, kind: string | null, missing: boolean) => {
+		const root = parseHtml(`<h2>${heading}</h2><p>Prices fell 8%.</p>`)
+		expect(buildContext(root, {}).sections.map((section: { kind: string | null }) => section.kind)).toEqual([kind])
+		const { defects } = checkStatic(root, {})
+		expect(defects.some((defect) => "code" in defect && defect.code === "missing_closing_section")).toBe(missing)
+		expect(defects.some((defect) => "code" in defect && defect.code === "unsourced_number")).toBe(missing)
+	})
+})

package/dist/skills/ulw-research/scripts/gates-static.mjs

@@ -14,6 +14,7 @@ import { figureGates } from "./gates-figures.mjs"
 import { structureGates } from "./gates-structure.mjs"
 import { textGates } from "./gates-text.mjs"
 import { PROSE_TAGS, proseText, selectorPath } from "./html-lite.mjs"
+import { sourcesClaims } from "./sources-citation.mjs"
 
 const CITATION_MARKER = /\[S\d+\]|\[Source\s*\d+\]|\[\uCD9C\uCC98[^\]]*\]|\[\^\d+\]/i
 const ANCHOR_HREF = /^#(?:s|src|source|ref|fn)/i
@@ -65,6 +66,11 @@ function hasCitation(node) {
 	return CITATION_MARKER.test(proseText(node)) || descendants(node).some(isCitationAnchor)
 }
 
+function proseScope(scope, owner) {
+	if (scope.section?.kind !== "sources" || !owner || sourcesClaims(owner).length === 0) return scope
+	return { ...scope, section: { ...scope.section, kind: null } }
+}
+
 function toRegExp(pattern) {
 	const re = pattern instanceof RegExp ? pattern : new RegExp(String(pattern), "i")
 	return new RegExp(re.source, re.flags.replace(/[gy]/g, ""))
@@ -181,7 +187,10 @@ export function checkStatic(doc, options = {}) {
 			continue
 		}
 		gatesRun.push(...gate.codes)
-		defects.push(...gate.run(ctx))
+		const context = gate.codes.includes("unsourced_number")
+			? { ...ctx, elements: ctx.elements.map(({ node, scope }) => ({ node, scope: proseScope(scope, node) })) }
+			: ctx
+		defects.push(...gate.run(context))
 	}
 	return { defects, gatesRun, gatesSkipped }
 }

package/dist/skills/ulw-research/scripts/html-lite.mjs

@@ -23,7 +23,7 @@ const PROSE_SKIP = new Set(["code", "pre", "kbd", "samp", "script", "style", "sv
 const SAFE_ID = /^[A-Za-z_][\w-]*$/
 
 export const PROSE_TAGS = new Set(["p", "li", ...HEADINGS, "td", "th", "figcaption", "blockquote", "dd", "dt"])
-const PROSE_BREAKS = new Set([...P_CLOSERS, ...PROSE_TAGS, "br", "tr"])
+export const PROSE_BREAKS = new Set([...P_CLOSERS, ...PROSE_TAGS, "br", "tr"])
 
 function makeNode(tag, attrs, parent, line) {
 	return { tag, attrs, children: [], parent, line, text: "", raw: null }
@@ -130,6 +130,8 @@ export function parseHtml(source) {
 	const flushText = (to) => {
 		if (to > textStart) {
 			const node = makeNode("#text", {}, top(), lineAt(starts, textStart))
+			node.source_start = textStart
+			node.source_end = to
 			node.text = decodeEntities(src.slice(textStart, to))
 			top().children.push(node)
 		}
@@ -157,7 +159,7 @@ export function parseHtml(source) {
 			const tag = readTag(src, lt + 2)
 			if (tag.name !== "body" && tag.name !== "html") {
 				for (let k = stack.length - 1; k > 0; k--) {
-					if (stack[k].tag === tag.name) { stack.length = k; break }
+					if (stack[k].tag === tag.name) { stack[k].source_end = lt; stack.length = k; break }
 				}
 			}
 			at = textStart = tag.end
@@ -169,6 +171,8 @@ export function parseHtml(source) {
 			if (P_CLOSERS.has(tag.name)) closeWithin("p", P_SCOPE)
 			if (tag.name === "li") closeWithin("li", LI_SCOPE)
 			const node = makeNode(tag.name, tag.attrs, top(), lineAt(starts, lt))
+			node.source_start = tag.end
+			node.source_end = src.length
 			top().children.push(node)
 			if (tag.name === "body") sawBody = true
 			if (tag.attrs.id) root.ids.set(tag.attrs.id, (root.ids.get(tag.attrs.id) ?? 0) + 1)

package/dist/skills/ulw-research/scripts/html-markup.mjs

@@ -61,10 +61,10 @@ function scriptEnd(src, from) {
 function closeOf(src, from, name) {
 	if (name === "script") return scriptEnd(src, from)
 	if (RAW.has(name)) {
-		const close = new RegExp(`</${name}(?=[\\s/>])[^>]*>`, "gi")
+		const close = new RegExp(`</${name}(?=[\\s/>])`, "gi")
 		close.lastIndex = from
 		const match = close.exec(src)
-		return match ? match.index + match[0].length : src.length
+		return match ? readTag(src, match.index + 2).end : src.length
 	}
 	let depth = 0
 	let stop = src.length