oh-my-opencode 5.1.48 → 5.1.54

package/packages/shared-skills/skills/ulw-research/scripts/conversion-probe.mjs

@@ -0,0 +1,229 @@
+// conversion-probe.mjs - numeric multiset differences after source conversion.
+// Inline markup may split one number; superscripts must not merge into it. Numbers are read with the
+// one grouped-number grammar (number-grammar.mjs). A cell, row, list-item, block or line break is a
+// hard boundary: a space never groups digits across it. A sign counts when no Latin, Greek or
+// Cyrillic letter, digit or underscore touches it (an en dash before a digit is a minus too), and a
+// "-" or U+2212 counts after ONE such letter ("\u0394\u22125") or after an exponent ("1e-5"); a run of
+// two or more ("cite_note-7", "AB-12", "UTF-8", "2019-2023") is an identifier or range and carries no
+// negative. Markdown markup a converter writes beside a sign never decides it: the closer run before it
+// (read with the CommonMark inline grammar in markdown-closers.mjs: emphasis, code, strike, titled and
+// parenthesised link and image targets, reference endings, backslash-escaped closers) is skipped to
+// the first character it closes ("**2019**-2023" and '[1990](/wiki/1990 "1990")-2000' are ranges), and
+// an opener run between the sign and its digits ("-**5**") keeps the sign. A raw-side character
+// reference (`−`, ` `) is decoded on BOTH sides with the one entities.mjs table, so a
+// verbatim copy probes clean. Raw inline HTML a converter leaves in the Markdown (pandoc's gfm writer
+// emits `<sup>`, `<sub>`, `<img ... width=320>`, `<span>`, `<br>`) is read with the html-lite tag
+// reader: sup, sub and break tags are boundaries, as on the raw side, and any other tag or comment is
+// dropped with its attributes, so it is a closer or opener like Markdown markup. Digits a converter
+// writes out from a link target, an image source, alt text, or a title cover a converted number only
+// inside Markdown link or image syntax (`](...)` with any title and balanced parentheses,
+// `![...](...)`, `<https://...>`, `[ref]: url "title"` with the title on the same or the next line),
+// never in prose; a footnote
+// definition's text ("[^1]: ...") is prose. Converter syntax is not a number: a line-start list marker
+// ("1. ", "2) ", "(3) ", indented or after "> ") is exempt only while the source has an <ol> item with
+// that number and the markers so far do not outnumber the <ol> items (so lazy "1. 1. 1." numbering
+// over three items passes), a numeric reference label ("[x][1]" with its "[1]: url") only while both
+// halves are present, and a footnote definition label ("[^1]:") only while its "[^1]" is used;
+// otherwise each is a candidate. A converter's hard wrap may split a space-grouped number over one line
+// break; the two halves read as one number only when the raw side holds that number and not both
+// halves, so raw "12 345 and 12 and 345" wrapped inside "12 345" reports both halves (it fails safe).
+// The built-in snapshot writes no attribute text, syntax, wraps or tags, and its text is already
+// decoded, so it gets none of these allowances. Constructs outside this grammar are declared out of
+// scope in references/fact-ledger.md: they report as altered (the safe direction).
+
+import { existsSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { CliError } from "./cli-support.mjs"
+import { decodeEntities, parseHtml, PROSE_BREAKS, readTag } from "./html-lite.mjs"
+import { escapedAt, inlineLinkSpans, stripClosers } from "./markdown-closers.mjs"
+import { NUMBER_SOURCE, plainDecimal } from "./number-grammar.mjs"
+
+const LETTER = "\\p{Script=Latin}\\p{Script=Greek}\\p{Script=Cyrillic}"
+const WORD = new RegExp(`[${LETTER}\\p{N}_]`, "u")
+const ALPHA = new RegExp(`[${LETTER}]`, "u")
+const EXPONENT = new RegExp(`(?<![${LETTER}\\p{N}_.,])\\d+(?:[.,]\\d+)?[eE]$`, "u")
+/** A dash, then any Markdown opener run, then a number; signed() decides whether the dash is a minus. */
+const NUMBERS = new RegExp(`(?:([-\\u2212\\u2013])([*_~\`[]*))?(${NUMBER_SOURCE})`, "gu")
+/** Sup, sub, datetime chunks and every break join no neighbour: neither a digit, a separator, nor whitespace. */
+const BOUNDARY = "\u0001"
+const CARRIED = ["href", "src", "alt", "title"]
+const LIST_MARKER = /(?:^|\u0001)(?: ?>)* ?(?:\d+[.)]|\(\d+\)) /gu
+const REF_USE = /\]\[(\d+)\]/gu
+const REF_DEF = /(?:^|\u0001)(?: ?>)* ?\[(\d+)\]:/gu
+const NOTE_USE = /\[\^(\d+)\](?!:)/gu
+const NOTE_DEF = /(?:^|\u0001)(?: ?>)* ?\[\^(\d+)\]:/gu
+const TITLE = `(?:"[^"\u0001]*"|'[^'\u0001]*'|\\([^)\u0001]*\\))`
+/** A reference definition's title may sit alone on the next line (only whitespace after it). */
+const REF_DEFINITION = `(?:^|\u0001) ?\\[(?!\\^)[^\\]\u0001]+\\]: ?[^\\s\u0001]*(?: ${TITLE}| ?\u0001 ?${TITLE}(?= ?(?:\u0001|$)))?`
+const LINK_SYNTAX = new RegExp(`!\\[[^\\]\u0001]*\\](?:\\([^)\u0001]*\\))?|\\]\\([^)\u0001]*\\)|<https?:\\/\\/[^>\\s\u0001]+>|${REF_DEFINITION}`, "gu")
+const TAG_NAME = /^[a-z][a-z0-9-]*$/u
+/** What a tag leaves in the numeric text on either side: sup, sub and every break join no neighbour. */
+const edgeOf = (tag) => (tag === "sup" || tag === "sub" || PROSE_BREAKS.has(tag) ? BOUNDARY : "")
+
+/** Prose text, attribute text and <ol> item numbers of an HTML document; a non-HTML document is prose only. */
+function numericText(source, html) {
+	if (!html) return { prose: decodeEntities(source), carried: "", listed: [] }
+	const chunks = []
+	const carried = []
+	const listed = []
+	const walk = (node) => {
+		if (node.tag === "#text") { chunks.push(node.text.replace(/\s+/gu, " ")); return }
+		for (const name of CARRIED) if (node.attrs?.[name] !== undefined) carried.push(node.attrs[name])
+		if (node.tag === "ol") {
+			const start = /^\s*\d+\s*$/u.test(node.attrs.start ?? "") ? Number(node.attrs.start) : 1
+			node.children.filter((child) => child.tag === "li").forEach((_, index) => listed.push(String(start + index)))
+		}
+		const edge = edgeOf(node.tag)
+		chunks.push(edge)
+		if (node.tag === "time" && node.attrs.datetime !== undefined) chunks.push(`${BOUNDARY}${node.attrs.datetime}${BOUNDARY}`)
+		for (const child of node.children) walk(child)
+		chunks.push(edge)
+	}
+	walk(parseHtml(source))
+	return { prose: chunks.join(""), carried: carried.join(BOUNDARY), listed }
+}
+
+/** Raw inline HTML in converted Markdown: sup, sub and break tags become boundaries, other tags and comments go. */
+function inlineHtml(text) {
+	let out = ""
+	let kept = 0
+	for (let lt = text.indexOf("<"); lt !== -1; lt = text.indexOf("<", Math.max(lt + 1, kept))) {
+		const comment = text.startsWith("<!--", lt)
+		const name = lt + (text[lt + 1] === "/" ? 2 : 1)
+		if (escapedAt(text, lt) || (!comment && !/[a-z]/iu.test(text[name] ?? ""))) continue
+		const tag = comment ? null : readTag(text, name)
+		const end = comment ? text.indexOf("-->", lt + 4) + 3 : tag.end
+		if (end <= lt || (tag && !(TAG_NAME.test(tag.name) && text[end - 1] === ">")) || /\n[ \t\r]*\n/u.test(text.slice(lt, end))) continue
+		out += text.slice(kept, lt) + (tag ? edgeOf(tag.name) : "")
+		kept = end
+	}
+	return out + text.slice(kept)
+}
+
+/**
+ * Start offsets of numeric reference labels whose use ("][1]") and definition ("[1]:") are both present,
+ * and of footnote definition labels ("[^1]:") whose "[^1]" is used (the use itself mirrors the source).
+ */
+function referenceLabels(flat) {
+	const uses = [...flat.matchAll(REF_USE)]
+	const defs = [...flat.matchAll(REF_DEF)]
+	const used = new Set(uses.map((match) => match[1]))
+	const defined = new Set(defs.map((match) => match[1]))
+	const paired = [...uses.filter((match) => defined.has(match[1])), ...defs.filter((match) => used.has(match[1]))]
+	const noted = new Set([...flat.matchAll(NOTE_USE)].map((match) => match[1]))
+	const notes = [...flat.matchAll(NOTE_DEF)].filter((match) => noted.has(match[1]))
+	return new Set([...paired, ...notes].map((match) => match.index + match[0].search(/\d/u)))
+}
+
+/** Whether the dash at `at` is a minus: Markdown closers (and a backslash escaping the dash) before it are skipped, as is an opener run before a lone letter. */
+function signed(flat, at, dash) {
+	const before = stripClosers(flat.slice(Math.max(0, at - 512), escapedAt(flat, at) ? at - 1 : at))
+	const last = before.at(-1) ?? ""
+	if (!WORD.test(last)) return true
+	if (dash === "\u2013") return false
+	if (EXPONENT.test(before)) return true
+	return ALPHA.test(last) && !WORD.test(before.slice(0, -1).replace(/[*_~`[]+$/u, "").at(-1) ?? "")
+}
+
+/**
+ * Line breaks become boundaries, then whitespace runs one space; `linked` marks tokens inside link
+ * syntax, `marker` a line-start list marker, `label` a paired reference label, `gap` the text since
+ * the previous number.
+ */
+function tokensOf(text, { links = false } = {}) {
+	const flat = text.replace(/\r\n?|\n|\f/gu, BOUNDARY).replace(/\s+/gu, " ")
+	const spans = links ? [...[...flat.matchAll(LINK_SYNTAX)].map((match) => [match.index, match.index + match[0].length]), ...inlineLinkSpans(flat)] : []
+	const markers = new Set([...flat.matchAll(LIST_MARKER)].map((match) => match.index + match[0].search(/\d/u)))
+	const labels = referenceLabels(flat)
+	let previousEnd = 0
+	return [...flat.matchAll(NUMBERS)].map((match, n) => {
+		const sign = match[1] !== undefined && signed(flat, match.index, match[1])
+		const at = sign ? match.index : match.index + match[0].length - match[3].length
+		const gap = flat.slice(previousEnd, at)
+		previousEnd = match.index + match[0].length
+		return {
+			n,
+			gap,
+			text: sign ? match[0] : match[3],
+			key: `${sign ? "-" : ""}${plainDecimal(match[3])}`,
+			linked: spans.some(([start, end]) => at >= start && at < end),
+			marker: markers.has(at),
+			label: labels.has(at),
+		}
+	})
+}
+
+/** The comparison key of one number token, so other tools can match a converted number against a probe record. */
+export const numberKey = (token) => tokensOf(String(token))[0]?.key ?? null
+
+/** The numbers in `texts` that the source's conversion probe recorded as altered (sources/<id>/conversion.json). */
+export function alteredIn(sourceDir, texts) {
+	const record = join(sourceDir, "conversion.json")
+	let altered = []
+	if (existsSync(record)) {
+		try {
+			altered = JSON.parse(readFileSync(record, "utf8")).conversion_altered_numbers ?? []
+		} catch (error) {
+			throw new CliError(`${record} is not a readable conversion record (${error.message}); snapshot the source again`)
+		}
+	}
+	const keys = new Set(altered.map(numberKey))
+	return texts.flatMap((text) => tokensOf(String(text ?? ""))).filter((token) => keys.has(token.key)).map((token) => token.text)
+}
+
+/** Rejoin a grouped number a hard wrap split over one line break, when only the joined reading is in the raw counts. */
+function joinWrapped(tokens, raw) {
+	const out = []
+	for (const token of tokens) {
+		const prev = out.at(-1)
+		const wrapped = prev?.n === token.n - 1 && /^ ?\u0001 ?$/u.test(token.gap) && !prev.linked && !token.linked
+		const [joined, ...more] = wrapped ? tokensOf(`${prev.text} ${token.text}`) : []
+		if (joined && more.length === 0 && raw.has(joined.key) && !(raw.has(prev.key) && raw.has(token.key))) {
+			out[out.length - 1] = { ...joined, n: token.n, text: `${prev.text}\n${token.text}` }
+			continue
+		}
+		out.push(token)
+	}
+	return out
+}
+
+const countOf = (tokens) => tokens.reduce((counts, token) => counts.set(token.key, (counts.get(token.key) ?? 0) + 1), new Map())
+
+/** The tokens `counts` cannot cover; each covered token uses up one count. */
+function uncovered(tokens, counts) {
+	return tokens.filter((token) => {
+		const remaining = counts.get(token.key) ?? 0
+		if (remaining === 0) return true
+		counts.set(token.key, remaining - 1)
+		return false
+	})
+}
+
+/**
+ * Preserve token spelling in diagnostics; the raw side is read as HTML, the converted side as Markdown
+ * with raw inline tags, both decoded with one entity table. `attributes: false`
+ * (the built-in snapshot) gives attribute digits no cover, converter syntax no exemption and no wrap rejoin. An exempt
+ * marker or label is not a converted number at all: it neither covers a raw number nor counts as altered.
+ */
+export function probeConversion(raw, converted, { attributes = true } = {}) {
+	const source = numericText(raw, /<[a-z!/]/i.test(raw))
+	const before = tokensOf(source.prose)
+	const listed = new Set(source.listed)
+	let items = attributes ? source.listed.length : 0
+	const syntax = tokensOf(attributes ? decodeEntities(inlineHtml(converted)) : converted, { links: attributes })
+		.filter((token) => !(attributes && token.label) && !(token.marker && listed.has(token.key) && items-- > 0))
+	const after = attributes ? joinWrapped(syntax, countOf(before)) : syntax
+	const counts = countOf(before)
+	const extra = new Set(uncovered(after.filter((token) => !token.linked), counts))
+	for (const token of tokensOf(source.carried)) counts.set(token.key, (counts.get(token.key) ?? 0) + 1)
+	for (const token of uncovered(after.filter((token) => token.linked), counts)) extra.add(token)
+	const lost = uncovered(before, countOf(after)).map((token) => token.text)
+	return { altered: after.filter((token) => extra.has(token)).map((token) => token.text), lost, raw_count: before.length, converted_count: after.length }
+}
+
+export function runConversionProbe(argv) {
+	if (argv.length !== 2) throw new CliError("usage: source probe-conversion <raw-file> <converted-file>")
+	const result = probeConversion(readFileSync(argv[0], "utf8"), readFileSync(argv[1], "utf8"))
+	return { json: result, summary: `source probe-conversion: ${result.altered.length} altered, ${result.lost.length} lost`, exitCode: result.altered.length === 0 ? 0 : 1 }
+}

package/packages/shared-skills/skills/ulw-research/scripts/conversion-probe.test.ts

@@ -0,0 +1,218 @@
+import { describe, expect, onTestFinished, test } from "bun:test"
+import { spawnSync } from "node:child_process"
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+
+import { numberKey } from "./conversion-probe.mjs"
+import { readFidelityRecord } from "./fidelity-records.mjs"
+import { writeSnapshot } from "./source-snapshot.mjs"
+
+const CLI = join(import.meta.dir, "report-tools.mjs")
+const FIXTURE = readFileSync(join(import.meta.dir, "tests/fixtures/conversion-probe.html"), "utf8")
+const FLATTENED = FIXTURE.replace(/<[^>]*>/g, "")
+// Superscript boundaries preserve 4.2 + 2, 27.3 + 1 and 10 + 9 in raw.
+const ALTERED = ["4.22", "27.31", "109"]
+
+function sandbox(): string {
+	const dir = mkdtempSync(join(tmpdir(), "conversion-probe-"))
+	onTestFinished(() => rmSync(dir, { recursive: true, force: true }))
+	return dir
+}
+
+function probe(raw: string, converted: string) {
+	const dir = sandbox()
+	const rawFile = join(dir, "raw.html")
+	const convertedFile = join(dir, "converted.txt")
+	writeFileSync(rawFile, raw)
+	writeFileSync(convertedFile, converted)
+	const proc = spawnSync("node", [CLI, "source", "probe-conversion", rawFile, convertedFile], { encoding: "utf8" })
+	expect(proc.error).toBeUndefined()
+	expect(proc.stdout, proc.stderr).not.toBe("")
+	return { exitCode: proc.status, json: JSON.parse(proc.stdout) }
+}
+
+describe("source probe-conversion", () => {
+	test("#given the probe page #when naively flattened #then only fabricated numbers are altered", () => {
+		const result = probe(FIXTURE, FLATTENED)
+		expect(result.exitCode).toBe(1)
+		// Stripping the adjacent table tags also joins Total's 2 to the first cell.
+		// The flatten glues Total's 2 onto 12,345 and 12,345 onto 15,678: "212,34515,678".
+		expect(result.json.altered).toEqual([...ALTERED, "212,34515", "678"])
+		expect(result.json.lost).toContain("2026")
+		expect(result.json.lost).not.toContain("99.9")
+		expect(result.json.altered).not.toContain("99.9")
+	})
+
+	test("#given a plain text #when compared to itself #then identity passes without alterations", () => {
+		const text = "USD 4.2 billion; 1,234.5 units; 1 234 567 people; -3.5 and 2019-2023."
+		const result = probe(text, text)
+		expect(result.exitCode).toBe(0)
+		expect(result.json.altered).toEqual([])
+		expect(result.json.lost).toEqual([])
+		expect(result.json.converted_count).toBe(result.json.raw_count)
+	})
+
+	test("#given a tag-split decimal #when compared with text #then each side has one number", () => {
+		const result = probe("<span>1</span><span>,</span><span>234.5</span>", "1,234.5")
+		expect(result).toEqual({ exitCode: 0, json: { altered: [], lost: [], raw_count: 1, converted_count: 1 } })
+	})
+
+	test.each(["\u202f", "\u00a0"])("#given %s thousands separators #when converted to spaces #then grouping is equivalent", (separator: string) => {
+		const result = probe(`1${separator}234${separator}567`, "1 234 567")
+		expect(result).toEqual({ exitCode: 0, json: { altered: [], lost: [], raw_count: 1, converted_count: 1 } })
+	})
+
+	test("#given a machine date #when its attribute is removed #then it is lost, not altered", () => {
+		const result = probe('<time datetime="2026-10-01T09:00:00Z">1 Oct</time>', "1 Oct")
+		expect(result.exitCode).toBe(0)
+		expect(result.json.altered).toEqual([])
+		expect(result.json.lost).toEqual(["2026", "10", "01", "09", "00", "00"])
+	})
+
+	test("#given repeated signed decimals #when one count increases #then multiset subtraction preserves the excess", () => {
+		const result = probe("-4.2 +7 2", "-4.2 +7 2; 2")
+		expect(result.exitCode).toBe(1)
+		expect(result.json.altered).toEqual(["2"])
+		expect(result.json.lost).toEqual([])
+	})
+
+	test("#given hidden text #when omitted by conversion #then the number is only lost", () => {
+		const result = probe('<span style="display:none">99.9</span>', "")
+		expect(result.exitCode).toBe(0)
+		expect(result.json.altered).toEqual([])
+		expect(result.json.lost).toEqual(["99.9"])
+	})
+})
+
+describe("one grouped-number grammar: a digit glued after the last group is an alteration", () => {
+	test("#given a footnote glued onto a grouped number #when probed #then it is altered", () => {
+		expect(probe("$1,234<sup>5</sup> million", "$1,2345 million").json.altered).toEqual(["1,2345"])
+		expect(probe("<table><tr><td>12,345<sup>1</sup></td></tr></table>", "12,3451").json.altered).toEqual(["12,3451"])
+	})
+
+	test("#given the same cell through the built-in snapshot #when written #then conversion.json lists it", () => {
+		const { dir } = writeSnapshot(join(sandbox(), "sources"), "S1", { path: "t.html", html: "<table><tr><td>12,345<sup>1</sup></td></tr></table>", extractor: "model", observed_at: "2026-10-25T00:00:00Z" })
+		expect(readFidelityRecord(join(dir, "conversion.json"), "source_conversion").conversion_altered_numbers).toEqual(["12,3451"])
+	})
+})
+
+describe("Markdown-shaped converter output reports only real artifacts", () => {
+	const RAW = [
+		"<p>Revenue was 4.2<sup>2</sup> billion.<sup><a href=\"#cite_note-7\">[7]</a></sup></p>",
+		"<p><a href=\"https://example.com/2024/03/report-4\">Report</a> covers q4-2023.</p>",
+		"<p><img src=\"chart-640x480.png\" alt=\"Chart 2023\" title=\"Fig 4\"></p>",
+		"<table><tr><td>12,345<sup>1</sup></td><td>1&#8201;234&#8201;567</td></tr></table>",
+		"<p>From 2019\u20132023 the index fell \u22123.5 points.</p>",
+	].join("\n")
+	const MARKDOWN = [
+		"Revenue was 4.22 billion.[[7]](#cite_note-7)",
+		"",
+		"[Report](https://example.com/2024/03/report-4) covers q4-2023.",
+		"",
+		"![Chart 2023](chart-640x480.png \"Fig 4\")",
+		"",
+		"| 12,3451 | 1\u2009234\u2009567 |",
+		"",
+		"From 2019-2023 the index fell -3.5 points.",
+	].join("\n")
+
+	test("#given markdownify-shaped text #when probed #then altered is exactly 4.22 and 12,3451", () => {
+		expect(probe(RAW, MARKDOWN).json.altered).toEqual(["4.22", "12,3451"])
+	})
+
+	test("#given the same page through the built-in snapshot #when written #then only the real artifacts are altered", () => {
+		const { dir } = writeSnapshot(join(sandbox(), "sources"), "S1", { path: "m.html", html: RAW, extractor: "model", observed_at: "2026-10-25T00:00:00Z" })
+		expect(readFidelityRecord(join(dir, "conversion.json"), "source_conversion").conversion_altered_numbers).toEqual(["4.22", "12,3451"])
+	})
+
+	test("#given a converted text that looks like a tag #when probed #then it is never parsed as HTML", () => {
+		expect(probe("<p>if a&lt;b then revenue was 4.2<sup>2</sup> billion</p>", "if a<b then revenue was 4.22 billion").json.altered).toEqual(["4.22"])
+	})
+})
+
+describe("whitespace and separators match the snapshot side", () => {
+	for (const [raw, converted] of [
+		["<p>1&#8201;234&#8201;567</p>", "1 234 567"],
+		["<p>1&#8199;234</p>", "1 234"],
+		["<table><tr><td>2,500<br>300</td></tr></table>", "2,500 300"],
+		["<ul><li><p>in 2023</p><p>500 sites</p></li></ul>", "in 2023 500 sites"],
+		["<p>1,250\n300</p>", "1,250 300"],
+		["<p><span>1</span>\n<span>234</span></p>", "1 234"],
+	]) {
+		test(`#given ${JSON.stringify(raw)} #when converted to ${JSON.stringify(converted)} #then nothing is altered`, () => {
+			const result = probe(raw, converted)
+			expect(result.json.altered).toEqual([])
+			expect(result.exitCode).toBe(0)
+		})
+	}
+})
+
+test("#given HTML snapshot conversion #when written #then source warnings use the fidelity envelope", () => {
+	const { dir } = writeSnapshot(join(sandbox(), "sources"), "S1", {
+		path: "fixture.html", html: FIXTURE, extractor: "model", observed_at: "2026-10-25T00:00:00Z",
+	})
+	const record = readFidelityRecord(join(dir, "conversion.json"), "source_conversion")
+	expect(record.conversion_altered_numbers).toEqual(ALTERED)
+	expect(record.record_kind).toBe("source_conversion")
+	expect(record.artifact_sha256).toMatch(/^[0-9a-f]{64}$/)
+	expect(record.report).toBe(join(dir, "source.html"))
+	expect(record.written_at).toBe("2026-10-25T00:00:00.000Z")
+	expect(record.defects).toEqual([])
+})
+
+describe("attribute digits cover only a number inside Markdown link or image syntax", () => {
+	const GLUED = "<p>Output reached 10<sup>9</sup> kWh</p>"
+	const BOOK = '<p><a href="https://books.example/b?id=x&amp;pg=PA109">book</a></p>'
+
+	test("#given an href page number 109 #when the prose shows a glued 10<sup>9</sup> #then 109 is altered in flat and Markdown output", () => {
+		expect(probe(`${GLUED}${BOOK}`, "Output reached 109 kWh\n\nbook").json.altered).toEqual(["109"])
+		expect(probe(`${GLUED}${BOOK}`, "Output reached 109 kWh\n\n[book](https://books.example/b?id=x&pg=PA109)").json.altered).toEqual(["109"])
+	})
+
+	test("#given #ref1..#ref45 links #when 4<sup>2</sup> is glued into 42 #then 42 is altered", () => {
+		const links = Array.from({ length: 45 }, (_, k) => `<a href="#ref${k + 1}">note</a>`).join(" ")
+		const markdown = Array.from({ length: 45 }, (_, k) => `[note](#ref${k + 1})`).join(" ")
+		expect(probe(`<p>Area grew 4<sup>2</sup> times.</p><p>${links}</p>`, `Area grew 42 times.\n\n${markdown}`).json.altered).toEqual(["42"])
+	})
+
+	test("#given an image named chart-109.png #when the built-in snapshot writes the page #then 109 is altered", () => {
+		const { dir } = writeSnapshot(join(sandbox(), "sources"), "S1", { path: "g.html", html: `${GLUED}<p><img src="chart-109.png"></p>`, extractor: "model", observed_at: "2026-10-25T00:00:00Z" })
+		expect(readFidelityRecord(join(dir, "conversion.json"), "source_conversion").conversion_altered_numbers).toEqual(["109"])
+	})
+})
+
+describe("cell, row, list-item and line breaks never group digits", () => {
+	for (const [raw, converted] of [
+		["<table><tr><td>3</td><td>250</td><td>100</td></tr></table>", "| 3 | 250 | 100 |"],
+		["<table><tr><td>12</td><td>345</td></tr></table>", "| 12 | 345 |"],
+		["<ul><li>3</li><li>250</li></ul>", "- 3\n- 250"],
+		["<p>3</p><p>250</p>", "3\n250"],
+	]) {
+		test(`#given ${JSON.stringify(raw)} #when converted to ${JSON.stringify(converted)} #then nothing is altered or lost`, () => {
+			expect(probe(raw, converted)).toMatchObject({ exitCode: 0, json: { altered: [], lost: [] } })
+		})
+	}
+
+	test("#given 1 234 567 inside one cell #when converted to a Markdown row #then it stays one grouped number", () => {
+		expect(probe("<table><tr><td>1 234 567</td><td>8</td></tr></table>", "| 1 234 567 | 8 |")).toEqual({ exitCode: 0, json: { altered: [], lost: [], raw_count: 2, converted_count: 2 } })
+	})
+})
+
+describe("a period group needs an unambiguous shape; a dropped minus is altered", () => {
+	test("#given 1.234 and 1,234 #when keyed #then they differ, and 1.234.567 still groups", () => {
+		expect(numberKey("1.234")).not.toBe(numberKey("1,234"))
+		expect(numberKey("1.234.567")).toBe(numberKey("1,234,567"))
+		expect(numberKey("1.234,5")).toBe(numberKey("1,234.5"))
+	})
+
+	test("#given a glued 12.34<sup>5</sup> beside a real 12,345 #when flattened #then the glued 12.345 is the altered token", () => {
+		expect(probe("<p>12.34<sup>5</sup> and 12,345</p>", "12.345 and 12,345").json.altered).toEqual(["12.345"])
+	})
+
+	test("#given a minus touching Hangul or an en dash minus #when the sign is dropped #then the unsigned number is altered", () => {
+		expect(probe("\uAC10\uC18C\uC728-3.5%", "\uAC10\uC18C\uC728 3.5%").json.altered).toEqual(["3.5"])
+		expect(probe("fell \u20135%", "fell 5%").json.altered).toEqual(["5"])
+		expect(probe("cite_note-7 and 2019\u20132023", "cite_note-7 and 2019-2023").json.altered).toEqual([])
+	})
+})

package/packages/shared-skills/skills/ulw-research/scripts/copy-returns-polish.test.ts

@@ -173,7 +173,7 @@ describe("fact polish-apply prints every rejection reason verbatim", () => {
 })
 
 describe("--changed-since stays fail-closed", () => {
-	const { pairsChangedSince } = polishCommand as unknown as { pairsChangedSince: (pairs: unknown[], mapPath: string, ledger: unknown, claims: unknown) => unknown[] }
+	const { pairsChangedSince } = polishCommand as unknown as { pairsChangedSince: (pairs: unknown[], mapPath: string, ledger: unknown, claims: unknown) => unknown }
 	const artifact = file("pre-polish.html", doc("<p>The span is 320 m [S1].</p>"))
 
 	test("#given a fill map without artifact_sha256 #when --changed-since reads it #then it refuses", () => {
@@ -181,9 +181,10 @@ describe("--changed-since stays fail-closed", () => {
 		expect(outcome(() => pairsChangedSince([], mapPath, LEDGER, null))).toContain("fill map has no artifact_sha256")
 	})
 
-	test("#given a fill map whose artifact changed since fill #when read #then it still refuses", () => {
+	test("#given a fill map whose artifact changed since fill #when read #then nothing is filtered out: every pair is re-checked", () => {
 		const mapPath = file("fill-map-stale.json", JSON.stringify({ artifact, artifact_sha256: "0".repeat(64), format: "html", entries: [] }))
-		expect(outcome(() => pairsChangedSince([], mapPath, LEDGER, null))).toContain("changed after fill")
+		const pairs = [{ kind: "quote", sentence_key: "s1", claim_id: null, sentence: "The span is 320 m [S1]." }]
+		expect(pairsChangedSince(pairs, mapPath, LEDGER, null)).toEqual({ pairs, fill_map: mapPath, narrowed: false })
 	})
 })
 

package/packages/shared-skills/skills/ulw-research/scripts/css-hiding.mjs

@@ -0,0 +1,158 @@
+// css-hiding.mjs - whether CSS hides an element of a page from every reader, from its inline style and
+// the page's own <style> rules (external stylesheets and scripts are not evaluated).
+//
+// Text counts as reader-visible when it is visible under at least one condition. So a rule can hide
+// only when it certainly applies: it sits outside every at-rule except @layer (and @media all), its
+// <style> element applies everywhere (no media other than all, CSS type, not in <noscript>), and its
+// selector is one tag (case-insensitive), one .class or one #id (case-sensitive). A rule that only
+// might apply (inside @media, @supports, @container, @scope, @starting-style, @document, @keyframes,
+// nested in another style rule, or with a compound, combinator, attribute or pseudo-class selector)
+// never hides, but it can make text visible when its subject could match and it outranks the hiding
+// declaration in the cascade (origin and importance, inline style, layer order, specificity, order).
+// Hiding properties: display:none and content-visibility:hidden remove the subtree;
+// visibility:hidden|collapse is inherited and a descendant's visibility:visible shows it again.
+// The hidden attribute (except hidden="until-found") and a closed <dialog> are the user-agent
+// display:none, which any author display declaration overrides. opacity, clip, zero size, font-size,
+// off-screen positions and colors do not hide: the text still renders or is read aloud.
+// Plain Node ESM, sibling imports only.
+
+import { scanRules } from "./css-lite.mjs"
+
+const PROPS = new Set(["display", "visibility", "content-visibility"])
+const INHERITED = new Set(["inherit", "unset", "revert", "revert-layer"])
+const IMPORTANT = /!\s*important\s*$/i
+const SIMPLE = /^(?:[.#][^\s.#:[\]>+~,()*"'\\]+|[a-z][a-z0-9-]*)$/iu
+const PSEUDO_ELEMENT = /::|:(?:before|after|first-line|first-letter)\b/i
+const NAME = "[^\\s.#:[\\]>+~,()*\"'\\\\]+"
+const STYLE_IN_SVG = /<style\b[^>]*>([\s\S]*?)<\/style\s*>/gi
+
+/** [tier (0 user agent, 1 author, 2 author !important), inline, layer, specificity, order]: higher wins. */
+const outranks = (a, b) => {
+	for (let k = 0; k < a.length; k++) if (a[k] !== b[k]) return a[k] > b[k]
+	return false
+}
+
+function specificity(selector) {
+	const core = selector.replace(/\[[^\]]*\]/g, " [] ")
+	const ids = core.match(new RegExp(`#${NAME}`, "gu"))?.length ?? 0
+	const classes = (core.match(new RegExp(`\\.${NAME}`, "gu"))?.length ?? 0) + (core.match(/\[\]|:(?!:)[\w-]+/g)?.length ?? 0)
+	const tags = core.match(/(?:^|[\s>+~(])[a-z][\w-]*/gi)?.length ?? 0
+	return ids * 10000 + classes * 100 + tags
+}
+
+/** The subject compound of a selector: its tag, classes and ids (attribute and pseudo-class parts dropped). */
+function subject(selector) {
+	const compound = selector.trim().match(/(?:[^\s>+~[\]()]|\[[^\]]*\]|\([^)]*\))+$/)?.[0] ?? ""
+	const core = compound.replace(/\([^)]*\)/g, "").replace(/\[[^\]]*\]/g, "").replace(/:+[\w-]+/g, "")
+	const tag = core.match(/^[a-z][a-z0-9-]*/i)?.[0].toLowerCase() ?? null
+	const classes = [...core.matchAll(new RegExp(`\\.(${NAME})`, "gu"))].map((match) => match[1])
+	const ids = [...core.matchAll(new RegExp(`#(${NAME})`, "gu"))].map((match) => match[1])
+	return { tag, classes, ids, key: ids[0] ? `#${ids[0]}` : classes[0] ? `.${classes[0]}` : tag ?? "*" }
+}
+
+const classList = (node) => String(node.attrs?.class ?? "").split(/[ \t\n\f\r]+/).filter(Boolean)
+
+function matches(sub, node) {
+	if (sub.tag !== null && sub.tag !== node.tag) return false
+	const names = classList(node)
+	return sub.classes.every((name) => names.includes(name)) && sub.ids.every((id) => id === node.attrs?.id)
+}
+
+/** Whether every enclosing at-rule leaves the rule unconditional (@layer, @media all), its layer name, and keyframe steps. */
+function context(atRule) {
+	const entries = atRule ? atRule.split(/\s(?=@)/) : []
+	const layers = entries.filter((entry) => /^@layer\b/i.test(entry)).map((entry) => entry.slice(6).trim())
+	const conditional = entries.some((entry) => !/^@layer\b/i.test(entry) && !/^@media(?:\s+all)?\s*$/i.test(entry))
+	return { conditional, layer: layers.length ? layers.join(".") : null, keyframe: entries.some((entry) => /^@(?:-[a-z]+-)?keyframes\b/i.test(entry)) }
+}
+
+const declaration = (value) => ({ value: value.replace(IMPORTANT, "").trim().toLowerCase(), important: IMPORTANT.test(value) })
+
+/** Every <style> text that applies to the page, with whether it applies only under a condition. */
+function sheets(root) {
+	const found = []
+	const visit = (node, conditional) => {
+		if (node.tag === "template") return
+		const inner = conditional || node.tag === "noscript"
+		if (node.tag === "style") {
+			const type = String(node.attrs?.type ?? "").trim().toLowerCase()
+			const media = String(node.attrs?.media ?? "").trim().toLowerCase()
+			if (type === "" || type === "text/css") found.push({ css: String(node.raw ?? ""), conditional: inner || (media !== "" && media !== "all") })
+		}
+		if (node.tag === "svg") for (const match of String(node.raw ?? "").matchAll(STYLE_IN_SVG)) found.push({ css: match[1], conditional: inner })
+		for (const child of node.children ?? []) visit(child, inner)
+	}
+	visit(root, false)
+	return found
+}
+
+/** Index every rule declaring display, visibility or content-visibility by its subject key. */
+function indexRules(root) {
+	const index = new Map()
+	const layers = []
+	const layerRank = (name, important) => {
+		if (name === null) return important ? -1 : 1e6
+		if (!layers.includes(name)) layers.push(name)
+		return important ? 1e6 - layers.indexOf(name) : layers.indexOf(name)
+	}
+	let order = 0
+	for (const sheet of sheets(root)) {
+		const css = sheet.css.replace(/<!--|-->/g, " ")
+		for (const match of css.replace(/\/\*[\s\S]*?(?:\*\/|$)/g, "").matchAll(/@layer\s+([^{;]*)[{;]/gi)) {
+			for (const name of match[1].split(",").map((part) => part.trim())) if (name && !layers.includes(name)) layers.push(name)
+		}
+		for (const rule of scanRules(css)) {
+			const decls = rule.declarations.filter((entry) => PROPS.has(entry.prop))
+			if (!decls.length || rule.selector.startsWith("@")) continue
+			const { conditional, layer, keyframe } = context(rule.atRule)
+			if (keyframe) continue
+			for (const selector of rule.selector.split(/,(?![^()[\]]*[)\]])/).map((part) => part.trim())) {
+				if (!selector || PSEUDO_ELEMENT.test(selector)) continue
+				const sub = subject(selector)
+				const certain = !sheet.conditional && !conditional && !rule.nested && SIMPLE.test(selector)
+				const spec = specificity(selector)
+				for (const entry of decls) {
+					const { value, important } = declaration(entry.value)
+					const rank = [important ? 2 : 1, 0, layerRank(layer, important), spec, order++]
+					if (!index.has(sub.key)) index.set(sub.key, [])
+					index.get(sub.key).push({ sub, prop: entry.prop, value, rank, certain })
+				}
+			}
+		}
+	}
+	return index
+}
+
+/** The values a property can take on the element: the certain winner, plus every possible rule that outranks it. */
+function possible(candidates, prop) {
+	let top = null
+	for (const entry of candidates) if (entry.prop === prop && entry.certain && (top === null || outranks(entry.rank, top.rank))) top = entry
+	const values = [top === null ? "unset" : top.value]
+	for (const entry of candidates) if (entry.prop === prop && !entry.certain && (top === null || outranks(entry.rank, top.rank))) values.push(entry.value)
+	return values
+}
+
+/**
+ * cssVisibility(root) -> (node, parentShown) => null when CSS removes the element and its subtree from every
+ * rendering, else whether its own text can be seen (visibility, inherited from parentShown).
+ */
+export function cssVisibility(root) {
+	const index = indexRules(root)
+	return (node, parentShown) => {
+		const attrs = node.attrs ?? {}
+		const keys = ["*", node.tag, ...classList(node).map((name) => `.${name}`), attrs.id === undefined ? null : `#${attrs.id}`]
+		const candidates = []
+		for (const key of new Set(keys)) for (const entry of index.get(key) ?? []) if (matches(entry.sub, node)) candidates.push(entry)
+		const hiddenAttr = attrs.hidden !== undefined && String(attrs.hidden).trim().toLowerCase() !== "until-found"
+		if (hiddenAttr || (node.tag === "dialog" && attrs.open === undefined)) candidates.push({ prop: "display", value: "none", rank: [0, 0, 0, 0, -1], certain: true })
+		const inline = attrs.style ? (scanRules(`x{${attrs.style}}`)[0]?.declarations ?? []) : []
+		inline.forEach((entry, at) => {
+			if (!PROPS.has(entry.prop)) return
+			const { value, important } = declaration(entry.value)
+			candidates.push({ prop: entry.prop, value, rank: [important ? 2 : 1, 1, 0, 0, at], certain: true })
+		})
+		if (possible(candidates, "display").every((value) => value === "none")) return null
+		if (possible(candidates, "content-visibility").every((value) => value === "hidden")) return null
+		return possible(candidates, "visibility").some((value) => (INHERITED.has(value) ? parentShown : value !== "hidden" && value !== "collapse"))
+	}
+}

package/packages/shared-skills/skills/ulw-research/scripts/css-lite.mjs

@@ -57,8 +57,9 @@ function parseDeclarations(text) {
 }
 
 /**
- * @returns {{ selector: string, declarations: { prop: string, value: string }[], atRule?: string }[]}
- * atRule is the chain of enclosing grouping at-rule preludes joined by a space.
+ * @returns {{ selector: string, declarations: { prop: string, value: string }[], atRule?: string, nested?: true }[]}
+ * atRule is the chain of enclosing grouping at-rule preludes joined by a space; nested is true for a
+ * rule written inside another style rule (CSS nesting).
  */
 export function scanRules(css) {
 	if (typeof css !== "string") return []
@@ -88,6 +89,7 @@ export function scanRules(css) {
 				const atChain = stack.filter((entry) => entry.group).map((entry) => entry.prelude)
 				const rule = { selector: frame.prelude, declarations: parseDeclarations(frame.body + buffer) }
 				if (atChain.length) rule.atRule = atChain.join(" ")
+				if (stack.some((entry) => !entry.group)) rule.nested = true
 				slots[frame.slot] = rule
 			}
 			buffer = ""