oh-my-opencode 5.1.47 → 5.1.51

package/packages/shared-skills/skills/ulw-research/scripts/conversion-probe.mjs

@@ -0,0 +1,229 @@
+// conversion-probe.mjs - numeric multiset differences after source conversion.
+// Inline markup may split one number; superscripts must not merge into it. Numbers are read with the
+// one grouped-number grammar (number-grammar.mjs). A cell, row, list-item, block or line break is a
+// hard boundary: a space never groups digits across it. A sign counts when no Latin, Greek or
+// Cyrillic letter, digit or underscore touches it (an en dash before a digit is a minus too), and a
+// "-" or U+2212 counts after ONE such letter ("\u0394\u22125") or after an exponent ("1e-5"); a run of
+// two or more ("cite_note-7", "AB-12", "UTF-8", "2019-2023") is an identifier or range and carries no
+// negative. Markdown markup a converter writes beside a sign never decides it: the closer run before it
+// (read with the CommonMark inline grammar in markdown-closers.mjs: emphasis, code, strike, titled and
+// parenthesised link and image targets, reference endings, backslash-escaped closers) is skipped to
+// the first character it closes ("**2019**-2023" and '[1990](/wiki/1990 "1990")-2000' are ranges), and
+// an opener run between the sign and its digits ("-**5**") keeps the sign. A raw-side character
+// reference (`−`, ` `) is decoded on BOTH sides with the one entities.mjs table, so a
+// verbatim copy probes clean. Raw inline HTML a converter leaves in the Markdown (pandoc's gfm writer
+// emits `<sup>`, `<sub>`, `<img ... width=320>`, `<span>`, `<br>`) is read with the html-lite tag
+// reader: sup, sub and break tags are boundaries, as on the raw side, and any other tag or comment is
+// dropped with its attributes, so it is a closer or opener like Markdown markup. Digits a converter
+// writes out from a link target, an image source, alt text, or a title cover a converted number only
+// inside Markdown link or image syntax (`](...)` with any title and balanced parentheses,
+// `![...](...)`, `<https://...>`, `[ref]: url "title"` with the title on the same or the next line),
+// never in prose; a footnote
+// definition's text ("[^1]: ...") is prose. Converter syntax is not a number: a line-start list marker
+// ("1. ", "2) ", "(3) ", indented or after "> ") is exempt only while the source has an <ol> item with
+// that number and the markers so far do not outnumber the <ol> items (so lazy "1. 1. 1." numbering
+// over three items passes), a numeric reference label ("[x][1]" with its "[1]: url") only while both
+// halves are present, and a footnote definition label ("[^1]:") only while its "[^1]" is used;
+// otherwise each is a candidate. A converter's hard wrap may split a space-grouped number over one line
+// break; the two halves read as one number only when the raw side holds that number and not both
+// halves, so raw "12 345 and 12 and 345" wrapped inside "12 345" reports both halves (it fails safe).
+// The built-in snapshot writes no attribute text, syntax, wraps or tags, and its text is already
+// decoded, so it gets none of these allowances. Constructs outside this grammar are declared out of
+// scope in references/fact-ledger.md: they report as altered (the safe direction).
+
+import { existsSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { CliError } from "./cli-support.mjs"
+import { decodeEntities, parseHtml, PROSE_BREAKS, readTag } from "./html-lite.mjs"
+import { escapedAt, inlineLinkSpans, stripClosers } from "./markdown-closers.mjs"
+import { NUMBER_SOURCE, plainDecimal } from "./number-grammar.mjs"
+
+const LETTER = "\\p{Script=Latin}\\p{Script=Greek}\\p{Script=Cyrillic}"
+const WORD = new RegExp(`[${LETTER}\\p{N}_]`, "u")
+const ALPHA = new RegExp(`[${LETTER}]`, "u")
+const EXPONENT = new RegExp(`(?<![${LETTER}\\p{N}_.,])\\d+(?:[.,]\\d+)?[eE]$`, "u")
+/** A dash, then any Markdown opener run, then a number; signed() decides whether the dash is a minus. */
+const NUMBERS = new RegExp(`(?:([-\\u2212\\u2013])([*_~\`[]*))?(${NUMBER_SOURCE})`, "gu")
+/** Sup, sub, datetime chunks and every break join no neighbour: neither a digit, a separator, nor whitespace. */
+const BOUNDARY = "\u0001"
+const CARRIED = ["href", "src", "alt", "title"]
+const LIST_MARKER = /(?:^|\u0001)(?: ?>)* ?(?:\d+[.)]|\(\d+\)) /gu
+const REF_USE = /\]\[(\d+)\]/gu
+const REF_DEF = /(?:^|\u0001)(?: ?>)* ?\[(\d+)\]:/gu
+const NOTE_USE = /\[\^(\d+)\](?!:)/gu
+const NOTE_DEF = /(?:^|\u0001)(?: ?>)* ?\[\^(\d+)\]:/gu
+const TITLE = `(?:"[^"\u0001]*"|'[^'\u0001]*'|\\([^)\u0001]*\\))`
+/** A reference definition's title may sit alone on the next line (only whitespace after it). */
+const REF_DEFINITION = `(?:^|\u0001) ?\\[(?!\\^)[^\\]\u0001]+\\]: ?[^\\s\u0001]*(?: ${TITLE}| ?\u0001 ?${TITLE}(?= ?(?:\u0001|$)))?`
+const LINK_SYNTAX = new RegExp(`!\\[[^\\]\u0001]*\\](?:\\([^)\u0001]*\\))?|\\]\\([^)\u0001]*\\)|<https?:\\/\\/[^>\\s\u0001]+>|${REF_DEFINITION}`, "gu")
+const TAG_NAME = /^[a-z][a-z0-9-]*$/u
+/** What a tag leaves in the numeric text on either side: sup, sub and every break join no neighbour. */
+const edgeOf = (tag) => (tag === "sup" || tag === "sub" || PROSE_BREAKS.has(tag) ? BOUNDARY : "")
+
+/** Prose text, attribute text and <ol> item numbers of an HTML document; a non-HTML document is prose only. */
+function numericText(source, html) {
+	if (!html) return { prose: decodeEntities(source), carried: "", listed: [] }
+	const chunks = []
+	const carried = []
+	const listed = []
+	const walk = (node) => {
+		if (node.tag === "#text") { chunks.push(node.text.replace(/\s+/gu, " ")); return }
+		for (const name of CARRIED) if (node.attrs?.[name] !== undefined) carried.push(node.attrs[name])
+		if (node.tag === "ol") {
+			const start = /^\s*\d+\s*$/u.test(node.attrs.start ?? "") ? Number(node.attrs.start) : 1
+			node.children.filter((child) => child.tag === "li").forEach((_, index) => listed.push(String(start + index)))
+		}
+		const edge = edgeOf(node.tag)
+		chunks.push(edge)
+		if (node.tag === "time" && node.attrs.datetime !== undefined) chunks.push(`${BOUNDARY}${node.attrs.datetime}${BOUNDARY}`)
+		for (const child of node.children) walk(child)
+		chunks.push(edge)
+	}
+	walk(parseHtml(source))
+	return { prose: chunks.join(""), carried: carried.join(BOUNDARY), listed }
+}
+
+/** Raw inline HTML in converted Markdown: sup, sub and break tags become boundaries, other tags and comments go. */
+function inlineHtml(text) {
+	let out = ""
+	let kept = 0
+	for (let lt = text.indexOf("<"); lt !== -1; lt = text.indexOf("<", Math.max(lt + 1, kept))) {
+		const comment = text.startsWith("<!--", lt)
+		const name = lt + (text[lt + 1] === "/" ? 2 : 1)
+		if (escapedAt(text, lt) || (!comment && !/[a-z]/iu.test(text[name] ?? ""))) continue
+		const tag = comment ? null : readTag(text, name)
+		const end = comment ? text.indexOf("-->", lt + 4) + 3 : tag.end
+		if (end <= lt || (tag && !(TAG_NAME.test(tag.name) && text[end - 1] === ">")) || /\n[ \t\r]*\n/u.test(text.slice(lt, end))) continue
+		out += text.slice(kept, lt) + (tag ? edgeOf(tag.name) : "")
+		kept = end
+	}
+	return out + text.slice(kept)
+}
+
+/**
+ * Start offsets of numeric reference labels whose use ("][1]") and definition ("[1]:") are both present,
+ * and of footnote definition labels ("[^1]:") whose "[^1]" is used (the use itself mirrors the source).
+ */
+function referenceLabels(flat) {
+	const uses = [...flat.matchAll(REF_USE)]
+	const defs = [...flat.matchAll(REF_DEF)]
+	const used = new Set(uses.map((match) => match[1]))
+	const defined = new Set(defs.map((match) => match[1]))
+	const paired = [...uses.filter((match) => defined.has(match[1])), ...defs.filter((match) => used.has(match[1]))]
+	const noted = new Set([...flat.matchAll(NOTE_USE)].map((match) => match[1]))
+	const notes = [...flat.matchAll(NOTE_DEF)].filter((match) => noted.has(match[1]))
+	return new Set([...paired, ...notes].map((match) => match.index + match[0].search(/\d/u)))
+}
+
+/** Whether the dash at `at` is a minus: Markdown closers (and a backslash escaping the dash) before it are skipped, as is an opener run before a lone letter. */
+function signed(flat, at, dash) {
+	const before = stripClosers(flat.slice(Math.max(0, at - 512), escapedAt(flat, at) ? at - 1 : at))
+	const last = before.at(-1) ?? ""
+	if (!WORD.test(last)) return true
+	if (dash === "\u2013") return false
+	if (EXPONENT.test(before)) return true
+	return ALPHA.test(last) && !WORD.test(before.slice(0, -1).replace(/[*_~`[]+$/u, "").at(-1) ?? "")
+}
+
+/**
+ * Line breaks become boundaries, then whitespace runs one space; `linked` marks tokens inside link
+ * syntax, `marker` a line-start list marker, `label` a paired reference label, `gap` the text since
+ * the previous number.
+ */
+function tokensOf(text, { links = false } = {}) {
+	const flat = text.replace(/\r\n?|\n|\f/gu, BOUNDARY).replace(/\s+/gu, " ")
+	const spans = links ? [...[...flat.matchAll(LINK_SYNTAX)].map((match) => [match.index, match.index + match[0].length]), ...inlineLinkSpans(flat)] : []
+	const markers = new Set([...flat.matchAll(LIST_MARKER)].map((match) => match.index + match[0].search(/\d/u)))
+	const labels = referenceLabels(flat)
+	let previousEnd = 0
+	return [...flat.matchAll(NUMBERS)].map((match, n) => {
+		const sign = match[1] !== undefined && signed(flat, match.index, match[1])
+		const at = sign ? match.index : match.index + match[0].length - match[3].length
+		const gap = flat.slice(previousEnd, at)
+		previousEnd = match.index + match[0].length
+		return {
+			n,
+			gap,
+			text: sign ? match[0] : match[3],
+			key: `${sign ? "-" : ""}${plainDecimal(match[3])}`,
+			linked: spans.some(([start, end]) => at >= start && at < end),
+			marker: markers.has(at),
+			label: labels.has(at),
+		}
+	})
+}
+
+/** The comparison key of one number token, so other tools can match a converted number against a probe record. */
+export const numberKey = (token) => tokensOf(String(token))[0]?.key ?? null
+
+/** The numbers in `texts` that the source's conversion probe recorded as altered (sources/<id>/conversion.json). */
+export function alteredIn(sourceDir, texts) {
+	const record = join(sourceDir, "conversion.json")
+	let altered = []
+	if (existsSync(record)) {
+		try {
+			altered = JSON.parse(readFileSync(record, "utf8")).conversion_altered_numbers ?? []
+		} catch (error) {
+			throw new CliError(`${record} is not a readable conversion record (${error.message}); snapshot the source again`)
+		}
+	}
+	const keys = new Set(altered.map(numberKey))
+	return texts.flatMap((text) => tokensOf(String(text ?? ""))).filter((token) => keys.has(token.key)).map((token) => token.text)
+}
+
+/** Rejoin a grouped number a hard wrap split over one line break, when only the joined reading is in the raw counts. */
+function joinWrapped(tokens, raw) {
+	const out = []
+	for (const token of tokens) {
+		const prev = out.at(-1)
+		const wrapped = prev?.n === token.n - 1 && /^ ?\u0001 ?$/u.test(token.gap) && !prev.linked && !token.linked
+		const [joined, ...more] = wrapped ? tokensOf(`${prev.text} ${token.text}`) : []
+		if (joined && more.length === 0 && raw.has(joined.key) && !(raw.has(prev.key) && raw.has(token.key))) {
+			out[out.length - 1] = { ...joined, n: token.n, text: `${prev.text}\n${token.text}` }
+			continue
+		}
+		out.push(token)
+	}
+	return out
+}
+
+const countOf = (tokens) => tokens.reduce((counts, token) => counts.set(token.key, (counts.get(token.key) ?? 0) + 1), new Map())
+
+/** The tokens `counts` cannot cover; each covered token uses up one count. */
+function uncovered(tokens, counts) {
+	return tokens.filter((token) => {
+		const remaining = counts.get(token.key) ?? 0
+		if (remaining === 0) return true
+		counts.set(token.key, remaining - 1)
+		return false
+	})
+}
+
+/**
+ * Preserve token spelling in diagnostics; the raw side is read as HTML, the converted side as Markdown
+ * with raw inline tags, both decoded with one entity table. `attributes: false`
+ * (the built-in snapshot) gives attribute digits no cover, converter syntax no exemption and no wrap rejoin. An exempt
+ * marker or label is not a converted number at all: it neither covers a raw number nor counts as altered.
+ */
+export function probeConversion(raw, converted, { attributes = true } = {}) {
+	const source = numericText(raw, /<[a-z!/]/i.test(raw))
+	const before = tokensOf(source.prose)
+	const listed = new Set(source.listed)
+	let items = attributes ? source.listed.length : 0
+	const syntax = tokensOf(attributes ? decodeEntities(inlineHtml(converted)) : converted, { links: attributes })
+		.filter((token) => !(attributes && token.label) && !(token.marker && listed.has(token.key) && items-- > 0))
+	const after = attributes ? joinWrapped(syntax, countOf(before)) : syntax
+	const counts = countOf(before)
+	const extra = new Set(uncovered(after.filter((token) => !token.linked), counts))
+	for (const token of tokensOf(source.carried)) counts.set(token.key, (counts.get(token.key) ?? 0) + 1)
+	for (const token of uncovered(after.filter((token) => token.linked), counts)) extra.add(token)
+	const lost = uncovered(before, countOf(after)).map((token) => token.text)
+	return { altered: after.filter((token) => extra.has(token)).map((token) => token.text), lost, raw_count: before.length, converted_count: after.length }
+}
+
+export function runConversionProbe(argv) {
+	if (argv.length !== 2) throw new CliError("usage: source probe-conversion <raw-file> <converted-file>")
+	const result = probeConversion(readFileSync(argv[0], "utf8"), readFileSync(argv[1], "utf8"))
+	return { json: result, summary: `source probe-conversion: ${result.altered.length} altered, ${result.lost.length} lost`, exitCode: result.altered.length === 0 ? 0 : 1 }
+}

package/packages/shared-skills/skills/ulw-research/scripts/conversion-probe.test.ts

@@ -0,0 +1,218 @@
+import { describe, expect, onTestFinished, test } from "bun:test"
+import { spawnSync } from "node:child_process"
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+
+import { numberKey } from "./conversion-probe.mjs"
+import { readFidelityRecord } from "./fidelity-records.mjs"
+import { writeSnapshot } from "./source-snapshot.mjs"
+
+const CLI = join(import.meta.dir, "report-tools.mjs")
+const FIXTURE = readFileSync(join(import.meta.dir, "tests/fixtures/conversion-probe.html"), "utf8")
+const FLATTENED = FIXTURE.replace(/<[^>]*>/g, "")
+// Superscript boundaries preserve 4.2 + 2, 27.3 + 1 and 10 + 9 in raw.
+const ALTERED = ["4.22", "27.31", "109"]
+
+function sandbox(): string {
+	const dir = mkdtempSync(join(tmpdir(), "conversion-probe-"))
+	onTestFinished(() => rmSync(dir, { recursive: true, force: true }))
+	return dir
+}
+
+function probe(raw: string, converted: string) {
+	const dir = sandbox()
+	const rawFile = join(dir, "raw.html")
+	const convertedFile = join(dir, "converted.txt")
+	writeFileSync(rawFile, raw)
+	writeFileSync(convertedFile, converted)
+	const proc = spawnSync("node", [CLI, "source", "probe-conversion", rawFile, convertedFile], { encoding: "utf8" })
+	expect(proc.error).toBeUndefined()
+	expect(proc.stdout, proc.stderr).not.toBe("")
+	return { exitCode: proc.status, json: JSON.parse(proc.stdout) }
+}
+
+describe("source probe-conversion", () => {
+	test("#given the probe page #when naively flattened #then only fabricated numbers are altered", () => {
+		const result = probe(FIXTURE, FLATTENED)
+		expect(result.exitCode).toBe(1)
+		// Stripping the adjacent table tags also joins Total's 2 to the first cell.
+		// The flatten glues Total's 2 onto 12,345 and 12,345 onto 15,678: "212,34515,678".
+		expect(result.json.altered).toEqual([...ALTERED, "212,34515", "678"])
+		expect(result.json.lost).toContain("2026")
+		expect(result.json.lost).not.toContain("99.9")
+		expect(result.json.altered).not.toContain("99.9")
+	})
+
+	test("#given a plain text #when compared to itself #then identity passes without alterations", () => {
+		const text = "USD 4.2 billion; 1,234.5 units; 1 234 567 people; -3.5 and 2019-2023."
+		const result = probe(text, text)
+		expect(result.exitCode).toBe(0)
+		expect(result.json.altered).toEqual([])
+		expect(result.json.lost).toEqual([])
+		expect(result.json.converted_count).toBe(result.json.raw_count)
+	})
+
+	test("#given a tag-split decimal #when compared with text #then each side has one number", () => {
+		const result = probe("<span>1</span><span>,</span><span>234.5</span>", "1,234.5")
+		expect(result).toEqual({ exitCode: 0, json: { altered: [], lost: [], raw_count: 1, converted_count: 1 } })
+	})
+
+	test.each(["\u202f", "\u00a0"])("#given %s thousands separators #when converted to spaces #then grouping is equivalent", (separator: string) => {
+		const result = probe(`1${separator}234${separator}567`, "1 234 567")
+		expect(result).toEqual({ exitCode: 0, json: { altered: [], lost: [], raw_count: 1, converted_count: 1 } })
+	})
+
+	test("#given a machine date #when its attribute is removed #then it is lost, not altered", () => {
+		const result = probe('<time datetime="2026-10-01T09:00:00Z">1 Oct</time>', "1 Oct")
+		expect(result.exitCode).toBe(0)
+		expect(result.json.altered).toEqual([])
+		expect(result.json.lost).toEqual(["2026", "10", "01", "09", "00", "00"])
+	})
+
+	test("#given repeated signed decimals #when one count increases #then multiset subtraction preserves the excess", () => {
+		const result = probe("-4.2 +7 2", "-4.2 +7 2; 2")
+		expect(result.exitCode).toBe(1)
+		expect(result.json.altered).toEqual(["2"])
+		expect(result.json.lost).toEqual([])
+	})
+
+	test("#given hidden text #when omitted by conversion #then the number is only lost", () => {
+		const result = probe('<span style="display:none">99.9</span>', "")
+		expect(result.exitCode).toBe(0)
+		expect(result.json.altered).toEqual([])
+		expect(result.json.lost).toEqual(["99.9"])
+	})
+})
+
+describe("one grouped-number grammar: a digit glued after the last group is an alteration", () => {
+	test("#given a footnote glued onto a grouped number #when probed #then it is altered", () => {
+		expect(probe("$1,234<sup>5</sup> million", "$1,2345 million").json.altered).toEqual(["1,2345"])
+		expect(probe("<table><tr><td>12,345<sup>1</sup></td></tr></table>", "12,3451").json.altered).toEqual(["12,3451"])
+	})
+
+	test("#given the same cell through the built-in snapshot #when written #then conversion.json lists it", () => {
+		const { dir } = writeSnapshot(join(sandbox(), "sources"), "S1", { path: "t.html", html: "<table><tr><td>12,345<sup>1</sup></td></tr></table>", extractor: "model", observed_at: "2026-10-25T00:00:00Z" })
+		expect(readFidelityRecord(join(dir, "conversion.json"), "source_conversion").conversion_altered_numbers).toEqual(["12,3451"])
+	})
+})
+
+describe("Markdown-shaped converter output reports only real artifacts", () => {
+	const RAW = [
+		"<p>Revenue was 4.2<sup>2</sup> billion.<sup><a href=\"#cite_note-7\">[7]</a></sup></p>",
+		"<p><a href=\"https://example.com/2024/03/report-4\">Report</a> covers q4-2023.</p>",
+		"<p><img src=\"chart-640x480.png\" alt=\"Chart 2023\" title=\"Fig 4\"></p>",
+		"<table><tr><td>12,345<sup>1</sup></td><td>1&#8201;234&#8201;567</td></tr></table>",
+		"<p>From 2019\u20132023 the index fell \u22123.5 points.</p>",
+	].join("\n")
+	const MARKDOWN = [
+		"Revenue was 4.22 billion.[[7]](#cite_note-7)",
+		"",
+		"[Report](https://example.com/2024/03/report-4) covers q4-2023.",
+		"",
+		"![Chart 2023](chart-640x480.png \"Fig 4\")",
+		"",
+		"| 12,3451 | 1\u2009234\u2009567 |",
+		"",
+		"From 2019-2023 the index fell -3.5 points.",
+	].join("\n")
+
+	test("#given markdownify-shaped text #when probed #then altered is exactly 4.22 and 12,3451", () => {
+		expect(probe(RAW, MARKDOWN).json.altered).toEqual(["4.22", "12,3451"])
+	})
+
+	test("#given the same page through the built-in snapshot #when written #then only the real artifacts are altered", () => {
+		const { dir } = writeSnapshot(join(sandbox(), "sources"), "S1", { path: "m.html", html: RAW, extractor: "model", observed_at: "2026-10-25T00:00:00Z" })
+		expect(readFidelityRecord(join(dir, "conversion.json"), "source_conversion").conversion_altered_numbers).toEqual(["4.22", "12,3451"])
+	})
+
+	test("#given a converted text that looks like a tag #when probed #then it is never parsed as HTML", () => {
+		expect(probe("<p>if a&lt;b then revenue was 4.2<sup>2</sup> billion</p>", "if a<b then revenue was 4.22 billion").json.altered).toEqual(["4.22"])
+	})
+})
+
+describe("whitespace and separators match the snapshot side", () => {
+	for (const [raw, converted] of [
+		["<p>1&#8201;234&#8201;567</p>", "1 234 567"],
+		["<p>1&#8199;234</p>", "1 234"],
+		["<table><tr><td>2,500<br>300</td></tr></table>", "2,500 300"],
+		["<ul><li><p>in 2023</p><p>500 sites</p></li></ul>", "in 2023 500 sites"],
+		["<p>1,250\n300</p>", "1,250 300"],
+		["<p><span>1</span>\n<span>234</span></p>", "1 234"],
+	]) {
+		test(`#given ${JSON.stringify(raw)} #when converted to ${JSON.stringify(converted)} #then nothing is altered`, () => {
+			const result = probe(raw, converted)
+			expect(result.json.altered).toEqual([])
+			expect(result.exitCode).toBe(0)
+		})
+	}
+})
+
+test("#given HTML snapshot conversion #when written #then source warnings use the fidelity envelope", () => {
+	const { dir } = writeSnapshot(join(sandbox(), "sources"), "S1", {
+		path: "fixture.html", html: FIXTURE, extractor: "model", observed_at: "2026-10-25T00:00:00Z",
+	})
+	const record = readFidelityRecord(join(dir, "conversion.json"), "source_conversion")
+	expect(record.conversion_altered_numbers).toEqual(ALTERED)
+	expect(record.record_kind).toBe("source_conversion")
+	expect(record.artifact_sha256).toMatch(/^[0-9a-f]{64}$/)
+	expect(record.report).toBe(join(dir, "source.html"))
+	expect(record.written_at).toBe("2026-10-25T00:00:00.000Z")
+	expect(record.defects).toEqual([])
+})
+
+describe("attribute digits cover only a number inside Markdown link or image syntax", () => {
+	const GLUED = "<p>Output reached 10<sup>9</sup> kWh</p>"
+	const BOOK = '<p><a href="https://books.example/b?id=x&amp;pg=PA109">book</a></p>'
+
+	test("#given an href page number 109 #when the prose shows a glued 10<sup>9</sup> #then 109 is altered in flat and Markdown output", () => {
+		expect(probe(`${GLUED}${BOOK}`, "Output reached 109 kWh\n\nbook").json.altered).toEqual(["109"])
+		expect(probe(`${GLUED}${BOOK}`, "Output reached 109 kWh\n\n[book](https://books.example/b?id=x&pg=PA109)").json.altered).toEqual(["109"])
+	})
+
+	test("#given #ref1..#ref45 links #when 4<sup>2</sup> is glued into 42 #then 42 is altered", () => {
+		const links = Array.from({ length: 45 }, (_, k) => `<a href="#ref${k + 1}">note</a>`).join(" ")
+		const markdown = Array.from({ length: 45 }, (_, k) => `[note](#ref${k + 1})`).join(" ")
+		expect(probe(`<p>Area grew 4<sup>2</sup> times.</p><p>${links}</p>`, `Area grew 42 times.\n\n${markdown}`).json.altered).toEqual(["42"])
+	})
+
+	test("#given an image named chart-109.png #when the built-in snapshot writes the page #then 109 is altered", () => {
+		const { dir } = writeSnapshot(join(sandbox(), "sources"), "S1", { path: "g.html", html: `${GLUED}<p><img src="chart-109.png"></p>`, extractor: "model", observed_at: "2026-10-25T00:00:00Z" })
+		expect(readFidelityRecord(join(dir, "conversion.json"), "source_conversion").conversion_altered_numbers).toEqual(["109"])
+	})
+})
+
+describe("cell, row, list-item and line breaks never group digits", () => {
+	for (const [raw, converted] of [
+		["<table><tr><td>3</td><td>250</td><td>100</td></tr></table>", "| 3 | 250 | 100 |"],
+		["<table><tr><td>12</td><td>345</td></tr></table>", "| 12 | 345 |"],
+		["<ul><li>3</li><li>250</li></ul>", "- 3\n- 250"],
+		["<p>3</p><p>250</p>", "3\n250"],
+	]) {
+		test(`#given ${JSON.stringify(raw)} #when converted to ${JSON.stringify(converted)} #then nothing is altered or lost`, () => {
+			expect(probe(raw, converted)).toMatchObject({ exitCode: 0, json: { altered: [], lost: [] } })
+		})
+	}
+
+	test("#given 1 234 567 inside one cell #when converted to a Markdown row #then it stays one grouped number", () => {
+		expect(probe("<table><tr><td>1 234 567</td><td>8</td></tr></table>", "| 1 234 567 | 8 |")).toEqual({ exitCode: 0, json: { altered: [], lost: [], raw_count: 2, converted_count: 2 } })
+	})
+})
+
+describe("a period group needs an unambiguous shape; a dropped minus is altered", () => {
+	test("#given 1.234 and 1,234 #when keyed #then they differ, and 1.234.567 still groups", () => {
+		expect(numberKey("1.234")).not.toBe(numberKey("1,234"))
+		expect(numberKey("1.234.567")).toBe(numberKey("1,234,567"))
+		expect(numberKey("1.234,5")).toBe(numberKey("1,234.5"))
+	})
+
+	test("#given a glued 12.34<sup>5</sup> beside a real 12,345 #when flattened #then the glued 12.345 is the altered token", () => {
+		expect(probe("<p>12.34<sup>5</sup> and 12,345</p>", "12.345 and 12,345").json.altered).toEqual(["12.345"])
+	})
+
+	test("#given a minus touching Hangul or an en dash minus #when the sign is dropped #then the unsigned number is altered", () => {
+		expect(probe("\uAC10\uC18C\uC728-3.5%", "\uAC10\uC18C\uC728 3.5%").json.altered).toEqual(["3.5"])
+		expect(probe("fell \u20135%", "fell 5%").json.altered).toEqual(["5"])
+		expect(probe("cite_note-7 and 2019\u20132023", "cite_note-7 and 2019-2023").json.altered).toEqual([])
+	})
+})

package/packages/shared-skills/skills/ulw-research/scripts/copy-returns-fidelity-policy.test.ts

@@ -0,0 +1,156 @@
+import { describe, expect, test } from "bun:test"
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+
+import { FILL_INPUT_FORMAT_RULES, FILL_INPUT_FORMATS } from "./fact-commands-fill.mjs"
+import { KIND_RULES } from "./fact-ledger-kinds.mjs"
+import { FILL_FORMATS } from "./fact-fill.mjs"
+import { meaningCheckEnabled as briefMeaningCheckEnabled } from "./fidelity-brief.mjs"
+import { FIDELITY_VALUE_RULES, meaningCheckEnabled, parseSourceFloor, policyErrors, SOURCE_FLOOR_RULES, sourceInsideFloor } from "./fidelity-policy.mjs"
+
+type PolicyRow = { fact_id: string; kind: string; status: string; source_id: string | null; operands: string[]; derived_from: string[] }
+const policyRow = (fact_id: string, kind: string, source_id: string | null, operands: string[] = []): PolicyRow => ({ fact_id, kind, status: "active", source_id, operands, derived_from: [] })
+
+/** policyErrors over one placeholder per row id under a parsed floor, as code and message pairs. */
+function floorRefusals(rows: PolicyRow[], ids: string[], floor: ReturnType<typeof parseSourceFloor>) {
+	const found = ids.map((id, index) => ({ token: `{{${id}}}`, start: index, parsed: { id } }))
+	const refused = policyErrors(ids.map((id) => `{{${id}}}`).join(""), found, { rows }, tmpdir(), { derivedNone: false, floor })
+	return refused.errors.map((error: { code: string; message: string }) => [error.code, error.message])
+}
+
+const referenceValues = async (field: string) => {
+	const row = (await Bun.file(REFERENCE).text()).split("\n").find((line) => line.startsWith(`| \`${field}\` |`)) ?? ""
+	return [...(row.split("|")[2] ?? "").matchAll(/`([^`]+)`/gu)].map((match) => match[1])
+}
+import { pointedFlag } from "./source-commands.mjs"
+
+const REFERENCE = join(import.meta.dir, "..", "references", "deliverable-phase.md")
+
+function briefWithFloor(floor: string) {
+	const dir = mkdtempSync(join(tmpdir(), "policy-registry-"))
+	const lines = ["as of: latest at run start, 2026-10-23", "meaning check: on", `source floor: ${floor}`, "derived numbers: allowed", "languages: report en; sources en; quotes original", "identifier classes: none", "fallback: block-quote"]
+	writeFileSync(join(dir, "brief.md"), `## Fidelity\n${lines.map((line) => `${line} (answered_by: default)`).join("\n")}\n`)
+	return dir
+}
+
+const floorOf = (text: string) => {
+	const parsed = parseSourceFloor(text)
+	if (parsed === null) throw new Error(`parseSourceFloor(${text}) read no floor`)
+	return parsed
+}
+
+const derivedRefusal = (kind: string) => {
+	const row = { fact_id: "F1", kind, status: "active", source_id: null, operands: [], derived_from: [] }
+	const found = [{ token: "{{F1}}", start: 0, parsed: { id: "F1" } }]
+	const refused = policyErrors("{{F1}}", found, { rows: [row] }, tmpdir(), { derivedNone: true, floor: null })
+	return refused.errors.some((error: { code: string }) => error.code === "derived_not_allowed")
+}
+
+describe("fidelity-policy registry: copies must not return", () => {
+	test("#given every ledger kind #when derived numbers none refuses #then policyErrors agrees with FIDELITY_VALUE_RULES and assumed is refused", () => {
+		const kinds = Object.keys(KIND_RULES)
+		for (const kind of kinds) expect({ kind, refused: derivedRefusal(kind) }).toEqual({ kind, refused: FIDELITY_VALUE_RULES.derivedRefused(kind) })
+		for (const kind of FIDELITY_VALUE_RULES.derivedKinds) expect(kinds).toContain(kind)
+		expect(FIDELITY_VALUE_RULES.derivedRefused("assumed")).toBe(true)
+	})
+
+	test("#given each source floor form #when parsed #then the rule matches the registry and an unknown form is a usage error", () => {
+		expect(parseSourceFloor("standard")).toBeNull()
+		expect(parseSourceFloor("Primary Only")).toEqual({ kind: "primary" })
+		expect(parseSourceFloor("pointed only: S4, https://a.example/x")).toEqual({ kind: "pointed", list: ["S4", "https://a.example/x"] })
+		expect(parseSourceFloor("classes: example.com, gov.uk")).toEqual({ kind: "classes", list: ["example.com", "gov.uk"] })
+		for (const rule of SOURCE_FLOOR_RULES) expect(Object.isFrozen(rule)).toBe(true)
+		expect(() => parseSourceFloor("only nice sources")).toThrow(/source floor/)
+	})
+
+	test("#given pointed and classes floors #when evaluated #then ids, urls, hosts, and suffixes match and notexample.com does not", () => {
+		const pointed = floorOf("pointed only: S4, https://a.example/x")
+		expect(sourceInsideFloor(pointed, "S4", { url: null })).toBe(true)
+		expect(sourceInsideFloor(pointed, "S9", { url: "https://a.example/x" })).toBe(true)
+		expect(sourceInsideFloor(pointed, "S9", { url: "https://a.example/y" })).toBe(false)
+		const classes = floorOf("classes: example.com")
+		for (const [url, inside] of [["https://example.com/a", true], ["https://docs.example.com/a", true], ["https://notexample.com/a", false], [null, false]] as const) {
+			expect({ url, inside: sourceInsideFloor(classes, "S1", { url }) }).toEqual({ url, inside })
+		}
+		expect(sourceInsideFloor({ kind: "primary" }, "S1", { provenance: "primary" })).toBe(true)
+		expect(sourceInsideFloor({ kind: "primary" }, "S1", { provenance: "secondary" })).toBe(false)
+	})
+
+	test("#given a brief floor #when pointedFlag reads it #then it agrees with parseSourceFloor for every form", () => {
+		const url = "https://a.example/x"
+		for (const floor of ["pointed only: S4, https://a.example/x", "pointed only: S4", "standard", "primary only", "classes: a.example"]) {
+			const dir = briefWithFloor(floor)
+			try {
+				const rule = (() => {
+					try {
+						return parseSourceFloor(floor)
+					} catch {
+						return null
+					}
+				})()
+				expect({ floor, flag: pointedFlag(dir, url) }).toEqual({ floor, flag: rule?.kind === "pointed" && rule.list.includes(url) })
+			} finally {
+				rmSync(dir, { recursive: true, force: true })
+			}
+		}
+	})
+
+	test("#given a malformed or pending floor #when pointedFlag reads it #then malformed is a usage error and pending is not pointed", () => {
+		const malformed = briefWithFloor("not a floor")
+		const pending = mkdtempSync(join(tmpdir(), "policy-registry-"))
+		writeFileSync(join(pending, "brief.md"), "## Fidelity\nsource floor: pending (answered_by: pending)\n")
+		try {
+			expect(() => pointedFlag(malformed, "https://a.example/x")).toThrow(/source floor "not a floor"/)
+			expect(pointedFlag(pending, "https://a.example/x")).toBe(false)
+		} finally {
+			for (const dir of [malformed, pending]) rmSync(dir, { recursive: true, force: true })
+		}
+	})
+
+	test("#given derived numbers and fallback values #when read #then only their table values pass, any case, and anything else names field and value", () => {
+		expect(["none", "None", " none "].map((value) => FIDELITY_VALUE_RULES.derivedNone(value))).toEqual([true, true, true])
+		expect(["allowed", "Allowed"].map((value) => FIDELITY_VALUE_RULES.derivedNone(value))).toEqual([false, false])
+		for (const value of ["nope", "no", "n/a", "none.", "measured only"]) expect(() => FIDELITY_VALUE_RULES.derivedNone(value)).toThrow(`derived numbers "${value}" is not allowed, none`)
+		expect(["block-quote", "Omit"].map((value) => FIDELITY_VALUE_RULES.fallback(value))).toEqual(["quote", "omit"])
+		for (const value of ["drop", "omit sentence", "quote"]) expect(() => FIDELITY_VALUE_RULES.fallback(value)).toThrow(`fallback "${value}" is not block-quote, omit`)
+	})
+
+	test("#given the deliverable reference #when its derived numbers and fallback rows are read #then they name exactly the registry tables", async () => {
+		for (const [field, table] of Object.entries(FIDELITY_VALUE_RULES.tables)) expect({ field, values: await referenceValues(field) }).toEqual({ field, values: Object.keys(table) })
+		expect(Object.keys(FIDELITY_VALUE_RULES.tables)).toEqual(["derived numbers", "fallback"])
+	})
+
+	test("#given sourceless rows #when judged under a strict floor and under standard #then strict refuses each, a derived row over one included, and standard refuses none", () => {
+		const rows = [policyRow("F1", "measured", "S1"), policyRow("A1", "assumed", null), policyRow("D1", "derived", null, ["F1", "A1"]), policyRow("D2", "derived", null, ["F9"])]
+		const ids = ["F1", "A1", "D1", "D2"]
+		expect(floorRefusals(rows, ids, floorOf("pointed only: S1"))).toEqual([
+			["source_outside_floor", "A1 rests on no source, outside the source floor"],
+			["source_outside_floor", "D1 rests on no source, outside the source floor"],
+			["source_outside_floor", "D2 rests on no source, outside the source floor"],
+		])
+		expect(floorRefusals(rows, ids, parseSourceFloor("standard"))).toEqual([])
+	})
+
+	test("#given the meaning check value rule #when read through the policy or the brief #then both are the same function", () => {
+		expect(meaningCheckEnabled).toBe(briefMeaningCheckEnabled)
+		for (const value of ["on", "off", "Off", "on; force strong: money", "offline"]) {
+			expect({ value, on: FIDELITY_VALUE_RULES.meaningCheck(value) }).toEqual({ value, on: meaningCheckEnabled({ fields: { "meaning check": value } }) })
+		}
+	})
+
+	test("#given the fill input vocabulary #when derived #then every input maps to a FILL_FORMATS member and every output format is reachable", () => {
+		const target = (format: string) => (FILL_INPUT_FORMAT_RULES.aliases as Record<string, string>)[format] ?? format
+		for (const format of FILL_INPUT_FORMATS) expect(FILL_FORMATS as readonly string[]).toContain(target(format))
+		for (const format of FILL_FORMATS) expect(FILL_INPUT_FORMATS as readonly string[]).toContain(format)
+		for (const format of Object.values(FILL_INPUT_FORMAT_RULES.extensions)) expect(FILL_INPUT_FORMATS as readonly string[]).toContain(format)
+		expect(FILL_INPUT_FORMAT_RULES.aliases).toEqual({ tex: "md" })
+	})
+
+	test("#given the deliverable reference #when its source floor row is read #then it names exactly the SOURCE_FLOOR_RULES forms", async () => {
+		const row = (await Bun.file(REFERENCE).text()).split("\n").find((line) => line.startsWith("| `source floor` |")) ?? ""
+		const values = row.split("|")[2] ?? ""
+		const forms = [...values.matchAll(/`([^`]+)`/gu)].map((match) => match[1])
+		expect(forms).toEqual(SOURCE_FLOOR_RULES.map((rule) => rule.form))
+	})
+})