oh-my-opencode 5.1.47 → 5.1.51

package/dist/skills/ulw-research/scripts/snapshot-readability.test.ts

@@ -0,0 +1,116 @@
+import { describe, expect, onTestFinished, test } from "bun:test"
+import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+
+import { sha256Of } from "./cli-support.mjs"
+import { DEFECT_CODES } from "./contracts.mjs"
+import { runFactVerify } from "./fact-commands-verify.mjs"
+import { valueChecks } from "./fidelity-values.mjs"
+import { unreadableSnapshots } from "./fidelity-snapshot-readability.mjs"
+import { buildContext } from "./gates-static.mjs"
+import { parseHtml } from "./html-lite.mjs"
+
+type Row = Record<string, unknown> & { fact_id: string; source_id?: string; locator?: Record<string, unknown> }
+type Defect = { code: string; fact_id?: string; message: string }
+const FIX = join(import.meta.dir, "tests", "fixtures", "fidelity")
+const BASE: { rows: Row[] } = JSON.parse(readFileSync(join(FIX, "ledger-verify.json"), "utf8"))
+const check = (html: string, ledger = BASE, sources = join(FIX, "sources")) =>
+	valueChecks(buildContext(parseHtml(`<h2>Findings</h2>${html}`), { baseDir: FIX }), ledger, null, sources) as Defect[]
+const data = (id: string, shown: string) => `<p><data data-fact="${id}">${shown}</data></p>`
+const verified = '<p>The suite reported <data data-fact="F7" data-claim-kind="verified">42 passed</data> [S5].</p>'
+const blocked = (defects: Defect[]) => defects.filter((d) => d.code === "snapshot_hash_mismatch")
+
+function sourcesCopy() {
+	const dir = mkdtempSync(join(tmpdir(), "snapshot-readability-"))
+	onTestFinished(() => rmSync(dir, { recursive: true, force: true }))
+	cpSync(join(FIX, "sources"), dir, { recursive: true })
+	return dir
+}
+
+describe("an unreadable snapshot blocks only the rows it backs", () => {
+	for (const damage of ["directory", "meta.json", "text.txt", "invalid metadata", "unreadable body", "raw.bin"]) {
+		test(`#given S4 with a damaged ${damage} #when verified #then every S4-backed row is blocked`, () => {
+			const dir = sourcesCopy()
+			const ledger = structuredClone(BASE)
+			if (damage === "directory") rmSync(dir, { recursive: true })
+			else if (damage === "invalid metadata") writeFileSync(join(dir, "S4/meta.json"), "{")
+			else if (damage === "unreadable body") {
+				rmSync(join(dir, "S4/text.txt"))
+				mkdirSync(join(dir, "S4/text.txt"))
+			} else if (damage === "raw.bin") {
+				for (const row of ledger.rows.filter((r) => r.source_id === "S4")) row.locator = { ...row.locator, artifact: "raw.bin", unit: "utf8-bytes" }
+			} else rmSync(join(dir, "S4", damage))
+			const defects = check(data("F1", "up to 320 m") + data("F2", "1,237"), ledger, dir)
+			expect(blocked(defects).map((d) => d.fact_id).sort()).toEqual(["F1", "F2"])
+			expect(DEFECT_CODES.snapshot_hash_mismatch).toMatchObject({ severity: "blocker", integrity: true })
+		})
+	}
+
+	test("#given S4's meta.json deleted #when a verified fact on the healthy command snapshot S5 is checked #then it stays clean", () => {
+		const dir = sourcesCopy()
+		rmSync(join(dir, "S4", "meta.json"))
+		const defects = check(data("F1", "up to 320 m") + verified, BASE, dir)
+		expect(blocked(defects).map((d) => d.fact_id)).toEqual(["F1"])
+		expect(defects.filter((d) => d.code !== "snapshot_hash_mismatch")).toEqual([])
+	})
+
+	test("#given a text edit and a deleted meta.json #when verified #then the same code names different causes", () => {
+		const edited = sourcesCopy()
+		writeFileSync(join(edited, "S4/text.txt"), `${readFileSync(join(edited, "S4/text.txt"), "utf8")}edited`)
+		const missing = sourcesCopy()
+		rmSync(join(missing, "S4", "meta.json"))
+		const [a] = blocked(check(data("F1", "up to 320 m"), BASE, edited))
+		const [b] = blocked(check(data("F1", "up to 320 m"), BASE, missing))
+		expect(a?.message).toContain("text.txt changed after its snapshot was pinned")
+		expect(b?.message).toContain("meta.json is missing")
+		expect(a?.message).not.toBe(b?.message)
+	})
+
+	test("#given an edited snapshot behind a row and one named only in prose #when verified #then both carry one wording of the cause", () => {
+		const dir = sourcesCopy()
+		for (const id of ["S4", "S5"]) writeFileSync(join(dir, id, "text.txt"), `${readFileSync(join(dir, id, "text.txt"), "utf8")}edited`)
+		const defects = blocked(check(`${data("F1", "up to 320 m")}<p>The suite ran [S5].</p>`, BASE, dir))
+		const cause = (id: string) => defects.find((d) => d.message.includes(`snapshot ${id} `))?.message.split(`snapshot ${id} `)[1]
+		expect(defects).toHaveLength(2)
+		expect(cause("S5")).toBe(cause("S4") as string)
+	})
+
+	test("#given source-free assumptions and intact snapshots #when verified #then nothing is blocked", () => {
+		expect(check(data("F2", "1,237"))).toEqual([])
+		const assumed = { ...BASE.rows.find((r) => r.fact_id === "F8"), fact_id: "F8", kind: "assumed", raw: "15%", value: "15", unit: "%", derived_from: [] }
+		expect(check('<data data-fact="F8" data-claim-kind="assumed">15%</data>', { ...BASE, rows: [assumed] }, "nonexistent")).toEqual([])
+	})
+
+	test("#given raw-body and explicit converted-only snapshots #when read #then both keep their source contract", () => {
+		const dir = mkdtempSync(join(tmpdir(), "snapshot-raw-"))
+		onTestFinished(() => rmSync(dir, { recursive: true, force: true }))
+		mkdirSync(join(dir, "S1"))
+		const ctx = buildContext(parseHtml(data("F1", "42")), { baseDir: FIX })
+		const cited = new Map([["F1", ctx.body]])
+		for (const kind of ["command", "converted-only"]) {
+			const body = kind === "command" ? "raw command body" : "converted quote"
+			const file = kind === "command" ? "raw.bin" : "text.txt"
+			writeFileSync(join(dir, "S1", file), body)
+			writeFileSync(join(dir, "S1/meta.json"), JSON.stringify({ kind, sha256: sha256Of(body) }))
+			const rows = new Map([["F1", { fact_id: "F1", source_id: "S1", kind: "quote", locator: { unit: "utf8-bytes" } }]])
+			const defects: object[] = []
+			expect([...unreadableSnapshots(ctx, rows, cited, dir, defects)]).toEqual([])
+			expect(defects).toEqual([])
+			rmSync(join(dir, "S1", file))
+		}
+	})
+
+	test("#given a missing sources directory #when fact verify runs #then exit 1 with a blocker per source-backed row", () => {
+		const dir = mkdtempSync(join(tmpdir(), "snapshot-verify-"))
+		onTestFinished(() => rmSync(dir, { recursive: true, force: true }))
+		writeFileSync(join(dir, "fact-ledger.json"), JSON.stringify(BASE))
+		cpSync(join(FIX, "..", "fidelity-brief-locked.md"), join(dir, "brief.md"))
+		const report = join(dir, "report.html")
+		writeFileSync(report, data("F1", "up to 320 m") + data("F2", "1,237"))
+		const result = runFactVerify({ positionals: [report], flags: { "session-dir": dir, sources: join(dir, "nonexistent") } })
+		expect(result.exitCode).toBe(1)
+		const defects = ("defects" in result.json ? result.json.defects : []) as Defect[]
+		expect(blocked(defects).map((d) => d.fact_id).sort()).toEqual(["F1", "F2"])
+	})
+})

package/dist/skills/ulw-research/scripts/source-commands.mjs

@@ -1,9 +1,13 @@
-// source-commands.mjs - the `source snapshot` and `claims index` subcommands of report-tools.
-// A source dir is immutable: the same id with identical text is a no-op, different text fails
-// (exit 1, snapshot_exists), because ledger rows pin the old hash. `--url` fetches with plain
-// fetch (30 s timeout, no browser) and records extractor tool:source-snapshot@1; `--html` and
-// `--text` import a browser-lane capture and record extractor model (lower trust), or the named
-// extractor for `--text --extractor NAME@VERSION`; `--command` runs an argv without a shell and
+// source-commands.mjs - the `source snapshot`, `source refresh` and `claims index` subcommands of
+// report-tools. A source dir is immutable: the same id with identical bytes is a no-op, different
+// bytes fail (exit 1, snapshot_exists), because ledger rows pin the old hash; `source refresh`
+// (source-refresh.mjs) is the one explicit repin. `--replace` swaps in a corrected capture under the
+// same id (the conversion_altered_number recovery) only while no active ledger row cites the id
+// (exit 1, snapshot_pinned, listing them). `--url` fetches with plain fetch (30 s timeout,
+// no browser) and records extractor tool:source-snapshot@1; `--html` imports a browser-lane page
+// capture and `--raw` alone a saved file's bytes (extractor model). `--text` alone registers a converted-only source (no raw bytes:
+// quotes only); `--raw <bytes> --text <extraction> --extractor NAME@VERSION` keeps the original
+// bytes and declares the text their extraction. `--command` runs an argv without a shell and
 // snapshots its stdout, with the argv, cwd, exit status, and output hashes in command.json.
 // node: builtins and sibling imports only.
 
@@ -13,13 +17,18 @@ import { join, resolve } from "node:path"
 
 import { indexClaims } from "./claims-index.mjs"
 import { CliError, parseArgs, sha256Of, writeJsonAtomic } from "./cli-support.mjs"
+import { runConversionProbe } from "./conversion-probe.mjs"
+import { readLedger } from "./fact-ledger.mjs"
 import { parseFidelityBlock } from "./fidelity-brief.mjs"
+import { parseSourceFloor } from "./fidelity-policy.mjs"
+import { encodingOf, headerCharset } from "./raw-decode.mjs"
 import { writeSnapshot } from "./source-snapshot.mjs"
+import { readSnapshot } from "./source-view.mjs"
 
 const FETCH_TIMEOUT_MS = 30_000
 const TOOL_EXTRACTOR = "tool:source-snapshot@1"
-const SNAPSHOT_USAGE =
-	"usage: source snapshot --id S<n> (--url <u> | --html <file> | --text <file> [--extractor NAME@VERSION] | --command '<argv json>' --cwd DIR) [--kind page|command|document] [--provenance primary|secondary] [--revision R] --session-dir D"
+export const SNAPSHOT_USAGE =
+	"usage: source probe-conversion <raw-file> <converted-file> | source snapshot --id S<n> (--url <u> | --html <file> | --raw <file> [--text <file> --extractor NAME@VERSION] | --text <file> (converted-only) | --command '<argv json>' --cwd DIR) [--charset NAME] [--kind page|command|document|converted-only] [--provenance primary|secondary] [--revision R] [--replace] --session-dir D"
 
 function readInput(path) {
 	try {
@@ -36,20 +45,27 @@ export function sessionDirOf(flags) {
 	return resolve(dir)
 }
 
-/** The snapshot of one source: meta.json, text.txt, and source.html when the source was a page. */
+/** The snapshot as source-view.mjs reads it ({ meta, raw, sha256, drift, view }); unreadable is a usage error. */
 export function snapshotOf(sessionDir, id) {
 	const dir = join(sessionDir, "sources", String(id))
 	if (!existsSync(join(dir, "meta.json"))) throw new CliError(`source ${id} has no snapshot; run source snapshot --id ${id} first`)
-	const html = join(dir, "source.html")
-	return { meta: JSON.parse(readInput(join(dir, "meta.json"))), text: readInput(join(dir, "text.txt")), html: existsSync(html) ? readInput(html) : null }
+	try {
+		return readSnapshot(dir)
+	} catch (error) {
+		throw new CliError(`source ${id}: ${error.message}`)
+	}
 }
 
-/** Whether the brief's source floor lists this source as pointed at (`pointed only: a, b`). */
+/**
+ * Whether the brief's source floor lists this source as pointed at (`pointed only: a, b`). An absent
+ * or pending floor points at nothing; a malformed floor is a usage error (parseSourceFloor's CliError).
+ */
 export function pointedFlag(sessionDir, url) {
 	const brief = join(sessionDir, "brief.md")
 	const floor = existsSync(brief) ? (parseFidelityBlock(readInput(brief)).fields?.["source floor"] ?? "") : ""
-	const match = /^pointed only:\s*(.+)$/i.exec(floor.trim())
-	return match ? match[1].split(",").map((entry) => entry.trim()).includes(url) : false
+	if (floor === "" || floor === "pending") return false
+	const rule = parseSourceFloor(floor)
+	return rule?.kind === "pointed" && rule.list.includes(url)
 }
 
 async function fetchSource(url) {
@@ -67,10 +83,10 @@ async function fetchSource(url) {
 	const type = response.headers.get("content-type") ?? ""
 	if (/pdf/i.test(type)) {
 		await response.body?.cancel()
-		throw new CliError(`pdf response: ${url}; extract its text and use --text <file> --extractor NAME@VERSION`)
+		throw new CliError(`pdf response: ${url}; save the bytes and use --raw <file.pdf> --text <extracted.txt> --extractor NAME@VERSION`)
 	}
-	const body = await response.text()
-	return /html|xml/i.test(type) || /^\s*</.test(body) ? { url, html: body } : { url, text: body }
+	const raw = Buffer.from(await response.arrayBuffer())
+	return { url, raw, charset: headerCharset(type), markup: /html|xml/i.test(type) || /^\s*</.test(raw.subarray(0, 1024).toString("latin1")) }
 }
 
 function runCommand(argvJson, cwd) {
@@ -84,35 +100,76 @@ function runCommand(argvJson, cwd) {
 		throw new CliError("--command must be a non-empty JSON array of non-empty strings")
 	}
 	if (!cwd || cwd === true) throw new CliError("--command needs --cwd DIR")
-	const proc = spawnSync(argv[0], argv.slice(1), { cwd: resolve(cwd), encoding: "utf8", shell: false, maxBuffer: 64 * 1024 * 1024 })
+	const proc = spawnSync(argv[0], argv.slice(1), { cwd: resolve(cwd), shell: false, maxBuffer: 64 * 1024 * 1024 })
 	if (proc.error) throw new CliError(`--command could not start: ${proc.error.message}`)
 	const receipt = { argv, cwd: resolve(cwd), exit_status: proc.status, stdout_sha256: sha256Of(proc.stdout), stderr_sha256: sha256Of(proc.stderr) }
-	return { path: `command:${argv.join(" ")}`, text: proc.stdout, kind: "command", receipt }
+	return { path: `command:${argv.join(" ")}`, raw: proc.stdout, kind: "command", receipt }
+}
+
+function readBytes(path) {
+	try {
+		return readFileSync(path)
+	} catch (error) {
+		throw new CliError(`cannot read ${path}: ${error.code ?? error.message}`)
+	}
 }
 
-async function acquire(flags) {
+/** The snapshot input the flags describe; shared by `source snapshot` and `source refresh`. */
+export async function acquire(flags) {
 	const given = ["url", "html", "text", "command"].filter((name) => flags[name] !== undefined)
-	if (given.length !== 1) throw new CliError(SNAPSHOT_USAGE)
+	if (flags.raw !== undefined && given.some((name) => name !== "text")) throw new CliError("--raw goes alone or with --text (--url, --html and --command already keep the raw bytes)")
+	if (given.length + (flags.raw !== undefined && flags.text === undefined ? 1 : 0) !== 1) throw new CliError(SNAPSHOT_USAGE)
 	if (flags.extractor !== undefined && flags.text === undefined) throw new CliError("--extractor goes with --text only")
-	if (flags.url !== undefined) return { ...(await fetchSource(flags.url)), extractor: TOOL_EXTRACTOR }
-	if (flags.html !== undefined) return { path: resolve(flags.html), html: readInput(flags.html), extractor: "model" }
-	if (flags.text !== undefined) return { path: resolve(flags.text), text: readInput(flags.text), extractor: flags.extractor === undefined ? "model" : `extractor:${flags.extractor}` }
+	if (flags.charset !== undefined && encodingOf(flags.charset) === null) throw new CliError(`--charset ${flags.charset} is not a known encoding`)
+	const charset = flags.charset === undefined ? {} : { charset: encodingOf(flags.charset) }
+	if (flags.url !== undefined) return { ...(await fetchSource(flags.url)), ...charset, extractor: TOOL_EXTRACTOR }
+	if (flags.html !== undefined) return { path: resolve(flags.html), raw: readBytes(flags.html), markup: true, ...charset, extractor: "model" }
+	if (flags.raw !== undefined && flags.text === undefined) return { path: resolve(flags.raw), raw: readBytes(flags.raw), ...charset, extractor: "model" }
+	if (flags.text !== undefined) {
+		const extractor = flags.extractor === undefined ? "model" : `extractor:${flags.extractor}`
+		if (flags.raw !== undefined) return { path: resolve(flags.raw), raw: readBytes(flags.raw), text: readInput(flags.text), ...charset, extractor }
+		if (flags.kind !== undefined && flags.kind !== "converted-only") throw new CliError("--text without --raw registers a converted-only source; pass --raw <original bytes> to keep a measurable snapshot")
+		return { path: resolve(flags.text), text: readInput(flags.text), kind: "converted-only", extractor }
+	}
 	return { ...runCommand(flags.command, flags.cwd), extractor: TOOL_EXTRACTOR }
 }
 
+/**
+ * `--replace` with `--text` alone (the conversion_altered_number recovery) keeps the source's raw.bin and
+ * declares the verified text its extraction, so value rows locate in text.txt while the original bytes stay
+ * pinned; plain `--text` without `--replace`, or over an id with no raw.bin, stays converted-only.
+ */
+function declareOverKeptRaw(sourcesDir, id, input) {
+	const dir = join(sourcesDir, String(id))
+	if (input.kind !== "converted-only" || !existsSync(join(dir, "raw.bin")) || !existsSync(join(dir, "meta.json"))) return
+	const before = JSON.parse(readFileSync(join(dir, "meta.json"), "utf8"))
+	Object.assign(input, { url: before.url, raw: readFileSync(join(dir, "raw.bin")), kind: before.kind, declared: true })
+	delete input.path
+}
+
 export async function runSource(argv) {
 	const [sub, ...rest] = argv
+	if (sub === "refresh") return (await import("./source-refresh.mjs")).runRefresh(rest)
+	if (sub === "probe-conversion") return runConversionProbe(rest)
 	if (sub !== "snapshot") throw new CliError(SNAPSHOT_USAGE)
-	const args = parseArgs(rest)
+	const args = parseArgs(rest, { booleans: ["replace"] })
 	const id = args.required("id")
-	const sourcesDir = join(sessionDirOf(args.flags), "sources")
+	const sessionDir = sessionDirOf(args.flags)
+	const sourcesDir = join(sessionDir, "sources")
+	const replace = args.flags.replace === true
+	const pinned = replace ? readLedger(sessionDir).rows.filter((row) => row.source_id === id && (row.status ?? "active") === "active").map((row) => row.fact_id) : []
+	if (pinned.length > 0) {
+		const message = `${id} is cited by active row(s) ${pinned.join(", ")}; retract them first, or snapshot the corrected text under a new id`
+		return { json: { command: "source snapshot", status: "fail", exitCode: 1, error: "snapshot_pinned", message, rows: pinned }, summary: `source snapshot: ${message}`, exitCode: 1 }
+	}
 	const { receipt, ...input } = await acquire(args.flags)
+	if (replace) declareOverKeptRaw(sourcesDir, id, input)
 	if (args.flags.kind !== undefined) input.kind = args.flags.kind
 	if (args.flags.provenance !== undefined) input.provenance = args.flags.provenance
 	if (args.flags.revision !== undefined) input.revision = args.flags.revision
 	let written
 	try {
-		written = writeSnapshot(sourcesDir, id, input)
+		written = writeSnapshot(sourcesDir, id, input, { replace })
 	} catch (error) {
 		if (String(error.message).startsWith("snapshot_exists")) {
 			return { json: { command: "source snapshot", status: "fail", exitCode: 1, error: "snapshot_exists", message: error.message }, summary: `source snapshot: ${error.message}`, exitCode: 1 }
@@ -122,8 +179,9 @@ export async function runSource(argv) {
 	}
 	if (receipt) writeFileSync(join(written.dir, "command.json"), `${JSON.stringify(receipt, null, 2)}\n`)
 	const meta = JSON.parse(readFileSync(join(written.dir, "meta.json"), "utf8"))
-	const data = { source_id: id, directory: written.dir, snapshot_sha256: written.sha256, extractor: meta.extractor, kind: meta.kind, provenance: meta.provenance }
-	return { json: { command: "source snapshot", status: "pass", exitCode: 0, data }, summary: `source snapshot: ${id} ${written.sha256.slice(0, 12)} (${meta.extractor})`, exitCode: 0 }
+	const data = { source_id: id, directory: written.dir, snapshot_sha256: written.sha256, extractor: meta.extractor, kind: meta.kind, provenance: meta.provenance, text_source: meta.text_source, ...(written.altered ? { conversion_altered_numbers: written.altered } : {}) }
+	const probe = written.altered ? `; conversion ${written.altered.length} altered${written.altered.length ? ` (${written.altered.join(", ")}): fact add refuses those numbers` : ""}` : ""
+	return { json: { command: "source snapshot", status: "pass", exitCode: 0, data }, summary: `source snapshot: ${id} ${written.sha256.slice(0, 12)} (${meta.extractor}, ${meta.text_source})${probe}`, exitCode: 0 }
 }
 
 export function runClaims(argv) {

package/dist/skills/ulw-research/scripts/source-integrity.test.ts

@@ -0,0 +1,243 @@
+import { describe, expect, onTestFinished, test } from "bun:test"
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+
+import { sha256Of } from "./cli-support.mjs"
+import { addRow, readLedger, writeLedger } from "./fact-ledger.mjs"
+import { run } from "./report-tools.mjs"
+
+const BRIEF = readFileSync(join(import.meta.dir, "tests/fixtures/fidelity-brief-locked.md"), "utf8")
+const PAGE = '<html><body><p>The main deck spans <span title="9 m">320</span> m in total.</p><table id="sizes"><tr><th>Name</th><th>Length</th></tr><tr><td>Deck</td><td>320 m</td></tr></table></body></html>'
+
+// Plain markup (no inline spans) so the pre-fold locator can build the same session: the fail-closed and refresh cases then fail for their own reason.
+const PLAIN = "<html><body><p>The main deck spans 320 m in total.</p></body></html>"
+
+type Defect = { code: string; message: string }
+
+/** {{F1}} -> {{F1:<its ledger label>}}: fill requires the label. */
+const labeled = (dir: string) => {
+	const labels = new Map(readLedger(dir).rows.map((row: { fact_id: string; label: string }) => [row.fact_id, row.label]))
+	return (placeholder: string) => placeholder.replace(/\{\{(\w+)\}\}/, (_, id) => `{{${id}:${labels.get(id)}}}`)
+}
+
+function session() {
+	const dir = mkdtempSync(join(tmpdir(), "source-integrity-"))
+	onTestFinished(() => {
+		for (const name of ["S1", "S2"]) if (existsSync(join(dir, "sources", name, "raw.bin"))) chmodSync(join(dir, "sources", name, "raw.bin"), 0o644)
+		rmSync(dir, { recursive: true, force: true })
+	})
+	writeFileSync(join(dir, "brief.md"), BRIEF)
+	return dir
+}
+
+function file(dir: string, name: string, body: string | Buffer) {
+	writeFileSync(join(dir, name), body)
+	return join(dir, name)
+}
+
+const snapshot = (dir: string, id: string, ...flags: string[]) => run(["source", "snapshot", "--id", id, ...flags, "--session-dir", dir])
+const fact = (dir: string, ...args: string[]) => run(["fact", ...args, "--session-dir", dir])
+
+async function report(dir: string, placeholders: string[], extra: string[] = []) {
+	const cite = (placeholder: string) => (placeholder.includes("S2") ? "S2" : "S1")
+	const body = [
+		'<!doctype html><html lang="en"><head><meta charset="utf-8"><title>Bridge</title></head><body><h1>Bridge</h1><h2>Findings</h2>',
+		...placeholders.map((placeholder) => `<p>The source states ${labeled(dir)(placeholder.replace(/:S\d\}\}$/, "}}"))} [${cite(placeholder)}].</p>`),
+		"<h2>How this report was made</h2><p>Collected from cited public pages [S1].</p>",
+		'<h2>Sources</h2><ol><li id="s1">[S1] Bridge page, retrieved 2026-10-01.</li><li id="s2">[S2] Second page, retrieved 2026-10-01.</li></ol></body></html>',
+	].join("\n")
+	file(dir, "report.src.html", body)
+	expect((await fact(dir, "fill", join(dir, "report.src.html"), "--out", join(dir, "report.html"))).exitCode).toBe(0)
+	return fact(dir, "verify", join(dir, "report.html"), ...extra)
+}
+
+/** A legacy text.txt snapshot S1 (no raw.bin, no byte locator) holding "<text>" with row F1, filled and verified clean. */
+async function legacySession(text = "Span is 320 m.") {
+	const dir = session()
+	const sources = join(dir, "sources/S1")
+	mkdirSync(sources, { recursive: true })
+	writeLegacy(sources, text)
+	const ledger = readLedger(dir)
+	addRow(ledger, { kind: "measured", value: "320", unit: "m", value_span: [8, 11], raw: "320", quote: text, source_id: "S1", locator: { artifact: "S1/text.txt", line: 1, char_start: 0, char_end: 14 }, snapshot_sha256: sha256Of(text), sentence_id: "S1:s1", extracted_by: "model", observed_at: "2026-01-01T00:00:00.000Z", label: "span" })
+	writeLedger(dir, ledger)
+	expect((await report(dir, ["{{F1}}"])).exitCode).toBe(0)
+	return dir
+}
+
+function writeLegacy(sources: string, text: string) {
+	writeFileSync(join(sources, "text.txt"), text)
+	writeFileSync(join(sources, "meta.json"), JSON.stringify({ source_id: "S1", url: "https://example.invalid/", observed_at: "2026-01-01T00:00:00.000Z", sha256: sha256Of(text), extractor: "model", revision: null, kind: "page", provenance: "unknown" }))
+}
+
+const reverify = (dir: string, ...extra: string[]) => fact(dir, "verify", join(dir, "report.html"), ...extra)
+const blockers = (json: { defects: Defect[] }) => json.defects.filter((defect) => defect.code === "snapshot_hash_mismatch")
+
+/** A page source S1 with F1 (--find) and Q1 (--match), filled and verified clean. */
+async function pageSession() {
+	const dir = session()
+	expect((await snapshot(dir, "S1", "--html", file(dir, "page.html", PLAIN))).exitCode).toBe(0)
+	expect((await fact(dir, "add", "--source", "S1", "--find", "main deck spans", "--json")).exitCode).toBe(0)
+	expect((await fact(dir, "quote", "--source", "S1", "--match", "main deck spans 320 m", "--json")).exitCode).toBe(0)
+	expect((await report(dir, ["{{F1}}", "{{Q1}}"])).exitCode).toBe(0)
+	return dir
+}
+
+describe("converted text is never a raw snapshot", () => {
+	for (const [name, extra] of [["extracted.txt", ["--extractor", "pdftotext@24.02"]], ["page.md", []]] as const) {
+		test(`#given --text ${name} without --raw #when snapshotted #then it is converted-only and refuses measured rows`, async () => {
+			const dir = session()
+			const snap = await snapshot(dir, "S1", "--text", file(dir, name, "The span is 320 m."), ...extra)
+			expect(snap.json.data).toMatchObject({ kind: "converted-only", text_source: "converted" })
+			expect(existsSync(join(dir, "sources/S1/raw.bin"))).toBe(false)
+			expect((await fact(dir, "add", "--source", "S1", "--find", "span is", "--json")).json.error).toBe("snapshot_not_raw")
+			const quoted = await fact(dir, "quote", "--source", "S1", "--match", "The span is 320 m.", "--json")
+			expect(quoted.json.defects.map((defect: Defect) => defect.code)).toEqual(["quote_converted_only"])
+			expect(quoted.json.data.row.locator).toMatchObject({ artifact: "text.txt", unit: "bytes", char_start: 0, char_end: 18 })
+			const text = await fact(dir, "quote", "--source", "S1", "--match", "span is 320")
+			expect(text.text).toContain("warning quote_converted_only")
+		})
+	}
+
+	test("#given --raw page bytes with --text #when --find and --table run #then they use the page's text nodes", async () => {
+		const dir = session()
+		const snap = await snapshot(dir, "S1", "--raw", file(dir, "page.html", PAGE), "--text", file(dir, "aid.txt", "aid"), "--extractor", "reader@1")
+		expect(snap.json.data.text_source).toBe("html")
+		expect((await fact(dir, "add", "--source", "S1", "--find", "main deck spans", "--json")).json.data.row.value).toBe("320")
+		const cell = await fact(dir, "add", "--source", "S1", "--table", "sizes", "--row", "Deck", "--col", "Length", "--json")
+		expect([cell.json.data.row.value, cell.json.data.row.quote]).toEqual(["320", "320 m"])
+	})
+
+	test("#given --raw non-HTML bytes with a declared --text extraction #when added, verified, then the extraction edited #then verify blocks", async () => {
+		const dir = session()
+		const snap = await snapshot(dir, "S1", "--raw", file(dir, "doc.pdf", Buffer.from([0x25, 0x50, 0x44, 0x46, 0x00, 0xff])), "--text", file(dir, "doc.txt", "The span is 320 m."), "--extractor", "pdftotext@24.02")
+		expect(snap.json.data.text_source).toBe("declared")
+		const added = await fact(dir, "add", "--source", "S1", "--find", "span is", "--json")
+		expect(added.json.data.row.locator).toMatchObject({ artifact: "text.txt", unit: "bytes" })
+		expect((await report(dir, ["{{F1}}"])).exitCode).toBe(0)
+		writeFileSync(join(dir, "sources/S1/text.txt"), "The span is 330 m.")
+		expect(blockers((await reverify(dir)).json).length).toBeGreaterThan(0)
+	})
+
+	test("#given --raw with --html or --url #when snapshotted #then it is a usage error, not silently ignored", async () => {
+		const dir = session()
+		const page = file(dir, "page.html", PAGE)
+		await expect(snapshot(dir, "S1", "--html", page, "--raw", page)).rejects.toThrow("--raw goes alone or with --text")
+		await expect(snapshot(dir, "S1", "--url", "https://example.invalid/", "--raw", page)).rejects.toThrow("--raw goes alone or with --text")
+	})
+})
+
+// chmod 000 does not stop the owner reading a file on Windows (no POSIX mode bits), so there the same
+// fail-closed condition is produced the way an unreadable path occurs on win32: raw.bin is replaced by
+// a directory, and reading it fails (EISDIR). Both paths must make verify exit 1 with the cause.
+const unreadableOn = (platform: string) => (dir: string) => {
+	const raw = join(dir, "sources/S1/raw.bin")
+	if (platform !== "win32") return chmodSync(raw, 0o000)
+	unlinkSync(raw)
+	mkdirSync(raw)
+}
+const unreadable = unreadableOn(process.platform)
+const UNREADABLE_CAUSE = process.platform === "win32" ? "raw.bin is not a readable file" : "raw.bin is unreadable (EACCES)"
+
+describe("fact verify fails closed when a cited snapshot cannot be read", () => {
+	const cases: [string, (dir: string) => void, string[], string][] = [
+		["the sources dir is removed", (dir) => rmSync(join(dir, "sources"), { recursive: true }), [], "meta.json is missing"],
+		["meta.json is removed", (dir) => unlinkSync(join(dir, "sources/S1/meta.json")), [], "meta.json is missing"],
+		["--sources names a missing dir", () => {}, ["--sources", "/nonexistent/sources"], "meta.json is missing"],
+		["raw.bin is unreadable", unreadable, [], UNREADABLE_CAUSE],
+	]
+	for (const [name, damage, extra, cause] of cases) {
+		test(`#given ${name} #when verified #then every cited row gets the blocker with the cause and a record is written`, async () => {
+			const dir = await pageSession()
+			rmSync(join(dir, "fidelity-defects.json"))
+			damage(dir)
+			const result = await reverify(dir, ...extra)
+			expect(result.exitCode).toBe(1)
+			expect(blockers(result.json).map((defect) => defect.message)).toEqual([expect.stringContaining(cause), expect.stringContaining(cause)])
+			expect(existsSync(join(dir, "fidelity-defects.json"))).toBe(true)
+		})
+	}
+
+	test("#given the win32 emulation of an unreadable raw.bin on any platform #when verified #then every cited row gets the blocker", async () => {
+		const dir = await pageSession()
+		unreadableOn("win32")(dir)
+		const result = await reverify(dir)
+		expect(result.exitCode).toBe(1)
+		expect(blockers(result.json).map((defect) => defect.message)).toEqual([expect.stringContaining("raw.bin is not a readable file"), expect.stringContaining("raw.bin is not a readable file")])
+	})
+
+	test("#given one of two sources unreadable #when verified #then only its rows are flagged and the healthy row stays clean", async () => {
+		const dir = await pageSession()
+		expect((await snapshot(dir, "S2", "--html", file(dir, "two.html", "<p>The side deck spans 210 m.</p>"))).exitCode).toBe(0)
+		expect((await fact(dir, "add", "--source", "S2", "--find", "side deck spans", "--json")).exitCode).toBe(0)
+		expect((await report(dir, ["{{F1}}", "{{F2:S2}}"])).exitCode).toBe(0)
+		unlinkSync(join(dir, "sources/S1/meta.json"))
+		const result = await reverify(dir)
+		expect(result.json.defects.map((defect: Defect) => defect.message.split(":")[0])).toEqual(["F1"])
+	})
+
+	test("#given a legacy text.txt row whose text.txt is deleted #when verified #then it blocks", async () => {
+		const dir = await legacySession()
+		const sources = join(dir, "sources/S1")
+		unlinkSync(join(sources, "text.txt"))
+		expect(blockers((await reverify(dir)).json)[0]?.message).toContain("text.txt is missing")
+	})
+
+	test("#given a byte-located row whose raw.bin is deleted #when verified #then it blocks naming raw.bin", async () => {
+		const dir = await pageSession()
+		unlinkSync(join(dir, "sources/S1/raw.bin"))
+		expect(blockers((await reverify(dir)).json).map((defect) => defect.message)).toEqual([expect.stringContaining("raw.bin is missing"), expect.stringContaining("raw.bin is missing")])
+	})
+
+	test("#given a converted-only quote whose text.txt is deleted #when verified #then it blocks", async () => {
+		const dir = session()
+		await snapshot(dir, "S1", "--text", file(dir, "aid.txt", "The span is 320 m."))
+		await fact(dir, "quote", "--source", "S1", "--match", "The span is 320 m.", "--json")
+		const clean = await report(dir, ["{{Q1}}"])
+		expect([clean.exitCode, clean.json.defects.map((defect: Defect) => defect.code)]).toEqual([0, ["quote_converted_only"]])
+		unlinkSync(join(dir, "sources/S1/text.txt"))
+		expect(blockers((await reverify(dir)).json)[0]?.message).toContain("text.txt is missing")
+	})
+})
+
+describe("source refresh is the one explicit repin", () => {
+	test("#given a re-captured page with the quote moved #when source refresh runs #then verify reports relocated with old and new offsets", async () => {
+		const dir = await pageSession()
+		const moved = PLAIN.replace("<body>", "<body><p>A new preface paragraph.</p>")
+		expect((await snapshot(dir, "S1", "--html", file(dir, "page2.html", moved))).json.error).toBe("snapshot_exists")
+		const refreshed = await run(["source", "refresh", "S1", "--html", join(dir, "page2.html"), "--session-dir", dir])
+		expect(refreshed.exitCode).toBe(0)
+		expect(refreshed.json.data.rows.map((row: { status: string }) => row.status)).toEqual(["relocated", "relocated"])
+		const result = await reverify(dir)
+		expect(result.exitCode).toBe(0)
+		const offset = Buffer.byteLength("<p>A new preface paragraph.</p>")
+		const before = readLedger(dir).rows[0].locator
+		expect(result.json.defects).toContainEqual(expect.objectContaining({ code: "relocated", severity: "minor", message: expect.stringContaining(`${before.char_start}:${before.char_end} -> ${before.char_start + offset}:${before.char_end + offset}`) }))
+	})
+
+	test("#given a re-captured page whose value changed #when source refresh runs #then the row is not re-anchored and verify blocks", async () => {
+		const dir = await pageSession()
+		const refreshed = await run(["source", "refresh", "S1", "--html", file(dir, "page3.html", PLAIN.replace("320", "330")), "--session-dir", dir])
+		expect(refreshed.exitCode).toBe(1)
+		expect(refreshed.json.defects.map((defect: Defect) => defect.code)).toEqual(["quote_not_verbatim", "quote_not_verbatim"])
+		expect(blockers((await reverify(dir)).json).length).toBe(2)
+	})
+
+	test("#given a legacy row whose source is refreshed to a page that no longer says it #when refresh then verify run #then both exit 1 naming F1", async () => {
+		const dir = await legacySession()
+		const refreshed = await run(["source", "refresh", "S1", "--html", file(dir, "page999.html", "<p>Span is 999 m.</p>"), "--session-dir", dir])
+		expect(refreshed.exitCode).toBe(1)
+		expect(refreshed.json.data.rows).toEqual([{ fact_id: "F1", status: "legacy_not_reanchored" }])
+		const result = await reverify(dir)
+		expect(result.exitCode).toBe(1)
+		expect(blockers(result.json).map((defect) => [defect.message.split(":")[0], defect.message.includes("no longer the legacy text.txt snapshot")])).toEqual([["F1", true]])
+	})
+
+	test("#given a legacy row whose legacy snapshot is replaced by another consistent one #when verified #then the pin mismatch blocks F1", async () => {
+		const dir = await legacySession()
+		writeLegacy(join(dir, "sources/S1"), "Span is 999 m.")
+		const result = await reverify(dir)
+		expect(result.exitCode).toBe(1)
+		expect(blockers(result.json).map((defect) => defect.message)).toEqual([expect.stringMatching(/^F1: snapshot S1 holds \w+ but the legacy row pins \w+/)])
+	})
+})

package/dist/skills/ulw-research/scripts/source-refresh.mjs

@@ -0,0 +1,82 @@
+// source-refresh.mjs - `source refresh <S<n>>`: the one explicit repin of a source snapshot.
+//
+// Re-fetches meta.url (or reads the new capture given with the source snapshot input flags),
+// replaces the snapshot, and re-anchors every active byte-located ledger row on that source by its
+// exact quote plus its stored context (raw-locator.mjs reanchor): a row found once gets the new
+// snapshot_sha256 and keeps its old locator, so `fact verify` then reports minor `relocated` with
+// the old and new byte offsets; a row whose quote is gone keeps its old pin, is reported as
+// quote_not_verbatim here, and stays a snapshot_hash_mismatch blocker until it is retracted or
+// re-added. A legacy text.txt row on the source (no byte locator) cannot be re-anchored: it is
+// listed as legacy_not_reanchored and stays a blocker. Exit 0 when every row re-anchored, 1 otherwise. node: builtins and sibling imports only.
+
+import { writeFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { CliError, parseArgs } from "./cli-support.mjs"
+import { DEFECT_CODES } from "./contracts.mjs"
+import { readLedger, writeLedger } from "./fact-ledger.mjs"
+import { reanchor } from "./raw-locator.mjs"
+import { acquire, sessionDirOf, snapshotOf } from "./source-commands.mjs"
+import { writeSnapshot } from "./source-snapshot.mjs"
+
+export const REFRESH_USAGE =
+	"usage: source refresh S<n> [--url <u> | --html <file> | --raw <file> [--text <file> --extractor NAME@VERSION] | --text <file> | --command '<argv json>' --cwd DIR] [--charset NAME] --session-dir D (no input flag re-fetches the source's http(s) url)"
+
+const INPUTS = ["url", "html", "raw", "text", "command"]
+
+const defect = (code, message) => ({ code, severity: DEFECT_CODES[code].severity, integrity: DEFECT_CODES[code].integrity, message })
+
+export async function runRefresh(argv) {
+	const args = parseArgs(argv)
+	const id = args.positionals[0] ?? args.flags.id
+	if (!id || id === true) throw new CliError(REFRESH_USAGE)
+	const sessionDir = sessionDirOf(args.flags)
+	const before = snapshotOf(sessionDir, id)
+	const flags = { ...args.flags }
+	if (!INPUTS.some((name) => flags[name] !== undefined)) {
+		if (!/^https?:\/\//i.test(String(before.meta.url))) throw new CliError(`${id} was not fetched from an http(s) url; pass its new capture. ${REFRESH_USAGE}`)
+		flags.url = before.meta.url
+	}
+	const { receipt, ...input } = await acquire(flags)
+	input.kind ??= flags.kind ?? before.meta.kind
+	input.provenance = flags.provenance ?? before.meta.provenance
+	input.revision = flags.revision ?? before.meta.revision ?? undefined
+	let written
+	try {
+		written = writeSnapshot(join(sessionDir, "sources"), id, input, { replace: true })
+	} catch (error) {
+		if (error instanceof TypeError) throw new CliError(error.message)
+		throw error
+	}
+	if (receipt) writeFileSync(join(written.dir, "command.json"), `${JSON.stringify(receipt, null, 2)}\n`)
+	const after = snapshotOf(sessionDir, id)
+	const ledger = readLedger(sessionDir)
+	const rows = []
+	const defects = []
+	for (const row of ledger.rows) {
+		if (row.source_id !== id || row.status !== "active") continue
+		if (row.locator?.unit === undefined) {
+			rows.push({ fact_id: row.fact_id, status: "legacy_not_reanchored" })
+			defects.push(defect("snapshot_hash_mismatch", `${row.fact_id}: a legacy text.txt row has no byte locator to re-anchor on refreshed ${id}; add it again from the new snapshot or retract it`))
+			continue
+		}
+		if (row.locator.unit !== "bytes") continue
+		const anchor = row.locator.artifact === after.view.artifact ? reanchor(after.view, row) : null
+		const span = `${row.locator.char_start}:${row.locator.char_end}`
+		if (!anchor) {
+			rows.push({ fact_id: row.fact_id, status: "not_found", from: span })
+			defects.push(defect("quote_not_verbatim", `${row.fact_id}: its quote and context are not found once in refreshed ${id}; retract it or add it again from the new snapshot`))
+			continue
+		}
+		row.snapshot_sha256 = after.sha256
+		rows.push({ fact_id: row.fact_id, status: anchor.relocated ? "relocated" : "unchanged", from: span, to: `${anchor.start}:${anchor.end}` })
+		if (anchor.relocated) defects.push(defect("relocated", `${row.fact_id}: ${span} -> ${anchor.start}:${anchor.end} (bytes of ${id}/${after.view.artifact})`))
+	}
+	writeLedger(sessionDir, ledger)
+	const failed = rows.filter((row) => row.status === "not_found" || row.status === "legacy_not_reanchored").length
+	const exitCode = failed > 0 ? 1 : 0
+	const data = { source_id: id, previous_sha256: before.sha256, snapshot_sha256: after.sha256, text_source: after.view.mode, rows }
+	const json = { command: "source refresh", status: exitCode === 0 ? "pass" : "fail", exitCode, defects, data }
+	const moved = rows.filter((row) => row.status === "relocated").length
+	return { json, summary: `source refresh: ${id} ${before.sha256.slice(0, 12)} -> ${after.sha256.slice(0, 12)}; ${rows.length - failed} row(s) re-anchored (${moved} relocated), ${failed} not re-anchored`, exitCode }
+}