oh-my-opencode 5.1.47 → 5.1.51

package/packages/omo-codex/plugin/skills/ulw-research/scripts/source-commands.mjs

@@ -1,9 +1,13 @@
-// source-commands.mjs - the `source snapshot` and `claims index` subcommands of report-tools.
-// A source dir is immutable: the same id with identical text is a no-op, different text fails
-// (exit 1, snapshot_exists), because ledger rows pin the old hash. `--url` fetches with plain
-// fetch (30 s timeout, no browser) and records extractor tool:source-snapshot@1; `--html` and
-// `--text` import a browser-lane capture and record extractor model (lower trust), or the named
-// extractor for `--text --extractor NAME@VERSION`; `--command` runs an argv without a shell and
+// source-commands.mjs - the `source snapshot`, `source refresh` and `claims index` subcommands of
+// report-tools. A source dir is immutable: the same id with identical bytes is a no-op, different
+// bytes fail (exit 1, snapshot_exists), because ledger rows pin the old hash; `source refresh`
+// (source-refresh.mjs) is the one explicit repin. `--replace` swaps in a corrected capture under the
+// same id (the conversion_altered_number recovery) only while no active ledger row cites the id
+// (exit 1, snapshot_pinned, listing them). `--url` fetches with plain fetch (30 s timeout,
+// no browser) and records extractor tool:source-snapshot@1; `--html` imports a browser-lane page
+// capture and `--raw` alone a saved file's bytes (extractor model). `--text` alone registers a converted-only source (no raw bytes:
+// quotes only); `--raw <bytes> --text <extraction> --extractor NAME@VERSION` keeps the original
+// bytes and declares the text their extraction. `--command` runs an argv without a shell and
 // snapshots its stdout, with the argv, cwd, exit status, and output hashes in command.json.
 // node: builtins and sibling imports only.
 
@@ -13,13 +17,18 @@ import { join, resolve } from "node:path"
 
 import { indexClaims } from "./claims-index.mjs"
 import { CliError, parseArgs, sha256Of, writeJsonAtomic } from "./cli-support.mjs"
+import { runConversionProbe } from "./conversion-probe.mjs"
+import { readLedger } from "./fact-ledger.mjs"
 import { parseFidelityBlock } from "./fidelity-brief.mjs"
+import { parseSourceFloor } from "./fidelity-policy.mjs"
+import { encodingOf, headerCharset } from "./raw-decode.mjs"
 import { writeSnapshot } from "./source-snapshot.mjs"
+import { readSnapshot } from "./source-view.mjs"
 
 const FETCH_TIMEOUT_MS = 30_000
 const TOOL_EXTRACTOR = "tool:source-snapshot@1"
-const SNAPSHOT_USAGE =
-	"usage: source snapshot --id S<n> (--url <u> | --html <file> | --text <file> [--extractor NAME@VERSION] | --command '<argv json>' --cwd DIR) [--kind page|command|document] [--provenance primary|secondary] [--revision R] --session-dir D"
+export const SNAPSHOT_USAGE =
+	"usage: source probe-conversion <raw-file> <converted-file> | source snapshot --id S<n> (--url <u> | --html <file> | --raw <file> [--text <file> --extractor NAME@VERSION] | --text <file> (converted-only) | --command '<argv json>' --cwd DIR) [--charset NAME] [--kind page|command|document|converted-only] [--provenance primary|secondary] [--revision R] [--replace] --session-dir D"
 
 function readInput(path) {
 	try {
@@ -36,20 +45,27 @@ export function sessionDirOf(flags) {
 	return resolve(dir)
 }
 
-/** The snapshot of one source: meta.json, text.txt, and source.html when the source was a page. */
+/** The snapshot as source-view.mjs reads it ({ meta, raw, sha256, drift, view }); unreadable is a usage error. */
 export function snapshotOf(sessionDir, id) {
 	const dir = join(sessionDir, "sources", String(id))
 	if (!existsSync(join(dir, "meta.json"))) throw new CliError(`source ${id} has no snapshot; run source snapshot --id ${id} first`)
-	const html = join(dir, "source.html")
-	return { meta: JSON.parse(readInput(join(dir, "meta.json"))), text: readInput(join(dir, "text.txt")), html: existsSync(html) ? readInput(html) : null }
+	try {
+		return readSnapshot(dir)
+	} catch (error) {
+		throw new CliError(`source ${id}: ${error.message}`)
+	}
 }
 
-/** Whether the brief's source floor lists this source as pointed at (`pointed only: a, b`). */
+/**
+ * Whether the brief's source floor lists this source as pointed at (`pointed only: a, b`). An absent
+ * or pending floor points at nothing; a malformed floor is a usage error (parseSourceFloor's CliError).
+ */
 export function pointedFlag(sessionDir, url) {
 	const brief = join(sessionDir, "brief.md")
 	const floor = existsSync(brief) ? (parseFidelityBlock(readInput(brief)).fields?.["source floor"] ?? "") : ""
-	const match = /^pointed only:\s*(.+)$/i.exec(floor.trim())
-	return match ? match[1].split(",").map((entry) => entry.trim()).includes(url) : false
+	if (floor === "" || floor === "pending") return false
+	const rule = parseSourceFloor(floor)
+	return rule?.kind === "pointed" && rule.list.includes(url)
 }
 
 async function fetchSource(url) {
@@ -67,10 +83,10 @@ async function fetchSource(url) {
 	const type = response.headers.get("content-type") ?? ""
 	if (/pdf/i.test(type)) {
 		await response.body?.cancel()
-		throw new CliError(`pdf response: ${url}; extract its text and use --text <file> --extractor NAME@VERSION`)
+		throw new CliError(`pdf response: ${url}; save the bytes and use --raw <file.pdf> --text <extracted.txt> --extractor NAME@VERSION`)
 	}
-	const body = await response.text()
-	return /html|xml/i.test(type) || /^\s*</.test(body) ? { url, html: body } : { url, text: body }
+	const raw = Buffer.from(await response.arrayBuffer())
+	return { url, raw, charset: headerCharset(type), markup: /html|xml/i.test(type) || /^\s*</.test(raw.subarray(0, 1024).toString("latin1")) }
 }
 
 function runCommand(argvJson, cwd) {
@@ -84,35 +100,76 @@ function runCommand(argvJson, cwd) {
 		throw new CliError("--command must be a non-empty JSON array of non-empty strings")
 	}
 	if (!cwd || cwd === true) throw new CliError("--command needs --cwd DIR")
-	const proc = spawnSync(argv[0], argv.slice(1), { cwd: resolve(cwd), encoding: "utf8", shell: false, maxBuffer: 64 * 1024 * 1024 })
+	const proc = spawnSync(argv[0], argv.slice(1), { cwd: resolve(cwd), shell: false, maxBuffer: 64 * 1024 * 1024 })
 	if (proc.error) throw new CliError(`--command could not start: ${proc.error.message}`)
 	const receipt = { argv, cwd: resolve(cwd), exit_status: proc.status, stdout_sha256: sha256Of(proc.stdout), stderr_sha256: sha256Of(proc.stderr) }
-	return { path: `command:${argv.join(" ")}`, text: proc.stdout, kind: "command", receipt }
+	return { path: `command:${argv.join(" ")}`, raw: proc.stdout, kind: "command", receipt }
+}
+
+function readBytes(path) {
+	try {
+		return readFileSync(path)
+	} catch (error) {
+		throw new CliError(`cannot read ${path}: ${error.code ?? error.message}`)
+	}
 }
 
-async function acquire(flags) {
+/** The snapshot input the flags describe; shared by `source snapshot` and `source refresh`. */
+export async function acquire(flags) {
 	const given = ["url", "html", "text", "command"].filter((name) => flags[name] !== undefined)
-	if (given.length !== 1) throw new CliError(SNAPSHOT_USAGE)
+	if (flags.raw !== undefined && given.some((name) => name !== "text")) throw new CliError("--raw goes alone or with --text (--url, --html and --command already keep the raw bytes)")
+	if (given.length + (flags.raw !== undefined && flags.text === undefined ? 1 : 0) !== 1) throw new CliError(SNAPSHOT_USAGE)
 	if (flags.extractor !== undefined && flags.text === undefined) throw new CliError("--extractor goes with --text only")
-	if (flags.url !== undefined) return { ...(await fetchSource(flags.url)), extractor: TOOL_EXTRACTOR }
-	if (flags.html !== undefined) return { path: resolve(flags.html), html: readInput(flags.html), extractor: "model" }
-	if (flags.text !== undefined) return { path: resolve(flags.text), text: readInput(flags.text), extractor: flags.extractor === undefined ? "model" : `extractor:${flags.extractor}` }
+	if (flags.charset !== undefined && encodingOf(flags.charset) === null) throw new CliError(`--charset ${flags.charset} is not a known encoding`)
+	const charset = flags.charset === undefined ? {} : { charset: encodingOf(flags.charset) }
+	if (flags.url !== undefined) return { ...(await fetchSource(flags.url)), ...charset, extractor: TOOL_EXTRACTOR }
+	if (flags.html !== undefined) return { path: resolve(flags.html), raw: readBytes(flags.html), markup: true, ...charset, extractor: "model" }
+	if (flags.raw !== undefined && flags.text === undefined) return { path: resolve(flags.raw), raw: readBytes(flags.raw), ...charset, extractor: "model" }
+	if (flags.text !== undefined) {
+		const extractor = flags.extractor === undefined ? "model" : `extractor:${flags.extractor}`
+		if (flags.raw !== undefined) return { path: resolve(flags.raw), raw: readBytes(flags.raw), text: readInput(flags.text), ...charset, extractor }
+		if (flags.kind !== undefined && flags.kind !== "converted-only") throw new CliError("--text without --raw registers a converted-only source; pass --raw <original bytes> to keep a measurable snapshot")
+		return { path: resolve(flags.text), text: readInput(flags.text), kind: "converted-only", extractor }
+	}
 	return { ...runCommand(flags.command, flags.cwd), extractor: TOOL_EXTRACTOR }
 }
 
+/**
+ * `--replace` with `--text` alone (the conversion_altered_number recovery) keeps the source's raw.bin and
+ * declares the verified text its extraction, so value rows locate in text.txt while the original bytes stay
+ * pinned; plain `--text` without `--replace`, or over an id with no raw.bin, stays converted-only.
+ */
+function declareOverKeptRaw(sourcesDir, id, input) {
+	const dir = join(sourcesDir, String(id))
+	if (input.kind !== "converted-only" || !existsSync(join(dir, "raw.bin")) || !existsSync(join(dir, "meta.json"))) return
+	const before = JSON.parse(readFileSync(join(dir, "meta.json"), "utf8"))
+	Object.assign(input, { url: before.url, raw: readFileSync(join(dir, "raw.bin")), kind: before.kind, declared: true })
+	delete input.path
+}
+
 export async function runSource(argv) {
 	const [sub, ...rest] = argv
+	if (sub === "refresh") return (await import("./source-refresh.mjs")).runRefresh(rest)
+	if (sub === "probe-conversion") return runConversionProbe(rest)
 	if (sub !== "snapshot") throw new CliError(SNAPSHOT_USAGE)
-	const args = parseArgs(rest)
+	const args = parseArgs(rest, { booleans: ["replace"] })
 	const id = args.required("id")
-	const sourcesDir = join(sessionDirOf(args.flags), "sources")
+	const sessionDir = sessionDirOf(args.flags)
+	const sourcesDir = join(sessionDir, "sources")
+	const replace = args.flags.replace === true
+	const pinned = replace ? readLedger(sessionDir).rows.filter((row) => row.source_id === id && (row.status ?? "active") === "active").map((row) => row.fact_id) : []
+	if (pinned.length > 0) {
+		const message = `${id} is cited by active row(s) ${pinned.join(", ")}; retract them first, or snapshot the corrected text under a new id`
+		return { json: { command: "source snapshot", status: "fail", exitCode: 1, error: "snapshot_pinned", message, rows: pinned }, summary: `source snapshot: ${message}`, exitCode: 1 }
+	}
 	const { receipt, ...input } = await acquire(args.flags)
+	if (replace) declareOverKeptRaw(sourcesDir, id, input)
 	if (args.flags.kind !== undefined) input.kind = args.flags.kind
 	if (args.flags.provenance !== undefined) input.provenance = args.flags.provenance
 	if (args.flags.revision !== undefined) input.revision = args.flags.revision
 	let written
 	try {
-		written = writeSnapshot(sourcesDir, id, input)
+		written = writeSnapshot(sourcesDir, id, input, { replace })
 	} catch (error) {
 		if (String(error.message).startsWith("snapshot_exists")) {
 			return { json: { command: "source snapshot", status: "fail", exitCode: 1, error: "snapshot_exists", message: error.message }, summary: `source snapshot: ${error.message}`, exitCode: 1 }
@@ -122,8 +179,9 @@ export async function runSource(argv) {
 	}
 	if (receipt) writeFileSync(join(written.dir, "command.json"), `${JSON.stringify(receipt, null, 2)}\n`)
 	const meta = JSON.parse(readFileSync(join(written.dir, "meta.json"), "utf8"))
-	const data = { source_id: id, directory: written.dir, snapshot_sha256: written.sha256, extractor: meta.extractor, kind: meta.kind, provenance: meta.provenance }
-	return { json: { command: "source snapshot", status: "pass", exitCode: 0, data }, summary: `source snapshot: ${id} ${written.sha256.slice(0, 12)} (${meta.extractor})`, exitCode: 0 }
+	const data = { source_id: id, directory: written.dir, snapshot_sha256: written.sha256, extractor: meta.extractor, kind: meta.kind, provenance: meta.provenance, text_source: meta.text_source, ...(written.altered ? { conversion_altered_numbers: written.altered } : {}) }
+	const probe = written.altered ? `; conversion ${written.altered.length} altered${written.altered.length ? ` (${written.altered.join(", ")}): fact add refuses those numbers` : ""}` : ""
+	return { json: { command: "source snapshot", status: "pass", exitCode: 0, data }, summary: `source snapshot: ${id} ${written.sha256.slice(0, 12)} (${meta.extractor}, ${meta.text_source})${probe}`, exitCode: 0 }
 }
 
 export function runClaims(argv) {

package/packages/omo-codex/plugin/skills/ulw-research/scripts/source-refresh.mjs

@@ -0,0 +1,82 @@
+// source-refresh.mjs - `source refresh <S<n>>`: the one explicit repin of a source snapshot.
+//
+// Re-fetches meta.url (or reads the new capture given with the source snapshot input flags),
+// replaces the snapshot, and re-anchors every active byte-located ledger row on that source by its
+// exact quote plus its stored context (raw-locator.mjs reanchor): a row found once gets the new
+// snapshot_sha256 and keeps its old locator, so `fact verify` then reports minor `relocated` with
+// the old and new byte offsets; a row whose quote is gone keeps its old pin, is reported as
+// quote_not_verbatim here, and stays a snapshot_hash_mismatch blocker until it is retracted or
+// re-added. A legacy text.txt row on the source (no byte locator) cannot be re-anchored: it is
+// listed as legacy_not_reanchored and stays a blocker. Exit 0 when every row re-anchored, 1 otherwise. node: builtins and sibling imports only.
+
+import { writeFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { CliError, parseArgs } from "./cli-support.mjs"
+import { DEFECT_CODES } from "./contracts.mjs"
+import { readLedger, writeLedger } from "./fact-ledger.mjs"
+import { reanchor } from "./raw-locator.mjs"
+import { acquire, sessionDirOf, snapshotOf } from "./source-commands.mjs"
+import { writeSnapshot } from "./source-snapshot.mjs"
+
+export const REFRESH_USAGE =
+	"usage: source refresh S<n> [--url <u> | --html <file> | --raw <file> [--text <file> --extractor NAME@VERSION] | --text <file> | --command '<argv json>' --cwd DIR] [--charset NAME] --session-dir D (no input flag re-fetches the source's http(s) url)"
+
+const INPUTS = ["url", "html", "raw", "text", "command"]
+
+const defect = (code, message) => ({ code, severity: DEFECT_CODES[code].severity, integrity: DEFECT_CODES[code].integrity, message })
+
+export async function runRefresh(argv) {
+	const args = parseArgs(argv)
+	const id = args.positionals[0] ?? args.flags.id
+	if (!id || id === true) throw new CliError(REFRESH_USAGE)
+	const sessionDir = sessionDirOf(args.flags)
+	const before = snapshotOf(sessionDir, id)
+	const flags = { ...args.flags }
+	if (!INPUTS.some((name) => flags[name] !== undefined)) {
+		if (!/^https?:\/\//i.test(String(before.meta.url))) throw new CliError(`${id} was not fetched from an http(s) url; pass its new capture. ${REFRESH_USAGE}`)
+		flags.url = before.meta.url
+	}
+	const { receipt, ...input } = await acquire(flags)
+	input.kind ??= flags.kind ?? before.meta.kind
+	input.provenance = flags.provenance ?? before.meta.provenance
+	input.revision = flags.revision ?? before.meta.revision ?? undefined
+	let written
+	try {
+		written = writeSnapshot(join(sessionDir, "sources"), id, input, { replace: true })
+	} catch (error) {
+		if (error instanceof TypeError) throw new CliError(error.message)
+		throw error
+	}
+	if (receipt) writeFileSync(join(written.dir, "command.json"), `${JSON.stringify(receipt, null, 2)}\n`)
+	const after = snapshotOf(sessionDir, id)
+	const ledger = readLedger(sessionDir)
+	const rows = []
+	const defects = []
+	for (const row of ledger.rows) {
+		if (row.source_id !== id || row.status !== "active") continue
+		if (row.locator?.unit === undefined) {
+			rows.push({ fact_id: row.fact_id, status: "legacy_not_reanchored" })
+			defects.push(defect("snapshot_hash_mismatch", `${row.fact_id}: a legacy text.txt row has no byte locator to re-anchor on refreshed ${id}; add it again from the new snapshot or retract it`))
+			continue
+		}
+		if (row.locator.unit !== "bytes") continue
+		const anchor = row.locator.artifact === after.view.artifact ? reanchor(after.view, row) : null
+		const span = `${row.locator.char_start}:${row.locator.char_end}`
+		if (!anchor) {
+			rows.push({ fact_id: row.fact_id, status: "not_found", from: span })
+			defects.push(defect("quote_not_verbatim", `${row.fact_id}: its quote and context are not found once in refreshed ${id}; retract it or add it again from the new snapshot`))
+			continue
+		}
+		row.snapshot_sha256 = after.sha256
+		rows.push({ fact_id: row.fact_id, status: anchor.relocated ? "relocated" : "unchanged", from: span, to: `${anchor.start}:${anchor.end}` })
+		if (anchor.relocated) defects.push(defect("relocated", `${row.fact_id}: ${span} -> ${anchor.start}:${anchor.end} (bytes of ${id}/${after.view.artifact})`))
+	}
+	writeLedger(sessionDir, ledger)
+	const failed = rows.filter((row) => row.status === "not_found" || row.status === "legacy_not_reanchored").length
+	const exitCode = failed > 0 ? 1 : 0
+	const data = { source_id: id, previous_sha256: before.sha256, snapshot_sha256: after.sha256, text_source: after.view.mode, rows }
+	const json = { command: "source refresh", status: exitCode === 0 ? "pass" : "fail", exitCode, defects, data }
+	const moved = rows.filter((row) => row.status === "relocated").length
+	return { json, summary: `source refresh: ${id} ${before.sha256.slice(0, 12)} -> ${after.sha256.slice(0, 12)}; ${rows.length - failed} row(s) re-anchored (${moved} relocated), ${failed} not re-anchored`, exitCode }
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/source-snapshot.mjs

@@ -1,77 +1,44 @@
 // source-snapshot.mjs - the per-source snapshot that fact locators read.
 //
-// snapshotFromHtml turns a page into text.txt form: one line per block (prose blocks via
-// proseText, one table cell per line; head, scripts, styles and code are dropped). writeSnapshot
-// writes <sourcesDir>/<S-id>/ with text.txt (pages joined by a form feed), source.html when html
-// was given, meta.json, sentences.jsonl (sid S<n>:s<k>) and acronyms.json. The sha256 covers
-// text.txt only. Nothing here touches the network: the caller supplies the html or text.
+// writeSnapshot writes <sourcesDir>/<S-id>/ with raw.bin (the fetched or captured bytes, absent
+// only for converted-only sources), meta.json (sha256 of the pinned body, text_source, charset),
+// text.txt (the locator text of source-view.mjs: the visible text nodes of a page, the decoded
+// bytes of a document, or a declared/converted extraction), source.html for pages (decoded, a
+// reading aid), sentences.jsonl (sid S<n>:s<k>, indexed over text.txt) and acronyms.json; a page
+// also gets conversion.json, the conversion probe's record of the numbers its text.txt altered.
 // Plain Node ESM, node: builtins and sibling imports only.
 
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
 import { join } from "node:path"
 
 import { sha256Of } from "./cli-support.mjs"
-import { parseHtml, PROSE_TAGS, proseText } from "./html-lite.mjs"
+import { probeConversion } from "./conversion-probe.mjs"
+import { writeFidelityRecord } from "./fidelity-records.mjs"
+import { extractHtml } from "./html-text-map.mjs"
 import { splitSentences } from "./sentence-split.mjs"
+import { buildView } from "./source-view.mjs"
 
-const SKIP = new Set(["head", "script", "style", "svg", "code", "pre", "kbd", "samp", "template", "noscript"])
-const INLINE = new Set([
-	"a", "abbr", "b", "bdi", "bdo", "cite", "data", "del", "dfn", "em", "font", "i", "ins", "label", "mark", "q", "s",
-	"small", "span", "strong", "sub", "sup", "time", "u", "var",
-])
 const SOURCE_ID = /^S[1-9]\d*$/
 const EXTRACTOR = /^(?:tool:source-snapshot@1|model|extractor:[\w.@+-]+)$/
-const KINDS = new Set(["page", "command", "document"])
+const KINDS = new Set(["page", "command", "document", "converted-only"])
 const PROVENANCE = new Set(["primary", "secondary", "unknown"])
 const ACRONYM_AFTER = /\(([A-Z]{2,6})\)/g
 const ACRONYM_BEFORE = /(?<![\p{L}\p{N}])([A-Z]{2,6})\s?\(([A-Z][^()\n]{2,79})\)/gu
 const NAME_WORD = /^[A-Z][\p{L}\p{N}-]*$/u
 const CONNECTOR = new Set(["of", "and", "for", "the", "in", "on", "to", "&"])
+const MARKUP = /^\s*</
 
 /**
- * Block lines of a page: { root, text, blocks: [{ node, start, end }] } where text joins the block
- * texts with "\n" and text.slice(start, end) is one block. fact-locate reads cells through `node`.
+ * Block lines of a page: { root, text, blocks: [{ node, start, end }], starts, ends } where text
+ * joins the visible block texts with "\n" (html-text-map.mjs). fact-locate reads cells through `node`.
  */
 export function snapshotBlocks(html) {
-	const root = parseHtml(html)
-	const lines = []
-	let buffer = ""
-	const flush = (node) => {
-		const line = buffer.replace(/\s+/g, " ").trim()
-		buffer = ""
-		if (line) lines.push({ node, text: line })
-	}
-	const walk = (node) => {
-		for (const child of node.children) {
-			if (child.tag === "#text") buffer += child.text
-			else if (SKIP.has(child.tag)) continue
-			else if (child.tag === "br") flush(node)
-			else if (INLINE.has(child.tag)) walk(child)
-			else if (PROSE_TAGS.has(child.tag)) {
-				flush(node)
-				const line = proseText(child)
-				if (line) lines.push({ node: child, text: line })
-			} else {
-				flush(node)
-				walk(child)
-				flush(child)
-			}
-		}
-	}
-	walk(root)
-	flush(root)
-	const blocks = []
-	let at = 0
-	for (const line of lines) {
-		blocks.push({ node: line.node, start: at, end: at + line.text.length })
-		at += line.text.length + 1
-	}
-	return { root, text: lines.map((line) => line.text).join("\n"), blocks }
+	return extractHtml(html)
 }
 
 /** text.txt form of an html page. */
 export function snapshotFromHtml(html) {
-	return snapshotBlocks(html).text
+	return extractHtml(html).text
 }
 
 /** Sentence index rows { sid: "S<n>:s<k>", start, end, text }; a form feed (page break) always ends a sentence. */
@@ -127,36 +94,61 @@ export function findAcronyms(text, sentences) {
 function snapshotText(input) {
 	if (Array.isArray(input.pages)) return input.pages.join("\f")
 	if (typeof input.text === "string") return input.text
-	if (typeof input.html === "string") return snapshotFromHtml(input.html)
-	throw new TypeError("writeSnapshot needs text, pages, or html")
+	throw new TypeError("writeSnapshot needs raw, html, text, or pages")
+}
+
+/** Which source-view text source the input makes, with the bytes it pins. */
+function bodiesOf(input) {
+	if (input.kind === "converted-only") return { mode: "converted", raw: null, textBody: Buffer.from(snapshotText(input)) }
+	if (input.declared === true) return { mode: "declared", raw: Buffer.from(input.raw), textBody: Buffer.from(snapshotText(input)) }
+	if (input.raw === undefined && typeof input.html === "string") return { mode: "html", raw: Buffer.from(input.html), textBody: null }
+	if (input.raw === undefined) return { mode: "decoded", raw: Buffer.from(snapshotText(input)), textBody: null }
+	const raw = Buffer.from(input.raw)
+	if (input.markup === true || typeof input.html === "string" || input.kind === "page" || MARKUP.test(raw.subarray(0, 1024).toString("latin1").replace(/^\u00ef\u00bb\u00bf/, ""))) return { mode: "html", raw, textBody: null }
+	if (input.text !== undefined || input.pages !== undefined) return { mode: "declared", raw, textBody: Buffer.from(snapshotText(input)) }
+	return { mode: "decoded", raw, textBody: null }
 }
 
 function checkInput(sourceId, input) {
 	if (!SOURCE_ID.test(String(sourceId))) throw new TypeError(`source id must be S<n>, got ${sourceId}`)
 	if (!EXTRACTOR.test(String(input.extractor))) throw new TypeError(`extractor must be tool:source-snapshot@1, model, or extractor:<name>, got ${input.extractor}`)
 	if (typeof (input.url ?? input.path) !== "string") throw new TypeError("writeSnapshot needs a url or a path")
-	if (input.kind !== undefined && !KINDS.has(input.kind)) throw new TypeError(`kind must be page, command, or document, got ${input.kind}`)
+	if (input.kind !== undefined && !KINDS.has(input.kind)) throw new TypeError(`kind must be page, command, document, or converted-only, got ${input.kind}`)
 	if (input.provenance !== undefined && !PROVENANCE.has(input.provenance)) throw new TypeError(`provenance must be primary, secondary, or unknown, got ${input.provenance}`)
+	if (input.observed_at !== undefined && (typeof input.observed_at !== "string" || Number.isNaN(new Date(input.observed_at).getTime()))) throw new TypeError(`observed_at must be an ISO timestamp, got ${input.observed_at}`)
+}
+
+function pinnedHash(dir) {
+	const metaPath = join(dir, "meta.json")
+	return existsSync(metaPath) ? JSON.parse(readFileSync(metaPath, "utf8")).sha256 : null
 }
 
 /**
- * Write <sourcesDir>/<sourceId>/ for one source; returns { dir, sha256 } with sha256 over text.txt.
- * input: { url | path, text | pages | html, extractor, observed_at?, revision?, kind?, provenance? }.
- * Re-writing the same text is a no-op change; a different text over an existing snapshot throws
- * snapshot_exists, because ledger rows pin the old hash.
+ * Write <sourcesDir>/<sourceId>/ for one source; returns { dir, sha256, altered } over the pinned body,
+ * where altered lists the numbers the page conversion altered (null when no page conversion ran).
+ * input: { url | path, raw?, markup?, html?, text? | pages?, charset?, extractor, observed_at?, revision?,
+ * kind?, provenance? } (see bodiesOf). Re-writing the same bytes is a no-op; different bytes over
+ * an existing snapshot throw snapshot_exists, because ledger rows pin the old hash, unless
+ * options.replace (`source refresh`, which re-anchors those rows, or `source snapshot --replace`, which
+ * has checked that no active row pins it) is set; a replacement drops the old capture's command.json.
  */
-export function writeSnapshot(sourcesDir, sourceId, input) {
+export function writeSnapshot(sourcesDir, sourceId, input, options = {}) {
 	checkInput(sourceId, input)
-	const text = snapshotText(input)
-	const sha256 = sha256Of(Buffer.from(text, "utf8"))
+	const { mode, raw, textBody } = bodiesOf(input)
+	const view = buildView(mode, { raw, textBody, charset: input.charset ?? null })
+	const sha256 = sha256Of(raw ?? textBody)
 	const dir = join(sourcesDir, sourceId)
-	const textPath = join(dir, "text.txt")
-	if (existsSync(textPath) && sha256Of(readFileSync(textPath)) !== sha256) {
-		throw new Error(`snapshot_exists: ${sourceId} already holds a different text.txt`)
+	const pinned = pinnedHash(dir)
+	if (pinned !== null && pinned !== sha256) {
+		if (options.replace !== true) throw new Error(`snapshot_exists: ${sourceId} already holds different snapshot bytes`)
+		rmSync(join(dir, "command.json"), { force: true })
 	}
 	mkdirSync(dir, { recursive: true })
-	writeFileSync(textPath, text)
-	if (typeof input.html === "string") writeFileSync(join(dir, "source.html"), input.html)
+	if (raw) writeFileSync(join(dir, "raw.bin"), raw)
+	else rmSync(join(dir, "raw.bin"), { force: true })
+	writeFileSync(join(dir, "text.txt"), textBody ?? view.text)
+	if (view.html !== null) writeFileSync(join(dir, "source.html"), view.html)
+	else rmSync(join(dir, "source.html"), { force: true })
 	const meta = {
 		source_id: sourceId,
 		url: input.url ?? input.path,
@@ -164,13 +156,25 @@ export function writeSnapshot(sourcesDir, sourceId, input) {
 		sha256,
 		extractor: input.extractor,
 		revision: input.revision ?? null,
-		kind: input.kind ?? (input.url !== undefined || typeof input.html === "string" ? "page" : "document"),
+		kind: input.kind ?? (input.url !== undefined || mode === "html" ? "page" : "document"),
 		provenance: input.provenance ?? "unknown",
+		text_source: mode,
+		...(raw ? { charset: view.encoding } : {}),
+		...(mode === "declared" ? { text_sha256: sha256Of(textBody) } : {}),
 	}
 	writeFileSync(join(dir, "meta.json"), `${JSON.stringify(meta, null, 2)}\n`)
-	const sentences = indexSentences(text, sourceId)
+	let altered = null
+	if (mode === "html") {
+		const { altered: changed, ...counts } = probeConversion(view.html, view.text, { attributes: false })
+		altered = changed
+		writeFidelityRecord(join(dir, "conversion.json"), "source_conversion", {
+			artifact_sha256: sha256Of(view.html), report: join(dir, "source.html"), defects: [],
+			conversion_altered_numbers: changed, ...counts,
+		}, new Date(meta.observed_at))
+	} else rmSync(join(dir, "conversion.json"), { force: true })
+	const sentences = indexSentences(view.text, sourceId)
 	writeFileSync(join(dir, "sentences.jsonl"), sentences.map((row) => `${JSON.stringify(row)}\n`).join(""))
-	const acronyms = { schemaVersion: 1, source_id: sourceId, snapshot_sha256: sha256, entries: findAcronyms(text, sentences) }
+	const acronyms = { schemaVersion: 1, source_id: sourceId, snapshot_sha256: sha256, entries: findAcronyms(view.text, sentences) }
 	writeFileSync(join(dir, "acronyms.json"), `${JSON.stringify(acronyms, null, 2)}\n`)
-	return { dir, sha256 }
+	return { dir, sha256, altered }
 }

package/packages/omo-codex/plugin/skills/ulw-research/scripts/source-view.mjs

@@ -0,0 +1,89 @@
+// source-view.mjs - the one reading of a source snapshot that locators, verify, and refresh share.
+//
+// A snapshot's meta.json `text_source` says where its locator text comes from:
+//   html      - text nodes of raw.bin (html-text-map.mjs), offsets are raw.bin bytes
+//   decoded   - raw.bin decoded in its charset, offsets are raw.bin bytes
+//   declared  - text.txt is a declared extraction (--raw + --text) of non-HTML raw.bin; offsets
+//               are text.txt bytes, and raw.bin and text.txt are both pinned
+//   converted - converted-only: no raw.bin, text.txt is all there is (quotes only)
+// A snapshot without text_source and without raw.bin is a legacy text.txt snapshot. byteSpan maps
+// a [start, end) range of `text` to [startByte, endByte) in `artifact`, never through a re-encode.
+// node: builtins and sibling imports only.
+
+import { readFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { sha256Of } from "./cli-support.mjs"
+import { extractHtml } from "./html-text-map.mjs"
+import { decodeRaw } from "./raw-decode.mjs"
+
+export const TEXT_SOURCES = Object.freeze(["html", "decoded", "declared", "converted"])
+
+/** Read one snapshot file; the error names the file and the cause (missing, EACCES, ...). */
+export function readPart(dir, name, encoding) {
+	try {
+		return readFileSync(join(dir, name), encoding)
+	} catch (error) {
+		const cause = error.code === "ENOENT" ? "is missing" : error.code === "EISDIR" ? "is not a readable file" : `is unreadable (${error.code ?? error.message})`
+		throw Object.assign(new Error(`${name} ${cause}`), { code: error.code, part: name })
+	}
+}
+
+/** The text source a meta.json declares, inferring it for snapshots written before the field. */
+export function textSourceOf(meta, hasRaw) {
+	if (TEXT_SOURCES.includes(meta.text_source)) return meta.text_source
+	if (!hasRaw) return meta.kind === "converted-only" ? "converted" : "legacy"
+	return meta.kind === "page" ? "html" : "decoded"
+}
+
+/**
+ * Locator text and byte map for one snapshot body. raw is raw.bin (null for converted/legacy),
+ * textBody is text.txt bytes (needed for declared, converted, legacy).
+ */
+export function buildView(mode, { raw = null, textBody = null, charset = null }) {
+	if (mode === "html") {
+		const decoded = decodeRaw(raw, charset, true)
+		const map = extractHtml(decoded.text)
+		const byteSpan = (start, end) => {
+			if (end <= start) return [decoded.bytes[map.ends[start - 1] ?? 0], decoded.bytes[map.ends[start - 1] ?? 0]]
+			return [decoded.bytes[map.starts[start]], decoded.bytes[map.ends[end - 1]]]
+		}
+		return { mode, artifact: "raw.bin", encoding: decoded.encoding, text: map.text, html: decoded.text, root: map.root, blocks: map.blocks, byteSpan }
+	}
+	if (mode === "legacy") return { mode, artifact: "text.txt", encoding: "utf-8", text: Buffer.from(textBody).toString("utf8"), html: null, root: null, blocks: null, byteSpan: null }
+	const body = mode === "decoded" ? raw : textBody
+	const decoded = decodeRaw(body, mode === "decoded" ? charset : "utf-8")
+	const byteSpan = (start, end) => [decoded.bytes[start], decoded.bytes[Math.max(start, end)]]
+	return { mode, artifact: mode === "decoded" ? "raw.bin" : "text.txt", encoding: decoded.encoding, text: decoded.text, html: null, root: null, blocks: null, byteSpan }
+}
+
+/**
+ * Read and hash a snapshot directory. Throws (with the file and cause) when meta.json, raw.bin, or
+ * text.txt cannot be read; hash drift is returned, not thrown: { meta, view, raw, sha256, drift }.
+ * sha256 is the hash of the pinned body (raw.bin, or text.txt for converted and legacy snapshots).
+ */
+export function readSnapshot(dir) {
+	let meta
+	try {
+		meta = JSON.parse(readPart(dir, "meta.json", "utf8"))
+	} catch (error) {
+		throw error.part ? error : new Error("meta.json is not valid JSON")
+	}
+	if (meta === null || typeof meta !== "object" || typeof meta.sha256 !== "string") throw new Error("has invalid snapshot metadata (meta.json has no sha256)")
+	let raw = null
+	if (meta.text_source === undefined) {
+		try {
+			raw = readPart(dir, "raw.bin")
+		} catch (error) {
+			if (error.code !== "ENOENT") throw error
+		}
+	}
+	const mode = textSourceOf(meta, raw !== null)
+	if (raw === null && ["html", "decoded", "declared"].includes(mode)) raw = readPart(dir, "raw.bin")
+	const textBody = ["declared", "converted", "legacy"].includes(mode) ? readPart(dir, "text.txt") : null
+	const sha256 = sha256Of(raw ?? textBody)
+	const drift = []
+	if (sha256 !== meta.sha256) drift.push(`hash mismatch: ${raw ? "raw.bin" : "text.txt"} changed after its snapshot was pinned`)
+	if (mode === "declared" && sha256Of(textBody) !== meta.text_sha256) drift.push("hash mismatch: text.txt (the declared extraction) changed after it was pinned")
+	return { meta, raw, sha256, drift, view: buildView(mode, { raw, textBody, charset: meta.charset ?? null }) }
+}