oh-my-opencode 5.1.48 → 5.1.54

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-originals.mjs

@@ -0,0 +1,161 @@
+// Check displayed conversion originals against their operands, not the mutable render field.
+import { sameValue, unitKey } from "./fidelity-binding.mjs"
+import { proseText } from "./html-lite.mjs"
+import { parseNumeral } from "./numerals.mjs"
+import { normalizeText } from "./text-normalize.mjs"
+import { display } from "./unit-precision.mjs"
+import { dimensionOf } from "./units.mjs"
+
+const DERIVED = new Set(["converted", "derived"])
+/**
+ * The one bracket grammar of the skill. After NFKC (fullwidth parentheses and square brackets fold
+ * to ASCII), an original is a () [] 【】 「」 or 〔〕 group. Inside a derived or converted element every
+ * group is an original, never prose; a dash separator is not an original.
+ */
+const PAIRS = [["(", ")"], ["[", "]"], ["\u3010", "\u3011"], ["\u300C", "\u300D"], ["\u3014", "\u3015"]]
+const [OPEN, CLOSE] = [PAIRS.map((pair) => pair[0]).join(""), PAIRS.map((pair) => pair[1]).join("")]
+const inClass = (chars) => chars.replace(/[\\\]\[^-]/g, "\\$&")
+const literal = (chars) => chars.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+const ANY = inClass(OPEN + CLOSE)
+const GROUP = new RegExp(PAIRS.map(([open, close]) => `${literal(open)}([^${ANY}]*)${literal(close)}`).join("|"), "gu")
+const BRACKET = new RegExp(`[${ANY}]`, "u")
+const folded = (text) => String(text).normalize("NFKC")
+const innerOf = (match) => match.slice(1).find((part) => part !== undefined)
+const mismatch = (message) => ["value_mismatch", message, "original"]
+
+const halfFits = (row, half) => {
+	const expected = normalizeText(folded(display(String(row.value), row.unit ?? null)))
+	const text = normalizeText(half)
+	return text === expected || (text.endsWith(` ${expected}`) && !/\d/.test(text.slice(0, -expected.length)))
+}
+
+/** The converted half (the text outside the originals) equals display(row.value, row.unit), rebuilt from the row. */
+export function convertedHalfFits(row, shown) {
+	if (!DERIVED.has(row.kind) || row.value === null || row.value === undefined) return false
+	return halfFits(row, folded(shown).replace(GROUP, ""))
+}
+
+/** A second number outside the value verify reads and its originals ("320,000 mm - 999 m"). */
+function strayNumber(row, half) {
+	if (row.value === null || row.value === undefined || halfFits(row, half)) return null
+	const read = parseNumeral(half)
+	if (!read || !sameValue(read.value, row.value)) return null
+	const rest = half.slice(0, read.span[0]) + half.slice(read.span[1])
+	return /\d/.test(row.unit ? rest.split(row.unit).join("") : rest) ? mismatch(`"${half.trim()}" carries a number besides the value and its originals`) : null
+}
+
+/** `valueMode`: the element shows the bare value ({{F:x:value}}), so no original is due. */
+export function originalProblem(row, shown, rows, valueMode = false) {
+	if (!DERIVED.has(row.kind)) return null
+	const text = folded(shown)
+	const groups = [...text.matchAll(GROUP)]
+	const half = text.replace(GROUP, "")
+	if (BRACKET.test(half)) return mismatch("original brackets are malformed or nested")
+	const operands = (row.derived_from ?? []).map((id) => rows.get(id))
+	for (const [index, match] of groups.entries()) {
+		const [inner, operand] = [innerOf(match), operands[index]]
+		const parsed = parseNumeral(inner)
+		if (!operand) return mismatch(`original "${inner}" has no operand of ${row.fact_id}`)
+		if (!parsed || !sameValue(parsed.value, operand.value)) return mismatch(`original "${inner}" differs from operand ${operand.fact_id}`)
+		if (unitKey(parsed.unitRaw ?? "") !== unitKey(operand.unit ?? "")) return ["unit_mismatch", `original "${inner}" is not in ${operand.unit}`, "original"]
+		const expected = normalizeText(folded(display(operand.value, operand.unit ?? null)))
+		if (normalizeText(inner) !== expected) return mismatch(`original "${inner}" differs from "${expected}"`)
+	}
+	if (groups.length > 0 && !/\d/.test(text.slice(0, groups[0].index))) return mismatch(`original "${innerOf(groups[0])}" precedes the value`)
+	if (row.kind === "converted" && !valueMode && /\d/.test(half) && groups.length < operands.length) return mismatch(`the original of ${row.derived_from[groups.length]} is missing after the value`)
+	return strayNumber(row, half)
+}
+
+/** Stands for a filled element or another fill-map value in the text after a fill site. */
+const FILLED = "\uFFFC"
+/** After a fill site the grammar also reads the brackets an element never carries as originals. */
+const FOLLOW = [...PAIRS, ["\u300E", "\u300F"], ["\u300A", "\u300B"], ["\u3008", "\u3009"], ["\u27E8", "\u27E9"]]
+const dimension = (unit) => dimensionOf(String(unit).replace(/-e\d+$/u, "")) ?? unitKey(unit)
+
+/** What the text after a fill site opens: a bracket group's content, or the clause after a separator (`open`); else null. */
+function openedGroup(text) {
+	const rest = folded(text).trimStart()
+	const pair = FOLLOW.find(([open]) => rest.startsWith(open))
+	const close = pair ? rest.indexOf(pair[1], 1) : -1
+	if (pair) return { group: rest.slice(1, close === -1 ? undefined : close).trim(), open: false }
+	return /^[-\u2013\u2014/]/u.test(rest) ? { group: rest.slice(1).split(/[,;:!?)\]]|\.(?!\d)/u)[0].trim(), open: true } : null
+}
+
+/**
+ * Whether the text right after a fill site of `row` opens another original: a bracket group or a
+ * separator clause that is exactly a numeral with a unit of the row's or an operand's dimension (a
+ * separator clause may end in a glued particle, as Korean writes "- 999 m\uC774\uB2E4").
+ * A filled element, a fill-map value (`fills`), a placeholder, a unitless number (a year, a
+ * footnote) and prose with digits are not originals; untraceable_number still reads a stray number.
+ */
+export const followsOriginal = (row, rows, fills = []) => (text) => {
+	if (!DERIVED.has(row?.kind)) return false
+	const after = [...fills].filter(Boolean).sort((a, b) => b.length - a.length).reduce((cur, fill) => cur.split(fill).join(FILLED), String(text))
+	const { group, open } = openedGroup(after) ?? { group: "" }
+	if (!group || group.includes(FILLED) || group.includes("{{")) return false
+	const units = [row.unit, ...(row.derived_from ?? []).map((id) => rows.get(id)?.unit)].filter(Boolean)
+	return [group, ...(open ? [group.replace(/[\uAC00-\uD7A3]+$/u, "").trim()] : [])].some((body) => {
+		const read = parseNumeral(body, 0)
+		return Boolean(read?.unitRaw) && read.span[0] === 0 && read.span[1] === body.length && units.some((unit) => dimension(unit) === dimension(read.unitRaw))
+	})
+}
+
+/** The prose after `node` in its parent, each filled element (data-fact) read as one FILLED mark. */
+export const textAfter = (node) => {
+	const siblings = node.parent?.children.slice(node.parent.children.indexOf(node) + 1) ?? []
+	return proseText({ children: siblings.map((child) => (child.attrs?.["data-fact"] === undefined ? child : { tag: "#text", text: FILLED })) })
+}
+
+const SPACE = "[\\s\\u3000]"
+const DIGIT = "[0-9\\uFF10-\\uFF19]"
+/** Each character with the fullwidth form NFKC folds onto it. */
+const wide = (chars) => [...chars].map((ch) => ch + (/[!-~]/.test(ch) ? String.fromCharCode(ch.charCodeAt(0) + 0xfee0) : "")).join("")
+const [WIDE_OPEN, WIDE_CLOSE] = [inClass(wide(OPEN)), inClass(wide(CLOSE))]
+
+/** A pattern reading `rendered` modulo the bracket grammar: NFKC fullwidth forms, bracket style, spacing around brackets. */
+function renderPattern(rendered) {
+	const text = folded(rendered).trim()
+	let body = ""
+	for (let at = 0; at < text.length; at += 1) {
+		const ch = text[at]
+		const beside = (OPEN + CLOSE).includes(text.slice(at).trimStart()[0]) || OPEN.includes(text[at - 1])
+		if (/\s/.test(ch)) body += beside ? "" : `${SPACE}+`
+		else if (OPEN.includes(ch)) body += `${SPACE}*([${WIDE_OPEN}])${SPACE}*`
+		else if (CLOSE.includes(ch)) body += `${SPACE}*([${WIDE_CLOSE}])`
+		else body += `[${inClass(wide(ch))}]`
+	}
+	// A digit may not run into a render's leading or trailing digit ("1,320 m" holds no "320 m"); a bracket ends the render.
+	return new RegExp(`${/^\d/.test(text) ? `(?<!${DIGIT})` : ""}${body}${/\d$/.test(text) ? `(?!${DIGIT})` : ""}`, "gu")
+}
+
+const pairIndex = (ch, side) => [...side].findIndex((open) => open === folded(ch))
+/** [start, end) of `rendered` in `text` from `from` under the bracket grammar, not followed by another original (`follows`); or null. */
+export function findRender(text, rendered, from = 0, follows = () => false) {
+	const pattern = renderPattern(rendered)
+	pattern.lastIndex = from
+	for (let match = pattern.exec(text); match; match = pattern.exec(text)) {
+		const marks = match.slice(1)
+		const paired = marks.every((mark, i) => i % 2 === 1 || pairIndex(mark, OPEN) === pairIndex(marks[i + 1] ?? "", CLOSE))
+		const end = match.index + match[0].length
+		if (paired && !follows(text.slice(end))) return [match.index, end]
+		pattern.lastIndex = match.index + 1
+	}
+	return null
+}
+
+/** Where a stale fill-map entry's text sits: derived and converted renders are read under the bracket grammar. */
+export const renderLocator = (rows, fillMap) => (entry, text, from) => {
+	const row = rows.get(entry.fact_id)
+	if (DERIVED.has(row?.kind)) return findRender(text, entry.rendered, from, followsOriginal(row, rows, (fillMap ?? []).map((other) => other.rendered)))
+	const hit = text.indexOf(entry.rendered, from)
+	return hit === -1 ? null : [hit, hit + entry.rendered.length]
+}
+
+/** Null when a stale derived render is still in the report; "original" when only its converted half is; "gone" otherwise. */
+export function staleRender(row, rendered, text, follows) {
+	if (findRender(text, rendered, 0, follows) !== null) return null
+	const half = folded(rendered).replace(GROUP, "").trim()
+	return half !== "" && /\d/.test(half) && findRender(text, half) !== null ? "original" : "gone"
+}
+
+export const isDerivedRow = (row) => DERIVED.has(row?.kind)

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-policy.mjs

@@ -137,11 +137,11 @@ function sourcesOf(row, rows, seen = new Set()) {
 /** sha256 of the locked `## Fidelity` block normalized: the seven fields in FIDELITY_FIELDS order, `field: value` per line. */
 export const briefHashOf = (parsed) => sha256Of(FIDELITY_FIELDS.map((field) => `${field}: ${parsed.fields[field]}`).join("\n"))
 
-/** brief_changed when a fill map was not written under the current brief (or predates the binding). */
+/** brief_changed when there is no fill map (map null) or it was not written under the current brief (or predates the binding). */
 export function briefBindingErrors(map, mapPath, parsed) {
-	if (map.brief_sha256 === briefHashOf(parsed)) return []
-	const why = typeof map.brief_sha256 === "string" ? "was filled under a different ## Fidelity block" : "records no brief_sha256"
-	return [{ code: "brief_changed", token: null, line: null, message: `${mapPath} ${why}; re-run fact fill under the current brief`, severity: DEFECT_CODES.brief_changed.severity, integrity: DEFECT_CODES.brief_changed.integrity, hint: DEFECT_CODES.brief_changed.hint }]
+	if (map?.brief_sha256 === briefHashOf(parsed)) return []
+	const why = map === null ? `no fill map at ${mapPath}` : `${mapPath} ${typeof map.brief_sha256 === "string" ? "was filled under a different ## Fidelity block" : "records no brief_sha256"}`
+	return [{ code: "brief_changed", token: null, line: null, message: `${why}; re-run fact fill under the current brief`, severity: DEFECT_CODES.brief_changed.severity, integrity: DEFECT_CODES.brief_changed.integrity, hint: DEFECT_CODES.brief_changed.hint }]
 }
 
 function metaOf(sessionDir, sourceId, cache) {

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-records.mjs

@@ -15,7 +15,7 @@
 // fidelity record exists, the barrier refuses the unverified edition regardless of it, and the same
 // exemption is the only way a required record may be absent.
 
-import { existsSync, readFileSync } from "node:fs"
+import { existsSync, readdirSync, readFileSync, realpathSync } from "node:fs"
 import { join } from "node:path"
 
 import { CliError, readJsonFile, sha256Of, writeJsonAtomic } from "./cli-support.mjs"
@@ -24,7 +24,10 @@ import { meaningCheckEnabled, requireLockedBrief } from "./fidelity-brief.mjs"
 import { blocksDelivery, deliveryBlockReason } from "./gates-fidelity.mjs"
 
 export const FIDELITY_RECORD = "fidelity-defects.json"
+/** The legacy session meaning record: a candidate for every edition whose bytes it claims, like any edition record (newest run wins). */
 export const MEANING_RECORD = "fidelity-meaning-defects.json"
+/** The meaning record of one delivered edition, named by its outcome format (`fact meaning-check --edition`). */
+export const meaningRecordName = (format) => `fidelity-meaning-defects.${format}.json`
 
 /** The fields every fidelity record carries, whatever its kind. */
 export const RECORD_ENVELOPE = Object.freeze(["record_kind", "artifact_sha256", "report", "written_at", "defects"])
@@ -39,13 +42,67 @@ function meaningRequired({ sessionDir }) {
 }
 
 const subjectOf = (row) => row.renderedFrom ?? row.path ?? null
+const hashOf = (path) => (existsSync(path) ? sha256Of(readFileSync(path)) : null)
 
 /** One record per delivered edition: the file its row names with --verified-by, bound to that edition's bytes. */
 const perEdition = (sessionDir, delivered) =>
 	delivered.map((row) => ({ path: row.verifiedBy ?? null, declared: row.verifiedBy !== undefined, format: row.format, at: `${row.format}: `, subjects: [subjectOf(row)].filter((subject) => subject !== null) }))
 
-/** One record per session, in the session directory, bound to any delivered edition's bytes. */
-const perSession = (file) => (sessionDir, delivered) => [{ path: join(sessionDir, file), declared: false, format: null, at: "", subjects: delivered.map(subjectOf).filter((subject) => subject !== null) }]
+const sessionTarget = (sessionDir, file) => ({ path: join(sessionDir, file), declared: false, format: null, at: "", subjects: [] })
+
+/** Include every existing session fidelity record, even when no delivered row attaches it. */
+const perEditionAndSession = (sessionDir, delivered) => [
+	...perEdition(sessionDir, delivered),
+	...(existsSync(join(sessionDir, FIDELITY_RECORD)) ? [sessionTarget(sessionDir, FIDELITY_RECORD)] : []),
+]
+
+/** Edition meaning records on disk (fidelity-meaning-defects.<format>.json). */
+const EDITION_MEANING = /^fidelity-meaning-defects\.[^.]+\.json$/u
+
+/** Verdict severity for equal times: a confirmed drift, then any other blocking defect (unverified), then clean. */
+const severityOf = (defects) =>
+	defects.some((defect) => defect?.code === "meaning_drift_confirmed") ? 2 : defects.some((defect) => typeof defect?.code === "string" && blocksDelivery(defect)) ? 1 : 0
+
+/**
+ * What selection reads from a meaning record: the bytes it claims, when it was written, how severe it is.
+ * An unreadable record matches no bytes (its own row still reads and refuses it); a record with no
+ * readable written_at sorts newest, so the barrier reads it and refuses it as malformed.
+ */
+function claimOf(path) {
+	let record
+	try {
+		record = readJsonFile(path)
+	} catch {
+		return null
+	}
+	const at = Date.parse(record?.written_at)
+	return { path, hash: record?.artifact_sha256 ?? null, at: Number.isNaN(at) ? Infinity : at, severity: severityOf(Array.isArray(record?.defects) ? record.defects : []) }
+}
+
+/** Newest written_at first; on equal times the more severe verdict first. File names never decide. */
+const newestFirst = (a, b) => (a.at === b.at ? b.severity - a.severity : b.at > a.at ? 1 : -1)
+
+/**
+ * One meaning record per delivered edition, bound to that edition's bytes. Every record whose
+ * artifact_sha256 equals the edition's current bytes (a PDF counts its --rendered-from bytes) is a
+ * candidate: its own fidelity-meaning-defects.<format>.json, another edition's record (a
+ * byte-identical copy, or the html edition a PDF renders), and the legacy session record. The
+ * newest reader run wins; on equal times the more severe verdict wins. When no record claims these
+ * bytes the target is its own record (refused as for a different edition), else the legacy record,
+ * else its own name, so the refusal names the file to write (meaning gate not run).
+ */
+function perEditionMeaning(sessionDir, delivered) {
+	if (delivered.length === 0) return [sessionTarget(sessionDir, MEANING_RECORD)]
+	const legacy = join(sessionDir, MEANING_RECORD)
+	const editions = readdirSync(sessionDir).filter((name) => EDITION_MEANING.test(name)).map((name) => join(sessionDir, name))
+	const claims = [...editions, ...(existsSync(legacy) ? [legacy] : [])].map(claimOf).filter((claim) => claim !== null)
+	return perEdition(sessionDir, delivered).map((target) => {
+		const own = join(sessionDir, meaningRecordName(target.format))
+		const bytes = target.subjects.length > 0 ? hashOf(target.subjects[0]) : null
+		const [newest] = bytes === null ? [] : claims.filter((claim) => claim.hash === bytes).sort(newestFirst)
+		return { ...target, declared: false, path: newest?.path ?? (existsSync(own) ? own : existsSync(legacy) ? legacy : own) }
+	})
+}
 
 /** Every fidelity record the barrier reads, in check order; each column is one rule the barrier applies. */
 export const FIDELITY_RECORDS = Object.freeze([
@@ -53,7 +110,7 @@ export const FIDELITY_RECORDS = Object.freeze([
 		kind: "fidelity",
 		file: FIDELITY_RECORD,
 		step: "fidelity gate",
-		targets: perEdition,
+		targets: perEditionAndSession,
 		requiredWhen: ({ hasFidelity }, target) => hasFidelity || GATED_FORMATS.includes(target.format),
 		remedy: (target) => `run fact verify on ${target.subjects[0] ?? "the edition"} and set it with --verified-by`,
 		bindsTo: "artifact_sha256",
@@ -64,17 +121,25 @@ export const FIDELITY_RECORDS = Object.freeze([
 		kind: "meaning",
 		file: MEANING_RECORD,
 		step: "meaning gate",
-		targets: perSession(MEANING_RECORD),
+		targets: perEditionMeaning,
 		requiredWhen: meaningRequired,
-		remedy: () => "run fact pairs and fact meaning-check before delivery",
+		remedy: (target) => `run fact pairs and fact meaning-check --edition ${target.format} on ${target.subjects[0] ?? "the edition"} before delivery`,
 		bindsTo: "artifact_sha256",
 		blocking: blocksDelivery,
 		payload: Object.freeze(["command", "status", "exitCode", "run_id", "pairs_sha256", "counts", "dropped", "unverified", "calibration_path", "calibration_written", "calibration_replaced", "promotion_eligibility"]),
 	}),
 ])
 
+// Collection warnings use the same envelope, but are not delivery-gate records.
+const SOURCE_RECORDS = Object.freeze([
+	Object.freeze({
+		kind: "source_conversion",
+		payload: Object.freeze(["conversion_altered_numbers", "lost", "raw_count", "converted_count"]),
+	}),
+])
+
 const entryOf = (kind) => {
-	const entry = FIDELITY_RECORDS.find((row) => row.kind === kind)
+	const entry = [...FIDELITY_RECORDS, ...SOURCE_RECORDS].find((row) => row.kind === kind)
 	if (entry === undefined) throw new CliError(`unknown fidelity record kind ${kind}`)
 	return entry
 }
@@ -124,19 +189,20 @@ export function readFidelityRecord(path, kind) {
 }
 
 const unresolved = (format, codes, path) => ({ code: "fidelity_unresolved", format, codes, message: codes.map((code) => deliveryBlockReason({ code })).join("; "), path })
-const hashOf = (path) => (existsSync(path) ? sha256Of(readFileSync(path)) : null)
 
 /** Barrier problems for `outcome verify` and `outcome finish`; empty when nothing blocks delivery. */
 export function fidelityProblems(sessionDir, manifest) {
 	const delivered = manifest.deliverables.filter((row) => row.status === "delivered")
-	const hasFidelity = existsSync(join(sessionDir, LEDGER_FILENAME)) || FIDELITY_RECORDS.some((entry) => existsSync(join(sessionDir, entry.file)))
+	const targets = FIDELITY_RECORDS.map((entry) => [entry, entry.targets(sessionDir, delivered)])
+	// Any session record a row did not name (a session fidelity record, a meaning record) is fidelity evidence.
+	const hasFidelity = existsSync(join(sessionDir, LEDGER_FILENAME)) || targets.some(([, list]) => list.some((target) => !target.declared && target.path !== null && existsSync(target.path)))
 	const lastFidelity = manifest.gateHistory?.findLast((entry) => entry.gate === "fidelity")
 	const exempt = !hasFidelity && manifest.gates.fidelity === "not_run" && lastFidelity?.status === "not_run" && Boolean(lastFidelity.reason?.trim())
 	const context = { sessionDir, hasFidelity }
 	const problems = []
 	const counted = new Set()
-	for (const entry of FIDELITY_RECORDS) {
-		for (const target of entry.targets(sessionDir, delivered)) {
+	for (const [entry, list] of targets) {
+		for (const target of list) {
 			const refuse = (message, path = target.path) => problems.push({ code: "edition_unverified", format: target.format, message: `${target.at}${message}`, path })
 			if (target.path === null || !existsSync(target.path)) {
 				if (target.declared) refuse(`${entry.kind} record ${target.path} is missing`)
@@ -151,10 +217,11 @@ export function fidelityProblems(sessionDir, manifest) {
 				continue
 			}
 			if (target.subjects.length > 0 && !target.subjects.some((subject) => hashOf(subject) === record[entry.bindsTo])) {
-				refuse(`${entry.kind} record ${target.path} is for a different edition than ${target.subjects.join(", ")}`)
+				refuse(`${entry.kind} record ${target.path} is for a different edition than ${target.subjects.join(", ")}; ${entry.remedy(target)}`)
 			}
-			if (counted.has(target.path)) continue
-			counted.add(target.path)
+			const path = realpathSync.native(target.path)
+			if (counted.has(path)) continue
+			counted.add(path)
 			const codes = [...new Set(record.defects.filter(entry.blocking).map((defect) => defect.code))].sort()
 			if (codes.length > 0) problems.push(unresolved(target.format, codes, target.path))
 		}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-snapshot-readability.mjs

@@ -0,0 +1,49 @@
+// Availability is checked before hash/locator checks, including the raw-byte snapshot contract.
+// One code (snapshot_hash_mismatch) for every unreadable or changed snapshot; the message names the cause.
+import { existsSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+import { sha256Of } from "./cli-support.mjs"
+
+function read(dir, file) {
+	try {
+		return readFileSync(join(dir, file))
+	} catch (error) {
+		if (error.code === "ENOENT") throw new Error(`${file} is missing`)
+		if (error.code === "EISDIR") throw new Error(`${file} is not a readable file`)
+		throw new Error(`${file} is unreadable (${error.code ?? error.message})`)
+	}
+}
+
+/** Why the snapshot behind `row` cannot be trusted, or null when it reads and matches its pinned hash. */
+export function snapshotCause(sourcesDir, row) {
+	if (!sourcesDir) return "has no sources directory"
+	const dir = join(sourcesDir, row.source_id)
+	try {
+		let meta
+		try {
+			meta = JSON.parse(read(dir, "meta.json").toString("utf8"))
+		} catch (error) {
+			throw error instanceof SyntaxError ? new Error("meta.json is not valid JSON") : error
+		}
+		if (!meta || typeof meta.sha256 !== "string") return "has invalid snapshot metadata (meta.json has no sha256)"
+		const needsRaw = ["bytes", "utf8-bytes"].includes(row.locator?.unit) && meta.kind !== "converted-only"
+		const file = existsSync(join(dir, "raw.bin")) || needsRaw ? "raw.bin" : "text.txt"
+		return sha256Of(read(dir, file)) === meta.sha256 ? null : `hash mismatch: ${file} changed after its snapshot was pinned`
+	} catch (error) {
+		return error.message
+	}
+}
+
+/** Flags each cited row whose own snapshot is unreadable; returns those source ids so healthy sources keep their metadata. */
+export function unreadableSnapshots(ctx, rows, cited, sourcesDir, out) {
+	const unreadable = new Set()
+	for (const [id, node] of cited) {
+		const row = rows.get(id)
+		if (!row?.source_id) continue
+		const cause = snapshotCause(sourcesDir, row)
+		if (cause === null) continue
+		unreadable.add(row.source_id)
+		out.push({ ...ctx.defect("snapshot_hash_mismatch", node, `${id}: snapshot ${row.source_id} ${cause}`), fact_id: id })
+	}
+	return unreadable
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-sources.mjs

@@ -0,0 +1,76 @@
+// Source integrity and raw quote anchoring, shared by the fidelity value checks. Fails closed:
+// a cited source-backed row whose sources dir, meta.json, raw.bin, or text.txt cannot be read, or
+// whose snapshot no longer matches its pin, gets snapshot_hash_mismatch (with fact_id) naming the
+// cause; healthy sources keep their metadata and checks. flag is (code, node, message, extra).
+import { join } from "node:path"
+
+import { proseText } from "./html-lite.mjs"
+import { reanchor } from "./raw-locator.mjs"
+import { readSnapshot } from "./source-view.mjs"
+
+const short = (hash) => String(hash ?? "none").slice(0, 12)
+
+function snapshotOf(sourcesDir, id) {
+	if (!sourcesDir) throw new Error("has no sources directory")
+	return readSnapshot(join(sourcesDir, id))
+}
+
+const blocked = (flag, cited, row, id, cause) => flag("snapshot_hash_mismatch", cited.get(row.fact_id), `${row.fact_id}: snapshot ${id} ${cause}`, { fact_id: row.fact_id })
+
+/** One source's checks for the rows that cite it; byte-located rows are re-anchored on the view. */
+function rowChecks(snap, id, referenced, cited, flag) {
+	const converted = snap.view.mode === "converted"
+	for (const row of referenced) {
+		const node = cited.get(row.fact_id)
+		// No locator unit marks a legacy text.txt row: it holds only on the legacy snapshot it pins.
+		if (row.locator?.unit === undefined) {
+			if (snap.view.mode !== "legacy") blocked(flag, cited, row, id, `is no longer the legacy text.txt snapshot the row was located on (now ${snap.view.mode}); add the row again from the new snapshot`)
+			else if (row.snapshot_sha256 !== snap.sha256) blocked(flag, cited, row, id, `holds ${short(snap.sha256)} but the legacy row pins ${short(row.snapshot_sha256)}`)
+			continue
+		}
+		if (snap.view.mode === "legacy") {
+			blocked(flag, cited, row, id, "raw.bin is missing (the row is byte-located)")
+			continue
+		}
+		if (converted && row.kind !== "quote") {
+			flag("snapshot_not_raw", node, `${row.fact_id} cannot measure converted-only source ${id}`)
+			continue
+		}
+		if (row.snapshot_sha256 !== snap.sha256) {
+			blocked(flag, cited, row, id, `holds ${short(snap.sha256)} but the row pins ${short(row.snapshot_sha256)}; run source refresh ${id} to re-anchor`)
+			continue
+		}
+		const anchor = row.locator.artifact === snap.view.artifact ? reanchor(snap.view, row) : null
+		if (!anchor) flag("quote_not_verbatim", node, `${row.fact_id}: no unique exact quote and context in ${id}/${snap.view.artifact}`)
+		else if (anchor.relocated) flag("relocated", node, `${row.fact_id}: ${row.locator.char_start}:${row.locator.char_end} -> ${anchor.start}:${anchor.end} (bytes of ${id}/${snap.view.artifact})`)
+		if (converted && row.kind === "quote") flag("quote_converted_only", node, `${row.fact_id} quotes converted-only source ${id}, not the original fetched bytes`)
+	}
+}
+
+export function sourceChecks(ctx, rows, cited, sourcesDir, flag, unreadable = new Set()) {
+	const referenced = new Map()
+	for (const key of cited.keys()) {
+		const row = rows.get(key)
+		if (row?.source_id) referenced.set(row.source_id, [...(referenced.get(row.source_id) ?? []), row])
+	}
+	const ids = new Set(referenced.keys())
+	for (const match of proseText(ctx.body).matchAll(/\[S(\d+)\]/g)) ids.add(`S${match[1]}`)
+	const meta = new Map()
+	for (const id of ids) {
+		if (unreadable.has(id)) continue
+		const backing = referenced.get(id) ?? []
+		let snap
+		try {
+			snap = snapshotOf(sourcesDir, id)
+		} catch (error) {
+			// A source only named in prose ([S<n>]) backs no row: an absent snapshot is not a finding.
+			for (const row of backing) blocked(flag, cited, row, id, error.message)
+			continue
+		}
+		meta.set(id, snap.meta)
+		if (snap.drift.length === 0) rowChecks(snap, id, backing, cited, flag)
+		else if (backing.length === 0) flag("snapshot_hash_mismatch", ctx.body, `snapshot ${id} ${snap.drift.join("; ")}`)
+		else for (const row of backing) blocked(flag, cited, row, id, snap.drift.join("; "))
+	}
+	return meta
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/fidelity-trace.mjs

@@ -13,9 +13,12 @@
 // so each entry, in array order, is matched to the next untraced occurrence of its rendered text in
 // document order that is a whole numeral and reads as the entry's row there (fits: "1,237 million"
 // is not F2's 1,237), and strips that one occurrence only. A copy of the same text that no entry
-// accounts for stays in the scan; an entry whose text is gone strips nothing.
+// accounts for stays in the scan; an entry whose text is gone strips nothing. A derived or
+// converted entry is located under the bracket grammar of fidelity-originals.mjs (renderLocator).
 
+import { PROSE_TAGS } from "./html-lite.mjs"
 import { NOT_CLAIMS } from "./numerals.mjs"
+import { sourcesClaims, textClaims } from "./sources-citation.mjs"
 
 /** Structural numerals: versions, figure/table/section references, times (NOT_CLAIMS 7-11), pages, citations, placeholders. */
 export const STRUCTURAL = [...NOT_CLAIMS.slice(7, 12), /\b(?:p|pp|page|pages)\.?\s*\d+(?:\s*[-\u2013]\s*\d+)?/gi, /\[(?:S|Source\s*)\d+\]|\[\^\d+\]/gi, /\{\{[^}]*\}\}/g]
@@ -29,7 +32,12 @@ function ownerTexts(ctx, charts) {
 	const owners = new Map()
 	for (const entry of ctx.texts) {
 		const kind = ctx.kindOf(entry.scope)
-		if (kind === "sources" || kind === "toc") continue
+		if (kind === "toc") continue
+		if (kind === "sources") {
+			if (!PROSE_TAGS.has(entry.owner.tag)) owners.set(entry.owner, `${owners.get(entry.owner) ?? ""} ${textClaims(entry.node.text).join(" ")}`)
+			else if (!owners.has(entry.owner)) owners.set(entry.owner, sourcesClaims(entry.owner).join(" "))
+			continue
+		}
 		if (charts.has(ctx.util.closest(entry.node, (cur) => cur.tag === "figure"))) continue
 		const hidden = ctx.util.closest(entry.node, (cur) => cur.attrs["data-fact"] !== undefined || cur.attrs["data-translates"] !== undefined || cur.tag === "q" || cur.tag === "blockquote")
 		owners.set(entry.owner, (owners.get(entry.owner) ?? "") + (hidden ? ` ${TRACED} ` : entry.node.text))
@@ -52,8 +60,13 @@ function scanOwner(ctx, owner, raw, flag) {
 	for (const match of rest.matchAll(NUMERAL)) flag("untraceable_number", owner, `number "${match[0]}" has no ledger row: "${clip(rest.slice(Math.max(0, match.index - 30), match.index + 30))}"`)
 }
 
+const plainSite = (entry, text, from) => {
+	const hit = text.indexOf(entry.rendered, from)
+	return hit === -1 ? null : [hit, hit + entry.rendered.length]
+}
+
 /** Strip each fill-map entry at its own fill site: the next untraced occurrence after the previous entry's site. */
-function traceFills(owners, fillMap, fits) {
+function traceFills(owners, fillMap, fits, locate = plainSite) {
 	const keys = [...owners.keys()]
 	const texts = [...owners.values()]
 	let [index, from] = [0, 0]
@@ -61,12 +74,12 @@ function traceFills(owners, fillMap, fits) {
 		const { rendered } = entry
 		if (!rendered) continue
 		for (let i = index, at = from; i < texts.length; i += 1, at = 0) {
-			let hit = texts[i].indexOf(rendered, at)
-			const glued = (pos) => /\d/.test(texts[i][pos - 1] ?? "") || /\d/.test(texts[i][pos + rendered.length] ?? "")
-			while (hit !== -1 && (glued(hit) || !fits(entry, texts[i].slice(hit)))) hit = texts[i].indexOf(rendered, hit + 1)
-			if (hit === -1) continue
-			texts[i] = `${texts[i].slice(0, hit)}${TRACED}${texts[i].slice(hit + rendered.length)}`
-			;[index, from] = [i, hit + TRACED.length]
+			let hit = locate(entry, texts[i], at)
+			const glued = ([start, end]) => (/^\d/.test(rendered) && /\d/.test(texts[i][start - 1] ?? "")) || (/\d$/.test(rendered) && /\d/.test(texts[i][end] ?? ""))
+			while (hit && (glued(hit) || !fits(entry, texts[i].slice(hit[0])))) hit = locate(entry, texts[i], hit[0] + 1)
+			if (!hit) continue
+			texts[i] = `${texts[i].slice(0, hit[0])}${TRACED}${texts[i].slice(hit[1])}`
+			;[index, from] = [i, hit[0] + TRACED.length]
 			break
 		}
 	}
@@ -79,7 +92,8 @@ function traceFills(owners, fillMap, fits) {
  * @param {{ fact_id: string, rendered: string, start?: number, end?: number }[] | null} fillMap Markdown fill map, or null
  * @param {(code: string, node: object, message: string) => void} flag
  * @param {(entry: object, tail: string) => boolean} fits whether the text from a candidate site on reads as the entry's row
+ * @param {(entry: object, text: string, from: number) => [number, number] | null} [locate] the next candidate site of an entry's text
  */
-export function traceChecks(ctx, charts, fillMap, flag, fits) {
-	for (const [owner, text] of traceFills(ownerTexts(ctx, charts), fillMap, fits)) scanOwner(ctx, owner, text, flag)
+export function traceChecks(ctx, charts, fillMap, flag, fits, locate) {
+	for (const [owner, text] of traceFills(ownerTexts(ctx, charts), fillMap, fits, locate)) scanOwner(ctx, owner, text, flag)
 }