oh-my-opencode 5.1.47 → 5.1.51

package/packages/shared-skills/skills/ulw-research/scripts/fidelity-policy.mjs

@@ -0,0 +1,183 @@
+// fidelity-policy.mjs - what the locked `## Fidelity` block of brief.md means to the CLI stages.
+//
+// lockedBrief is the gate the six reader-facing stages (fact fill, verify, pairs, meaning-check,
+// polish-apply, polish-diff) run before reading or writing anything: an unlocked brief returns
+// one fidelity_interview_incomplete defect (exit 1), a malformed block is a usage error (exit 2).
+// fillPolicy reads the fields fact fill consumes (a value off its table is a usage error, exit 2),
+// and policyErrors refuses a placeholder whose row the brief forbids: derived_not_allowed
+// (`derived numbers: none` against a derived, converted, or assumed row) and source_outside_floor
+// (the source floor is evaluated live against each source's meta.json at every fill and verify;
+// under any floor but `standard` a row resting on no source is refused too; a row's outside_floor
+// is only the cached result). briefHashOf binds the locked block into the fill map, and
+// briefBindingErrors refuses an edition whose map was filled under another brief (brief_changed).
+// Collection stages never call this module. node: builtins and sibling imports only.
+
+import { existsSync, readFileSync } from "node:fs"
+import { join, resolve } from "node:path"
+
+import { CliError, sha256Of } from "./cli-support.mjs"
+import { DEFECT_CODES } from "./contracts.mjs"
+import { KIND_RULES } from "./fact-ledger-kinds.mjs"
+import { FIDELITY_FIELDS, meaningCheckEnabled, requireLockedBrief } from "./fidelity-brief.mjs"
+
+export { meaningCheckEnabled }
+
+/**
+ * Ledger kinds a brief with `derived numbers: none` refuses. A policy list, not the ledger's
+ * derivation set: it also names assumed. Every name must be a KIND_RULES kind (checked at load).
+ */
+const DERIVED_KINDS = Object.freeze(["derived", "converted", "assumed"])
+for (const kind of DERIVED_KINDS) {
+	if (!Object.hasOwn(KIND_RULES, kind)) throw new Error(`fidelity-policy: ${kind} is not a KIND_RULES kind`)
+}
+
+/** Fields whose only legal values are a closed table: value (any case) -> its reading. */
+const VALUE_TABLES = Object.freeze({
+	"derived numbers": Object.freeze({ allowed: false, none: true }),
+	fallback: Object.freeze({ "block-quote": "quote", omit: "omit" }),
+})
+
+/** The reading of a table value; any other text is a usage error naming the field and the value. */
+function tableValue(field, value) {
+	const table = VALUE_TABLES[field]
+	const key = value.trim().toLowerCase()
+	if (Object.hasOwn(table, key)) return table[key]
+	throw new CliError(`${field} "${value}" is not ${Object.keys(table).join(", ")}`)
+}
+
+/** How each field value the fill stages consume is read; the single owner of those readings. */
+export const FIDELITY_VALUE_RULES = Object.freeze({
+	tables: VALUE_TABLES,
+	derivedKinds: DERIVED_KINDS,
+	derivedRefused: (kind) => DERIVED_KINDS.includes(kind),
+	derivedNone: (value) => tableValue("derived numbers", value),
+	fallback: (value) => tableValue("fallback", value),
+	translations: (value) => /with translation beside/iu.test(value),
+	meaningCheck: (value) => meaningCheckEnabled({ fields: { "meaning check": value } }),
+})
+
+/** The `source floor` forms: `pattern` reads one, `listed` forms carry a comma list. */
+export const SOURCE_FLOOR_RULES = Object.freeze([
+	Object.freeze({ kind: "standard", form: "standard", pattern: /^standard$/iu, listed: false }),
+	Object.freeze({ kind: "primary", form: "primary only", pattern: /^primary only$/iu, listed: false }),
+	Object.freeze({ kind: "pointed", form: "pointed only: <list>", pattern: /^pointed only:\s*(.+)$/iu, listed: true }),
+	Object.freeze({ kind: "classes", form: "classes: <list>", pattern: /^classes:\s*(.+)$/iu, listed: true }),
+])
+const lineOf = (text, at) => text.slice(0, at).split("\n").length
+const listOf = (text) => text.split(",").map((entry) => entry.trim()).filter(Boolean)
+
+/** brief.md from --brief, else the session's; a usage error when neither is known. */
+export function briefPathOf(flags) {
+	if (flags.brief !== undefined && flags.brief !== true) return resolve(flags.brief)
+	const dir = flags["session-dir"] ?? process.env.SESSION_DIR
+	if (!dir || dir === true) throw new CliError("--brief is required without --session-dir (or SESSION_DIR)")
+	return join(resolve(dir), "brief.md")
+}
+
+/**
+ * @returns {{ parsed: object, refusal: null } | { parsed: null, refusal: { json: object, summary: string, exitCode: 1 } }}
+ */
+export function lockedBrief(command, flags) {
+	const path = briefPathOf(flags)
+	const lock = requireLockedBrief(path)
+	if (lock.exitCode === 2) throw new CliError(`${path}: ${lock.errors.join("; ")}`)
+	if (lock.locked) return { parsed: lock.parsed, refusal: null }
+	const json = { command, status: "fail", exitCode: 1, defects: lock.defects }
+	return { parsed: null, refusal: { json, summary: `${command}: ${lock.defects[0].message}`, exitCode: 1 } }
+}
+
+/**
+ * The source floor as a rule, or null for `standard`; a usage error for any other text.
+ * @returns {null | { kind: "primary" } | { kind: "pointed" | "classes", list: string[] }}
+ */
+export function parseSourceFloor(value) {
+	const text = value.trim()
+	for (const rule of SOURCE_FLOOR_RULES) {
+		const match = rule.pattern.exec(text)
+		if (!match) continue
+		if (rule.kind === "standard") return null
+		return rule.listed ? { kind: rule.kind, list: listOf(match[1]) } : { kind: rule.kind }
+	}
+	throw new CliError(`source floor "${value}" is not ${SOURCE_FLOOR_RULES.map((rule) => rule.form).join(", ")}`)
+}
+
+function hostOf(url) {
+	return typeof url === "string" && URL.canParse(url) ? new URL(url).hostname.toLowerCase() : null
+}
+
+/** Whether one source (id and meta.json) satisfies a parsed floor rule. */
+export function sourceInsideFloor(rule, sourceId, meta) {
+	if (rule.kind === "primary") return meta.provenance === "primary"
+	if (rule.kind === "pointed") return rule.list.includes(sourceId) || rule.list.includes(meta.url)
+	const host = hostOf(meta.url)
+	return host !== null && rule.list.some((entry) => host === entry.toLowerCase() || host.endsWith(`.${entry.toLowerCase()}`))
+}
+
+/** The fields fact fill consumes: the fallback form and whether a translation binds to its quote. */
+export function fillPolicy(parsed) {
+	const field = (name) => parsed.fields[name]
+	return {
+		fallback: FIDELITY_VALUE_RULES.fallback(field("fallback")),
+		translations: FIDELITY_VALUE_RULES.translations(field("languages")),
+		derivedNone: FIDELITY_VALUE_RULES.derivedNone(field("derived numbers")),
+		floor: parseSourceFloor(field("source floor")),
+	}
+}
+
+/** Source ids behind a row: its own, or for a derived row its operands' sources; null marks a leaf with no source. */
+function sourcesOf(row, rows, seen = new Set()) {
+	if (seen.has(row.fact_id)) return []
+	seen.add(row.fact_id)
+	if (row.source_id) return [row.source_id]
+	const inputs = [...(row.operands ?? []), ...(row.derived_from ?? [])]
+	if (inputs.length === 0) return [null]
+	return inputs.flatMap((id) => (rows.has(id) ? sourcesOf(rows.get(id), rows, seen) : [null]))
+}
+
+/** sha256 of the locked `## Fidelity` block normalized: the seven fields in FIDELITY_FIELDS order, `field: value` per line. */
+export const briefHashOf = (parsed) => sha256Of(FIDELITY_FIELDS.map((field) => `${field}: ${parsed.fields[field]}`).join("\n"))
+
+/** brief_changed when there is no fill map (map null) or it was not written under the current brief (or predates the binding). */
+export function briefBindingErrors(map, mapPath, parsed) {
+	if (map?.brief_sha256 === briefHashOf(parsed)) return []
+	const why = map === null ? `no fill map at ${mapPath}` : `${mapPath} ${typeof map.brief_sha256 === "string" ? "was filled under a different ## Fidelity block" : "records no brief_sha256"}`
+	return [{ code: "brief_changed", token: null, line: null, message: `${why}; re-run fact fill under the current brief`, severity: DEFECT_CODES.brief_changed.severity, integrity: DEFECT_CODES.brief_changed.integrity, hint: DEFECT_CODES.brief_changed.hint }]
+}
+
+function metaOf(sessionDir, sourceId, cache) {
+	if (!cache.has(sourceId)) {
+		const file = join(sessionDir, "sources", sourceId, "meta.json")
+		cache.set(sourceId, existsSync(file) ? JSON.parse(readFileSync(file, "utf8")) : { url: null, provenance: "unknown" })
+	}
+	return cache.get(sourceId)
+}
+
+const defect = (code, pointer, text, message) => ({ code, token: pointer.token, line: lineOf(text, pointer.start), message, severity: DEFECT_CODES[code].severity, hint: DEFECT_CODES[code].hint })
+
+/**
+ * Refusals for the placeholders of one fill source (or the data-fact elements of one edition at
+ * verify). Sets each referenced row's outside_floor to this evaluation and reports whether any
+ * cached value changed.
+ * @param {{ token: string, start: number, parsed: { id?: string } }[]} found findPlaceholders output
+ * @returns {{ errors: object[], cacheChanged: boolean }}
+ */
+export function policyErrors(text, found, ledger, sessionDir, policy) {
+	const rows = new Map(ledger.rows.map((row) => [row.fact_id, row]))
+	const metas = new Map()
+	const errors = []
+	let cacheChanged = false
+	for (const pointer of found) {
+		const row = rows.get(pointer.parsed?.id)
+		if (row === undefined || row.status !== "active") continue
+		if (policy.derivedNone && FIDELITY_VALUE_RULES.derivedRefused(row.kind)) {
+			errors.push(defect("derived_not_allowed", pointer, text, `${row.fact_id} is ${row.kind}; the brief records derived numbers: none`))
+		}
+		const sources = sourcesOf(row, rows)
+		const outside = policy.floor === null ? [] : [...new Set(sources.filter((id) => id === null || !sourceInsideFloor(policy.floor, id, metaOf(sessionDir, id, metas))))]
+		if (row.outside_floor !== outside.length > 0) cacheChanged = true
+		row.outside_floor = outside.length > 0
+		const named = outside.map((id) => id ?? "no source").join(", ")
+		if (outside.length > 0) errors.push(defect("source_outside_floor", pointer, text, `${row.fact_id} rests on ${named}, outside the source floor`))
+	}
+	return { errors, cacheChanged }
+}

package/packages/shared-skills/skills/ulw-research/scripts/fidelity-records.mjs

@@ -15,7 +15,7 @@
 // fidelity record exists, the barrier refuses the unverified edition regardless of it, and the same
 // exemption is the only way a required record may be absent.
 
-import { existsSync, readFileSync } from "node:fs"
+import { existsSync, readdirSync, readFileSync, realpathSync } from "node:fs"
 import { join } from "node:path"
 
 import { CliError, readJsonFile, sha256Of, writeJsonAtomic } from "./cli-support.mjs"
@@ -24,7 +24,10 @@ import { meaningCheckEnabled, requireLockedBrief } from "./fidelity-brief.mjs"
 import { blocksDelivery, deliveryBlockReason } from "./gates-fidelity.mjs"
 
 export const FIDELITY_RECORD = "fidelity-defects.json"
+/** The legacy session meaning record: a candidate for every edition whose bytes it claims, like any edition record (newest run wins). */
 export const MEANING_RECORD = "fidelity-meaning-defects.json"
+/** The meaning record of one delivered edition, named by its outcome format (`fact meaning-check --edition`). */
+export const meaningRecordName = (format) => `fidelity-meaning-defects.${format}.json`
 
 /** The fields every fidelity record carries, whatever its kind. */
 export const RECORD_ENVELOPE = Object.freeze(["record_kind", "artifact_sha256", "report", "written_at", "defects"])
@@ -39,13 +42,67 @@ function meaningRequired({ sessionDir }) {
 }
 
 const subjectOf = (row) => row.renderedFrom ?? row.path ?? null
+const hashOf = (path) => (existsSync(path) ? sha256Of(readFileSync(path)) : null)
 
 /** One record per delivered edition: the file its row names with --verified-by, bound to that edition's bytes. */
 const perEdition = (sessionDir, delivered) =>
 	delivered.map((row) => ({ path: row.verifiedBy ?? null, declared: row.verifiedBy !== undefined, format: row.format, at: `${row.format}: `, subjects: [subjectOf(row)].filter((subject) => subject !== null) }))
 
-/** One record per session, in the session directory, bound to any delivered edition's bytes. */
-const perSession = (file) => (sessionDir, delivered) => [{ path: join(sessionDir, file), declared: false, format: null, at: "", subjects: delivered.map(subjectOf).filter((subject) => subject !== null) }]
+const sessionTarget = (sessionDir, file) => ({ path: join(sessionDir, file), declared: false, format: null, at: "", subjects: [] })
+
+/** Include every existing session fidelity record, even when no delivered row attaches it. */
+const perEditionAndSession = (sessionDir, delivered) => [
+	...perEdition(sessionDir, delivered),
+	...(existsSync(join(sessionDir, FIDELITY_RECORD)) ? [sessionTarget(sessionDir, FIDELITY_RECORD)] : []),
+]
+
+/** Edition meaning records on disk (fidelity-meaning-defects.<format>.json). */
+const EDITION_MEANING = /^fidelity-meaning-defects\.[^.]+\.json$/u
+
+/** Verdict severity for equal times: a confirmed drift, then any other blocking defect (unverified), then clean. */
+const severityOf = (defects) =>
+	defects.some((defect) => defect?.code === "meaning_drift_confirmed") ? 2 : defects.some((defect) => typeof defect?.code === "string" && blocksDelivery(defect)) ? 1 : 0
+
+/**
+ * What selection reads from a meaning record: the bytes it claims, when it was written, how severe it is.
+ * An unreadable record matches no bytes (its own row still reads and refuses it); a record with no
+ * readable written_at sorts newest, so the barrier reads it and refuses it as malformed.
+ */
+function claimOf(path) {
+	let record
+	try {
+		record = readJsonFile(path)
+	} catch {
+		return null
+	}
+	const at = Date.parse(record?.written_at)
+	return { path, hash: record?.artifact_sha256 ?? null, at: Number.isNaN(at) ? Infinity : at, severity: severityOf(Array.isArray(record?.defects) ? record.defects : []) }
+}
+
+/** Newest written_at first; on equal times the more severe verdict first. File names never decide. */
+const newestFirst = (a, b) => (a.at === b.at ? b.severity - a.severity : b.at > a.at ? 1 : -1)
+
+/**
+ * One meaning record per delivered edition, bound to that edition's bytes. Every record whose
+ * artifact_sha256 equals the edition's current bytes (a PDF counts its --rendered-from bytes) is a
+ * candidate: its own fidelity-meaning-defects.<format>.json, another edition's record (a
+ * byte-identical copy, or the html edition a PDF renders), and the legacy session record. The
+ * newest reader run wins; on equal times the more severe verdict wins. When no record claims these
+ * bytes the target is its own record (refused as for a different edition), else the legacy record,
+ * else its own name, so the refusal names the file to write (meaning gate not run).
+ */
+function perEditionMeaning(sessionDir, delivered) {
+	if (delivered.length === 0) return [sessionTarget(sessionDir, MEANING_RECORD)]
+	const legacy = join(sessionDir, MEANING_RECORD)
+	const editions = readdirSync(sessionDir).filter((name) => EDITION_MEANING.test(name)).map((name) => join(sessionDir, name))
+	const claims = [...editions, ...(existsSync(legacy) ? [legacy] : [])].map(claimOf).filter((claim) => claim !== null)
+	return perEdition(sessionDir, delivered).map((target) => {
+		const own = join(sessionDir, meaningRecordName(target.format))
+		const bytes = target.subjects.length > 0 ? hashOf(target.subjects[0]) : null
+		const [newest] = bytes === null ? [] : claims.filter((claim) => claim.hash === bytes).sort(newestFirst)
+		return { ...target, declared: false, path: newest?.path ?? (existsSync(own) ? own : existsSync(legacy) ? legacy : own) }
+	})
+}
 
 /** Every fidelity record the barrier reads, in check order; each column is one rule the barrier applies. */
 export const FIDELITY_RECORDS = Object.freeze([
@@ -53,7 +110,7 @@ export const FIDELITY_RECORDS = Object.freeze([
 		kind: "fidelity",
 		file: FIDELITY_RECORD,
 		step: "fidelity gate",
-		targets: perEdition,
+		targets: perEditionAndSession,
 		requiredWhen: ({ hasFidelity }, target) => hasFidelity || GATED_FORMATS.includes(target.format),
 		remedy: (target) => `run fact verify on ${target.subjects[0] ?? "the edition"} and set it with --verified-by`,
 		bindsTo: "artifact_sha256",
@@ -64,17 +121,25 @@ export const FIDELITY_RECORDS = Object.freeze([
 		kind: "meaning",
 		file: MEANING_RECORD,
 		step: "meaning gate",
-		targets: perSession(MEANING_RECORD),
+		targets: perEditionMeaning,
 		requiredWhen: meaningRequired,
-		remedy: () => "run fact pairs and fact meaning-check before delivery",
+		remedy: (target) => `run fact pairs and fact meaning-check --edition ${target.format} on ${target.subjects[0] ?? "the edition"} before delivery`,
 		bindsTo: "artifact_sha256",
 		blocking: blocksDelivery,
 		payload: Object.freeze(["command", "status", "exitCode", "run_id", "pairs_sha256", "counts", "dropped", "unverified", "calibration_path", "calibration_written", "calibration_replaced", "promotion_eligibility"]),
 	}),
 ])
 
+// Collection warnings use the same envelope, but are not delivery-gate records.
+const SOURCE_RECORDS = Object.freeze([
+	Object.freeze({
+		kind: "source_conversion",
+		payload: Object.freeze(["conversion_altered_numbers", "lost", "raw_count", "converted_count"]),
+	}),
+])
+
 const entryOf = (kind) => {
-	const entry = FIDELITY_RECORDS.find((row) => row.kind === kind)
+	const entry = [...FIDELITY_RECORDS, ...SOURCE_RECORDS].find((row) => row.kind === kind)
 	if (entry === undefined) throw new CliError(`unknown fidelity record kind ${kind}`)
 	return entry
 }
@@ -124,19 +189,20 @@ export function readFidelityRecord(path, kind) {
 }
 
 const unresolved = (format, codes, path) => ({ code: "fidelity_unresolved", format, codes, message: codes.map((code) => deliveryBlockReason({ code })).join("; "), path })
-const hashOf = (path) => (existsSync(path) ? sha256Of(readFileSync(path)) : null)
 
 /** Barrier problems for `outcome verify` and `outcome finish`; empty when nothing blocks delivery. */
 export function fidelityProblems(sessionDir, manifest) {
 	const delivered = manifest.deliverables.filter((row) => row.status === "delivered")
-	const hasFidelity = existsSync(join(sessionDir, LEDGER_FILENAME)) || FIDELITY_RECORDS.some((entry) => existsSync(join(sessionDir, entry.file)))
+	const targets = FIDELITY_RECORDS.map((entry) => [entry, entry.targets(sessionDir, delivered)])
+	// Any session record a row did not name (a session fidelity record, a meaning record) is fidelity evidence.
+	const hasFidelity = existsSync(join(sessionDir, LEDGER_FILENAME)) || targets.some(([, list]) => list.some((target) => !target.declared && target.path !== null && existsSync(target.path)))
 	const lastFidelity = manifest.gateHistory?.findLast((entry) => entry.gate === "fidelity")
 	const exempt = !hasFidelity && manifest.gates.fidelity === "not_run" && lastFidelity?.status === "not_run" && Boolean(lastFidelity.reason?.trim())
 	const context = { sessionDir, hasFidelity }
 	const problems = []
 	const counted = new Set()
-	for (const entry of FIDELITY_RECORDS) {
-		for (const target of entry.targets(sessionDir, delivered)) {
+	for (const [entry, list] of targets) {
+		for (const target of list) {
 			const refuse = (message, path = target.path) => problems.push({ code: "edition_unverified", format: target.format, message: `${target.at}${message}`, path })
 			if (target.path === null || !existsSync(target.path)) {
 				if (target.declared) refuse(`${entry.kind} record ${target.path} is missing`)
@@ -151,10 +217,11 @@ export function fidelityProblems(sessionDir, manifest) {
 				continue
 			}
 			if (target.subjects.length > 0 && !target.subjects.some((subject) => hashOf(subject) === record[entry.bindsTo])) {
-				refuse(`${entry.kind} record ${target.path} is for a different edition than ${target.subjects.join(", ")}`)
+				refuse(`${entry.kind} record ${target.path} is for a different edition than ${target.subjects.join(", ")}; ${entry.remedy(target)}`)
 			}
-			if (counted.has(target.path)) continue
-			counted.add(target.path)
+			const path = realpathSync.native(target.path)
+			if (counted.has(path)) continue
+			counted.add(path)
 			const codes = [...new Set(record.defects.filter(entry.blocking).map((defect) => defect.code))].sort()
 			if (codes.length > 0) problems.push(unresolved(target.format, codes, target.path))
 		}

package/packages/shared-skills/skills/ulw-research/scripts/fidelity-snapshot-readability.mjs

@@ -0,0 +1,49 @@
+// Availability is checked before hash/locator checks, including the raw-byte snapshot contract.
+// One code (snapshot_hash_mismatch) for every unreadable or changed snapshot; the message names the cause.
+import { existsSync, readFileSync } from "node:fs"
+import { join } from "node:path"
+import { sha256Of } from "./cli-support.mjs"
+
+function read(dir, file) {
+	try {
+		return readFileSync(join(dir, file))
+	} catch (error) {
+		if (error.code === "ENOENT") throw new Error(`${file} is missing`)
+		if (error.code === "EISDIR") throw new Error(`${file} is not a readable file`)
+		throw new Error(`${file} is unreadable (${error.code ?? error.message})`)
+	}
+}
+
+/** Why the snapshot behind `row` cannot be trusted, or null when it reads and matches its pinned hash. */
+export function snapshotCause(sourcesDir, row) {
+	if (!sourcesDir) return "has no sources directory"
+	const dir = join(sourcesDir, row.source_id)
+	try {
+		let meta
+		try {
+			meta = JSON.parse(read(dir, "meta.json").toString("utf8"))
+		} catch (error) {
+			throw error instanceof SyntaxError ? new Error("meta.json is not valid JSON") : error
+		}
+		if (!meta || typeof meta.sha256 !== "string") return "has invalid snapshot metadata (meta.json has no sha256)"
+		const needsRaw = ["bytes", "utf8-bytes"].includes(row.locator?.unit) && meta.kind !== "converted-only"
+		const file = existsSync(join(dir, "raw.bin")) || needsRaw ? "raw.bin" : "text.txt"
+		return sha256Of(read(dir, file)) === meta.sha256 ? null : `hash mismatch: ${file} changed after its snapshot was pinned`
+	} catch (error) {
+		return error.message
+	}
+}
+
+/** Flags each cited row whose own snapshot is unreadable; returns those source ids so healthy sources keep their metadata. */
+export function unreadableSnapshots(ctx, rows, cited, sourcesDir, out) {
+	const unreadable = new Set()
+	for (const [id, node] of cited) {
+		const row = rows.get(id)
+		if (!row?.source_id) continue
+		const cause = snapshotCause(sourcesDir, row)
+		if (cause === null) continue
+		unreadable.add(row.source_id)
+		out.push({ ...ctx.defect("snapshot_hash_mismatch", node, `${id}: snapshot ${row.source_id} ${cause}`), fact_id: id })
+	}
+	return unreadable
+}

package/packages/shared-skills/skills/ulw-research/scripts/fidelity-sources.mjs

@@ -0,0 +1,76 @@
+// Source integrity and raw quote anchoring, shared by the fidelity value checks. Fails closed:
+// a cited source-backed row whose sources dir, meta.json, raw.bin, or text.txt cannot be read, or
+// whose snapshot no longer matches its pin, gets snapshot_hash_mismatch (with fact_id) naming the
+// cause; healthy sources keep their metadata and checks. flag is (code, node, message, extra).
+import { join } from "node:path"
+
+import { proseText } from "./html-lite.mjs"
+import { reanchor } from "./raw-locator.mjs"
+import { readSnapshot } from "./source-view.mjs"
+
+const short = (hash) => String(hash ?? "none").slice(0, 12)
+
+function snapshotOf(sourcesDir, id) {
+	if (!sourcesDir) throw new Error("has no sources directory")
+	return readSnapshot(join(sourcesDir, id))
+}
+
+const blocked = (flag, cited, row, id, cause) => flag("snapshot_hash_mismatch", cited.get(row.fact_id), `${row.fact_id}: snapshot ${id} ${cause}`, { fact_id: row.fact_id })
+
+/** One source's checks for the rows that cite it; byte-located rows are re-anchored on the view. */
+function rowChecks(snap, id, referenced, cited, flag) {
+	const converted = snap.view.mode === "converted"
+	for (const row of referenced) {
+		const node = cited.get(row.fact_id)
+		// No locator unit marks a legacy text.txt row: it holds only on the legacy snapshot it pins.
+		if (row.locator?.unit === undefined) {
+			if (snap.view.mode !== "legacy") blocked(flag, cited, row, id, `is no longer the legacy text.txt snapshot the row was located on (now ${snap.view.mode}); add the row again from the new snapshot`)
+			else if (row.snapshot_sha256 !== snap.sha256) blocked(flag, cited, row, id, `holds ${short(snap.sha256)} but the legacy row pins ${short(row.snapshot_sha256)}`)
+			continue
+		}
+		if (snap.view.mode === "legacy") {
+			blocked(flag, cited, row, id, "raw.bin is missing (the row is byte-located)")
+			continue
+		}
+		if (converted && row.kind !== "quote") {
+			flag("snapshot_not_raw", node, `${row.fact_id} cannot measure converted-only source ${id}`)
+			continue
+		}
+		if (row.snapshot_sha256 !== snap.sha256) {
+			blocked(flag, cited, row, id, `holds ${short(snap.sha256)} but the row pins ${short(row.snapshot_sha256)}; run source refresh ${id} to re-anchor`)
+			continue
+		}
+		const anchor = row.locator.artifact === snap.view.artifact ? reanchor(snap.view, row) : null
+		if (!anchor) flag("quote_not_verbatim", node, `${row.fact_id}: no unique exact quote and context in ${id}/${snap.view.artifact}`)
+		else if (anchor.relocated) flag("relocated", node, `${row.fact_id}: ${row.locator.char_start}:${row.locator.char_end} -> ${anchor.start}:${anchor.end} (bytes of ${id}/${snap.view.artifact})`)
+		if (converted && row.kind === "quote") flag("quote_converted_only", node, `${row.fact_id} quotes converted-only source ${id}, not the original fetched bytes`)
+	}
+}
+
+export function sourceChecks(ctx, rows, cited, sourcesDir, flag, unreadable = new Set()) {
+	const referenced = new Map()
+	for (const key of cited.keys()) {
+		const row = rows.get(key)
+		if (row?.source_id) referenced.set(row.source_id, [...(referenced.get(row.source_id) ?? []), row])
+	}
+	const ids = new Set(referenced.keys())
+	for (const match of proseText(ctx.body).matchAll(/\[S(\d+)\]/g)) ids.add(`S${match[1]}`)
+	const meta = new Map()
+	for (const id of ids) {
+		if (unreadable.has(id)) continue
+		const backing = referenced.get(id) ?? []
+		let snap
+		try {
+			snap = snapshotOf(sourcesDir, id)
+		} catch (error) {
+			// A source only named in prose ([S<n>]) backs no row: an absent snapshot is not a finding.
+			for (const row of backing) blocked(flag, cited, row, id, error.message)
+			continue
+		}
+		meta.set(id, snap.meta)
+		if (snap.drift.length === 0) rowChecks(snap, id, backing, cited, flag)
+		else if (backing.length === 0) flag("snapshot_hash_mismatch", ctx.body, `snapshot ${id} ${snap.drift.join("; ")}`)
+		else for (const row of backing) blocked(flag, cited, row, id, snap.drift.join("; "))
+	}
+	return meta
+}

package/packages/shared-skills/skills/ulw-research/scripts/fidelity-trace.mjs

@@ -13,9 +13,12 @@
 // so each entry, in array order, is matched to the next untraced occurrence of its rendered text in
 // document order that is a whole numeral and reads as the entry's row there (fits: "1,237 million"
 // is not F2's 1,237), and strips that one occurrence only. A copy of the same text that no entry
-// accounts for stays in the scan; an entry whose text is gone strips nothing.
+// accounts for stays in the scan; an entry whose text is gone strips nothing. A derived or
+// converted entry is located under the bracket grammar of fidelity-originals.mjs (renderLocator).
 
+import { PROSE_TAGS } from "./html-lite.mjs"
 import { NOT_CLAIMS } from "./numerals.mjs"
+import { sourcesClaims, textClaims } from "./sources-citation.mjs"
 
 /** Structural numerals: versions, figure/table/section references, times (NOT_CLAIMS 7-11), pages, citations, placeholders. */
 export const STRUCTURAL = [...NOT_CLAIMS.slice(7, 12), /\b(?:p|pp|page|pages)\.?\s*\d+(?:\s*[-\u2013]\s*\d+)?/gi, /\[(?:S|Source\s*)\d+\]|\[\^\d+\]/gi, /\{\{[^}]*\}\}/g]
@@ -29,7 +32,12 @@ function ownerTexts(ctx, charts) {
 	const owners = new Map()
 	for (const entry of ctx.texts) {
 		const kind = ctx.kindOf(entry.scope)
-		if (kind === "sources" || kind === "toc") continue
+		if (kind === "toc") continue
+		if (kind === "sources") {
+			if (!PROSE_TAGS.has(entry.owner.tag)) owners.set(entry.owner, `${owners.get(entry.owner) ?? ""} ${textClaims(entry.node.text).join(" ")}`)
+			else if (!owners.has(entry.owner)) owners.set(entry.owner, sourcesClaims(entry.owner).join(" "))
+			continue
+		}
 		if (charts.has(ctx.util.closest(entry.node, (cur) => cur.tag === "figure"))) continue
 		const hidden = ctx.util.closest(entry.node, (cur) => cur.attrs["data-fact"] !== undefined || cur.attrs["data-translates"] !== undefined || cur.tag === "q" || cur.tag === "blockquote")
 		owners.set(entry.owner, (owners.get(entry.owner) ?? "") + (hidden ? ` ${TRACED} ` : entry.node.text))
@@ -52,8 +60,13 @@ function scanOwner(ctx, owner, raw, flag) {
 	for (const match of rest.matchAll(NUMERAL)) flag("untraceable_number", owner, `number "${match[0]}" has no ledger row: "${clip(rest.slice(Math.max(0, match.index - 30), match.index + 30))}"`)
 }
 
+const plainSite = (entry, text, from) => {
+	const hit = text.indexOf(entry.rendered, from)
+	return hit === -1 ? null : [hit, hit + entry.rendered.length]
+}
+
 /** Strip each fill-map entry at its own fill site: the next untraced occurrence after the previous entry's site. */
-function traceFills(owners, fillMap, fits) {
+function traceFills(owners, fillMap, fits, locate = plainSite) {
 	const keys = [...owners.keys()]
 	const texts = [...owners.values()]
 	let [index, from] = [0, 0]
@@ -61,12 +74,12 @@ function traceFills(owners, fillMap, fits) {
 		const { rendered } = entry
 		if (!rendered) continue
 		for (let i = index, at = from; i < texts.length; i += 1, at = 0) {
-			let hit = texts[i].indexOf(rendered, at)
-			const glued = (pos) => /\d/.test(texts[i][pos - 1] ?? "") || /\d/.test(texts[i][pos + rendered.length] ?? "")
-			while (hit !== -1 && (glued(hit) || !fits(entry, texts[i].slice(hit)))) hit = texts[i].indexOf(rendered, hit + 1)
-			if (hit === -1) continue
-			texts[i] = `${texts[i].slice(0, hit)}${TRACED}${texts[i].slice(hit + rendered.length)}`
-			;[index, from] = [i, hit + TRACED.length]
+			let hit = locate(entry, texts[i], at)
+			const glued = ([start, end]) => (/^\d/.test(rendered) && /\d/.test(texts[i][start - 1] ?? "")) || (/\d$/.test(rendered) && /\d/.test(texts[i][end] ?? ""))
+			while (hit && (glued(hit) || !fits(entry, texts[i].slice(hit[0])))) hit = locate(entry, texts[i], hit[0] + 1)
+			if (!hit) continue
+			texts[i] = `${texts[i].slice(0, hit[0])}${TRACED}${texts[i].slice(hit[1])}`
+			;[index, from] = [i, hit[0] + TRACED.length]
 			break
 		}
 	}
@@ -79,7 +92,8 @@ function traceFills(owners, fillMap, fits) {
  * @param {{ fact_id: string, rendered: string, start?: number, end?: number }[] | null} fillMap Markdown fill map, or null
  * @param {(code: string, node: object, message: string) => void} flag
  * @param {(entry: object, tail: string) => boolean} fits whether the text from a candidate site on reads as the entry's row
+ * @param {(entry: object, text: string, from: number) => [number, number] | null} [locate] the next candidate site of an entry's text
  */
-export function traceChecks(ctx, charts, fillMap, flag, fits) {
-	for (const [owner, text] of traceFills(ownerTexts(ctx, charts), fillMap, fits)) scanOwner(ctx, owner, text, flag)
+export function traceChecks(ctx, charts, fillMap, flag, fits, locate) {
+	for (const [owner, text] of traceFills(ownerTexts(ctx, charts), fillMap, fits, locate)) scanOwner(ctx, owner, text, flag)
 }