lazycodex-ai 5.1.44 → 5.1.46

package/packages/omo-codex/plugin/skills/ulw-research/scripts/polish-rules.mjs

@@ -0,0 +1,170 @@
+// polish-rules.mjs - the one rule set behind polish-apply (per edit) and polish-diff (whole document).
+//
+// polishDefects(beforeHtml, afterHtml, ledger, {ordered}) compares the visible projections
+// (html-markup.markupParts, then polish-project.project) and returns {code, message} per failure.
+// Polish may change text nodes only. The primary rule is the whole-document skeleton comparison
+// (html-markup.skeletonChange): every markup token (tags, comments, bogus comments and frozen
+// script/style/template content) and every character reference of a text node must be byte-identical
+// in document order; an edited span holds no skeleton token, so any difference is polish_diff_fail
+// naming the first differing token and its offset. The remaining checks compare the projections:
+//   polish_diff_fail: fact spans (id + rendered markup), numerals outside fact spans (with units),
+//     tickers (TICKER: 1-5 capitals with an optional ".XX" class, or a 6-digit KRX code), or a
+//     table's row count changed; any tag, attribute, comment, declaration, or script/style/template
+//     content changed; protected tokens also stay within their original text-node boundaries;
+//   polish_hedging_decreased: a hedge or qualifier the before text holds and must keep is gone;
+//   polish_text_churn: more than POLISH_CHURN_LIMIT of the normalized visible prose changed;
+//   polish_link_changed: href values changed or an internal #target stopped resolving;
+//   polish_name_changed: a proper noun (POLISH_NAME) of a ledger quote, present before, is gone.
+// A document holding a construct html-markup.ambiguousMarkup lists (</ + non-letter, <!-- inside
+// script/style, <![CDATA[, <?, --!>, or < inside a tag) is refused first with polish_diff_fail
+// "ambiguous markup": the byte comparison is only trusted where the tokenizer is exact.
+// ordered (polish-apply) also refuses a reordering of fact spans, numerals, tickers and links;
+// without it they compare as multisets. Severity lives only in contracts.DEFECT_CODES.
+// No IO, no model. Plain Node ESM.
+
+import { ambiguousMarkup, markupParts, skeletonChange, tagAttributes } from "./html-markup.mjs"
+import { missingTerms, numeralKeys, project, requiredTerms, termCounts, textOf } from "./polish-project.mjs"
+import { splitSentences } from "./sentence-split.mjs"
+import { normalizeText } from "./text-normalize.mjs"
+
+/** The codes polishDefects can emit, in report order. */
+export const POLISH_CODES = Object.freeze(["polish_diff_fail", "polish_hedging_decreased", "polish_text_churn", "polish_link_changed", "polish_name_changed"])
+export const TICKER = /(?<![\p{L}\p{N}])(?:[A-Z]{1,5}(?:\.[A-Z]{1,2})?|\d{6})(?![\p{L}\p{N}])/gu
+/** The capitalized-token heuristic for proper nouns in ledger quotes. */
+export const POLISH_NAME = /\p{Lu}[\p{L}\p{N}&'-]+/gu
+/** Above this share of changed normalized prose a polish composed instead of proofreading. */
+export const POLISH_CHURN_LIMIT = 0.5
+/** Above this many cells a gap counts as fully rewritten instead of running the quadratic distance. */
+const DISTANCE_CELLS = 25_000_000
+
+const sorted = (list) => [...list].sort()
+const same = (a, b) => a.length === b.length && a.every((item, k) => item === b[k])
+
+/** null when unchanged; a reorder counts only when ordered. */
+function listChange(label, before, after, ordered) {
+	if (same(before, after)) return null
+	const remaining = sorted(before)
+	const added = []
+	for (const item of after) {
+		const at = remaining.indexOf(item)
+		if (at === -1) added.push(item)
+		else remaining.splice(at, 1)
+	}
+	if (remaining.length > 0 || added.length > 0) return `${label} changed: removed [${remaining.join(", ")}], added [${added.join(", ")}]`
+	return ordered ? `${label} reordered: [${before.join(", ")}] became [${after.join(", ")}]` : null
+}
+
+const tableRows = (html) => html.split(/<table\b/i).slice(1).map((part) => (part.split(/<\/table\s*>/i)[0].match(/<tr\b/gi) ?? []).length)
+
+function levenshtein(a, b) {
+	if (a === b) return 0
+	if (a.length * b.length > DISTANCE_CELLS) return Math.max(a.length, b.length)
+	let previous = Array.from({ length: b.length + 1 }, (_, k) => k)
+	for (let i = 1; i <= a.length; i++) {
+		const current = [i]
+		for (let j = 1; j <= b.length; j++) {
+			current[j] = Math.min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1))
+		}
+		previous = current
+	}
+	return previous[b.length]
+}
+
+/** Changed share of normalized prose: edit distance over unmatched sentence runs / longer length. */
+export function churnRatio(beforeText, afterText) {
+	const [a, b] = [normalizeText(beforeText), normalizeText(afterText)]
+	const longest = Math.max(a.length, b.length)
+	if (longest === 0) return 0
+	const [sa, sb] = [splitSentences(a).map((s) => s.text), splitSentences(b).map((s) => s.text)]
+	const lcs = Array.from({ length: sa.length + 1 }, () => new Array(sb.length + 1).fill(0))
+	for (let i = sa.length - 1; i >= 0; i--) {
+		for (let j = sb.length - 1; j >= 0; j--) lcs[i][j] = sa[i] === sb[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1])
+	}
+	let distance = 0
+	let gap = [[], []]
+	const flush = () => {
+		distance += levenshtein(gap[0].join(" "), gap[1].join(" "))
+		gap = [[], []]
+	}
+	let [i, j] = [0, 0]
+	while (i < sa.length || j < sb.length) {
+		if (i < sa.length && j < sb.length && sa[i] === sb[j]) {
+			flush()
+			i++
+			j++
+		} else if (j >= sb.length || (i < sa.length && lcs[i + 1][j] >= lcs[i][j + 1])) gap[0].push(sa[i++])
+		else gap[1].push(sb[j++])
+	}
+	flush()
+	return distance / longest
+}
+
+function linkChange(from, to, before, after, ordered) {
+	const change = listChange("href and data-claim values", from.links, to.links, ordered)
+	if (change) return change
+	const ids = (html) => new Set(tagAttributes(html).flatMap(({ attrs }) => (Object.hasOwn(attrs, "id") ? [attrs.id] : [])))
+	const [idsBefore, idsAfter] = [ids(before), ids(after)]
+	const targets = to.links.filter((link) => link.startsWith("href=#")).map((link) => link.slice(5))
+	const broken = targets.filter((href) => idsBefore.has(href.slice(1)) && !idsAfter.has(href.slice(1)))
+	return broken.length > 0 ? `internal link targets stopped resolving: [${[...new Set(broken)].join(", ")}]` : null
+}
+
+/** Capitalized tokens of the ledger quotes, minus each sentence's first word. */
+function ledgerNames(rows) {
+	const names = new Set()
+	for (const row of rows) {
+		for (const { text } of splitSentences(row.quote ?? "")) {
+			const first = text.match(/^\P{L}*\p{L}+/u)?.[0].length ?? 0
+			for (const match of text.matchAll(POLISH_NAME)) if (match.index >= first) names.add(match[0])
+		}
+	}
+	return names
+}
+
+const hasWord = (text, word) => new RegExp(`(?<![\\p{L}\\p{N}])${word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![\\p{L}\\p{N}])`, "u").test(text)
+
+/**
+ * @param {string} beforeHtml
+ * @param {string} afterHtml
+ * @param {{ rows: object[] }} ledger
+ * @param {{ ordered?: boolean }} [options]
+ * @returns {{ code: string, message: string }[]} in POLISH_CODES order
+ */
+export function polishDefects(beforeHtml, afterHtml, ledger, { ordered = false } = {}) {
+	const ambiguous = [...new Set([...ambiguousMarkup(beforeHtml), ...ambiguousMarkup(afterHtml)])]
+	if (ambiguous.length > 0) return [{ code: "polish_diff_fail", message: `ambiguous markup: ${ambiguous.join(", ")}` }]
+	const [beforeParts, afterParts] = [markupParts(beforeHtml), markupParts(afterHtml)]
+	const [before, after] = [beforeParts.source, afterParts.source]
+	const [from, to] = [project(before), project(after)]
+	const rows = ledger?.rows ?? []
+	const defects = []
+	const note = (code, message) => message && defects.push({ code, message })
+	const factKey = (fact) => `${fact.id} ${fact.html}`
+	const tickers = (text) => text.match(TICKER) ?? []
+	const markupEqual = same(beforeParts.markup, afterParts.markup)
+	note("polish_diff_fail", markupEqual ? null : "markup changed")
+	note("polish_diff_fail", skeletonChange(beforeHtml, afterHtml))
+	note("polish_diff_fail", listChange("fact spans", from.facts.map(factKey), to.facts.map(factKey), ordered))
+	note("polish_diff_fail", listChange("numerals outside fact spans", numeralKeys(from.masked), numeralKeys(to.masked), ordered))
+	note("polish_diff_fail", listChange("tickers", tickers(from.masked), tickers(to.masked), ordered))
+	const [rowsBefore, rowsAfter] = [tableRows(before), tableRows(after)]
+	if (!same(rowsBefore, rowsAfter)) note("polish_diff_fail", `table row counts changed: [${rowsBefore}] became [${rowsAfter}]`)
+	const required = requiredTerms(from, new Map(rows.map((row) => [row.fact_id, row])))
+	if (markupEqual) {
+		for (const [index, raw] of beforeParts.text.entries()) {
+			const [a, b] = [raw, afterParts.text[index]].map(textOf)
+			note("polish_diff_fail", listChange("text-node numerals", numeralKeys(a), numeralKeys(b), ordered))
+			note("polish_diff_fail", listChange("text-node tickers", tickers(a), tickers(b), ordered))
+			const lost = missingTerms(termCounts(a, [...required.keys()]), b)
+			if (lost.length) note("polish_hedging_decreased", `text-node hedges removed: ${lost.join(", ")}`)
+		}
+	}
+	const missing = missingTerms(required, to.visible)
+	if (missing.length > 0) note("polish_hedging_decreased", `hedges or qualifiers removed: ${missing.map((term) => `"${term}"`).join(", ")}`)
+	const ratio = churnRatio(from.visible, to.visible)
+	if (ratio > POLISH_CHURN_LIMIT) note("polish_text_churn", `${(ratio * 100).toFixed(1)} percent of the normalized prose changed`)
+	note("polish_link_changed", linkChange(from, to, before, after, ordered))
+	const lost = [...ledgerNames(rows)].filter((name) => hasWord(from.visible, name) && !hasWord(to.visible, name))
+	if (lost.length > 0) note("polish_name_changed", `ledger-quote names missing after polish: ${lost.join(", ")}`)
+	return defects.sort((a, b) => POLISH_CODES.indexOf(a.code) - POLISH_CODES.indexOf(b.code))
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/rational.mjs

@@ -0,0 +1,139 @@
+// rational.mjs - exact rational arithmetic over BigInt numerator/denominator pairs.
+//
+// A rational is { num: bigint, den: bigint }, reduced, with den > 0. Numerals enter as decimal
+// strings and never pass through Number(), so no value is ever rounded by floating point.
+// Plain Node ESM, no imports.
+
+const DECIMAL = /^([+-]?)(\d+)(?:\.(\d+))?$/
+
+const abs = (n) => (n < 0n ? -n : n)
+
+function gcd(a, b) {
+	let x = abs(a)
+	let y = abs(b)
+	while (y !== 0n) [x, y] = [y, x % y]
+	return x
+}
+
+function make(num, den) {
+	if (den === 0n) throw new RangeError("rational: zero denominator")
+	const sign = den < 0n ? -1n : 1n
+	const g = gcd(num, den) || 1n
+	return Object.freeze({ num: (sign * num) / g, den: (sign * den) / g })
+}
+
+/** Accepts a rational, a bigint, a safe integer, or a decimal string. */
+function coerce(value) {
+	if (typeof value === "bigint") return make(value, 1n)
+	if (typeof value === "number") {
+		if (!Number.isSafeInteger(value)) throw new TypeError(`rational: ${value} is not a safe integer; pass a decimal string`)
+		return make(BigInt(value), 1n)
+	}
+	if (typeof value === "string") return fromDecimal(value)
+	if (value && typeof value.num === "bigint" && typeof value.den === "bigint") return make(value.num, value.den)
+	throw new TypeError("rational: expected a rational, bigint, integer, or decimal string")
+}
+
+/** "12.50" -> 25/2. Accepts an optional sign, digits, and an optional fraction; nothing else. */
+export function fromDecimal(str) {
+	const match = DECIMAL.exec(String(str).trim())
+	if (!match) throw new SyntaxError(`rational: "${str}" is not a plain decimal`)
+	const [, sign, whole, frac = ""] = match
+	const num = BigInt(whole + frac) * (sign === "-" ? -1n : 1n)
+	return make(num, 10n ** BigInt(frac.length))
+}
+
+export function add(a, b) {
+	const x = coerce(a)
+	const y = coerce(b)
+	return make(x.num * y.den + y.num * x.den, x.den * y.den)
+}
+
+export function sub(a, b) {
+	const x = coerce(a)
+	const y = coerce(b)
+	return make(x.num * y.den - y.num * x.den, x.den * y.den)
+}
+
+export function mul(a, b) {
+	const x = coerce(a)
+	const y = coerce(b)
+	return make(x.num * y.num, x.den * y.den)
+}
+
+export function div(a, b) {
+	const x = coerce(a)
+	const y = coerce(b)
+	if (y.num === 0n) throw new RangeError("rational: division by zero")
+	return make(x.num * y.den, x.den * y.num)
+}
+
+export function eq(a, b) {
+	const x = coerce(a)
+	const y = coerce(b)
+	return x.num === y.num && x.den === y.den
+}
+
+/** Digits of a non-negative integer scaled by 10^-scale, as a decimal string. */
+function placeDecimal(digits, scale) {
+	if (scale <= 0) return digits + "0".repeat(-scale)
+	const padded = digits.padStart(scale + 1, "0")
+	return `${padded.slice(0, padded.length - scale)}.${padded.slice(padded.length - scale)}`
+}
+
+/** Exact decimal; throws RangeError when the expansion does not terminate. */
+function exactDecimal(r) {
+	let den = r.den
+	let twos = 0
+	let fives = 0
+	while (den % 2n === 0n) [den, twos] = [den / 2n, twos + 1]
+	while (den % 5n === 0n) [den, fives] = [den / 5n, fives + 1]
+	if (den !== 1n) throw new RangeError(`rational: ${r.num}/${r.den} has no finite decimal; pass sigFigs`)
+	const scale = Math.max(twos, fives)
+	const digits = (abs(r.num) * 10n ** BigInt(scale)) / r.den
+	return (r.num < 0n ? "-" : "") + placeDecimal(digits.toString(), scale)
+}
+
+/** Number of decimal digits in the integer part of |num/den| (0 when below 1). */
+function integerDigits(num, den) {
+	const whole = num / den
+	if (whole > 0n) return whole.toString().length
+	let exp = 0
+	let scaled = num
+	while (scaled < den) [scaled, exp] = [scaled * 10n, exp - 1]
+	return exp + 1
+}
+
+/**
+ * Decimal string. Without sigFigs the value must terminate and is rendered exactly. With sigFigs it
+ * is rounded half away from zero to that many significant figures; integers are never written
+ * with an exponent, so 319999 at 2 figures renders "320000".
+ */
+export function toDecimal(value, sigFigs) {
+	const r = coerce(value)
+	if (sigFigs === undefined) return exactDecimal(r)
+	if (!Number.isInteger(sigFigs) || sigFigs < 1) throw new RangeError("rational: sigFigs must be a positive integer")
+	if (r.num === 0n) return sigFigs > 1 ? `0.${"0".repeat(sigFigs - 1)}` : "0"
+	const num = abs(r.num)
+	const scale = sigFigs - integerDigits(num, r.den)
+	const numer = scale >= 0 ? num * 10n ** BigInt(scale) : num
+	const denom = scale >= 0 ? r.den : r.den * 10n ** BigInt(-scale)
+	let digits = numer / denom
+	let places = scale
+	if ((numer % denom) * 2n >= denom) digits += 1n
+	if (digits.toString().length > sigFigs) [digits, places] = [digits / 10n, places - 1]
+	return (r.num < 0n ? "-" : "") + placeDecimal(digits.toString(), places)
+}
+
+/**
+ * Significant figures of a decimal string. With a decimal point: digits from the first nonzero
+ * digit to the end, trailing zeros included ("3.2" -> 2, "0.050" -> 2). Without one: every digit
+ * after leading zeros counts ("320" -> 3, "100" -> 3, "20" -> 2).
+ */
+export function sigFigsOf(decimalStr) {
+	const match = DECIMAL.exec(String(decimalStr).trim())
+	if (!match) throw new SyntaxError(`rational: "${decimalStr}" is not a plain decimal`)
+	const [, , whole, frac] = match
+	const digits = (whole + (frac ?? "")).replace(/^0+/, "")
+	return Math.max(digits.length, 1)
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/report-tools-commands.mjs

@@ -5,7 +5,7 @@ import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs"
 import { dirname, join, resolve } from "node:path"
 
 import { CliError, parseArgs, readJsonFile, writeJsonAtomic } from "./cli-support.mjs"
-import { REPAIR_DEFAULTS } from "./contracts.mjs"
+import { GATE_NAMES, GATE_STATUSES, REPAIR_DEFAULTS } from "./contracts.mjs"
 import { parseLineageMode, parsePalette, renderDesignSpec } from "./design-spec.mjs"
 import { extractFromHtml, extractFromPath, fetchHtml } from "./format-extract.mjs"
 import { checkLayout } from "./gates-layout.mjs"
@@ -80,12 +80,16 @@ export function runLayoutProbe(argv) {
 }
 
 export function runRepairDecide(argv) {
-	const args = parseArgs(argv)
+	const args = parseArgs(argv, { repeatable: ["defects"] })
 	const statePath = args.required("state")
 	const now = nowFrom(args.flags)
-	const defectsInput = readJsonFile(args.required("defects"))
-	const defects = Array.isArray(defectsInput) ? defectsInput : defectsInput.defects
-	if (!Array.isArray(defects)) throw new CliError("--defects must be a JSON array or a check result with a defects array")
+	// Every --defects file (check result, fidelity record, or bare array) joins one defect list.
+	const defects = args.required("defects").flatMap((path) => {
+		const input = readJsonFile(path)
+		const list = Array.isArray(input) ? input : input.defects
+		if (!Array.isArray(list)) throw new CliError(`--defects ${path} must be a JSON array or a check result with a defects array`)
+		return list
+	})
 	const state = existsSync(statePath)
 		? deserializeState(readText(statePath))
 		: createTracker({
@@ -122,7 +126,7 @@ function manifestPath(flags) {
 	return join(sessionDirFrom(flags), outcome.MANIFEST_FILENAME)
 }
 
-export function runOutcome(argv) {
+export async function runOutcome(argv) {
 	const [sub, ...rest] = argv
 	const args = parseArgs(rest, { booleans: ["json"] })
 	const file = sub === "state" ? null : manifestPath(args.flags)
@@ -135,7 +139,7 @@ export function runOutcome(argv) {
 		}
 		case "set": {
 			const [format, status] = args.positionals
-			if (!format || !status) throw new CliError("usage: outcome set <format> <status> [--path p] [--reason r] [--pages n]")
+			if (!format || !status) throw new CliError("usage: outcome set <format> <status> [--path p] [--reason r] [--pages n] [--verified-by fidelity-defects.json] [--rendered-from source]")
 			const manifest = outcome.readManifest(file)
 			const detail = {}
 			if (args.flags.path !== undefined) {
@@ -144,13 +148,14 @@ export function runOutcome(argv) {
 			}
 			if (args.flags.reason !== undefined) detail.reason = args.flags.reason
 			if (args.flags.pages !== undefined) detail.pages = args.int("pages")
+			for (const [flag, key] of [["verified-by", "verifiedBy"], ["rendered-from", "renderedFrom"]]) if (args.flags[flag] !== undefined) detail[key] = resolve(args.flags[flag])
 			outcome.writeManifest(file, outcome.setDeliverable(manifest, format, status, { ...detail, now: nowFrom(args.flags) }))
 			return { json: { command: "outcome set", format, status }, exitCode: 0 }
 		}
 		case "gate": {
 			const [gate, status] = args.positionals
-			if (!gate || !status) throw new CliError("usage: outcome gate <static|layout|visual|proofread> <pass|fail|not_run>")
-			outcome.writeManifest(file, outcome.setGate(outcome.readManifest(file), gate, status))
+			if (!gate || !status) throw new CliError(`usage: outcome gate <${GATE_NAMES.join("|")}> <${GATE_STATUSES.join("|")}>`)
+			outcome.writeManifest(file, outcome.setGate(outcome.readManifest(file), gate, status, { reason: args.flags.reason, now: nowFrom(args.flags) }))
 			return { json: { command: "outcome gate", gate, status }, exitCode: 0 }
 		}
 		case "render": {
@@ -169,11 +174,17 @@ export function runOutcome(argv) {
 			return { json: { command: "outcome state", state }, summary: `state: ${state}`, exitCode: 0 }
 		}
 		case "verify": {
-			const result = outcome.verifyManifest(outcome.readManifest(file))
+			const manifest = outcome.readManifest(file)
+			const base = outcome.verifyManifest(manifest)
+			const { fidelityProblems } = await import("./fidelity-records.mjs")
+			const problems = [...base.problems, ...fidelityProblems(sessionDirFrom(args.flags), manifest)]
+			const result = { ok: problems.length === 0, problems }
 			return { json: { command: "outcome verify", ...result }, summary: `verify: ${result.ok ? "OK" : `${result.problems.length} problem(s)`}`, exitCode: result.ok ? 0 : 1 }
 		}
 		case "finish": {
 			const manifest = outcome.readManifest(file)
+			const refused = (await import("./fidelity-records.mjs")).fidelityProblems(sessionDirFrom(args.flags), manifest)
+			if (refused.length > 0) return { json: { command: "outcome finish", refused: true, problems: refused }, summary: `finish: refused (${refused.map((p) => p.code).join(", ")})`, exitCode: 1 }
 			const ledgerPath = args.flags.ledger ?? join(sessionDirFrom(args.flags), "sources-ledger.md")
 			const now = nowFrom(args.flags)
 			const facts = outcome.briefingFacts(manifest, existsSync(ledgerPath) ? readText(ledgerPath) : "", now)

package/packages/omo-codex/plugin/skills/ulw-research/scripts/report-tools.mjs

@@ -20,6 +20,15 @@ export const COMMANDS = Object.freeze([
 	"outcome finish",
 	"outcome briefing",
 	"format-extract",
+	"fact add",
+	"fact quote",
+	"fact retract",
+	"fact export",
+	"fact fill",
+	"fact verify",
+	"fact derive",
+	"source snapshot",
+	"claims index",
 ])
 
 export async function run(argv) {
@@ -36,6 +45,12 @@ export async function run(argv) {
 			return runOutcome(rest)
 		case "format-extract":
 			return runFormatExtract(rest)
+		case "fact":
+			return (await import("./fact-commands.mjs")).runFact(rest)
+		case "source":
+			return (await import("./source-commands.mjs")).runSource(rest)
+		case "claims":
+			return (await import("./source-commands.mjs")).runClaims(rest)
 		case "--help":
 		case "help":
 			if (rest.includes("--json")) return { json: { commands: COMMANDS }, exitCode: 0 }

package/packages/omo-codex/plugin/skills/ulw-research/scripts/sentence-split.mjs

@@ -0,0 +1,77 @@
+// sentence-split.mjs - deterministic sentence index for snapshot text.
+//
+// splitSentences(text) returns [{ sid, start, end, text }] with sid "s<k>" (1-based) and
+// start/end as offsets into the raw text, so text.slice(start, end) === record.text.
+// Boundaries: a line break; ". ! ?" followed by whitespace or end of line (known
+// abbreviations such as "e.g." excepted); and the Korean endings da. yo. nida. kka? even
+// with no following space, unless closing punctuation follows (quoted speech such as
+// '...da."rago' then needs whitespace like any other terminator). "no." is an abbreviation
+// only before a number ("No. 5"). Plain Node ESM, no dependencies.
+
+import { normalizeText } from "./text-normalize.mjs"
+
+const TERMINATORS = ".!?"
+const CLOSERS = "\"')]}\u201D\u2019\u00BB\u300D\u300F"
+const KOREAN_PERIOD_ENDINGS = new Set(["\uB2E4", "\uC694"]) // da, yo (nida ends in da)
+const KOREAN_QUESTION_ENDING = "\uAE4C" // kka
+const ABBREVIATIONS = new Set([
+	"e.g", "i.e", "etc", "vs", "cf", "approx", "al", "fig", "no", "mr", "mrs", "ms", "dr", "prof", "inc", "ltd", "jr", "sr",
+])
+const SPACE = /\s/u
+
+function isKoreanEnding(line, at) {
+	const prev = line[at - 1]
+	if (line[at] === ".") return KOREAN_PERIOD_ENDINGS.has(prev)
+	return line[at] === "?" && prev === KOREAN_QUESTION_ENDING
+}
+
+function isAbbreviation(line, from, at) {
+	let k = at
+	while (k > from && !SPACE.test(line[k - 1])) k--
+	const word = line.slice(k, at).replace(/^[("'[]+/u, "").toLowerCase()
+	if (word === "no") return /^\s*\d/u.test(line.slice(at + 1))
+	return ABBREVIATIONS.has(word)
+}
+
+function pushSegment(line, base, from, to, out) {
+	let start = from
+	let end = to
+	while (start < end && SPACE.test(line[start])) start++
+	while (end > start && SPACE.test(line[end - 1])) end--
+	const text = line.slice(start, end)
+	if (normalizeText(text) === "") return
+	out.push({ sid: `s${out.length + 1}`, start: base + start, end: base + end, text })
+}
+
+function splitLine(line, base, out) {
+	let from = 0
+	for (let i = 0; i < line.length; i++) {
+		if (!TERMINATORS.includes(line[i])) continue
+		let j = i + 1
+		while (j < line.length && (TERMINATORS.includes(line[j]) || CLOSERS.includes(line[j]))) j++
+		const atBreak = j >= line.length || SPACE.test(line[j])
+		const closed = [...line.slice(i + 1, j)].some((c) => CLOSERS.includes(c))
+		const korean = !closed && isKoreanEnding(line, i)
+		const abbreviation = line[i] === "." && !korean && isAbbreviation(line, from, i)
+		if ((atBreak || korean) && !abbreviation) {
+			pushSegment(line, base, from, j, out)
+			from = j
+		}
+		i = j - 1
+	}
+	pushSegment(line, base, from, line.length, out)
+}
+
+export function splitSentences(text) {
+	const source = String(text ?? "")
+	const out = []
+	let base = 0
+	while (base <= source.length) {
+		const newline = source.indexOf("\n", base)
+		const stop = newline === -1 ? source.length : newline
+		splitLine(source.slice(base, stop), base, out)
+		if (newline === -1) break
+		base = newline + 1
+	}
+	return out
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/source-commands.mjs

@@ -0,0 +1,140 @@
+// source-commands.mjs - the `source snapshot` and `claims index` subcommands of report-tools.
+// A source dir is immutable: the same id with identical text is a no-op, different text fails
+// (exit 1, snapshot_exists), because ledger rows pin the old hash. `--url` fetches with plain
+// fetch (30 s timeout, no browser) and records extractor tool:source-snapshot@1; `--html` and
+// `--text` import a browser-lane capture and record extractor model (lower trust), or the named
+// extractor for `--text --extractor NAME@VERSION`; `--command` runs an argv without a shell and
+// snapshots its stdout, with the argv, cwd, exit status, and output hashes in command.json.
+// node: builtins and sibling imports only.
+
+import { spawnSync } from "node:child_process"
+import { existsSync, readFileSync, writeFileSync } from "node:fs"
+import { join, resolve } from "node:path"
+
+import { indexClaims } from "./claims-index.mjs"
+import { CliError, parseArgs, sha256Of, writeJsonAtomic } from "./cli-support.mjs"
+import { parseFidelityBlock } from "./fidelity-brief.mjs"
+import { writeSnapshot } from "./source-snapshot.mjs"
+
+const FETCH_TIMEOUT_MS = 30_000
+const TOOL_EXTRACTOR = "tool:source-snapshot@1"
+const SNAPSHOT_USAGE =
+	"usage: source snapshot --id S<n> (--url <u> | --html <file> | --text <file> [--extractor NAME@VERSION] | --command '<argv json>' --cwd DIR) [--kind page|command|document] [--provenance primary|secondary] [--revision R] --session-dir D"
+
+function readInput(path) {
+	try {
+		return readFileSync(path, "utf8")
+	} catch (error) {
+		throw new CliError(`cannot read ${path}: ${error.code ?? error.message}`)
+	}
+}
+
+/** The session dir from --session-dir or SESSION_DIR; a usage error when neither is set. */
+export function sessionDirOf(flags) {
+	const dir = flags["session-dir"] ?? process.env.SESSION_DIR
+	if (!dir || dir === true) throw new CliError("--session-dir is required (or set SESSION_DIR)")
+	return resolve(dir)
+}
+
+/** The snapshot of one source: meta.json, text.txt, and source.html when the source was a page. */
+export function snapshotOf(sessionDir, id) {
+	const dir = join(sessionDir, "sources", String(id))
+	if (!existsSync(join(dir, "meta.json"))) throw new CliError(`source ${id} has no snapshot; run source snapshot --id ${id} first`)
+	const html = join(dir, "source.html")
+	return { meta: JSON.parse(readInput(join(dir, "meta.json"))), text: readInput(join(dir, "text.txt")), html: existsSync(html) ? readInput(html) : null }
+}
+
+/** Whether the brief's source floor lists this source as pointed at (`pointed only: a, b`). */
+export function pointedFlag(sessionDir, url) {
+	const brief = join(sessionDir, "brief.md")
+	const floor = existsSync(brief) ? (parseFidelityBlock(readInput(brief)).fields?.["source floor"] ?? "") : ""
+	const match = /^pointed only:\s*(.+)$/i.exec(floor.trim())
+	return match ? match[1].split(",").map((entry) => entry.trim()).includes(url) : false
+}
+
+async function fetchSource(url) {
+	if (!URL.canParse(url)) throw new CliError(`--url is not a URL: ${url}`)
+	let response
+	try {
+		response = await fetch(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) })
+	} catch (error) {
+		throw new CliError(`fetch failed for ${url}: ${error.message}`)
+	}
+	if (!response.ok) {
+		await response.body?.cancel()
+		throw new CliError(`fetch failed: HTTP ${response.status} for ${url}`)
+	}
+	const type = response.headers.get("content-type") ?? ""
+	if (/pdf/i.test(type)) {
+		await response.body?.cancel()
+		throw new CliError(`pdf response: ${url}; extract its text and use --text <file> --extractor NAME@VERSION`)
+	}
+	const body = await response.text()
+	return /html|xml/i.test(type) || /^\s*</.test(body) ? { url, html: body } : { url, text: body }
+}
+
+function runCommand(argvJson, cwd) {
+	let argv
+	try {
+		argv = JSON.parse(argvJson)
+	} catch (error) {
+		throw new CliError(`--command must be a JSON array of strings: ${error.message}`)
+	}
+	if (!Array.isArray(argv) || argv.length === 0 || !argv.every((part) => typeof part === "string" && part.length > 0)) {
+		throw new CliError("--command must be a non-empty JSON array of non-empty strings")
+	}
+	if (!cwd || cwd === true) throw new CliError("--command needs --cwd DIR")
+	const proc = spawnSync(argv[0], argv.slice(1), { cwd: resolve(cwd), encoding: "utf8", shell: false, maxBuffer: 64 * 1024 * 1024 })
+	if (proc.error) throw new CliError(`--command could not start: ${proc.error.message}`)
+	const receipt = { argv, cwd: resolve(cwd), exit_status: proc.status, stdout_sha256: sha256Of(proc.stdout), stderr_sha256: sha256Of(proc.stderr) }
+	return { path: `command:${argv.join(" ")}`, text: proc.stdout, kind: "command", receipt }
+}
+
+async function acquire(flags) {
+	const given = ["url", "html", "text", "command"].filter((name) => flags[name] !== undefined)
+	if (given.length !== 1) throw new CliError(SNAPSHOT_USAGE)
+	if (flags.extractor !== undefined && flags.text === undefined) throw new CliError("--extractor goes with --text only")
+	if (flags.url !== undefined) return { ...(await fetchSource(flags.url)), extractor: TOOL_EXTRACTOR }
+	if (flags.html !== undefined) return { path: resolve(flags.html), html: readInput(flags.html), extractor: "model" }
+	if (flags.text !== undefined) return { path: resolve(flags.text), text: readInput(flags.text), extractor: flags.extractor === undefined ? "model" : `extractor:${flags.extractor}` }
+	return { ...runCommand(flags.command, flags.cwd), extractor: TOOL_EXTRACTOR }
+}
+
+export async function runSource(argv) {
+	const [sub, ...rest] = argv
+	if (sub !== "snapshot") throw new CliError(SNAPSHOT_USAGE)
+	const args = parseArgs(rest)
+	const id = args.required("id")
+	const sourcesDir = join(sessionDirOf(args.flags), "sources")
+	const { receipt, ...input } = await acquire(args.flags)
+	if (args.flags.kind !== undefined) input.kind = args.flags.kind
+	if (args.flags.provenance !== undefined) input.provenance = args.flags.provenance
+	if (args.flags.revision !== undefined) input.revision = args.flags.revision
+	let written
+	try {
+		written = writeSnapshot(sourcesDir, id, input)
+	} catch (error) {
+		if (String(error.message).startsWith("snapshot_exists")) {
+			return { json: { command: "source snapshot", status: "fail", exitCode: 1, error: "snapshot_exists", message: error.message }, summary: `source snapshot: ${error.message}`, exitCode: 1 }
+		}
+		if (error instanceof TypeError) throw new CliError(error.message)
+		throw error
+	}
+	if (receipt) writeFileSync(join(written.dir, "command.json"), `${JSON.stringify(receipt, null, 2)}\n`)
+	const meta = JSON.parse(readFileSync(join(written.dir, "meta.json"), "utf8"))
+	const data = { source_id: id, directory: written.dir, snapshot_sha256: written.sha256, extractor: meta.extractor, kind: meta.kind, provenance: meta.provenance }
+	return { json: { command: "source snapshot", status: "pass", exitCode: 0, data }, summary: `source snapshot: ${id} ${written.sha256.slice(0, 12)} (${meta.extractor})`, exitCode: 0 }
+}
+
+export function runClaims(argv) {
+	const [sub, ...rest] = argv
+	const args = parseArgs(rest)
+	const [graph] = args.positionals
+	if (sub !== "index" || !graph) throw new CliError("usage: claims index <claim-graph.md> --out claims.json")
+	const out = args.required("out")
+	const index = indexClaims(readInput(graph))
+	if (index.errors.length > 0) throw new CliError(`claims index ${graph}: ${index.errors.join("; ")}`)
+	writeJsonAtomic(out, { schemaVersion: 1, graph: resolve(graph), graph_sha256: sha256Of(readFileSync(graph)), ...index })
+	const verified = index.claims.filter((claim) => claim.in_verified).length
+	return { json: { command: "claims index", out: resolve(out), claims: index.claims.length, verified }, summary: `claims index: ${index.claims.length} claim(s), ${verified} verified`, exitCode: 0 }
+}