oh-my-opencode 5.1.48 → 5.1.51

package/packages/omo-codex/plugin/skills/ulw-research/scripts/source-snapshot.mjs

@@ -1,77 +1,44 @@
 // source-snapshot.mjs - the per-source snapshot that fact locators read.
 //
-// snapshotFromHtml turns a page into text.txt form: one line per block (prose blocks via
-// proseText, one table cell per line; head, scripts, styles and code are dropped). writeSnapshot
-// writes <sourcesDir>/<S-id>/ with text.txt (pages joined by a form feed), source.html when html
-// was given, meta.json, sentences.jsonl (sid S<n>:s<k>) and acronyms.json. The sha256 covers
-// text.txt only. Nothing here touches the network: the caller supplies the html or text.
+// writeSnapshot writes <sourcesDir>/<S-id>/ with raw.bin (the fetched or captured bytes, absent
+// only for converted-only sources), meta.json (sha256 of the pinned body, text_source, charset),
+// text.txt (the locator text of source-view.mjs: the visible text nodes of a page, the decoded
+// bytes of a document, or a declared/converted extraction), source.html for pages (decoded, a
+// reading aid), sentences.jsonl (sid S<n>:s<k>, indexed over text.txt) and acronyms.json; a page
+// also gets conversion.json, the conversion probe's record of the numbers its text.txt altered.
 // Plain Node ESM, node: builtins and sibling imports only.
 
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
 import { join } from "node:path"
 
 import { sha256Of } from "./cli-support.mjs"
-import { parseHtml, PROSE_TAGS, proseText } from "./html-lite.mjs"
+import { probeConversion } from "./conversion-probe.mjs"
+import { writeFidelityRecord } from "./fidelity-records.mjs"
+import { extractHtml } from "./html-text-map.mjs"
 import { splitSentences } from "./sentence-split.mjs"
+import { buildView } from "./source-view.mjs"
 
-const SKIP = new Set(["head", "script", "style", "svg", "code", "pre", "kbd", "samp", "template", "noscript"])
-const INLINE = new Set([
-	"a", "abbr", "b", "bdi", "bdo", "cite", "data", "del", "dfn", "em", "font", "i", "ins", "label", "mark", "q", "s",
-	"small", "span", "strong", "sub", "sup", "time", "u", "var",
-])
 const SOURCE_ID = /^S[1-9]\d*$/
 const EXTRACTOR = /^(?:tool:source-snapshot@1|model|extractor:[\w.@+-]+)$/
-const KINDS = new Set(["page", "command", "document"])
+const KINDS = new Set(["page", "command", "document", "converted-only"])
 const PROVENANCE = new Set(["primary", "secondary", "unknown"])
 const ACRONYM_AFTER = /\(([A-Z]{2,6})\)/g
 const ACRONYM_BEFORE = /(?<![\p{L}\p{N}])([A-Z]{2,6})\s?\(([A-Z][^()\n]{2,79})\)/gu
 const NAME_WORD = /^[A-Z][\p{L}\p{N}-]*$/u
 const CONNECTOR = new Set(["of", "and", "for", "the", "in", "on", "to", "&"])
+const MARKUP = /^\s*</
 
 /**
- * Block lines of a page: { root, text, blocks: [{ node, start, end }] } where text joins the block
- * texts with "\n" and text.slice(start, end) is one block. fact-locate reads cells through `node`.
+ * Block lines of a page: { root, text, blocks: [{ node, start, end }], starts, ends } where text
+ * joins the visible block texts with "\n" (html-text-map.mjs). fact-locate reads cells through `node`.
  */
 export function snapshotBlocks(html) {
-	const root = parseHtml(html)
-	const lines = []
-	let buffer = ""
-	const flush = (node) => {
-		const line = buffer.replace(/\s+/g, " ").trim()
-		buffer = ""
-		if (line) lines.push({ node, text: line })
-	}
-	const walk = (node) => {
-		for (const child of node.children) {
-			if (child.tag === "#text") buffer += child.text
-			else if (SKIP.has(child.tag)) continue
-			else if (child.tag === "br") flush(node)
-			else if (INLINE.has(child.tag)) walk(child)
-			else if (PROSE_TAGS.has(child.tag)) {
-				flush(node)
-				const line = proseText(child)
-				if (line) lines.push({ node: child, text: line })
-			} else {
-				flush(node)
-				walk(child)
-				flush(child)
-			}
-		}
-	}
-	walk(root)
-	flush(root)
-	const blocks = []
-	let at = 0
-	for (const line of lines) {
-		blocks.push({ node: line.node, start: at, end: at + line.text.length })
-		at += line.text.length + 1
-	}
-	return { root, text: lines.map((line) => line.text).join("\n"), blocks }
+	return extractHtml(html)
 }
 
 /** text.txt form of an html page. */
 export function snapshotFromHtml(html) {
-	return snapshotBlocks(html).text
+	return extractHtml(html).text
 }
 
 /** Sentence index rows { sid: "S<n>:s<k>", start, end, text }; a form feed (page break) always ends a sentence. */
@@ -127,36 +94,61 @@ export function findAcronyms(text, sentences) {
 function snapshotText(input) {
 	if (Array.isArray(input.pages)) return input.pages.join("\f")
 	if (typeof input.text === "string") return input.text
-	if (typeof input.html === "string") return snapshotFromHtml(input.html)
-	throw new TypeError("writeSnapshot needs text, pages, or html")
+	throw new TypeError("writeSnapshot needs raw, html, text, or pages")
+}
+
+/** Which source-view text source the input makes, with the bytes it pins. */
+function bodiesOf(input) {
+	if (input.kind === "converted-only") return { mode: "converted", raw: null, textBody: Buffer.from(snapshotText(input)) }
+	if (input.declared === true) return { mode: "declared", raw: Buffer.from(input.raw), textBody: Buffer.from(snapshotText(input)) }
+	if (input.raw === undefined && typeof input.html === "string") return { mode: "html", raw: Buffer.from(input.html), textBody: null }
+	if (input.raw === undefined) return { mode: "decoded", raw: Buffer.from(snapshotText(input)), textBody: null }
+	const raw = Buffer.from(input.raw)
+	if (input.markup === true || typeof input.html === "string" || input.kind === "page" || MARKUP.test(raw.subarray(0, 1024).toString("latin1").replace(/^\u00ef\u00bb\u00bf/, ""))) return { mode: "html", raw, textBody: null }
+	if (input.text !== undefined || input.pages !== undefined) return { mode: "declared", raw, textBody: Buffer.from(snapshotText(input)) }
+	return { mode: "decoded", raw, textBody: null }
 }
 
 function checkInput(sourceId, input) {
 	if (!SOURCE_ID.test(String(sourceId))) throw new TypeError(`source id must be S<n>, got ${sourceId}`)
 	if (!EXTRACTOR.test(String(input.extractor))) throw new TypeError(`extractor must be tool:source-snapshot@1, model, or extractor:<name>, got ${input.extractor}`)
 	if (typeof (input.url ?? input.path) !== "string") throw new TypeError("writeSnapshot needs a url or a path")
-	if (input.kind !== undefined && !KINDS.has(input.kind)) throw new TypeError(`kind must be page, command, or document, got ${input.kind}`)
+	if (input.kind !== undefined && !KINDS.has(input.kind)) throw new TypeError(`kind must be page, command, document, or converted-only, got ${input.kind}`)
 	if (input.provenance !== undefined && !PROVENANCE.has(input.provenance)) throw new TypeError(`provenance must be primary, secondary, or unknown, got ${input.provenance}`)
+	if (input.observed_at !== undefined && (typeof input.observed_at !== "string" || Number.isNaN(new Date(input.observed_at).getTime()))) throw new TypeError(`observed_at must be an ISO timestamp, got ${input.observed_at}`)
+}
+
+function pinnedHash(dir) {
+	const metaPath = join(dir, "meta.json")
+	return existsSync(metaPath) ? JSON.parse(readFileSync(metaPath, "utf8")).sha256 : null
 }
 
 /**
- * Write <sourcesDir>/<sourceId>/ for one source; returns { dir, sha256 } with sha256 over text.txt.
- * input: { url | path, text | pages | html, extractor, observed_at?, revision?, kind?, provenance? }.
- * Re-writing the same text is a no-op change; a different text over an existing snapshot throws
- * snapshot_exists, because ledger rows pin the old hash.
+ * Write <sourcesDir>/<sourceId>/ for one source; returns { dir, sha256, altered } over the pinned body,
+ * where altered lists the numbers the page conversion altered (null when no page conversion ran).
+ * input: { url | path, raw?, markup?, html?, text? | pages?, charset?, extractor, observed_at?, revision?,
+ * kind?, provenance? } (see bodiesOf). Re-writing the same bytes is a no-op; different bytes over
+ * an existing snapshot throw snapshot_exists, because ledger rows pin the old hash, unless
+ * options.replace (`source refresh`, which re-anchors those rows, or `source snapshot --replace`, which
+ * has checked that no active row pins it) is set; a replacement drops the old capture's command.json.
  */
-export function writeSnapshot(sourcesDir, sourceId, input) {
+export function writeSnapshot(sourcesDir, sourceId, input, options = {}) {
 	checkInput(sourceId, input)
-	const text = snapshotText(input)
-	const sha256 = sha256Of(Buffer.from(text, "utf8"))
+	const { mode, raw, textBody } = bodiesOf(input)
+	const view = buildView(mode, { raw, textBody, charset: input.charset ?? null })
+	const sha256 = sha256Of(raw ?? textBody)
 	const dir = join(sourcesDir, sourceId)
-	const textPath = join(dir, "text.txt")
-	if (existsSync(textPath) && sha256Of(readFileSync(textPath)) !== sha256) {
-		throw new Error(`snapshot_exists: ${sourceId} already holds a different text.txt`)
+	const pinned = pinnedHash(dir)
+	if (pinned !== null && pinned !== sha256) {
+		if (options.replace !== true) throw new Error(`snapshot_exists: ${sourceId} already holds different snapshot bytes`)
+		rmSync(join(dir, "command.json"), { force: true })
 	}
 	mkdirSync(dir, { recursive: true })
-	writeFileSync(textPath, text)
-	if (typeof input.html === "string") writeFileSync(join(dir, "source.html"), input.html)
+	if (raw) writeFileSync(join(dir, "raw.bin"), raw)
+	else rmSync(join(dir, "raw.bin"), { force: true })
+	writeFileSync(join(dir, "text.txt"), textBody ?? view.text)
+	if (view.html !== null) writeFileSync(join(dir, "source.html"), view.html)
+	else rmSync(join(dir, "source.html"), { force: true })
 	const meta = {
 		source_id: sourceId,
 		url: input.url ?? input.path,
@@ -164,13 +156,25 @@ export function writeSnapshot(sourcesDir, sourceId, input) {
 		sha256,
 		extractor: input.extractor,
 		revision: input.revision ?? null,
-		kind: input.kind ?? (input.url !== undefined || typeof input.html === "string" ? "page" : "document"),
+		kind: input.kind ?? (input.url !== undefined || mode === "html" ? "page" : "document"),
 		provenance: input.provenance ?? "unknown",
+		text_source: mode,
+		...(raw ? { charset: view.encoding } : {}),
+		...(mode === "declared" ? { text_sha256: sha256Of(textBody) } : {}),
 	}
 	writeFileSync(join(dir, "meta.json"), `${JSON.stringify(meta, null, 2)}\n`)
-	const sentences = indexSentences(text, sourceId)
+	let altered = null
+	if (mode === "html") {
+		const { altered: changed, ...counts } = probeConversion(view.html, view.text, { attributes: false })
+		altered = changed
+		writeFidelityRecord(join(dir, "conversion.json"), "source_conversion", {
+			artifact_sha256: sha256Of(view.html), report: join(dir, "source.html"), defects: [],
+			conversion_altered_numbers: changed, ...counts,
+		}, new Date(meta.observed_at))
+	} else rmSync(join(dir, "conversion.json"), { force: true })
+	const sentences = indexSentences(view.text, sourceId)
 	writeFileSync(join(dir, "sentences.jsonl"), sentences.map((row) => `${JSON.stringify(row)}\n`).join(""))
-	const acronyms = { schemaVersion: 1, source_id: sourceId, snapshot_sha256: sha256, entries: findAcronyms(text, sentences) }
+	const acronyms = { schemaVersion: 1, source_id: sourceId, snapshot_sha256: sha256, entries: findAcronyms(view.text, sentences) }
 	writeFileSync(join(dir, "acronyms.json"), `${JSON.stringify(acronyms, null, 2)}\n`)
-	return { dir, sha256 }
+	return { dir, sha256, altered }
 }

package/packages/omo-codex/plugin/skills/ulw-research/scripts/source-view.mjs

@@ -0,0 +1,89 @@
+// source-view.mjs - the one reading of a source snapshot that locators, verify, and refresh share.
+//
+// A snapshot's meta.json `text_source` says where its locator text comes from:
+//   html      - text nodes of raw.bin (html-text-map.mjs), offsets are raw.bin bytes
+//   decoded   - raw.bin decoded in its charset, offsets are raw.bin bytes
+//   declared  - text.txt is a declared extraction (--raw + --text) of non-HTML raw.bin; offsets
+//               are text.txt bytes, and raw.bin and text.txt are both pinned
+//   converted - converted-only: no raw.bin, text.txt is all there is (quotes only)
+// A snapshot without text_source and without raw.bin is a legacy text.txt snapshot. byteSpan maps
+// a [start, end) range of `text` to [startByte, endByte) in `artifact`, never through a re-encode.
+// node: builtins and sibling imports only.
+
+import { readFileSync } from "node:fs"
+import { join } from "node:path"
+
+import { sha256Of } from "./cli-support.mjs"
+import { extractHtml } from "./html-text-map.mjs"
+import { decodeRaw } from "./raw-decode.mjs"
+
+export const TEXT_SOURCES = Object.freeze(["html", "decoded", "declared", "converted"])
+
+/** Read one snapshot file; the error names the file and the cause (missing, EACCES, ...). */
+export function readPart(dir, name, encoding) {
+	try {
+		return readFileSync(join(dir, name), encoding)
+	} catch (error) {
+		const cause = error.code === "ENOENT" ? "is missing" : error.code === "EISDIR" ? "is not a readable file" : `is unreadable (${error.code ?? error.message})`
+		throw Object.assign(new Error(`${name} ${cause}`), { code: error.code, part: name })
+	}
+}
+
+/** The text source a meta.json declares, inferring it for snapshots written before the field. */
+export function textSourceOf(meta, hasRaw) {
+	if (TEXT_SOURCES.includes(meta.text_source)) return meta.text_source
+	if (!hasRaw) return meta.kind === "converted-only" ? "converted" : "legacy"
+	return meta.kind === "page" ? "html" : "decoded"
+}
+
+/**
+ * Locator text and byte map for one snapshot body. raw is raw.bin (null for converted/legacy),
+ * textBody is text.txt bytes (needed for declared, converted, legacy).
+ */
+export function buildView(mode, { raw = null, textBody = null, charset = null }) {
+	if (mode === "html") {
+		const decoded = decodeRaw(raw, charset, true)
+		const map = extractHtml(decoded.text)
+		const byteSpan = (start, end) => {
+			if (end <= start) return [decoded.bytes[map.ends[start - 1] ?? 0], decoded.bytes[map.ends[start - 1] ?? 0]]
+			return [decoded.bytes[map.starts[start]], decoded.bytes[map.ends[end - 1]]]
+		}
+		return { mode, artifact: "raw.bin", encoding: decoded.encoding, text: map.text, html: decoded.text, root: map.root, blocks: map.blocks, byteSpan }
+	}
+	if (mode === "legacy") return { mode, artifact: "text.txt", encoding: "utf-8", text: Buffer.from(textBody).toString("utf8"), html: null, root: null, blocks: null, byteSpan: null }
+	const body = mode === "decoded" ? raw : textBody
+	const decoded = decodeRaw(body, mode === "decoded" ? charset : "utf-8")
+	const byteSpan = (start, end) => [decoded.bytes[start], decoded.bytes[Math.max(start, end)]]
+	return { mode, artifact: mode === "decoded" ? "raw.bin" : "text.txt", encoding: decoded.encoding, text: decoded.text, html: null, root: null, blocks: null, byteSpan }
+}
+
+/**
+ * Read and hash a snapshot directory. Throws (with the file and cause) when meta.json, raw.bin, or
+ * text.txt cannot be read; hash drift is returned, not thrown: { meta, view, raw, sha256, drift }.
+ * sha256 is the hash of the pinned body (raw.bin, or text.txt for converted and legacy snapshots).
+ */
+export function readSnapshot(dir) {
+	let meta
+	try {
+		meta = JSON.parse(readPart(dir, "meta.json", "utf8"))
+	} catch (error) {
+		throw error.part ? error : new Error("meta.json is not valid JSON")
+	}
+	if (meta === null || typeof meta !== "object" || typeof meta.sha256 !== "string") throw new Error("has invalid snapshot metadata (meta.json has no sha256)")
+	let raw = null
+	if (meta.text_source === undefined) {
+		try {
+			raw = readPart(dir, "raw.bin")
+		} catch (error) {
+			if (error.code !== "ENOENT") throw error
+		}
+	}
+	const mode = textSourceOf(meta, raw !== null)
+	if (raw === null && ["html", "decoded", "declared"].includes(mode)) raw = readPart(dir, "raw.bin")
+	const textBody = ["declared", "converted", "legacy"].includes(mode) ? readPart(dir, "text.txt") : null
+	const sha256 = sha256Of(raw ?? textBody)
+	const drift = []
+	if (sha256 !== meta.sha256) drift.push(`hash mismatch: ${raw ? "raw.bin" : "text.txt"} changed after its snapshot was pinned`)
+	if (mode === "declared" && sha256Of(textBody) !== meta.text_sha256) drift.push("hash mismatch: text.txt (the declared extraction) changed after it was pinned")
+	return { meta, raw, sha256, drift, view: buildView(mode, { raw, textBody, charset: meta.charset ?? null }) }
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/sources-citation.mjs

@@ -0,0 +1,203 @@
+// sources-citation.mjs - the one Sources predicate shared by the static, trace, and claim gates.
+//
+// Under a Sources heading an element is split at <br>/block breaks and at sentence ends, and every
+// sentence holding a numeral is checked exactly as under any other heading, unless EVERY numeral in
+// it is a citation-field number. That is decided by the number's type and position only: a year
+// (1900-2099), a full date, or a month/day (10/26, Oct 26, 2026. 10. 26.; a day count is not one);
+// a page, volume, issue, number, or edition marker; a number inside a URL, DOI, arXiv id, ISBN or
+// ISSN; a number inside a quoted title ("...", \u201C...\u201D, \u300C...\u300D,
+// \u300E...\u300F, <cite>) or an <i>/<em> title directly before a URL or a date; a number inside link
+// text; or a citation tag's digits. There is no word list: any other numeral makes the sentence a
+// claim. A digit run glued to letters is a name, not a numeral (a hash, #12, GPT-4, v2.3.1, Q3, FY27),
+// unless the letters are a unit, multiplier, or currency of units.mjs / numerals.mjs (50M, 10k, 3x,
+// 2.5GB, 320m, 12%, 40percent, x3). A unit, multiplier, or percent on the right is checked before the
+// letters on the left, so a word glued in front of a measure does not make it a name. A dotted version
+// is a name, a European digit grouping (1.234.567) a numeral; a Korean ordinal (the je prefix, the
+// jjae suffix) is a name and a Korean quarter a date. numeralKinds names the class of every digit run
+// (citation-field, date, name, numeral); only a numeral is checked. A heading is not a prose sentence. Pure.
+// Korean markers are \u escapes.
+
+import { DOT_GROUPED, UNIT } from "./numerals.mjs"
+import { dimensionOf, normalizeUnit } from "./units.mjs"
+
+const SKIP = new Set(["script", "style", "template"])
+const BREAK = new Set(["br", "p", "li", "div", "tr", "dd", "dt"])
+// Abbreviations whose period never ends a citation sentence: pp., p., vol., no., ed., eds., et al., J. (initials).
+const NOT_END = /(?:\b(?:pp?|vol|vols|no|nos|ed|eds|al|ch|sec|fig|vs|e\.g|i\.e|[A-Z])\.)$/i
+const FACT = "\u0000"
+const LINK = "\u0002"
+const [CITE_OPEN, CITE_CLOSE, EM_OPEN, EM_CLOSE, A_OPEN, A_CLOSE] = ["\u0003", "\u0004", "\u0005", "\u0006", "\u0007", "\u0008"]
+const MONTH = "(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\\b\\.?"
+const YEAR = "(?:19|20)\\d{2}"
+const URL = `(?:https?://|www\\.)[^\\s<>"${LINK}${A_CLOSE}]+?(?=[.,;:)\\]]?(?:\\s|$))`
+/** A date at the start of a string: ISO/dotted/slashed, month-name, or Korean nyeon-wol-il. */
+const DATE_AHEAD = new RegExp(`^(?:${YEAR}[-./]\\d{1,2}(?:[-./]\\d{1,2})?|\\d{1,2}[-./]\\d{1,2}[-./]${YEAR}|${MONTH}\\s+\\d{1,2},?\\s+${YEAR}|\\d{1,2}\\s+${MONTH}\\s+${YEAR}|${MONTH}\\s+${YEAR}|${YEAR}\\s*\\uB144|${YEAR}\\b)`, "i")
+/** Citation fields a number can sit INSIDE (a checked sentence is traced without them, so a title is never a quotation). */
+const CONTAINERS = [
+	new RegExp(URL, "gi"),
+	/\b(?:doi:\s*)?10\.\d{4,9}\/[^\s<>"]+/gi,
+	/\b(?:arXiv:\s*)?\d{4}\.\d{4,5}(?:v\d+)?\b/gi,
+	/\bIS[BS]N(?:-1[03])?:?\s*[\dXx][\dXx\s-]{6,16}[\dXx]/g,
+	/\[(?:S|Source\s*)?\^?\d+\]/gi,
+	/"[^"]*"|\u201C[^\u201D]*\u201D|\u300C[^\u300D]*\u300D|\u300E[^\u300F]*\u300F/g,
+	new RegExp(`${CITE_OPEN}[^${CITE_CLOSE}]*${CITE_CLOSE}`, "g"),
+	new RegExp(`${A_OPEN}[^${A_CLOSE}]*${A_CLOSE}`, "g"),
+]
+/** Date fields: full dates, month/day, a Korean quarter, dotted dates (with spaces, or DD.MM.YY), years and year ranges. */
+const DATES = [
+	new RegExp(`${YEAR}[-./]\\d{1,2}(?:[-./]\\d{1,2})?|\\d{1,2}[-./]\\d{1,2}[-./]${YEAR}`, "g"),
+	new RegExp(`\\b${MONTH}\\s+\\d{1,2}(?:st|nd|rd|th)?\\b(?:,?\\s+${YEAR})?|\\b\\d{1,2}(?:st|nd|rd|th)?\\s+${MONTH}`, "gi"),
+	/\d{1,2}\s*\uC6D4(?:\s*\d{1,2}\s*\uC77C)?|(?<!\d)[1-4]\s*\uBD84\uAE30/g,
+	new RegExp(`${YEAR}\\.\\s+\\d{1,2}\\.\\s+\\d{1,2}\\.?`, "g"),
+	/(?<![\d/.,])(?:1[0-2]|0?[1-9])\/(?:3[01]|[12]\d|0?[1-9])(?![\d/]|[.,]\d)/g,
+	/(?<![\d.])\d{2}\.\d{2}\.\d{2}(?!\d|\.\d)/g,
+	new RegExp(`(?<![\\d.,])${YEAR}(?:\\s*[-\\u2013~]\\s*${YEAR})?(?![\\d]|[.,]\\d)`, "g"),
+]
+/** Marker fields: page, volume, issue, number, and edition markers. */
+const MARKERS = [
+	/\b(?:pp?|pg|pages?)\.?\s*\d+(?:\s*[-\u2013]\s*\d+)?/gi,
+	/\b(?:vols?|nos?|iss|ed)\.\s*\d+(?:\s*[-\u2013/]\s*\d+)*/gi,
+	/\b(?:Vol|Volume|No|Issue|Number|volume|issue)\s+\d+(?:\s*[-\u2013/]\s*\d+)*/g,
+	/\b\d+(?:st|nd|rd|th)\s+(?:ed\.|edition)|\bedition\s+\d+/gi,
+	/\uC81C?\s*\d+(?:\s*[-\u2013~]\s*\d+)?\s*(?:\uCABD|\uBA74|\uAD8C|\uD638|\uD310)/g,
+]
+const TYPED = [...DATES, ...MARKERS]
+const NUMERAL = /\d+(?:[.,]\d+)*/g
+const UNIT_WORD = new RegExp(`^${UNIT}$`, "iu")
+const UNIT_LEAD = new RegExp(`^${UNIT}`, "iu")
+const LETTERS_AFTER = /^(?:[A-Za-z]+|[\uAC00-\uD7A3]+|\p{L}+)/u
+/** A unit or currency symbol right after the digits (numerals.mjs SYMBOL_UNIT plus the currency signs). */
+const SYMBOL_AFTER = /^(?:%|\u2030|\u00D7|\u00B0[CF]|\u2103|\u2109|[$\u20AC\u00A3\u00A5\u20A9])/
+/** Korean ordinals: the prefix je ("je 3 hoe") and a counter followed by jjae ("3 beon jjae"). */
+const KO_ORDINAL_BEFORE = /(?:^|[^\p{L}])\uC81C$/u
+const KO_ORDINAL_AFTER = /^[\uAC00-\uD7A3]?\uC9F8/
+const GROUPED = new RegExp(`^${DOT_GROUPED}$`)
+
+/** Letters that make a glued digit run a quantity; a Korean run may carry a particle after its unit. */
+function isUnit(letters) {
+	if (!/^[\uAC00-\uD7A3]/.test(letters)) return UNIT_WORD.test(letters) || normalizeUnit(letters) !== null || normalizeUnit(letters.toLowerCase()) !== null
+	if (UNIT_LEAD.test(letters)) return true
+	for (let size = letters.length; size > 0; size--) if (normalizeUnit(letters.slice(0, size))) return true
+	return false
+}
+/** A multiplier written before its number ("x3", "x1.5"): a standalone x, never a label ("w1:x02") or a leading zero. */
+const isMultiplierBefore = (before, left, run) => dimensionOf(left) === "multiple" && /(?:^|[\s(])\p{L}+$/u.test(before) && !/^0\d/.test(run)
+const isCurrency = (letters) => /^currency:/.test(dimensionOf(letters) ?? dimensionOf(letters.toUpperCase()) ?? "")
+
+/**
+ * The digit run [start, end) is part of a name: a dotted version ("0.30.14", not the grouping
+ * "1.234.567"), a run after # or inside a word (letters on its left that are not a currency or a
+ * standalone multiplier x: "W3C", "eb2b", "Q3", "FY27"; or glued on the left through - . _ /: "GPT-4", "H.15"),
+ * or a run whose right-hand letters are not a unit, multiplier, or currency ("3D", "21st", "10-Q",
+ * "1H25"). A unit, multiplier, or percent on the right makes the run a numeral first, whatever is
+ * glued on its left ("revenue40%", "about300 people" written solid), unless Latin letters sit on both
+ * sides, which is an identifier ("W3C", "eb2b").
+ */
+function isName(text, start, end) {
+	const run = text.slice(start, end)
+	if (/^\d+(?:\.\d+){2,}$/.test(run) && !GROUPED.test(run)) return true
+	const before = text.slice(0, start)
+	const rest = text.slice(end)
+	if (KO_ORDINAL_BEFORE.test(before) || KO_ORDINAL_AFTER.test(rest)) return true
+	const left = /(?<![A-Za-z])[A-Za-z]+$/.exec(before)?.[0] ?? /\p{L}+$/u.exec(before)?.[0]
+	const right = LETTERS_AFTER.exec(rest)?.[0]
+	const measure = right ? !/^\d/.test(rest.slice(right.length)) && isUnit(right) : SYMBOL_AFTER.test(rest)
+	if (measure && !(right && /^[A-Za-z]/.test(right) && /[A-Za-z]$/.test(before))) return false
+	if (left) return !isCurrency(left) && !(!right && isMultiplierBefore(before, left, run))
+	if (right) return true
+	if (before.endsWith("#") || /\p{L}/u.test(/([\p{L}\p{N}]+)[-._/]$/u.exec(before)?.[1] ?? "")) return true
+	const joined = /^-(\p{L}[\p{L}\p{N}]*)/u.exec(rest)?.[1]
+	return joined !== undefined && !isUnit(joined)
+}
+
+/** A fact element is bound by its ledger row, not traced here; it reads as one opaque token. */
+const isFact = (node) => node.attrs?.["data-fact"] !== undefined || node.attrs?.["data-translates"] !== undefined || node.tag === "q" || node.tag === "blockquote"
+
+function lines(node, out = [""]) {
+	for (const child of node.children ?? []) {
+		if (child.tag === "#text") out[out.length - 1] += child.text
+		else if (SKIP.has(child.tag)) continue
+		else if (isFact(child)) out[out.length - 1] += ` ${FACT} `
+		else if (child.tag === "cite" || child.tag === "i" || child.tag === "em" || child.tag === "a") {
+			const [open, close] = child.tag === "cite" ? [CITE_OPEN, CITE_CLOSE] : child.tag === "a" ? [A_OPEN, A_CLOSE] : [EM_OPEN, EM_CLOSE]
+			out[out.length - 1] += open
+			lines(child, out)
+			out[out.length - 1] += close
+			if (child.tag === "a" && /^https?:\/\//i.test(child.attrs?.href ?? "")) out[out.length - 1] += ` ${LINK} `
+		} else {
+			if (BREAK.has(child.tag)) out.push("")
+			lines(child, out)
+			if (BREAK.has(child.tag)) out.push("")
+		}
+	}
+	return out
+}
+
+/** Spans [start, end) of the citation fields in `text`; an <i>/<em> run is a title only right before a URL, a link, or a date. */
+function containerSpans(text) {
+	const spans = []
+	for (const re of CONTAINERS) for (const match of text.matchAll(re)) spans.push([match.index, match.index + match[0].length])
+	for (const match of text.matchAll(new RegExp(`${EM_OPEN}[^${EM_CLOSE}]*${EM_CLOSE}`, "g"))) {
+		const after = text.slice(match.index + match[0].length).replace(/^[\s,.:;()\u2013-]*/, "")
+		if (after.startsWith(LINK) || after.startsWith(A_OPEN) || /^(?:https?:\/\/|www\.)/i.test(after) || DATE_AHEAD.test(after)) spans.push([match.index, match.index + match[0].length])
+	}
+	return spans
+}
+
+const typedSpans = (text, list) => list.flatMap((re) => [...text.matchAll(re)].map((match) => [match.index, match.index + match[0].length]))
+
+const fieldSpans = (text) => [...containerSpans(text), ...typedSpans(text, TYPED)]
+
+const inside = (spans, start, end) => spans.some(([s, e]) => start >= s && end <= e)
+
+/** Sentences of one line; a period after an abbreviation, an initial, or inside a field never ends one. */
+function sentences(line) {
+	const spans = fieldSpans(line)
+	const out = []
+	let start = 0
+	for (const match of line.matchAll(/[.!?](?=\s+\S|$)/g)) {
+		const end = match.index + 1
+		if (NOT_END.test(line.slice(start, end)) || inside(spans, match.index, end)) continue
+		out.push(line.slice(start, end))
+		start = end
+	}
+	out.push(line.slice(start))
+	return out.map((text) => text.replace(/\s+/g, " ").trim()).filter(Boolean)
+}
+
+/** Each digit run of `text` with its class: "citation-field" (a container or a marker), "date", "name", or "numeral". */
+export function numeralKinds(text) {
+	const [containers, markers, dates] = [containerSpans(text), typedSpans(text, MARKERS), typedSpans(text, DATES)]
+	return [...text.matchAll(NUMERAL)].map((match) => {
+		const [start, end] = [match.index, match.index + match[0].length]
+		const kind = inside(containers, start, end) || inside(markers, start, end) ? "citation-field" : inside(dates, start, end) ? "date" : isName(text, start, end) ? "name" : "numeral"
+		return { text: match[0], kind }
+	})
+}
+
+/** The sentence holds a numeral that is not a citation-field number, so the gates check it. */
+export const isClaimSegment = (text) => numeralKinds(text).some(({ kind }) => kind === "numeral")
+
+/** A checked sentence as the gates read it: container fields (URL, DOI, title, tag) removed, every other number kept. */
+function plain(text) {
+	let out = ""
+	let at = 0
+	for (const [start, end] of containerSpans(text).sort((a, b) => a[0] - b[0])) {
+		if (end <= at) continue
+		out += `${text.slice(at, Math.max(at, start))} `
+		at = end
+	}
+	return `${out}${text.slice(at)}`.replace(/[\u0002-\u0008]/g, " ").replace(/\s+/g, " ").trim()
+}
+const lineClaims = (line) => sentences(line).filter(isClaimSegment).map(plain)
+
+/** The checked sentences of an element under Sources ([] when every numeral is a citation field). */
+export function sourcesClaims(node) {
+	if (/^h[1-6]$/.test(node.tag)) return []
+	return lines(node).flatMap(lineClaims)
+}
+
+/** The checked sentences of loose text (a text node owned by a container element rather than a prose element). */
+export function textClaims(text) {
+	return lines({ children: [{ tag: "#text", text }] }).flatMap(lineClaims)
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/units.mjs

@@ -21,6 +21,9 @@ const TABLE = [
 	["s", "time", "1", "open", ["sec", "secs", "second", "seconds", "\uCD08"]], // cho
 	["min", "time", "60", "closed", ["mins", "minute", "minutes", "\uBD84"]], // bun
 	["h", "time", "3600", "closed", ["hr", "hrs", "hour", "hours", "\uC2DC\uAC04"]], // sigan
+	// Months and years are a calendar dimension of their own (a year is no fixed number of seconds).
+	["mo", "calendar", "1", "l", []], // gaewol
+	["yr", "calendar", "12", "closed", ["yrs"]], // nyeon
 	["B", "data", "1", "open", ["byte", "bytes", "\uBC14\uC774\uD2B8"]], // baiteu
 	["KB", "data", "1000", "open", ["\uD0AC\uB85C\uBC14\uC774\uD2B8"]],
 	["MB", "data", "1000000", "open", ["\uBA54\uAC00\uBC14\uC774\uD2B8"]],
@@ -28,13 +31,13 @@ const TABLE = [
 	["TB", "data", "1000000000000", "open", ["\uD14C\uB77C\uBC14\uC774\uD2B8"]],
 	["C", "temperature", "1", "open", ["\u00B0C", "\u2103", "celsius", "\uC12D\uC528"], "0"], // do
 	["F", "temperature", "5/9", "open", ["\u00B0F", "\u2109", "fahrenheit", "\uD654\uC528"], "-160/9"], // do
-	["%", "percent", "1", "open", ["percent", "\uD37C\uC13C\uD2B8", "\uD504\uB85C"]], // peosenteu
+	["%", "percent", "1", "open", ["percent", "pct", "\uD37C\uC13C\uD2B8", "\uD504\uB85C"]], // peosenteu
 	["\u2030", "percent", "0.1", "l", ["permille", "\uD37C\uBC00"]], // peomil
-	["%p", "percentage-point", "1", "open", ["pp", "%pt", "percentage point", "percentage points", "\uD37C\uC13C\uD2B8\uD3EC\uC778\uD2B8", "%\uD3EC\uC778\uD2B8"]],
+	["%p", "percentage-point", "1", "open", ["pp", "%pt", "percentage point", "percentage points", "percent point", "percent points", "% point", "% points", "\uD37C\uC13C\uD2B8\uD3EC\uC778\uD2B8", "%\uD3EC\uC778\uD2B8"]],
 	["x", "multiple", "1", "open", ["\u00D7", "times", "\uBC30"]], // bae
 	["e3", "scale", "1000", "closed", ["k", "thousand", "\uCC9C"]], // cheon
 	["e4", "scale", "10000", "closed", ["\uB9CC"]], // man
-	["e6", "scale", "1000000", "closed", ["M", "million", "\uBC31\uB9CC"]], // baengman
+	["e6", "scale", "1000000", "closed", ["M", "mn", "million", "\uBC31\uB9CC"]], // baengman
 	["e8", "scale", "100000000", "closed", ["\uC5B5"]], // eok
 	["e9", "scale", "1000000000", "closed", ["bn", "billion", "\uC2ED\uC5B5"]], // sibeok
 	["e12", "scale", "1000000000000", "open", ["trillion", "\uC870"]], // jo
@@ -59,6 +62,8 @@ const ENTRIES = new Map()
 const EXACT = new Map()
 /** Case-insensitive keys: multi-letter ASCII aliases only, so m/M and B/b never collide. */
 const WORD = new Map()
+/** Aliases written with their own spacing ("percentage points"), by lower-case text. */
+const SPACED = new Map()
 for (const [token, dimension, factor, reading, aliases, offset] of TABLE) {
 	const entry = Object.freeze({ token, dimension, factor: parse(factor), offset: parse(offset ?? "0"), reading })
 	ENTRIES.set(token, entry)
@@ -66,6 +71,7 @@ for (const [token, dimension, factor, reading, aliases, offset] of TABLE) {
 		const key = squash(alias)
 		EXACT.set(key, entry)
 		if (alias !== token && key.length > 2 && /^[a-z]+$/.test(key) && !WORD.has(key.toLowerCase())) WORD.set(key.toLowerCase(), entry)
+		if (/\s/.test(alias)) SPACED.set(alias.toLowerCase(), entry)
 	}
 }
 
@@ -81,6 +87,11 @@ export function normalizeUnit(raw) {
 	return entryOf(raw)?.token ?? null
 }
 
+/** Canonical token for a multi-word unit written with its alias's own spacing, or null (never a squashed join). */
+export function spacedAliasUnit(raw) {
+	return SPACED.get(String(raw).trim().replace(/\s+/g, " ").toLowerCase())?.token ?? null
+}
+
 /** Dimension of a unit ("length", "currency:KRW", ...), or null. */
 export function dimensionOf(token) {
 	return entryOf(token)?.dimension ?? null

package/packages/omo-codex/plugin/test/payload-equivalence.test.mjs

@@ -1,5 +1,6 @@
 import assert from "node:assert/strict";
-import { mkdtempSync, rmSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
 import { readFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -59,9 +60,11 @@ test("payload-equivalence: #given telemetry-core and component shim #when sessio
 			stateDir,
 			transportFactory: componentRecorder.factory,
 		});
-		const expectedDistinctId = componentPostHog.getPostHogDistinctIdForTesting(createOsProvider());
-		componentClient.trackActive(expectedDistinctId, "session_start");
+		componentClient.trackActive("session_start");
 		await componentClient.shutdown();
+		const installId = readFileSync(join(stateDir, "install-id"), "utf8").trim();
+		assert.match(installId, /^[0-9a-f]{64}$/);
+		const expectedDistinctId = createHash("sha256").update(installId).digest("hex");
 
 		assert.deepEqual(componentMessages, [
 			{
@@ -104,6 +107,5 @@ test("payload-equivalence: #given built telemetry CLI #when inspected #then tele
 	assert.doesNotMatch(bundle, new RegExp(`from ["']${TELEMETRY_CORE_PACKAGE}`));
 	assert.doesNotMatch(bundle, new RegExp(`import\\(["']${TELEMETRY_CORE_PACKAGE}`));
 	assert.match(bundle, /omo_codex_daily_active/);
-	assert.match(bundle, /omo-codex:/);
 	assert.match(bundle, /posthog-activity\.json/);
 });

package/packages/omo-codex/plugin/test/sync-skills.test.mjs

@@ -146,10 +146,11 @@ test("#given shared skill source tests #when aggregate Codex skills are synced #
 		for (const file of await listSkillFiles(join(aggregateSkillsRoot, skillName))) {
 			const normalized = file.replaceAll("\\", "/");
 			const segments = normalized.split("/");
-			const scriptsIndex = segments.lastIndexOf("scripts");
-			const hasPythonTestDir = scriptsIndex !== -1 && segments[scriptsIndex + 1] === "tests";
+			const hasNestedTestDir = segments
+				.slice(0, -1)
+				.some((segment, index) => /^(?:tests?|__tests__|.+_test)$/.test(segment) && !(index === 0 && (segment === "test" || segment === "tests")));
 			const isSourceMetadata = normalized === ".gitignore" || normalized === ".npmignore" || normalized === "pyrightconfig.json";
-			if (normalized.endsWith(".test.ts") || hasPythonTestDir || segments.includes("__pycache__") || normalized.endsWith(".pyc") || isSourceMetadata) {
+			if (normalized.endsWith(".test.ts") || hasNestedTestDir || segments.includes("__pycache__") || normalized.endsWith(".pyc") || isSourceMetadata) {
 				forbiddenFiles.push(`${skillName}/${normalized}`);
 			}
 		}