oh-my-opencode 5.1.48 → 5.1.54

package/packages/omo-codex/plugin/skills/ulw-research/scripts/markdown-closers.mjs

@@ -0,0 +1,126 @@
+// markdown-closers.mjs - the Markdown closer runs a converter writes after a piece of text, read with
+// the CommonMark inline grammar instead of a hand-written pattern. A closer run is any sequence of:
+// an inline link ending `](dest "title")` (the destination either in <...> or bare with balanced
+// parentheses, the optional title in "...", '...' or (...), backslash escapes inside both), a full or
+// collapsed reference ending `][ref]` / `][]`, a whole image `![alt](...)` or `![alt][ref]` (its alt
+// text is not context), the `[` that opened a link whose text is all markup (`[![x](i.png)](/p)`),
+// an emphasis, strike or code delimiter, a bare `)`, `]` or `>`, and a backslash-escaped closer
+// (`\*`, `\_`, `\]`, `\)`, `\~`, a backtick), which stands for the raw character and is skipped like
+// it. An escaped opener (`\[`, `\(`) is a real character and ends the run. Shared by the conversion
+// probe's sign reading and its link-syntax spans. Pure; no imports.
+
+const SIMPLE = new Set(["*", "_", "~", "`", ")", "]", ">"])
+const BREAK = "\u0001"
+
+/** Whether the character at `index` is backslash-escaped (an odd run of backslashes before it). */
+export function escapedAt(text, index) {
+	let count = 0
+	for (let i = index - 1; i >= 0 && text[i] === "\\"; i--) count++
+	return count % 2 === 1
+}
+
+/** The end (exclusive) of an inline link target `dest "title")` that starts right after its "(", or -1. */
+export function inlineTargetEnd(text, from) {
+	let i = from
+	const space = () => {
+		while (text[i] === " " || text[i] === "\t") i++
+	}
+	space()
+	if (text[i] === "<") {
+		for (i++; i < text.length && text[i] !== ">"; i++) {
+			if (text[i] === "\\") i++
+			else if (text[i] === "<" || text[i] === BREAK) return -1
+		}
+		if (text[i++] !== ">") return -1
+	} else {
+		let depth = 0
+		for (; i < text.length; i++) {
+			const ch = text[i]
+			if (ch === "\\") i++
+			else if (ch === "(") depth++
+			else if (ch === ")" && depth === 0) break
+			else if (ch === ")") depth--
+			else if (ch === " " || ch === "\t" || ch === BREAK) break
+		}
+		if (depth !== 0) return -1
+	}
+	const destEnd = i
+	space()
+	const open = text[i]
+	if (i > destEnd && (open === '"' || open === "'" || open === "(")) {
+		const close = open === "(" ? ")" : open
+		for (i++; i < text.length && text[i] !== close; i++) {
+			if (text[i] === "\\") i++
+			else if (text[i] === BREAK || (open === "(" && text[i] === "(")) return -1
+		}
+		if (text[i++] !== close) return -1
+		space()
+	}
+	return text[i] === ")" ? i + 1 : -1
+}
+
+/** The unescaped "[" that the unescaped "]" at `close` closes (brackets balance), or -1. */
+function openerOf(text, close) {
+	let depth = 0
+	for (let i = close; i >= 0 && text[i] !== BREAK; i--) {
+		if ((text[i] !== "[" && text[i] !== "]") || escapedAt(text, i)) continue
+		depth += text[i] === "]" ? 1 : -1
+		if (depth === 0) return i
+	}
+	return -1
+}
+
+/** The "](" positions whose inline target ends exactly at `end`, leftmost first (so a title holding "](" is not cut). */
+function inlineLinkClose(text, end) {
+	const from = Math.max(0, text.lastIndexOf(BREAK, end - 1))
+	for (let k = text.indexOf("](", from); k !== -1 && k < end; k = text.indexOf("](", k + 1)) {
+		if (!escapedAt(text, k) && inlineTargetEnd(text, k + 2) === end) return k
+	}
+	return -1
+}
+
+/** Where the link or image tail ending at `end` starts, and whether its "[" is still open; null when none ends there. */
+function linkTail(text, end) {
+	let close = -1
+	if (text[end - 1] === ")") close = inlineLinkClose(text, end)
+	else if (text[end - 1] === "]" && !escapedAt(text, end - 1)) {
+		const ref = openerOf(text, end - 1)
+		if (ref > 0 && text[ref - 1] === "]" && !escapedAt(text, ref - 1)) close = ref - 1
+	}
+	if (close === -1) return null
+	const open = openerOf(text, close)
+	if (open > 0 && text[open - 1] === "!" && !escapedAt(text, open - 1)) return { start: open - 1, opened: 0 }
+	return { start: close, opened: open === -1 ? 0 : 1 }
+}
+
+/** `text` without the Markdown closer run it ends with. */
+export function stripClosers(text) {
+	let end = text.length
+	let owed = 0
+	for (;;) {
+		const tail = linkTail(text, end)
+		if (tail) {
+			end = tail.start
+			owed += tail.opened
+			continue
+		}
+		const ch = text[end - 1]
+		if (SIMPLE.has(ch)) end -= escapedAt(text, end - 1) ? 2 : 1
+		else if (ch === "[" && owed > 0 && !escapedAt(text, end - 1)) {
+			end--
+			owed--
+		} else return text.slice(0, end)
+	}
+}
+
+/** [start, end) spans of every inline link or image with its target (an image from its "!"), for link-syntax cover. */
+export function inlineLinkSpans(text) {
+	const spans = []
+	for (let k = text.indexOf("]("); k !== -1; k = text.indexOf("](", k + 1)) {
+		const end = escapedAt(text, k) ? -1 : inlineTargetEnd(text, k + 2)
+		if (end === -1) continue
+		const open = openerOf(text, k)
+		spans.push([open > 0 && text[open - 1] === "!" ? open - 1 : k, end])
+	}
+	return spans
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/md-lite.mjs

@@ -113,7 +113,7 @@ export function parseMarkdown(text) {
 			const codeStart = Math.min(line.next, source.length)
 			let close = lines.length
 			for (let k = i + 1; k < lines.length; k++) {
-				const candidate = lines[k].raw.trim()
+				const candidate = lines[k].raw.replace(/^ {0,3}/, "").trimEnd()
 				if (candidate.startsWith(open[0].repeat(open.length)) && /^(`+|~+)$/.test(candidate)) {
 					close = k
 					break

package/packages/omo-codex/plugin/skills/ulw-research/scripts/number-grammar.mjs

@@ -0,0 +1,33 @@
+// number-grammar.mjs - the one grouped-number grammar. A grouped number is a 1-3 digit leading
+// group, then one or more 3-digit groups joined by ONE separator type, with no digit right after the
+// last group, then an optional decimal part. "12,3451" is therefore not "12,345" plus "1". A period
+// groups only when the reading is unambiguous: two or more period groups ("1.234.567") or one group
+// before a decimal comma ("1.234,5"); a lone "1.234" is a decimal, as numerals.mjs reads it. Shared by
+// numerals.mjs (comma grouping), the conversion probe (every separator), the value-mode unit check,
+// and the converted-half display check. Pure; no imports.
+
+/** Separators a thousands group may use: comma, period, space, NBSP, narrow NBSP, thin space, figure space. */
+export const GROUP_SEPARATORS = Object.freeze([",", ".", " ", "\u00A0", "\u202F", "\u2009", "\u2007"])
+
+const literal = (sep) => (sep === "." ? "\\." : sep === " " ? " " : sep === "," ? "," : `\\u${sep.charCodeAt(0).toString(16).padStart(4, "0")}`)
+const decimalAfter = (sep) => (sep === "," ? "\\." : sep === "." ? "," : "[.,]")
+
+/** Regex source of one grouped number using any of `separators` (one type per number); no flags needed. */
+export function groupedNumberSource(separators = GROUP_SEPARATORS) {
+	const one = (sep) => sep === "."
+		? "\\d{1,3}(?:(?:\\.\\d{3}(?!\\d)){2,}(?:,\\d+)?|\\.\\d{3}(?!\\d),\\d+)"
+		: `\\d{1,3}(?:${literal(sep)}\\d{3}(?!\\d))+(?:${decimalAfter(sep)}\\d+)?`
+	return `(?<!\\d)(?:${separators.map(one).join("|")})`
+}
+
+/** A grouped number or a plain one ("4.22", "12,3451" read as 12.3451), as regex source. */
+export const NUMBER_SOURCE = `(?:${groupedNumberSource()}|(?<!\\d)\\d+(?:[.,]\\d+)?)`
+
+const WHOLE = new Map(GROUP_SEPARATORS.map((sep) => [sep, new RegExp(`^${groupedNumberSource([sep])}$`)]))
+
+/** The plain decimal a number token spells: group separators dropped, a decimal comma read as a point. */
+export function plainDecimal(token) {
+	const sep = GROUP_SEPARATORS.find((candidate) => WHOLE.get(candidate).test(token))
+	const ungrouped = sep === undefined ? token : token.split(sep).join("")
+	return ungrouped.replace(",", ".")
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/numerals.mjs

@@ -6,9 +6,12 @@
 // English number words one to twelve, an adjacent currency symbol or unit, and an adjacent bounding
 // qualifier. No numeral is ever passed through Number(). Plain Node ESM, sibling imports only.
 
+import { groupedNumberSource } from "./number-grammar.mjs"
 import { add, fromDecimal, mul, toDecimal } from "./rational.mjs"
 import { normalizeUnit } from "./units.mjs"
 
+/** European digit grouping ("1.234.567"): two or more dot groups of three digits, never a version. */
+export const DOT_GROUPED = "\\d{1,3}(?:\\.\\d{3}){2,}"
 const MONTH = "(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\\.?"
 /** Stripped before claim detection: dates, versions, figure refs, times, bare years. */
 export const NOT_CLAIMS = [
@@ -19,7 +22,7 @@ export const NOT_CLAIMS = [
 	/(?:19|20)\d{2}\s*\uB144/g,
 	new RegExp(`\\b${MONTH}\\s+\\d{1,2}(?:st|nd|rd|th)?(?:,?\\s+\\d{4})?`, "gi"),
 	new RegExp(`\\b\\d{1,2}(?:st|nd|rd|th)?\\s+${MONTH}(?:\\s+\\d{4})?`, "gi"),
-	/\bv\d+(?:\.\d+)*\b|\b\d+(?:\.\d+){2,}\b/gi,
+	new RegExp(`\\bv\\d+(?:\\.\\d+)*\\b|\\b(?!${DOT_GROUPED}(?![\\d,]|\\.\\d))\\d+(?:\\.\\d+){2,}\\b`, "gi"),
 	/\b(?:version|ver\.?|release)\s*\d+(?:\.\d+)*/gi,
 	/\b(?:fig(?:ure)?\.?|table|chart|section|step|phase|wave|part)\s*\d+(?:\.\d+)*/gi,
 	/(?:\uADF8\uB9BC|\uB3C4\uD45C|\uD45C|\uBC84\uC804)\s*\d+(?:\.\d+)*/g,
@@ -45,8 +48,8 @@ const BIG = new Map([["\uB9CC", 10000n], ["\uC5B5", 100000000n], ["\uC870", 1000
 /** A scale word that starts a longer word is not a scale: jogak (pieces), mankeum (as much as). */
 const KO_GROUP = /^(\s?)(?:(\uCC9C)?([\uB9CC\uC5B5\uC870])|(\uCC9C))(?![\uAC01\uD07C])/
 const KO_NEXT = new RegExp(`^\\s?(${NUM})(\uCC9C)?([\uB9CC\uC5B5\uC870])?`, "u")
-const LATIN_SCALE = /^(?:(k|K|M|bn|B)|\s(thousand|million|billion|trillion|bn))(?![\p{L}\d])/u
-const LATIN_FACTOR = { k: 1000n, K: 1000n, M: 1000000n, bn: 1000000000n, B: 1000000000n, thousand: 1000n, million: 1000000n, billion: 1000000000n, trillion: 1000000000000n }
+const LATIN_SCALE = /^(?:(k|K|M|mn|bn|B)|\s(thousand|million|billion|trillion|mn|bn))(?![\p{L}\d])/u
+const LATIN_FACTOR = { k: 1000n, K: 1000n, M: 1000000n, mn: 1000000n, bn: 1000000000n, B: 1000000000n, thousand: 1000n, million: 1000000n, billion: 1000000000n, trillion: 1000000000000n }
 const CURRENCY = "$\u20AC\u00A3\u00A5\u20A9"
 const SYMBOL_UNIT = /^\s?(%p|%|\u2030|\u00D7|\u00B0C|\u00B0F|\u2103|\u2109)/
 const LATIN_UNIT = /^\s?([A-Za-z]+)(?![\p{L}\d])/u
@@ -60,15 +63,23 @@ const RANGE_SEP = /^(?:\s?[-\u2013~]\s?|\sto\s)/
 const SIGN = "-\u2212"
 /** Bounding qualifiers: before the value (Korean and English) and right after it (Korean). */
 const QUALIFIERS_BEFORE = ["up to", "at least", "at most", "more than", "less than", "about", "approximately", "nearly", "over", "under", "\uCD5C\uB300", "\uCD5C\uC18C", "\uC57D"]
-const QUALIFIER_AFTER = /^\s?(\uC774\uC0C1|\uC774\uD558|\uBBF8\uB9CC|\uCD08\uACFC|\uB0B4\uC678|\uAE4C\uC9C0)/
+/** Korean bounding qualifiers written right after a value (isang, iha, miman, chogwa, naeoe, kkaji). */
+export const QUALIFIERS_AFTER = Object.freeze(["\uC774\uC0C1", "\uC774\uD558", "\uBBF8\uB9CC", "\uCD08\uACFC", "\uB0B4\uC678", "\uAE4C\uC9C0"])
+const QUALIFIER_AFTER = new RegExp(`^\\s?(${QUALIFIERS_AFTER.join("|")})`)
 
-/** "1,237" -> "1237"; a malformed group ("1,23") keeps only the leading plain number. */
+/** Comma grouping from the one grouped-number grammar (number-grammar.mjs). */
+const COMMA_GROUPED = new RegExp(`^${groupedNumberSource([","])}$`)
+
+/** "1,237" -> "1237"; a malformed group ("1,23", "12,3451") keeps only the leading plain number. */
 function cleanDigits(raw) {
-	if (/^\d{1,3}(?:,\d{3})+(?:\.\d+)?$/.test(raw)) return { digits: raw.replace(/,/g, ""), length: raw.length }
+	if (COMMA_GROUPED.test(raw)) return { digits: raw.replace(/,/g, ""), length: raw.length }
 	const plain = /^\d+(?:\.\d+)?/.exec(raw)[0]
 	return { digits: plain, length: plain.length }
 }
 
+/** A numeral's digits, reading a European period grouping ("1.234.567") as one integer. */
+const digitsOf = (raw) => (new RegExp(`^${DOT_GROUPED}$`).test(raw) ? { digits: raw.replace(/\./g, ""), length: raw.length } : cleanDigits(raw))
+
 /** The bare-year and four-digit-date patterns: a numeral they cover can still be a value. */
 const YEAR_SHAPED = new Set([NOT_CLAIMS[0], NOT_CLAIMS[12]])
 
@@ -95,7 +106,7 @@ function findStart(text, from, ranges) {
 	let digit = null
 	for (const match of text.slice(from).matchAll(new RegExp(NUM, "gu"))) {
 		const at = from + match.index
-		const candidate = { at, ...cleanDigits(match[0]) }
+		const candidate = { at, ...digitsOf(match[0]) }
 		const end = at + candidate.length
 		const cover = covering(at)
 		const lifted = cover.every(([s, e, yearShaped]) => yearShaped && s === at && e <= end)

package/packages/omo-codex/plugin/skills/ulw-research/scripts/outcome-start.mjs

@@ -0,0 +1,66 @@
+// outcome-start.mjs - the run clock (outcome.json startedAt). `outcome init` is its one writer and
+// records it once: --started-at from the orchestrator, else the startedAt of a valid manifest an
+// earlier init wrote (a re-init keeps the clock), else terminal-state.json.started_at when a
+// supervisor wrote one, else the YYYYMMDD-HHMMSS session dir name. A recorded value must be a real
+// ISO time with Z or an offset, not after `now`, and within a day of a stamped dir name; otherwise
+// init refuses instead of falling back. Never the interview's as-of date. No clock reads.
+import { readFileSync } from "node:fs"
+import { basename, join, resolve } from "node:path"
+
+const ISO = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-](\d{2}):(\d{2}))$/
+const SESSION_STAMP = /^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$/
+const DAY_MS = 86_400_000
+
+/** A real calendar time (no 2026-02-30, no T24:00), or null. */
+function strictIso(value) {
+	const match = typeof value === "string" ? ISO.exec(value) : null
+	if (!match) return null
+	const [y, mo, d, h, mi, s, oh = 0, om = 0] = match.slice(1).map((part) => Number(part ?? 0))
+	const day = new Date(Date.UTC(y, mo - 1, d))
+	if (day.getUTCMonth() !== mo - 1 || day.getUTCDate() !== d || h > 23 || mi > 59 || s > 59 || oh > 23 || om > 59) return null
+	const date = new Date(value)
+	return Number.isNaN(date.getTime()) ? null : date
+}
+
+/** Local start time from a YYYYMMDD-HHMMSS dir name, null when the name is not a stamp. */
+function dirStamp(sessionDir, fail) {
+	const match = SESSION_STAMP.exec(basename(sessionDir))
+	if (!match) return null
+	const [y, mo, d, h, mi, s] = match.slice(1).map(Number)
+	const date = new Date(y, mo - 1, d, h, mi, s)
+	if (date.getMonth() !== mo - 1 || date.getDate() !== d || date.getHours() !== h) fail(`session dir basename is not a real local time: ${sessionDir}`)
+	return date
+}
+
+function readJson(sessionDir, file) {
+	try {
+		return { data: JSON.parse(readFileSync(join(sessionDir, file), "utf8")) }
+	} catch (error) {
+		if (error.code === "ENOENT" || error.code === "ENOTDIR") return null
+		if (error instanceof SyntaxError) return { corrupt: `${join(sessionDir, file)} is not valid JSON (${error.message})` }
+		throw error
+	}
+}
+
+/**
+ * @param {{ sessionDir: string, startedAt?: string, now?: Date, validManifest: (data: unknown) => boolean, fail: (message: string) => never }} input
+ * @returns {Date}
+ */
+export function runStart({ sessionDir, startedAt, now, validManifest, fail }) {
+	const dir = dirStamp(sessionDir, fail)
+	const check = (value, location) => {
+		const date = strictIso(value)
+		if (!date) fail(`invalid run timestamp in ${location}: ${JSON.stringify(value)} is not a real ISO time with Z or an offset`)
+		if (now && date > now) fail(`run timestamp in ${location} (${value}) is after now (${now.toISOString()})`)
+		if (dir && Math.abs(date - dir) > DAY_MS) fail(`run timestamp in ${location} (${value}) is not within a day of the session dir name ${basename(sessionDir)}: a copied or stale value`)
+		return date
+	}
+	if (startedAt !== undefined) return check(startedAt, "--started-at")
+	const manifest = readJson(sessionDir, "outcome.json")
+	if (manifest?.data && validManifest(manifest.data) && resolve(String(manifest.data.sessionDir)) === resolve(sessionDir)) return check(manifest.data.startedAt, "outcome.json.startedAt")
+	const terminal = readJson(sessionDir, "terminal-state.json")
+	if (terminal?.corrupt) fail(terminal.corrupt)
+	if (terminal?.data?.started_at !== undefined) return check(terminal.data.started_at, "terminal-state.json.started_at")
+	if (dir) return dir
+	return fail(`missing run timestamp: pass --started-at <ISO>, record terminal-state.json.started_at, or name the session dir YYYYMMDD-HHMMSS (${sessionDir})`)
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/outcome.mjs

@@ -1,14 +1,13 @@
 // outcome.mjs - the per-run outcome manifest (`outcome.json`) and the closing briefing. Manifest functions are pure (they return a new manifest).
 // The briefing reads only the manifest, the sources-ledger text, and an injected `now`. node: builtins only.
-
 import * as nodeFs from "node:fs"
 import { basename, dirname, join } from "node:path"
 
 import { DELIVERABLE_FORMATS, DELIVERABLE_STATUSES, GATE_NAMES, GATE_STATUSES, LANES, MANIFEST_CODES, normalizeManifest, SCHEMA_VERSION, validateOutcomeManifest } from "./contracts.mjs"
+import { runStart } from "./outcome-start.mjs"
 
 export const MANIFEST_FILENAME = "outcome.json"
 const MISSING = MANIFEST_CODES[0]
-const SESSION_STAMP = /^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$/
 const SOURCE_LINE = /^\[(S|Source )\d+\]/
 const URL_PATTERN = /https?:\/\/[^\s)<>\]"'`]+/g
 const DRAFT_MARKER = "STATUS: draft"
@@ -33,26 +32,20 @@ function assertOneOf(code, label, value, allowed) {
 
 const clone = (manifest) => structuredClone(manifest)
 const isMissing = (error) => error?.code === "ENOENT" || error?.code === "ENOTDIR"
-
 function stamp(now) {
 	return (now ?? new Date()).toISOString()
 }
-
-/** Local wall-clock start time from a `YYYYMMDD-HHMMSS` session dir basename. */
-export function parseSessionStart(sessionDir) {
-	const match = SESSION_STAMP.exec(basename(sessionDir))
-	if (!match) throw new OutcomeError("invalid_session_dir", `session dir basename must be YYYYMMDD-HHMMSS: ${sessionDir}`)
-	const [y, mo, d, h, mi, s] = match.slice(1).map(Number)
-	const date = new Date(y, mo - 1, d, h, mi, s)
-	if (date.getMonth() !== mo - 1 || date.getDate() !== d || date.getHours() !== h) {
-		throw new OutcomeError("invalid_session_dir", `session dir basename is not a real local time: ${sessionDir}`)
+/** The run clock outcome init records (outcome-start.mjs owns the contract). */
+export function parseSessionStart(sessionDir, { startedAt, now } = {}) {
+	const fail = (message) => {
+		throw new OutcomeError("invalid_session_dir", message)
 	}
-	return date
+	return runStart({ sessionDir, startedAt, now, validManifest: (data) => validateOutcomeManifest(normalizeManifest(data)).ok, fail })
 }
 
-/** @param {{ sessionDir: string, promised: string[], lane: string|null, interview?: object|null, now?: Date }} input */
-export function initManifest({ sessionDir, promised, lane, interview = null, now }) {
-	const startedAt = parseSessionStart(sessionDir).toISOString()
+/** @param {{ sessionDir: string, promised: string[], lane: string|null, interview?: object|null, now?: Date, startedAt?: string }} input */
+export function initManifest({ sessionDir, promised, lane, interview = null, now, startedAt: recorded }) {
+	const startedAt = parseSessionStart(sessionDir, { startedAt: recorded, now }).toISOString()
 	if (lane !== null && lane !== undefined) assertOneOf("unknown_lane", "lane", lane, LANES)
 	const formats = [...new Set(promised)]
 	for (const format of formats) assertOneOf("unknown_format", "deliverable format", format, DELIVERABLE_FORMATS)
@@ -74,9 +67,7 @@ export function initManifest({ sessionDir, promised, lane, interview = null, now
 	}
 }
 
-/**
- * Set one format's row; every other row is kept as is. An unpromised format gets a new
- * row with promised=false. The row's detail fields are replaced by the ones given.
+/** Set one format's row, keeping other rows. New formats have promised=false.
  * @param {{ path?: string, reason?: string, pages?: number, bytes?: number, now?: Date }} [detail]
  */
 export function setDeliverable(manifest, format, status, detail = {}) {
@@ -143,9 +134,7 @@ export function verifyManifest(manifest, fs = nodeFs) {
 	return { ok: problems.length === 0, problems }
 }
 
-/**
- * none = missing or empty file; partial = `STATUS: draft` in the first 5 lines, or the
- * manifest (given, else a sibling outcome.json) still has a pending row; else complete.
+/** Missing/empty = none; draft marker or pending manifest = partial; else complete.
  * @returns {"none"|"partial"|"complete"}
  */
 export function deliverableState(deliverablePath, manifestOrNull) {

package/packages/omo-codex/plugin/skills/ulw-research/scripts/placeholder.mjs

@@ -1,14 +1,16 @@
 // placeholder.mjs - the fact pointer grammar of the fill step and its render helpers.
 // Value pointers: {{F12:label}}, {{F12:label:value}}, {{F12:label:unit}}, {{F12:label:q}}.
+// Claim prefixes from CLAIM_KINDS (V, D, A) precede any one pointer: {{D:F12:label:value}}, {{V:Q7:label:lead=18}}.
 // Quote pointers: {{Q7:label}}, {{Q7:label:lead=18}}, {{Q7:label:from="x":to="y"}} (anchors are
 // JSON strings, so they may hold a colon or an escaped quote). Labels are compared exactly by the
 // caller; this module never guesses an id. Pure functions; node: builtins only.
 
 import { parseNumeral } from "./numerals.mjs"
+import { CLAIM_KINDS } from "./claim-kinds.mjs"
 
 export const ELLIPSIS = "\u2026"
 
-const ID = /^([FQ])\d+$/
+const ID = /^([FQ])[1-9]\d*$/
 const LABEL = /^[^\s:{}"]+$/
 const VALUE_MODES = new Set(["value", "unit", "q"])
 const LEAD = /^lead=([1-9]\d*)$/
@@ -45,6 +47,12 @@ function parseAnchors(tail) {
 export function parsePlaceholder(token) {
 	if (!token.startsWith("{{") || !token.endsWith("}}")) return { error: "a placeholder is {{ID:label...}}" }
 	const body = token.slice(2, -2)
+	const marked = Object.entries(CLAIM_KINDS).find(([, entry]) => entry.prefix && body.startsWith(`${entry.prefix}:`))
+	if (marked) {
+		const [claimKind, entry] = marked
+		const parsed = parsePlaceholder(`{{${body.slice(entry.prefix.length + 1)}}}`)
+		return parsed.error || parsed.claimKind ? { error: "a claim prefix must precede one fact or quote pointer" } : { ...parsed, claimKind }
+	}
 	const colon = body.indexOf(":")
 	const id = colon === -1 ? body : body.slice(0, colon)
 	const idMatch = ID.exec(id)
@@ -130,7 +138,7 @@ export function lead(quote, n) {
 /** The value and unit parts of a row's raw text ("4.2 eok won" -> value "4.2 eok", unit "won"). */
 export function valueParts(row) {
 	const unitText = parseNumeral(row.raw)?.unitRaw ?? row.unit
-	const valueText = unitText && row.raw.endsWith(unitText) ? row.raw.slice(0, -unitText.length).trimEnd() : (row.value ?? row.raw)
+	const valueText = unitText && row.raw.endsWith(unitText) ? row.raw.slice(0, -unitText.length).trimEnd() : row.raw
 	return { valueText, unitText: unitText ?? "" }
 }
 
@@ -156,15 +164,19 @@ export function escapeHtml(text) {
 /** The elements the renderers below emit as fact spans; recognizers build their patterns from this. */
 export const FACT_ELEMENT_TAGS = Object.freeze(["data", "q", "blockquote"])
 
-export function renderValueHtml(row, text) {
+/** `mode` "value" marks a bare value whose unit verify checks beside it; `claimKind` is the claim mark. */
+export function renderValueHtml(row, text, { mode = "full", claimKind = null } = {}) {
 	const unit = row.unit === null ? "" : ` data-unit="${escapeHtml(row.unit)}"`
-	return `<data class="fact" value="${escapeHtml(row.value)}" data-fact="${row.fact_id}"${unit} data-lineage="${LINEAGE[row.kind]}">${escapeHtml(text)}</data>`
+	const marker = mode === "value" ? ' data-fact-mode="value"' : ""
+	const mark = claimKind === null ? "" : ` data-claim-kind="${escapeHtml(claimKind)}"`
+	return `<data class="fact" value="${escapeHtml(row.value)}" data-fact="${row.fact_id}"${unit}${marker}${mark} data-lineage="${LINEAGE[row.kind]}">${escapeHtml(text)}</data>`
 }
 
-export function renderQuote(row, selector, text, format) {
+export function renderQuote(row, selector, text, format, claimKind = null) {
 	const marker = `[${row.source_id}]`
 	if (format === "md") return `"${text}" ${marker}`
-	return `<q data-fact="${row.fact_id}" data-quote-sel="${escapeHtml(selector)}">${escapeHtml(text)}</q> ${marker}`
+	const mark = claimKind === null ? "" : ` data-claim-kind="${escapeHtml(claimKind)}"`
+	return `<q data-fact="${row.fact_id}"${mark} data-quote-sel="${escapeHtml(selector)}">${escapeHtml(text)}</q> ${marker}`
 }
 
 const RENDERED_MD_QUOTE = /^"([\s\S]*)" (\[S\d+\])$/u

package/packages/omo-codex/plugin/skills/ulw-research/scripts/polish-apply.mjs

@@ -54,9 +54,10 @@ export const POLISH_REFERENCE_REASON = "edit touches a character reference: rout
 /** True when [start, end) overlaps a character reference of html or starts inside one. */
 export function touchesReference(html, start, end) {
 	for (const match of String(html).matchAll(CHARACTER_REFERENCE)) {
-		if (match.index < end && match.index + match[0].length > start) return true
+		const stop = match.index + match[0].length
+		if (match.index < end && (stop > start || (stop === start && !match[0].endsWith(";")))) return true
 	}
-	return /&[#\w]*$/.test(html.slice(Math.max(0, start - 16), start))
+	return false
 }
 
 const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")

package/packages/omo-codex/plugin/skills/ulw-research/scripts/raw-decode.mjs

@@ -0,0 +1,110 @@
+// raw-decode.mjs - decode a source's raw bytes in their declared encoding and keep a byte map.
+//
+// decodeRaw returns { text, encoding, bytes } where bytes[i] is the raw byte offset at which UTF-16
+// unit i of `text` starts and bytes[text.length] === raw.length, so any text range maps back to the
+// bytes the source actually holds (never to a lossy UTF-8 re-encode). The encoding comes from a
+// BOM, then the caller's declared charset (HTTP header or meta.json), then an HTML <meta> charset,
+// and defaults to UTF-8. node: builtins only (TextDecoder).
+
+const SNIFF_BYTES = 1024
+const META_CHARSET = /<meta\b[^>]*?charset\s*=\s*["']?\s*([\w.:-]+)/i
+
+/** The WHATWG encoding name for a label, or null when the runtime does not know it. */
+export function encodingOf(label) {
+	if (typeof label !== "string" || label.trim() === "") return null
+	try {
+		return new TextDecoder(label.trim()).encoding
+	} catch {
+		return null
+	}
+}
+
+/** charset=... from a Content-Type header value, as an encoding name, or null. */
+export function headerCharset(contentType) {
+	const match = /charset\s*=\s*["']?([\w.:-]+)/i.exec(String(contentType ?? ""))
+	return match ? encodingOf(match[1]) : null
+}
+
+function bomOf(raw) {
+	if (raw[0] === 0xef && raw[1] === 0xbb && raw[2] === 0xbf) return { encoding: "utf-8", skip: 3 }
+	if (raw[0] === 0xff && raw[1] === 0xfe) return { encoding: "utf-16le", skip: 2 }
+	if (raw[0] === 0xfe && raw[1] === 0xff) return { encoding: "utf-16be", skip: 2 }
+	return null
+}
+
+/** BOM, then the declared charset, then (for markup) a <meta> charset in the first 1 KiB, then UTF-8. */
+export function detectEncoding(raw, declared = null, markup = false) {
+	const bom = bomOf(raw)
+	if (bom) return bom.encoding
+	const given = encodingOf(declared)
+	if (given) return given
+	if (markup) {
+		const sniffed = META_CHARSET.exec(Buffer.from(raw.subarray(0, SNIFF_BYTES)).toString("latin1"))
+		const named = sniffed ? encodingOf(sniffed[1]) : null
+		// A page that is already decoded to UTF-16 cannot declare a UTF-16 charset in ASCII bytes.
+		if (named && !named.startsWith("utf-16")) return named
+	}
+	return "utf-8"
+}
+
+function utf8Map(text, bytes, from) {
+	let at = from
+	for (let i = 0; i < text.length; i++) {
+		bytes[i] = at
+		const code = text.charCodeAt(i)
+		if (code >= 0xd800 && code <= 0xdbff && i + 1 < text.length) {
+			bytes[++i] = at
+			at += 4
+		} else at += code < 0x80 ? 1 : code < 0x800 ? 2 : 3
+	}
+	return at
+}
+
+/** Byte-at-a-time streaming decode: correct for any encoding, used when no shortcut applies. */
+function streamMap(raw, encoding, from) {
+	const decoder = new TextDecoder(encoding)
+	const units = []
+	const starts = []
+	let pending = from
+	for (let i = from; i < raw.length; i++) {
+		const out = decoder.decode(raw.subarray(i, i + 1), { stream: true })
+		if (out.length === 0) continue
+		// A replacement for an earlier broken sequence can arrive with this byte's own character.
+		const own = out.length > 1 && raw[i] < 0x80 ? 1 : 0
+		for (let k = 0; k < out.length; k++) starts.push(k >= out.length - own ? i : pending)
+		units.push(out)
+		pending = i + 1
+	}
+	const tail = decoder.decode()
+	for (let k = 0; k < tail.length; k++) starts.push(pending)
+	units.push(tail)
+	return { text: units.join(""), starts }
+}
+
+/** Decode raw bytes; bytes is an Int32Array of length text.length + 1 (see the file comment). */
+export function decodeRaw(raw, declared = null, markup = false) {
+	const buffer = Buffer.from(raw)
+	const encoding = detectEncoding(buffer, declared, markup)
+	const skip = bomOf(buffer)?.encoding === encoding ? bomOf(buffer).skip : 0
+	if (encoding === "utf-8") {
+		try {
+			const text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(buffer.subarray(skip))
+			const bytes = new Int32Array(text.length + 1)
+			bytes[text.length] = utf8Map(text, bytes, skip)
+			return { text, encoding, bytes }
+		} catch {
+			// invalid UTF-8: fall through to the streaming map, which counts each broken byte once
+		}
+	}
+	if (encoding !== "utf-8" && !encoding.startsWith("utf-16")) {
+		const text = new TextDecoder(encoding).decode(buffer.subarray(skip))
+		if (text.length === buffer.length - skip) {
+			const bytes = new Int32Array(text.length + 1)
+			for (let i = 0; i <= text.length; i++) bytes[i] = skip + i
+			return { text, encoding, bytes }
+		}
+	}
+	const { text, starts } = streamMap(buffer, encoding, skip)
+	const bytes = Int32Array.from([...starts, buffer.length])
+	return { text, encoding, bytes }
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/raw-locator.mjs

@@ -0,0 +1,47 @@
+// Byte locators over a source view (source-view.mjs) and exact, context-bound re-anchoring.
+// Matching always happens on the view's text (text nodes of a page, never markup); only the
+// stored offsets are bytes of the view's artifact. value_span remains quote-relative text.
+
+const chars = (text) => [...text]
+
+/** Context is copied before normalization, around the value (or the whole quote for Q rows). */
+export function quoteContext(text, start, end) {
+	return { prefix: chars(text.slice(0, start)).slice(-32).join(""), suffix: chars(text.slice(end)).slice(0, 32).join("") }
+}
+
+/** Convert a text-space locator from fact-locate.mjs into artifact byte offsets. */
+export function rawLocated(view, found) {
+	if (found.error) return found
+	const { char_start: start, char_end: end } = found.locator
+	const from = start + (found.value_span?.[0] ?? 0)
+	const to = found.value_span ? start + found.value_span[1] : end
+	const [byteStart, byteEnd] = view.byteSpan(start, end)
+	return {
+		...found,
+		context: quoteContext(view.text, from, to),
+		locator: { ...found.locator, artifact: view.artifact, unit: "bytes", char_start: byteStart, char_end: byteEnd },
+	}
+}
+
+function contextMatches(text, row, start, end) {
+	if (!row.context) return false
+	const from = row.value_span ? start + row.value_span[0] : start
+	const to = row.value_span ? start + row.value_span[1] : end
+	const { prefix, suffix } = row.context
+	return from >= prefix.length && text.slice(from - prefix.length, from) === prefix && text.slice(to, to + suffix.length) === suffix
+}
+
+/**
+ * The row's quote in the view: unchanged when an exact occurrence sits at the pinned byte range,
+ * else exactly one occurrence whose context matches (relocated), else null. Never rewrites the row.
+ */
+export function reanchor(view, row) {
+	const quote = String(row.quote ?? "")
+	if (quote === "" || !view.byteSpan) return null
+	const { char_start: start, char_end: end } = row.locator
+	const hits = []
+	for (let at = view.text.indexOf(quote); at !== -1; at = view.text.indexOf(quote, at + 1)) hits.push({ at, span: view.byteSpan(at, at + quote.length) })
+	if (hits.some(({ span }) => span[0] === start && span[1] === end)) return { start, end, relocated: false }
+	const matching = hits.filter(({ at }) => contextMatches(view.text, row, at, at + quote.length))
+	return matching.length === 1 ? { start: matching[0].span[0], end: matching[0].span[1], relocated: true } : null
+}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/report-tools-commands.mjs

@@ -133,7 +133,7 @@ export async function runOutcome(argv) {
 	switch (sub) {
 		case "init": {
 			const promised = args.required("promised").split(",").map((s) => s.trim()).filter(Boolean)
-			const manifest = outcome.initManifest({ sessionDir: sessionDirFrom(args.flags), promised, lane: args.flags.lane ?? null, now: nowFrom(args.flags) })
+			const manifest = outcome.initManifest({ sessionDir: sessionDirFrom(args.flags), promised, lane: args.flags.lane ?? null, now: nowFrom(args.flags), startedAt: args.flags["started-at"] })
 			outcome.writeManifest(file, manifest)
 			return { json: { command: "outcome init", path: file, deliverables: manifest.deliverables }, exitCode: 0 }
 		}

package/packages/omo-codex/plugin/skills/ulw-research/scripts/report-tools.mjs

@@ -28,6 +28,8 @@ export const COMMANDS = Object.freeze([
 	"fact verify",
 	"fact derive",
 	"source snapshot",
+	"source refresh",
+	"source probe-conversion",
 	"claims index",
 ])
 
@@ -53,7 +55,11 @@ export async function run(argv) {
 			return (await import("./source-commands.mjs")).runClaims(rest)
 		case "--help":
 		case "help":
-			if (rest.includes("--json")) return { json: { commands: COMMANDS }, exitCode: 0 }
+			if (rest.includes("--json")) {
+				const { SNAPSHOT_USAGE } = await import("./source-commands.mjs")
+				const { REFRESH_USAGE } = await import("./source-refresh.mjs")
+				return { json: { commands: COMMANDS, usage: { "source snapshot": SNAPSHOT_USAGE, "source refresh": REFRESH_USAGE } }, exitCode: 0 }
+			}
 			return { text: `report-tools <command>\n${COMMANDS.map((name) => `  ${name}`).join("\n")}`, exitCode: 0 }
 		default:
 			throw new CliError(`usage: report-tools <${COMMANDS.join(" | ")}> (run --help)`)