lazycodex-ai 5.1.44 → 5.1.45

package/packages/shared-skills/skills/ulw-research/scripts/text-normalize.mjs

@@ -0,0 +1,82 @@
+// text-normalize.mjs - text normalization and raw-offset matching for fidelity checks.
+//
+// normalizeText applies NFC, maps curly and CJK quote glyphs to straight quotes, collapses
+// whitespace runs to one space, and trims. findNormalized matches a needle against the
+// normalized haystack and returns offsets into the RAW haystack through an offset map built
+// during normalization. Plain Node ESM, no dependencies.
+
+const DOUBLE_QUOTES = new Set([
+	"\u201C", "\u201D", "\u201E", "\u201F", "\u00AB", "\u00BB", "\u300C", "\u300D", "\u300E", "\u300F",
+	"\u301D", "\u301E", "\u301F", "\uFF02", "\u2033",
+])
+const SINGLE_QUOTES = new Set(["\u2018", "\u2019", "\u201A", "\u201B", "\u2039", "\u203A", "\uFF07", "\u2032"])
+// Combining marks and conjoining Hangul medial/final jamo attach to the preceding code point,
+// so a decomposed syllable is normalized as one unit.
+const JOINS_PREVIOUS = /^[\p{M}\u1160-\u11FF\uD7B0-\uD7FF]$/u
+const WHITESPACE = /^\s+$/u
+const TOKEN_SEPARATOR = /[^\p{L}\p{N}\p{M}]+/u
+
+function mapQuote(ch) {
+	if (DOUBLE_QUOTES.has(ch)) return '"'
+	if (SINGLE_QUOTES.has(ch)) return "'"
+	return ch
+}
+
+function clusters(raw) {
+	const out = []
+	let at = 0
+	for (const ch of raw) {
+		const last = out[out.length - 1]
+		if (last && JOINS_PREVIOUS.test(ch)) last[1] += ch.length
+		else out.push([at, at + ch.length])
+		at += ch.length
+	}
+	return out
+}
+
+// Returns { text, starts, ends }: text[k] came from raw.slice(starts[k], ends[k]).
+function normalizeWithMap(raw) {
+	const source = String(raw ?? "")
+	let text = ""
+	const starts = []
+	const ends = []
+	for (const [start, end] of clusters(source)) {
+		const piece = Array.from(source.slice(start, end).normalize("NFC"), mapQuote).join("")
+		if (WHITESPACE.test(piece)) {
+			if (text !== "" && !text.endsWith(" ")) {
+				text += " "
+				starts.push(start)
+				ends.push(end)
+			}
+			continue
+		}
+		for (let k = 0; k < piece.length; k++) {
+			starts.push(start)
+			ends.push(end)
+		}
+		text += piece
+	}
+	if (text.endsWith(" ")) {
+		text = text.slice(0, -1)
+		starts.pop()
+		ends.pop()
+	}
+	return { text, starts, ends }
+}
+
+export function normalizeText(s) {
+	return normalizeWithMap(s).text
+}
+
+export function tokenize(s) {
+	return normalizeText(s).split(TOKEN_SEPARATOR).filter((token) => token !== "")
+}
+
+export function findNormalized(hay, needle) {
+	const target = normalizeText(needle)
+	if (target === "") return null
+	const map = normalizeWithMap(hay)
+	const at = map.text.indexOf(target)
+	if (at === -1) return null
+	return { start: map.starts[at], end: map.ends[at + target.length - 1] }
+}

package/packages/shared-skills/skills/ulw-research/scripts/text-normalize.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, test } from "bun:test"
+
+import { findNormalized, normalizeText, tokenize } from "./text-normalize.mjs"
+
+describe("normalizeText", () => {
+	test("#given NFD and NFC Korean #when normalized #then both are equal NFC strings", () => {
+		// given
+		const nfc = "매출은 한국어로 적었다"
+		const nfd = nfc.normalize("NFD")
+		// when
+		const a = normalizeText(nfd)
+		const b = normalizeText(nfc)
+		// then
+		expect(nfd).not.toBe(nfc)
+		expect(a).toBe(b)
+		expect(a).toBe(nfc)
+	})
+
+	test("#given curly and CJK quote glyphs #when normalized #then they become straight quotes", () => {
+		// given
+		const raw = "\u201Cgrowth\u201D \u2018ok\u2019 \u300Cquoted\u300D \u300Etitle\u300F \u00ABfr\u00BB"
+		// when
+		const out = normalizeText(raw)
+		// then
+		expect(out).toBe("\"growth\" 'ok' \"quoted\" \"title\" \"fr\"")
+	})
+
+	test("#given runs of mixed whitespace #when normalized #then they collapse to single spaces and are trimmed", () => {
+		// when
+		const out = normalizeText("  a \t\n b\u00A0\u00A0c  ")
+		// then
+		expect(out).toBe("a b c")
+	})
+})
+
+describe("tokenize", () => {
+	test("#given Latin words and Hangul eojeol with punctuation #when tokenized #then tokens split on whitespace and punctuation", () => {
+		// when
+		const tokens = tokenize("Revenue rose, 매출은 100억 원이다. (\u201Cok\u201D)")
+		// then
+		expect(tokens).toEqual(["Revenue", "rose", "매출은", "100억", "원이다", "ok"])
+	})
+})
+
+describe("findNormalized", () => {
+	test("#given raw text with double spaces and curly quotes #when a straight-quoted needle is found #then RAW offsets slice back to the raw substring", () => {
+		// given
+		const hay = "He said  \u201Cgrowth  is\n strong\u201D  today."
+		// when
+		const hit = findNormalized(hay, "\"growth is strong\"")
+		// then
+		if (hit === null) throw new Error("needle not found")
+		expect(hay.slice(hit.start, hit.end)).toBe("\u201Cgrowth  is\n strong\u201D")
+	})
+
+	test("#given NFD Korean raw text with double spaces #when an NFC needle is found #then RAW offsets slice the NFD substring", () => {
+		// given
+		const rawNfc = "앞 문장. 매출은  100억 원이다. 끝"
+		const hay = rawNfc.normalize("NFD")
+		const expected = "매출은  100억 원이다".normalize("NFD")
+		// when
+		const hit = findNormalized(hay, "매출은 100억 원이다")
+		// then
+		if (hit === null) throw new Error("needle not found")
+		expect(hay.slice(hit.start, hit.end)).toBe(expected)
+	})
+
+	test("#given a needle absent from the text #when searched #then null is returned", () => {
+		// when / then
+		expect(findNormalized("abc", "zzz")).toBeNull()
+		expect(findNormalized("abc", "   ")).toBeNull()
+	})
+})

package/packages/shared-skills/skills/ulw-research/scripts/text-similarity.mjs

@@ -0,0 +1,28 @@
+// text-similarity.mjs - similarity in [0, 1] from normalized Levenshtein distance.
+//
+// Both inputs go through normalizeText first (NFC, quote glyphs, whitespace), then the
+// edit distance is computed over code points and divided by the longer length.
+// Plain Node ESM, no dependencies.
+
+import { normalizeText } from "./text-normalize.mjs"
+
+function levenshtein(a, b) {
+	let previous = Array.from({ length: b.length + 1 }, (_, k) => k)
+	for (let i = 1; i <= a.length; i++) {
+		const current = [i]
+		for (let k = 1; k <= b.length; k++) {
+			const cost = a[i - 1] === b[k - 1] ? 0 : 1
+			current.push(Math.min(previous[k] + 1, current[k - 1] + 1, previous[k - 1] + cost))
+		}
+		previous = current
+	}
+	return previous[b.length]
+}
+
+export function similarity(a, b) {
+	const left = Array.from(normalizeText(a))
+	const right = Array.from(normalizeText(b))
+	const longest = Math.max(left.length, right.length)
+	if (longest === 0) return 1
+	return 1 - levenshtein(left, right) / longest
+}

package/packages/shared-skills/skills/ulw-research/scripts/text-similarity.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, test } from "bun:test"
+
+import { similarity } from "./text-similarity.mjs"
+
+describe("similarity", () => {
+	test("#given identical strings #when compared #then similarity is exactly 1", () => {
+		// when / then
+		expect(similarity("a", "a")).toBe(1)
+		expect(similarity("", "")).toBe(1)
+	})
+
+	test("#given an OCR-noisy copy of a sentence #when compared #then similarity is at least 0.95", () => {
+		// given
+		const clean = "The company reported revenue of 100 billion won in the third quarter of 2025."
+		const noisy = "The  cornpany reported revenue of 100 bi1lion won in the third quarter of 2025. "
+		// when
+		const score = similarity(clean, noisy)
+		// then
+		expect(score).toBeGreaterThanOrEqual(0.95)
+		expect(score).toBeLessThan(1)
+	})
+
+	test("#given NFD vs NFC and curly vs straight quotes #when compared #then normalization makes them equal", () => {
+		// when / then
+		expect(similarity("\u201C매출은 늘었다\u201D".normalize("NFD"), "\"매출은 늘었다\"")).toBe(1)
+	})
+
+	test("#given a different sentence #when compared #then similarity is below 0.8", () => {
+		// given
+		const a = "The company reported revenue of 100 billion won in the third quarter of 2025."
+		const b = "Analysts expect margins to shrink next year as input costs keep rising."
+		// when
+		const score = similarity(a, b)
+		// then
+		expect(score).toBeLessThan(0.8)
+		expect(score).toBeGreaterThanOrEqual(0)
+	})
+})

package/packages/shared-skills/skills/ulw-research/scripts/units.mjs

@@ -0,0 +1,105 @@
+// units.mjs - unit table: normalization, dimensions, exact conversion factors, Korean readings.
+//
+// normalizeUnit maps a raw unit (any spacing or spelling variant) to one canonical token. Each token
+// has a dimension, an exact factor to the dimension's base unit (base = value * factor + offset), and
+// the final-sound class of its Korean reading for the particle rule: "open" (no final consonant),
+// "l" (final rieul), or "closed". Currency is its own dimension per currency, so no exchange rate is
+// ever implied. Data units are decimal (1 KB = 1000 B). Plain Node ESM, sibling imports only.
+
+import { div, fromDecimal, sub } from "./rational.mjs"
+
+/** [token, dimension, factor, reading, aliases, offset?]; the Korean reading is noted per row. */
+const TABLE = [
+	["mm", "length", "0.001", "open", ["millimeter", "millimeters", "\uBC00\uB9AC\uBBF8\uD130"]], // millimiteo
+	["cm", "length", "0.01", "open", ["centimeter", "centimeters", "\uC13C\uD2F0\uBBF8\uD130"]], // sentimiteo
+	["m", "length", "1", "open", ["meter", "meters", "metre", "metres", "\uBBF8\uD130"]], // miteo
+	["km", "length", "1000", "open", ["kilometer", "kilometers", "\uD0AC\uB85C\uBBF8\uD130"]], // killomiteo
+	["g", "mass", "1", "closed", ["gram", "grams", "\uADF8\uB7A8"]], // geuraem
+	["kg", "mass", "1000", "closed", ["kilogram", "kilograms", "\uD0AC\uB85C\uADF8\uB7A8"]], // killogeuraem
+	["t", "mass", "1000000", "closed", ["ton", "tons", "tonne", "tonnes", "\uD1A4"]], // ton
+	["ms", "time", "0.001", "open", ["millisecond", "milliseconds", "\uBC00\uB9AC\uCD08"]], // millicho
+	["s", "time", "1", "open", ["sec", "secs", "second", "seconds", "\uCD08"]], // cho
+	["min", "time", "60", "closed", ["mins", "minute", "minutes", "\uBD84"]], // bun
+	["h", "time", "3600", "closed", ["hr", "hrs", "hour", "hours", "\uC2DC\uAC04"]], // sigan
+	["B", "data", "1", "open", ["byte", "bytes", "\uBC14\uC774\uD2B8"]], // baiteu
+	["KB", "data", "1000", "open", ["\uD0AC\uB85C\uBC14\uC774\uD2B8"]],
+	["MB", "data", "1000000", "open", ["\uBA54\uAC00\uBC14\uC774\uD2B8"]],
+	["GB", "data", "1000000000", "open", ["\uAE30\uAC00\uBC14\uC774\uD2B8"]],
+	["TB", "data", "1000000000000", "open", ["\uD14C\uB77C\uBC14\uC774\uD2B8"]],
+	["C", "temperature", "1", "open", ["\u00B0C", "\u2103", "celsius", "\uC12D\uC528"], "0"], // do
+	["F", "temperature", "5/9", "open", ["\u00B0F", "\u2109", "fahrenheit", "\uD654\uC528"], "-160/9"], // do
+	["%", "percent", "1", "open", ["percent", "\uD37C\uC13C\uD2B8", "\uD504\uB85C"]], // peosenteu
+	["\u2030", "percent", "0.1", "l", ["permille", "\uD37C\uBC00"]], // peomil
+	["%p", "percentage-point", "1", "open", ["pp", "%pt", "percentage point", "percentage points", "\uD37C\uC13C\uD2B8\uD3EC\uC778\uD2B8", "%\uD3EC\uC778\uD2B8"]],
+	["x", "multiple", "1", "open", ["\u00D7", "times", "\uBC30"]], // bae
+	["e3", "scale", "1000", "closed", ["k", "thousand", "\uCC9C"]], // cheon
+	["e4", "scale", "10000", "closed", ["\uB9CC"]], // man
+	["e6", "scale", "1000000", "closed", ["M", "million", "\uBC31\uB9CC"]], // baengman
+	["e8", "scale", "100000000", "closed", ["\uC5B5"]], // eok
+	["e9", "scale", "1000000000", "closed", ["bn", "billion", "\uC2ED\uC5B5"]], // sibeok
+	["e12", "scale", "1000000000000", "open", ["trillion", "\uC870"]], // jo
+	["person", "count:person", "1", "closed", ["persons", "people", "\uBA85"]], // myeong
+	["case", "count:case", "1", "closed", ["cases", "\uAC74"]], // geon
+	["KRW", "currency:KRW", "1", "closed", ["\uC6D0", "\u20A9", "won"]], // won
+	["KRW-e3", "currency:KRW", "1000", "closed", ["\uCC9C\uC6D0"]],
+	["KRW-e4", "currency:KRW", "10000", "closed", ["\uB9CC\uC6D0"]],
+	["KRW-e8", "currency:KRW", "100000000", "closed", ["\uC5B5\uC6D0"]],
+	["KRW-e12", "currency:KRW", "1000000000000", "closed", ["\uC870\uC6D0"]],
+	["USD", "currency:USD", "1", "open", ["$", "US$", "dollar", "dollars", "\uB2EC\uB7EC"]], // dalleo
+	["EUR", "currency:EUR", "1", "open", ["\u20AC", "euro", "euros", "\uC720\uB85C"]], // yuro
+	["JPY", "currency:JPY", "1", "closed", ["\u00A5", "yen", "\uC5D4"]], // en
+	["GBP", "currency:GBP", "1", "open", ["\u00A3", "pound", "pounds", "\uD30C\uC6B4\uB4DC"]], // paundeu
+]
+
+const parse = (text) => (text.includes("/") ? div(fromDecimal(text.split("/")[0]), fromDecimal(text.split("/")[1])) : fromDecimal(text))
+const squash = (raw) => raw.replace(/\s+/g, "")
+
+const ENTRIES = new Map()
+/** Exact-case keys: every token and alias with whitespace removed. */
+const EXACT = new Map()
+/** Case-insensitive keys: multi-letter ASCII aliases only, so m/M and B/b never collide. */
+const WORD = new Map()
+for (const [token, dimension, factor, reading, aliases, offset] of TABLE) {
+	const entry = Object.freeze({ token, dimension, factor: parse(factor), offset: parse(offset ?? "0"), reading })
+	ENTRIES.set(token, entry)
+	for (const alias of [token, ...aliases]) {
+		const key = squash(alias)
+		EXACT.set(key, entry)
+		if (key.length > 1 && /^[A-Za-z]+$/.test(key) && !WORD.has(key.toLowerCase())) WORD.set(key.toLowerCase(), entry)
+	}
+}
+
+function entryOf(raw) {
+	if (typeof raw !== "string") return null
+	const key = squash(raw)
+	if (!key) return null
+	return EXACT.get(key) ?? WORD.get(key.toLowerCase()) ?? null
+}
+
+/** Canonical token for a raw unit, or null when the unit is not in the table. */
+export function normalizeUnit(raw) {
+	return entryOf(raw)?.token ?? null
+}
+
+/** Dimension of a unit ("length", "currency:KRW", ...), or null. */
+export function dimensionOf(token) {
+	return entryOf(token)?.dimension ?? null
+}
+
+/**
+ * Exact factor from one unit to another of the same dimension: to = from * num / den (+ offset for
+ * affine scales). Null across dimensions, across currencies, or for an unknown unit.
+ */
+export function convertFactor(from, to) {
+	const a = entryOf(from)
+	const b = entryOf(to)
+	if (!a || !b || a.dimension !== b.dimension) return null
+	const { num, den } = div(a.factor, b.factor)
+	const offset = div(sub(a.offset, b.offset), b.factor)
+	return offset.num === 0n ? { num, den } : { num, den, offset: { num: offset.num, den: offset.den } }
+}
+
+/** Final-sound class of the unit's Korean reading: "open", "closed", or "l"; null when unknown. */
+export function koreanReadingOf(token) {
+	return entryOf(token)?.reading ?? null
+}

package/packages/shared-skills/skills/ulw-research/scripts/units.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, test } from "bun:test"
+
+import { convertFactor, dimensionOf, koreanReadingOf, normalizeUnit } from "./units.mjs"
+
+describe("normalizeUnit", () => {
+	test("ignores whitespace and spelling variants", () => {
+		expect(normalizeUnit("억 원")).toBe(normalizeUnit("억원"))
+		expect(normalizeUnit("억원")).not.toBeNull()
+		expect(normalizeUnit("million")).toBe(normalizeUnit("M"))
+		expect(normalizeUnit("USD")).toBe(normalizeUnit("$"))
+		expect(normalizeUnit("달러")).toBe(normalizeUnit("USD"))
+		expect(normalizeUnit("%")).not.toBe(normalizeUnit("%p"))
+		expect(normalizeUnit("m")).not.toBe(normalizeUnit("M"))
+		expect(normalizeUnit("furlongs")).toBeNull()
+	})
+
+	test("a normalized token normalizes to itself", () => {
+		for (const raw of ["억 원", "m", "kg", "GB", "%p", "USD", "C", "F"]) {
+			const token = normalizeUnit(raw)
+			expect(normalizeUnit(token as string)).toBe(token)
+		}
+	})
+})
+
+describe("dimensionOf and convertFactor", () => {
+	test("length, mass, time and data convert exactly", () => {
+		expect(convertFactor("m", "mm")).toEqual({ num: 1000n, den: 1n })
+		expect(convertFactor("mm", "m")).toEqual({ num: 1n, den: 1000n })
+		expect(convertFactor("km", "cm")).toEqual({ num: 100000n, den: 1n })
+		expect(convertFactor("kg", "t")).toEqual({ num: 1n, den: 1000n })
+		expect(convertFactor("h", "s")).toEqual({ num: 3600n, den: 1n })
+		expect(convertFactor("GB", "MB")).toEqual({ num: 1000n, den: 1n })
+		expect(dimensionOf(normalizeUnit("cm") as string)).toBe("length")
+	})
+
+	test("the Korean currency scales convert within one currency", () => {
+		expect(convertFactor("억 원", "원")).toEqual({ num: 100000000n, den: 1n })
+		expect(convertFactor("조원", "억원")).toEqual({ num: 10000n, den: 1n })
+	})
+
+	test("currency is never converted across currencies, and dimensions never mix", () => {
+		expect(convertFactor("USD", "KRW")).toBeNull()
+		expect(convertFactor("m", "kg")).toBeNull()
+		expect(convertFactor("%", "%p")).toBeNull()
+		expect(convertFactor("m", "furlongs")).toBeNull()
+	})
+
+	test("temperature is affine", () => {
+		expect(convertFactor("C", "F")).toEqual({ num: 9n, den: 5n, offset: { num: 32n, den: 1n } })
+		expect(convertFactor("F", "C")).toEqual({ num: 5n, den: 9n, offset: { num: -160n, den: 9n } })
+	})
+})
+
+describe("koreanReadingOf", () => {
+	const cases: Array<[string, string]> = [
+		["%", "open"],
+		["%p", "open"],
+		["m", "open"],
+		["km", "open"],
+		["원", "closed"],
+		["달러", "open"],
+		["유로", "open"],
+		["엔", "closed"],
+		["배", "open"],
+		["건", "closed"],
+		["명", "closed"],
+		["조", "open"],
+		["억", "closed"],
+		["만", "closed"],
+		["천", "closed"],
+		["kg", "closed"],
+		["GB", "open"],
+	]
+	for (const [raw, reading] of cases) {
+		test(`${raw} reads ${reading}`, () => {
+			expect(koreanReadingOf(raw)).toBe(reading)
+		})
+	}
+
+	test("unknown units have no reading", () => {
+		expect(koreanReadingOf("furlongs")).toBeNull()
+	})
+})