lazycodex-ai 5.1.44 → 5.1.46

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/ledger-verify.json

@@ -0,0 +1,420 @@
+{
+	"schemaVersion": 1,
+	"rows": [
+		{
+			"fact_id": "F1",
+			"kind": "measured",
+			"raw": "320 m",
+			"quote": "The bridge span is up to 320 m long.",
+			"extracted_by": "tool:source-snapshot@1",
+			"observed_at": "2026-10-01T00:00:00Z",
+			"label": "span_len",
+			"value": "320",
+			"unit": "m",
+			"value_span": [
+				25,
+				30
+			],
+			"source_id": "S4",
+			"locator": {
+				"artifact": "text.txt",
+				"char_start": 0,
+				"char_end": 36
+			},
+			"locate_hint": null,
+			"snapshot_sha256": "89abfafc79d0e2c13a5ccad6843c020dc65e86f1eca385f85e7f58153960cdfe",
+			"valid_at": "2026",
+			"qualifier": "upper_inclusive",
+			"entity": "Bridge",
+			"sentence_id": "S4:s1",
+			"observation_id": "O1",
+			"fuzzy": false,
+			"outside_floor": false,
+			"population": null,
+			"missing_rule": null,
+			"accounting_basis": null,
+			"snapshot_window": null,
+			"revision": null,
+			"exact": {
+				"num": "320",
+				"den": "1"
+			},
+			"render": null,
+			"operation": null,
+			"operands": [],
+			"derived_from": [],
+			"formula": null,
+			"factor": null,
+			"rationale": null,
+			"status": "active"
+		},
+		{
+			"fact_id": "F2",
+			"kind": "measured",
+			"raw": "1,237",
+			"quote": "Units sold reached 1,237 in 2026.",
+			"extracted_by": "tool:source-snapshot@1",
+			"observed_at": "2026-10-01T00:00:00Z",
+			"label": "units_sold",
+			"value": "1237",
+			"unit": null,
+			"value_span": [
+				19,
+				24
+			],
+			"source_id": "S4",
+			"locator": {
+				"artifact": "text.txt",
+				"char_start": 37,
+				"char_end": 70
+			},
+			"locate_hint": null,
+			"snapshot_sha256": "89abfafc79d0e2c13a5ccad6843c020dc65e86f1eca385f85e7f58153960cdfe",
+			"valid_at": "2026",
+			"qualifier": null,
+			"entity": null,
+			"sentence_id": "S4:s2",
+			"observation_id": "O1",
+			"fuzzy": false,
+			"outside_floor": false,
+			"population": null,
+			"missing_rule": null,
+			"accounting_basis": null,
+			"snapshot_window": null,
+			"revision": null,
+			"exact": {
+				"num": "1237",
+				"den": "1"
+			},
+			"render": null,
+			"operation": null,
+			"operands": [],
+			"derived_from": [],
+			"formula": null,
+			"factor": null,
+			"rationale": null,
+			"status": "active"
+		},
+		{
+			"fact_id": "F3",
+			"kind": "measured",
+			"raw": "12%",
+			"quote": "Revenue grew 12% year over year.",
+			"extracted_by": "tool:source-snapshot@1",
+			"observed_at": "2026-10-01T00:00:00Z",
+			"label": "rate_yoy",
+			"value": "12",
+			"unit": "%",
+			"value_span": [
+				13,
+				16
+			],
+			"source_id": "S4",
+			"locator": {
+				"artifact": "text.txt",
+				"char_start": 71,
+				"char_end": 103
+			},
+			"locate_hint": null,
+			"snapshot_sha256": "89abfafc79d0e2c13a5ccad6843c020dc65e86f1eca385f85e7f58153960cdfe",
+			"valid_at": "2026 YoY",
+			"qualifier": null,
+			"entity": null,
+			"sentence_id": "S4:s3",
+			"observation_id": "O2",
+			"fuzzy": false,
+			"outside_floor": false,
+			"population": null,
+			"missing_rule": null,
+			"accounting_basis": null,
+			"snapshot_window": null,
+			"revision": null,
+			"exact": {
+				"num": "12",
+				"den": "1"
+			},
+			"render": null,
+			"operation": null,
+			"operands": [],
+			"derived_from": [],
+			"formula": null,
+			"factor": null,
+			"rationale": null,
+			"status": "active"
+		},
+		{
+			"fact_id": "F4",
+			"kind": "measured",
+			"raw": "12%",
+			"quote": "Revenue grew 12% quarter over quarter.",
+			"extracted_by": "tool:source-snapshot@1",
+			"observed_at": "2026-10-01T00:00:00Z",
+			"label": "rate_qoq",
+			"value": "12",
+			"unit": "%",
+			"value_span": [
+				13,
+				16
+			],
+			"source_id": "S4",
+			"locator": {
+				"artifact": "text.txt",
+				"char_start": 104,
+				"char_end": 142
+			},
+			"locate_hint": null,
+			"snapshot_sha256": "89abfafc79d0e2c13a5ccad6843c020dc65e86f1eca385f85e7f58153960cdfe",
+			"valid_at": "2026 Q3 QoQ",
+			"qualifier": null,
+			"entity": null,
+			"sentence_id": "S4:s4",
+			"observation_id": "O3",
+			"fuzzy": false,
+			"outside_floor": false,
+			"population": null,
+			"missing_rule": null,
+			"accounting_basis": null,
+			"snapshot_window": null,
+			"revision": null,
+			"exact": {
+				"num": "12",
+				"den": "1"
+			},
+			"render": null,
+			"operation": null,
+			"operands": [],
+			"derived_from": [],
+			"formula": null,
+			"factor": null,
+			"rationale": null,
+			"status": "active"
+		},
+		{
+			"fact_id": "F5",
+			"kind": "measured",
+			"raw": "12%",
+			"quote": "Segment A held 12% of the market and segment B held 27%.",
+			"extracted_by": "tool:source-snapshot@1",
+			"observed_at": "2026-10-01T00:00:00Z",
+			"label": "seg_a",
+			"value": "12",
+			"unit": "%",
+			"value_span": [
+				15,
+				18
+			],
+			"source_id": "S4",
+			"locator": {
+				"artifact": "text.txt",
+				"char_start": 143,
+				"char_end": 199
+			},
+			"locate_hint": null,
+			"snapshot_sha256": "89abfafc79d0e2c13a5ccad6843c020dc65e86f1eca385f85e7f58153960cdfe",
+			"valid_at": null,
+			"qualifier": null,
+			"entity": null,
+			"sentence_id": "S4:s5",
+			"observation_id": null,
+			"fuzzy": false,
+			"outside_floor": false,
+			"population": null,
+			"missing_rule": null,
+			"accounting_basis": null,
+			"snapshot_window": null,
+			"revision": null,
+			"exact": {
+				"num": "12",
+				"den": "1"
+			},
+			"render": null,
+			"operation": null,
+			"operands": [],
+			"derived_from": [],
+			"formula": null,
+			"factor": null,
+			"rationale": null,
+			"status": "active"
+		},
+		{
+			"fact_id": "F6",
+			"kind": "measured",
+			"raw": "27%",
+			"quote": "Segment A held 12% of the market and segment B held 27%.",
+			"extracted_by": "tool:source-snapshot@1",
+			"observed_at": "2026-10-01T00:00:00Z",
+			"label": "seg_b",
+			"value": "27",
+			"unit": "%",
+			"value_span": [
+				52,
+				55
+			],
+			"source_id": "S4",
+			"locator": {
+				"artifact": "text.txt",
+				"char_start": 143,
+				"char_end": 199
+			},
+			"locate_hint": null,
+			"snapshot_sha256": "89abfafc79d0e2c13a5ccad6843c020dc65e86f1eca385f85e7f58153960cdfe",
+			"valid_at": null,
+			"qualifier": null,
+			"entity": null,
+			"sentence_id": "S4:s5",
+			"observation_id": null,
+			"fuzzy": false,
+			"outside_floor": false,
+			"population": null,
+			"missing_rule": null,
+			"accounting_basis": null,
+			"snapshot_window": null,
+			"revision": null,
+			"exact": {
+				"num": "27",
+				"den": "1"
+			},
+			"render": null,
+			"operation": null,
+			"operands": [],
+			"derived_from": [],
+			"formula": null,
+			"factor": null,
+			"rationale": null,
+			"status": "active"
+		},
+		{
+			"fact_id": "F7",
+			"kind": "measured",
+			"raw": "42 passed",
+			"quote": "42 passed",
+			"extracted_by": "tool:source-snapshot@1",
+			"observed_at": "2026-10-01T00:00:00Z",
+			"label": "test_pass",
+			"value": "42",
+			"unit": null,
+			"value_span": [
+				0,
+				9
+			],
+			"source_id": "S5",
+			"locator": {
+				"artifact": "text.txt",
+				"char_start": 0,
+				"char_end": 9
+			},
+			"locate_hint": null,
+			"snapshot_sha256": "e6d47cc01ac7d20e7ba7acd6d1ad113573c1991f02c5af5f0f64a0572e0607f1",
+			"valid_at": null,
+			"qualifier": null,
+			"entity": null,
+			"sentence_id": "S5:s1",
+			"observation_id": null,
+			"fuzzy": false,
+			"outside_floor": false,
+			"population": null,
+			"missing_rule": null,
+			"accounting_basis": null,
+			"snapshot_window": null,
+			"revision": null,
+			"exact": {
+				"num": "42",
+				"den": "1"
+			},
+			"render": null,
+			"operation": null,
+			"operands": [],
+			"derived_from": [],
+			"formula": null,
+			"factor": null,
+			"rationale": null,
+			"status": "active"
+		},
+		{
+			"fact_id": "Q1",
+			"kind": "quote",
+			"raw": "we estimate completion no earlier than 2027",
+			"quote": "we estimate completion no earlier than 2027",
+			"extracted_by": "tool:source-snapshot@1",
+			"observed_at": "2026-10-01T00:00:00Z",
+			"label": "completion",
+			"value": null,
+			"unit": null,
+			"value_span": null,
+			"source_id": "S4",
+			"locator": {
+				"artifact": "text.txt",
+				"char_start": 215,
+				"char_end": 258
+			},
+			"locate_hint": null,
+			"snapshot_sha256": "89abfafc79d0e2c13a5ccad6843c020dc65e86f1eca385f85e7f58153960cdfe",
+			"valid_at": null,
+			"qualifier": null,
+			"entity": "Operator",
+			"sentence_id": "S4:s6",
+			"observation_id": null,
+			"fuzzy": false,
+			"outside_floor": false,
+			"population": null,
+			"missing_rule": null,
+			"accounting_basis": null,
+			"snapshot_window": null,
+			"revision": null,
+			"exact": null,
+			"render": null,
+			"operation": null,
+			"operands": [],
+			"derived_from": [],
+			"formula": null,
+			"factor": null,
+			"rationale": null,
+			"status": "active"
+		},
+		{
+			"fact_id": "F8",
+			"kind": "converted",
+			"raw": "320 m",
+			"quote": "",
+			"extracted_by": "tool:source-snapshot@1",
+			"observed_at": "2026-10-01T00:00:00Z",
+			"label": "span_mm",
+			"value": "320000",
+			"unit": "mm",
+			"value_span": null,
+			"source_id": null,
+			"locator": null,
+			"locate_hint": null,
+			"snapshot_sha256": null,
+			"valid_at": null,
+			"qualifier": null,
+			"entity": null,
+			"sentence_id": null,
+			"observation_id": null,
+			"fuzzy": false,
+			"outside_floor": false,
+			"population": null,
+			"missing_rule": null,
+			"accounting_basis": null,
+			"snapshot_window": null,
+			"revision": null,
+			"exact": {
+				"num": "320000",
+				"den": "1"
+			},
+			"render": "320,000 mm (320 m)",
+			"operation": "convert",
+			"operands": [
+				"F1"
+			],
+			"derived_from": [
+				"F1"
+			],
+			"formula": "F1 * 1000",
+			"factor": "1000",
+			"rationale": null,
+			"status": "active"
+		}
+	],
+	"events": []
+}

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/report-bad.html

@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="en">
+<head><meta charset="utf-8"><title>Bridge report</title></head>
+<body>
+<h1>Bridge report</h1>
+<h2>Findings</h2>
+<p>Revenue grew 12% [S4].</p>
+<p>The span is <data class="fact" value="320" data-fact="F1" data-unit="m" data-lineage="MEASURED">up to 320 mm</data> long [S4].</p>
+<p>Units sold reached <data class="fact" value="1237" data-fact="F2" data-lineage="MEASURED">1,273</data> [S4].</p>
+<p>The operator wrote <q data-fact="Q1" data-quote-sel="full">we estimate completion no earlier than 2028</q> [S4].</p>
+<p>Next year brings {{F9:growth}} more traffic [S4].</p>
+<p>The bridge is popular with tourists.</p>
+<p data-claim="C99">The design is the safest in the region.</p>
+<h2>Sources</h2>
+<ol>
+<li id="s4">[S4] Bridge authority page, retrieved 2026-10-01.</li>
+</ol>
+</body>
+</html>

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/report-good.html

@@ -0,0 +1,39 @@
+<!doctype html>
+<html lang="en">
+<head><meta charset="utf-8"><title>Bridge report</title></head>
+<body>
+<h1>Bridge report</h1>
+<h2>Findings</h2>
+<p>The span is <data class="fact" value="320" data-fact="F1" data-unit="m" data-lineage="MEASURED">up to 320 m</data> long [S4].</p>
+<p>In millimetres the span is <data class="fact" value="320000" data-fact="F8" data-unit="mm" data-lineage="DERIVED">320,000 mm (320 m)</data> [S4].</p>
+<p>Units sold reached <data class="fact" value="1237" data-fact="F2" data-lineage="MEASURED">1,237</data> [S4].</p>
+<p>The operator wrote <q data-fact="Q1" data-quote-sel="full">we estimate completion no earlier than 2027</q> [S4].</p>
+<p>The schedule note opens with <q data-fact="Q1" data-quote-sel="lead=4">we estimate completion no…</q> [S4].</p>
+<p>The opening date is <q data-fact="Q1" data-quote-sel="from=no earlier;to=2027">…no earlier than 2027</q> [S4].</p>
+<p>The test suite ran with <data class="fact" value="42" data-fact="F7" data-claim-kind="verified" data-lineage="MEASURED">42 passed</data> [S5].</p>
+<p data-kind="opinion">This looks like a sound plan.</p>
+<p data-claim="C1">The design follows the published standard.</p>
+<figure data-chart-source="assets/share.csv">
+<svg viewBox="0 0 200 100" role="img"><g class="ticks" data-tick-step="10" data-tick-min="0" data-tick-max="30"><text x="0" y="90">0</text><text x="0" y="60">10</text><text x="0" y="30">20</text><text x="0" y="5">30</text></g><text x="40" y="50">12%</text><text x="120" y="20">27%</text></svg>
+<figcaption>Market share by segment, in percent [S4].</figcaption>
+</figure>
+<table>
+<thead><tr><th>Metric</th><th>Value</th></tr></thead>
+<tbody>
+<tr><td>Span</td><td><data class="fact" value="320" data-fact="F1" data-unit="m" data-lineage="MEASURED">320 m</data></td></tr>
+<tr><td>Units sold</td><td><data class="fact" value="1237" data-fact="F2" data-lineage="MEASURED">1,237</data></td></tr>
+</tbody>
+</table>
+<h2>Appendix</h2>
+<p>Units sold in the base year were <data class="fact" value="1237" data-fact="F2" data-lineage="MEASURED">1,237</data> [S4].</p>
+<h2>Gaps and contradictions</h2>
+<p data-claim="C3">An early schedule claim was refuted.</p>
+<h2>How this report was made</h2>
+<p>Collected from the cited public page and one recorded test run [S4] [S5].</p>
+<h2>Sources</h2>
+<ol>
+<li id="s4">[S4] Bridge authority page, retrieved 2026-10-01.</li>
+<li id="s5">[S5] Test run output, recorded 2026-10-01.</li>
+</ol>
+</body>
+</html>

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4/acronyms.json

@@ -0,0 +1,3 @@
+{
+	"HBM": "High Bandwidth Memory"
+}

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4/meta.json

@@ -0,0 +1,9 @@
+{
+	"source_id": "S4",
+	"url": "https://example.org/bridge",
+	"observed_at": "2026-10-01T00:00:00Z",
+	"sha256": "89abfafc79d0e2c13a5ccad6843c020dc65e86f1eca385f85e7f58153960cdfe",
+	"extractor": "tool:source-snapshot@1",
+	"revision": null,
+	"kind": "page"
+}

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4/sentences.jsonl

@@ -0,0 +1,7 @@
+{"sid":"S4:s1","start":0,"end":36,"text":"The bridge span is up to 320 m long."}
+{"sid":"S4:s2","start":37,"end":70,"text":"Units sold reached 1,237 in 2026."}
+{"sid":"S4:s3","start":71,"end":103,"text":"Revenue grew 12% year over year."}
+{"sid":"S4:s4","start":104,"end":142,"text":"Revenue grew 12% quarter over quarter."}
+{"sid":"S4:s5","start":143,"end":199,"text":"Segment A held 12% of the market and segment B held 27%."}
+{"sid":"S4:s6","start":200,"end":258,"text":"Operators said we estimate completion no earlier than 2027"}
+{"sid":"S4:s7","start":259,"end":299,"text":"High Bandwidth Memory (HBM) demand rose."}

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4/text.txt

@@ -0,0 +1,7 @@
+The bridge span is up to 320 m long.
+Units sold reached 1,237 in 2026.
+Revenue grew 12% year over year.
+Revenue grew 12% quarter over quarter.
+Segment A held 12% of the market and segment B held 27%.
+Operators said we estimate completion no earlier than 2027
+High Bandwidth Memory (HBM) demand rose.

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S4.html

@@ -0,0 +1,19 @@
+<!doctype html>
+<html>
+<head><title>Harbor Bridge progress note</title><style>p { margin: 0 }</style></head>
+<body>
+<h1>Harbor Bridge progress note</h1>
+<p>Span length: 320 m</p>
+<p>The deck uses High Bandwidth Memory (HBM) sensors for strain logging.</p>
+<p>Main span cost: 12 million USD, and approach span cost: 4 million USD.</p>
+<!-- internal draft comment -->
+<p>After the delay, we estimate completion no earlier than 2027.</p>
+<table id="dimensions">
+<caption>Bridge dimensions</caption>
+<tr><th>Metric</th><th>Length</th></tr>
+<tr><td>Span</td><td>320 m</td></tr>
+<tr><td>Approach</td><td>85 m</td></tr>
+</table>
+<script>console.log("not prose")</script>
+</body>
+</html>

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S5/acronyms.json

@@ -0,0 +1 @@
+{}

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S5/meta.json

@@ -0,0 +1,9 @@
+{
+	"source_id": "S5",
+	"url": null,
+	"observed_at": "2026-10-01T00:00:00Z",
+	"sha256": "e6d47cc01ac7d20e7ba7acd6d1ad113573c1991f02c5af5f0f64a0572e0607f1",
+	"extractor": "tool:source-snapshot@1",
+	"revision": null,
+	"kind": "command"
+}

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S5/sentences.jsonl

@@ -0,0 +1 @@
+{"sid":"S5:s1","start":0,"end":9,"text":"42 passed"}

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S5/text.txt

@@ -0,0 +1 @@
+42 passed

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity/sources/S9.txt

@@ -0,0 +1 @@
+북미 클라우드향 HBM3E 매출은 2026년 3분기 연결 기준 최대 4조 2,000억 원에 달할 것으로 전망된다.

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity-brief-locked.md

@@ -0,0 +1,17 @@
+# Brief
+
+## Deliverable
+format: pdf (answered_by: request)
+
+## Fidelity
+as of: latest at run start, 2026-10-23 (answered_by: default)
+meaning check: on (answered_by: default)
+source floor: standard (answered_by: default)
+derived numbers: allowed (answered_by: user)
+languages: report ko; sources ko, en; quotes original with translation beside (answered_by: default)
+identifier classes: version strings, package names, commit ids (answered_by: default)
+fallback: block-quote (answered_by: request)
+asked: derived numbers
+
+## Notes
+free text that is not part of the block

package/packages/shared-skills/skills/ulw-research/scripts/tests/fixtures/fidelity-brief-pending.md

@@ -0,0 +1,17 @@
+# Brief
+
+## Deliverable
+format: pdf (answered_by: request)
+
+## Fidelity
+as of: latest at run start, 2026-10-23 (answered_by: default)
+meaning check: on (answered_by: default)
+source floor: standard (answered_by: default)
+derived numbers: pending (answered_by: pending)
+languages: report ko; sources ko, en; quotes original with translation beside (answered_by: default)
+identifier classes: version strings, package names, commit ids (answered_by: default)
+fallback: block-quote (answered_by: request)
+asked: derived numbers
+
+## Notes
+free text that is not part of the block

package/packages/shared-skills/skills/ulw-research/scripts/text-normalize.mjs

@@ -0,0 +1,82 @@
+// text-normalize.mjs - text normalization and raw-offset matching for fidelity checks.
+//
+// normalizeText applies NFC, maps curly and CJK quote glyphs to straight quotes, collapses
+// whitespace runs to one space, and trims. findNormalized matches a needle against the
+// normalized haystack and returns offsets into the RAW haystack through an offset map built
+// during normalization. Plain Node ESM, no dependencies.
+
+const DOUBLE_QUOTES = new Set([
+	"\u201C", "\u201D", "\u201E", "\u201F", "\u00AB", "\u00BB", "\u300C", "\u300D", "\u300E", "\u300F",
+	"\u301D", "\u301E", "\u301F", "\uFF02", "\u2033",
+])
+const SINGLE_QUOTES = new Set(["\u2018", "\u2019", "\u201A", "\u201B", "\u2039", "\u203A", "\uFF07", "\u2032"])
+// Combining marks and conjoining Hangul medial/final jamo attach to the preceding code point,
+// so a decomposed syllable is normalized as one unit.
+const JOINS_PREVIOUS = /^[\p{M}\u1160-\u11FF\uD7B0-\uD7FF]$/u
+const WHITESPACE = /^\s+$/u
+const TOKEN_SEPARATOR = /[^\p{L}\p{N}\p{M}]+/u
+
+function mapQuote(ch) {
+	if (DOUBLE_QUOTES.has(ch)) return '"'
+	if (SINGLE_QUOTES.has(ch)) return "'"
+	return ch
+}
+
+function clusters(raw) {
+	const out = []
+	let at = 0
+	for (const ch of raw) {
+		const last = out[out.length - 1]
+		if (last && JOINS_PREVIOUS.test(ch)) last[1] += ch.length
+		else out.push([at, at + ch.length])
+		at += ch.length
+	}
+	return out
+}
+
+// Returns { text, starts, ends }: text[k] came from raw.slice(starts[k], ends[k]).
+function normalizeWithMap(raw) {
+	const source = String(raw ?? "")
+	let text = ""
+	const starts = []
+	const ends = []
+	for (const [start, end] of clusters(source)) {
+		const piece = Array.from(source.slice(start, end).normalize("NFC"), mapQuote).join("")
+		if (WHITESPACE.test(piece)) {
+			if (text !== "" && !text.endsWith(" ")) {
+				text += " "
+				starts.push(start)
+				ends.push(end)
+			}
+			continue
+		}
+		for (let k = 0; k < piece.length; k++) {
+			starts.push(start)
+			ends.push(end)
+		}
+		text += piece
+	}
+	if (text.endsWith(" ")) {
+		text = text.slice(0, -1)
+		starts.pop()
+		ends.pop()
+	}
+	return { text, starts, ends }
+}
+
+export function normalizeText(s) {
+	return normalizeWithMap(s).text
+}
+
+export function tokenize(s) {
+	return normalizeText(s).split(TOKEN_SEPARATOR).filter((token) => token !== "")
+}
+
+export function findNormalized(hay, needle) {
+	const target = normalizeText(needle)
+	if (target === "") return null
+	const map = normalizeWithMap(hay)
+	const at = map.text.indexOf(target)
+	if (at === -1) return null
+	return { start: map.starts[at], end: map.ends[at + target.length - 1] }
+}