@ai-matrx/content-ir 0.18.0 → 0.18.2

package/CHANGELOG.md

@@ -1,5 +1,49 @@
 # Changelog
 
+## 0.18.2 — 2026-09-25
+
+**Carries: RC-B10 only.** Three helpers on THE one code-range rule, so every
+consumer strips or reads code in one line instead of a private regex:
+`mapCodeRanges(text, fn)` (rewrite each fence/span, prose kept byte for
+byte), `codeSpanText(raw)` (CommonMark span text: runs off, one padding space
+each side), `fenceParts(raw)` (opener, body, closed — list-marker line,
+blockquote and the renderer's line-end closer handled).
+- Consumer action: none — refresh to latest.
+
+## 0.18.1 — 2026-09-25
+
+**Carries: RC-B10 only** (the one code-range rule). No RC-B3 work is pending
+after the 0.18.0 tag; 0.18.0 itself (RC-B3's single-dollar release) ALSO
+carried RC-B10 stages a and b, swept onto main before that tag — recorded
+here because its entry does not mention them.
+
+**`/source`: THE one code-range rule is the renderer's rule** (RC-B10 ruling,
+REGISTER rulings ledger). `source/code-ranges.ts` owns it: a backtick fence
+opens where a line's trimmed text starts with ``` at ANY indent (list items at
+any depth), after stripping blockquote `>` prefixes; a tilde fence opens at
+≤ 3 spaces relative to its container (list content column); a backtick fence
+closes by the nested-fence rule (with the strict retry) or the renderer's
+line-end closer (`code````), and inside ```json a backtick run in a JSON
+string is text; a blockquoted fence ends with its quote. Strict CommonMark's
+"4+ spaces is not a fence" is rejected on purpose — users see what the
+renderer's splitter draws. Phases follow the renderer: the splitter's
+any-indent backtick fences first, then remark inside each prose segment
+(blockquote, list-marker-line and tilde fences, code spans).
+- New exports: `fenceOpenerOf`, `closeFence`, `FenceReader` (the streaming
+  closer), `BacktickRuns` (the code-span pairing table), `fenceLineKinds`
+  (per-line prose/open/body/close), types `FenceOpener`, `FenceLines`,
+  `FenceEnd`, `FenceLineKind`.
+- The tokenizer (0.18.0) and `XmlContainerTracker` (this release) read fences
+  only through these; their private copies are deleted.
+- Corpus: 136,662/136,662 byte-identical; every tokenizer backtick-fence
+  island (17,518) is exactly one `findCodeRanges` fence (the 0.17 rule missed
+  92); display-math pairing unchanged on every row. Renderer blocks ⊆
+  tokenizer islands (matrx-frontend `source-covers-renderer`, container
+  fences added). Vectors: `__tests__/fixtures/code-range-vectors.json` (25,
+  incl. stored-row shapes with every word replaced) and the Python twin's
+  `fence-nesting-vectors.json` (container + stored-row documents).
+- Consumer action: none — refresh to latest.
+
 ## 0.18.0 — 2026-09-25
 
 **`/source` `isSingleDollarMath`: `$FC$` is math.** (RC-B9 verifier, RC-B3.)

package/dist/index.cjs

@@ -5367,7 +5367,7 @@ function scan(text) {
   }
   const count = starts.length;
   const raw = (k) => text.slice(starts[k], ends[k]);
-  const out = { ranges: [], paragraphs: [], endsInFence: false, pendingPara: -1 };
+  const out = { ranges: [], fences: [], paragraphs: [], endsInFence: false, pendingPara: -1 };
   const rawLines = { count, line: raw };
   let segment = 0;
   for (let k = 0; k < count; k++) {
@@ -5376,6 +5376,7 @@ function scan(text) {
     prose(text, starts, ends, raw, segment, k, out);
     const end = closeFence(rawLines, k, opener);
     out.ranges.push({ start: starts[k], end: ends[end.line], kind: "fence" });
+    out.fences.push([k, end.line, end.closed]);
     out.pendingPara = -1;
     if (!end.closed && end.line === count - 1) out.endsInFence = true;
     k = end.line;
@@ -5441,6 +5442,7 @@ function prose(text, starts, ends, raw, from, to, out) {
     };
     const end = closeFence(src, k, opener);
     out.ranges.push({ start: starts[k], end: ends[end.line], kind: "fence" });
+    out.fences.push([k, end.line, end.closed]);
     if (!end.closed && end.line === starts.length - 1) out.endsInFence = true;
     k = end.line;
   }
@@ -5504,6 +5506,45 @@ function findCodeRanges(text) {
   for (const [from, to] of s.paragraphs) codeSpans(text, from, to, ranges);
   return ranges.sort((a, b) => a.start - b.start);
 }
+function mapCodeRanges(text, fn) {
+  let out = "";
+  let at = 0;
+  for (const r of findCodeRanges(text)) {
+    if (r.start < at) continue;
+    out += text.slice(at, r.start) + fn(r, text.slice(r.start, r.end));
+    at = r.end;
+  }
+  return out + text.slice(at);
+}
+function codeSpanText(raw) {
+  const ticks = backtickRunLength(raw, 0);
+  const inner = raw.slice(ticks, raw.length - ticks);
+  return inner.length >= 2 && inner.startsWith(" ") && inner.endsWith(" ") && inner.trim() !== "" ? inner.slice(1, -1) : inner;
+}
+function fenceParts(raw) {
+  const lines = raw.split("\n").map((l) => l.endsWith("\r") ? l.slice(0, -1) : l);
+  const { offset } = quotePrefix(lines[0] ?? "");
+  const opener = fenceOpenerOf((lines[0] ?? "").slice(offset).replace(/^ *(?:[-*+]|\d{1,9}[.)])(?= |\t|$) ?/, ""));
+  if (!opener) return null;
+  const rest = lines.slice(1);
+  const closed = rest.length > 0 && new FenceReader(opener, { nesting: false }).feed(rest[rest.length - 1]);
+  if (!closed) return { opener, body: rest.join("\n"), closed };
+  const last = rest[rest.length - 1];
+  const before = trimFenceLine(last).startsWith("```") ? "" : last.slice(0, last.indexOf("```"));
+  return { opener, body: [...rest.slice(0, -1), ...before.trim() ? [before] : []].join("\n"), closed };
+}
+function fenceLineKinds(text) {
+  const s = scan(text);
+  let count = 1;
+  for (let i = text.indexOf("\n"); i !== -1; i = text.indexOf("\n", i + 1)) count++;
+  const kinds = new Array(count).fill("prose");
+  for (const [open, last, closed] of s.fences) {
+    kinds[open] = "open";
+    for (let k = open + 1; k <= last; k++) kinds[k] = "body";
+    if (closed && last > open) kinds[last] = "close";
+  }
+  return kinds;
+}
 function pendingCodeSpanStart(text) {
   const n = text.length;
   if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
@@ -5802,25 +5843,20 @@ var XmlContainerTracker = class {
   inComment = false;
   inCdata = false;
   inlineTicks = 0;
-  fenceMarker = null;
+  /** The open fence, read by THE one closer (source/code-ranges.ts). */
+  fence = null;
   consumeLine(line, startOffset = 0) {
-    const fence = /^[ \t]*(`{3,}|~{3,})(.*)$/.exec(line);
-    if (!this.inComment && !this.inCdata && this.inlineTicks === 0 && fence) {
-      const marker = fence[1];
-      const char = marker[0];
-      const suffix = fence[2];
-      if (marker.length >= 3) {
-        if (!this.fenceMarker) {
-          this.fenceMarker = { char, ticks: marker.length };
-          return null;
-        }
-        if (char === this.fenceMarker.char && marker.length >= this.fenceMarker.ticks && suffix.trim() === "") {
-          this.fenceMarker = null;
-        }
+    if (this.fence) {
+      if (this.fence.feed(line)) this.fence = null;
+      return null;
+    }
+    if (!this.inComment && !this.inCdata && this.inlineTicks === 0) {
+      const opener = fenceOpenerOf(line);
+      if (opener) {
+        this.fence = new FenceReader(opener);
         return null;
       }
     }
-    if (this.fenceMarker) return null;
     for (let i = startOffset; i < line.length; i++) {
       if (this.inlineTicks > 0) {
         if (line[i] === "`") {
@@ -7311,6 +7347,7 @@ function mapRange(changes, start, end, unit = "utf16") {
 }
 
 exports.ACTION_BY_CLASS = ACTION_BY_CLASS;
+exports.BacktickRuns = BacktickRuns;
 exports.CALLOUT_NAMES = CALLOUT_NAMES;
 exports.CAPABILITY_BY_CLASS = CAPABILITY_BY_CLASS;
 exports.CLASSES = CLASSES;
@@ -7318,6 +7355,7 @@ exports.DIRECTIVE_VERSION = DIRECTIVE_VERSION;
 exports.DirectiveDecodeError = DirectiveDecodeError;
 exports.FENCE_OPENER_WHITESPACE = FENCE_OPENER_WHITESPACE;
 exports.FENCE_WHITESPACE = FENCE_WHITESPACE;
+exports.FenceReader = FenceReader;
 exports.IN_CONTENT_CLASSES = IN_CONTENT_CLASSES;
 exports.IR_ENVELOPE_CACHE_VERSION = IR_ENVELOPE_CACHE_VERSION;
 exports.IR_ENVELOPE_KEY = IR_ENVELOPE_KEY;
@@ -7352,6 +7390,8 @@ exports.buildKindDirective = buildKindDirective;
 exports.capabilityOf = capabilityOf;
 exports.classifyInboundEnvelopeMetadata = classifyInboundEnvelopeMetadata;
 exports.classifyInnerFenceLine = classifyInnerFenceLine;
+exports.closeFence = closeFence;
+exports.codeSpanText = codeSpanText;
 exports.collectReferencedKinds = collectReferencedKinds;
 exports.collectSchemaReferencedKinds = collectSchemaReferencedKinds;
 exports.compareWithExistingKindSchema = compareWithExistingKindSchema;
@@ -7371,7 +7411,10 @@ exports.envelopeCacheFromEnvelopes = envelopeCacheFromEnvelopes;
 exports.envelopeFromCompleteValue = envelopeFromCompleteValue;
 exports.executesAtOutputRoot = executesAtOutputRoot;
 exports.fenceDiscriminator = fenceDiscriminator;
+exports.fenceLineKinds = fenceLineKinds;
 exports.fenceNestsInnerFences = fenceNestsInnerFences;
+exports.fenceOpenerOf = fenceOpenerOf;
+exports.fenceParts = fenceParts;
 exports.fieldsToDbPayload = fieldsToDbPayload;
 exports.findCodeRanges = findCodeRanges;
 exports.fingerprintText = fingerprintText;
@@ -7411,6 +7454,7 @@ exports.looksLikeDirectiveHead = looksLikeDirectiveHead;
 exports.looksLikeDisplayMath = looksLikeDisplayMath;
 exports.looksLikeOpenInlineMath = looksLikeOpenInlineMath;
 exports.makePartialKindStalenessGate = makePartialKindStalenessGate;
+exports.mapCodeRanges = mapCodeRanges;
 exports.mapPosition = mapPosition;
 exports.mapRange = mapRange;
 exports.mergeResidueIntoValue = mergeResidueIntoValue;