@ai-matrx/content-ir 0.19.0 → 0.19.2

package/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## 0.19.2
+### Unreleased
+
+- `/source`: **THE GFM table rule** (`source/gfm-table.ts`, verify-RC-B4) —
+  `splitRowSegments` / `rowCells` (a `|` is a boundary unless an odd backslash
+  run precedes it), `unescapeCellPipes`, `isGfmDelimiterRow`, `isPipeLedRow`,
+  `startsPipelessTable`, `continuesTable`, `opensTable`, `tableStartsAt`,
+  `findTableStart`, `findTableEnd`. Held to the shared vectors
+  (`__tests__/gfm-table-vectors.json`, byte copy of matrx-frontend's and of the
+  Python block detector's fixture). Consumers: `@ai-matrx/print` now; the
+  matrx-frontend readers move onto it once this version is on npm.
+
+## 0.19.1
+
+Automatic republish — despite the generated note, this version carries source
+changes: **the RC-B3 round 3 fix round** (verify-RC-B3 § round 3, RC-B3 lane;
+aidream a8506eeeca, 4b92192527, aaef42333d).
+
+- `/source` streaming: a trailing single `$` may be half of a `$$` (B3); a lone
+  CR is half of a line break still arriving (B4); a block that opens in
+  provisional text may still dissolve (a footnote or paragraph does not end
+  there); front matter still waiting for its closing fence is undecided.
+  Lone CR, tab, list-item and quote atoms joined the LLM-fragment generator.
+- Holdbacks R4 (`$$` block may dissolve), R8 (spilled region), R10 (`\(`/`\[`
+  opener) each pinned by a reproducer that fails when it alone is removed; the
+  JSON open-tail holdback (R12) and a provisional-tree-root holdback were
+  deleted as redundant — planted-mutation differentials over 12,000 documents
+  showed 64 and 972 differing prefixes but 0 settled-island changes.
+- Front matter behind a leading byte-order mark (`\uFEFF---`) is front matter.
+- `isSingleDollarMath`: PHP/Perl member access is not math (`$this->$prop`,
+  `$obj->$method`, `$class::$instance`). Measured read-only over 86,136
+  assistant messages + 4,329 notes: 0 gained, 8 lost (6 shell-variable-chain,
+  2 php-perl-member), every one intended.
+- No consumer action required.
+
 ## 0.19.0 — kind validation behind a schema-source port (Matrx Alchemy ALC-13)
 
 `/registry` (and the root) now export `createKindValidator(source, options?)`,
@@ -976,3 +1011,4 @@ unchanged. Two things are worth doing at your next working session:
 - Added ESM and declaration builds, package linting, packed-artifact analysis, and an isolated consumer canary.
 - Preserved all parser, session, normalization, schema conversion, and structural-validation behavior with the existing shared fixtures and fingerprint vectors.
 
+

package/dist/index.cjs

@@ -5648,7 +5648,7 @@ function pendingCodeSpanStart(text) {
   const n = text.length;
   if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
   const lastLine = text.slice(text.lastIndexOf("\n") + 1);
-  if (lastLine && /^(?: {0,3}>[ \t]?)*[ \t]*$/.test(lastLine)) return pendingCodeSpanStart(`${text}x`);
+  if (lastLine && /^(?: {0,3}>[ \t]?)*[ \t\r]*$/.test(lastLine)) return pendingCodeSpanStart(`${text}x`);
   const s = scan(text);
   const para = s.pendingPara;
   if (s.endsInFence || para === -1) return -1;
@@ -5812,7 +5812,7 @@ function isVariableChain(content, after) {
   const run = /^[A-Za-z0-9_]+/.exec(after)?.[0];
   if (!run) return false;
   if (after.length === 1) return false;
-  if (/^[A-Za-z_][A-Za-z0-9_]*[.@-]$/.test(content)) return true;
+  if (/^[A-Za-z_][A-Za-z0-9_]*(?:[.@-]|->|::)$/.test(content)) return true;
   return IDENTIFIER.test(content) && !MATH_SUFFIX.test(run);
 }
 var FUNCTION_NAMES = /* @__PURE__ */ new Set([
@@ -6217,7 +6217,7 @@ var Tokenizer = class {
           if (close >= span) this.provisionalFrom = Math.min(this.provisionalFrom, open);
         }
       }
-      for (const pad of ["`", "``", "```", "~~~"]) {
+      for (const pad of ["`", "``", "```", "~~~", "$"]) {
         const alt = pairDisplayMath(text + pad);
         for (const [open, close] of this.mathPairs) {
           if (alt.get(open) !== close) this.provisionalFrom = Math.min(this.provisionalFrom, open);
@@ -6249,19 +6249,6 @@ var Tokenizer = class {
   provisionalFrom = Number.POSITIVE_INFINITY;
   /** Streaming: the text ends in a lone CR whose LF has not arrived yet. */
   pendingCR = false;
-  /**
-   * Streaming: an open last line that begins with `{` is undecided — each new
-   * byte can balance or unbalance its braces and flip the whole line between
-   * prose and a JSON island — so it is provisional until it ends (G5).
-   * (A lone HTML tag on that line is held by {@link holdUndecidedTail}.)
-   */
-  markOpenJsonTail() {
-    const last = this.lineStarts.length - 1;
-    if (!this.isOpenTail(last)) return;
-    if (this.lineText(last).trim().startsWith("{")) {
-      this.provisionalFrom = Math.min(this.provisionalFrom, this.lineStarts[last]);
-    }
-  }
   /**
    * `text.indexOf(needle, from)` in O(log k): the first query for a needle
    * indexes all of its occurrences once, so repeated searches for a closer
@@ -6296,7 +6283,7 @@ var Tokenizer = class {
     const rest = this.n - pe;
     if (rest === 0) return true;
     const tail = this.text.slice(pe);
-    return /^(?:\r\n|\r|\n)[ \t]*$/.test(tail);
+    return /^(?:\r\n|\r|\n)[ \t\r]*$/.test(tail);
   }
   /**
    * Streaming: an open block that may still turn back into prose — a `$$`
@@ -6305,7 +6292,9 @@ var Tokenizer = class {
    * or any opener that is itself still arriving on the open last line.
    */
   mayDissolve(block) {
-    if (!this.streaming || block.complete) return false;
+    if (!this.streaming) return false;
+    if (block.start >= this.provisionalFrom) return true;
+    if (block.complete) return false;
     if (block.type === "math_block" || block.type === "json") return true;
     return this.isOpenTail(this.lineOf(block.start));
   }
@@ -6344,7 +6333,6 @@ var Tokenizer = class {
     return this.text.slice(this.lineStarts[li], this.contentEnds[li]);
   }
   run() {
-    if (this.streaming) this.markOpenJsonTail();
     let pos = 0;
     while (pos < this.n) {
       const li = this.lineOf(pos);
@@ -6582,7 +6570,7 @@ var Tokenizer = class {
     return { type, start, end, complete, meta };
   }
   frontMatter(li, t) {
-    const fence = this.lineText(li);
+    const fence = this.lineText(li).replace(/^\uFEFF/, "");
     if (fence !== "---" && fence !== "+++" || t !== fence) return null;
     const toml = fence === "+++";
     let sawContent = false;
@@ -6593,9 +6581,13 @@ var Tokenizer = class {
       }
       if (!NOT_WHITESPACE.test(line)) continue;
       const shaped = toml ? /^\s*(?:[A-Za-z0-9_."-]+\s*=|\[|#)/.test(line) : /^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s|#)/.test(line);
-      if (!shaped) return null;
+      if (!shaped) {
+        if (this.isOpenTail(k)) break;
+        return null;
+      }
       sawContent = true;
     }
+    if (this.streaming) return this.island("front_matter", 0, this.n, false, { format: toml ? "toml" : "yaml" });
     return null;
   }
   /**
@@ -6996,7 +6988,7 @@ var Tokenizer = class {
       const block = this.detectBlock(this.lineStarts[k], k, false);
       if (block) {
         breakLine = k;
-        if (this.mayDissolve(block) || this.streaming && block.start >= this.provisionalFrom) {
+        if (this.mayDissolve(block)) {
           this.undecidedBreaks.add(end);
         }
         break;
@@ -7627,6 +7619,76 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
+// source/gfm-table.ts
+function splitRowSegments(line) {
+  const segments = [];
+  let start = 0;
+  for (let i = 0; i < line.length; i += 1) {
+    const ch = line[i];
+    if (ch === "\\") {
+      i += 1;
+      continue;
+    }
+    if (ch === "|") {
+      segments.push(line.slice(start, i));
+      start = i + 1;
+    }
+  }
+  segments.push(line.slice(start));
+  return segments;
+}
+function rowCells(line) {
+  const cells = splitRowSegments(line).map((cell) => cell.trim());
+  if (cells.length > 1 && cells[0] === "") cells.shift();
+  if (cells.length > 1 && cells[cells.length - 1] === "") cells.pop();
+  else if (cells.length === 1 && cells[0] === "") cells.pop();
+  return cells;
+}
+function unescapeCellPipes(cell) {
+  return cell.replace(/\\\|/g, "|");
+}
+var BLOCK_START = /^(?:[-+*](?:\s|$)|\d{1,9}[.)](?:\s|$)|>|#{1,6}(?:\s|$)|```|~~~|(?:-\s*){3,}$|(?:\*\s*){3,}$|(?:_\s*){3,}$)/;
+var DELIMITER_CELL = /^:?-+:?$/;
+function isGfmDelimiterRow(line) {
+  const trimmed = line.trim();
+  if (!trimmed.includes("|")) return false;
+  const cells = rowCells(trimmed);
+  return cells.length > 0 && cells.every((cell) => DELIMITER_CELL.test(cell));
+}
+function isPipeLedRow(line) {
+  const trimmed = line.trim();
+  return trimmed.startsWith("|") && trimmed.includes("|", 1);
+}
+function startsPipelessTable(header, delimiter) {
+  const head = header.trim();
+  if (!head || head.startsWith("|") || BLOCK_START.test(head)) return false;
+  if (splitRowSegments(head).length < 2) return false;
+  if (delimiter === void 0 || !isGfmDelimiterRow(delimiter)) return false;
+  return rowCells(head).length === rowCells(delimiter.trim()).length;
+}
+function continuesTable(line) {
+  const trimmed = line.trim();
+  if (!trimmed) return false;
+  if (trimmed.startsWith("|")) return trimmed.includes("|", 1);
+  return splitRowSegments(trimmed).length > 1 && !BLOCK_START.test(trimmed);
+}
+function opensTable(lines, index) {
+  const line = lines[index] ?? "";
+  return isPipeLedRow(line) || startsPipelessTable(line, lines[index + 1]);
+}
+function tableStartsAt(lines, index) {
+  return opensTable(lines, index) && isGfmDelimiterRow(lines[index + 1] ?? "");
+}
+function findTableStart(lines, from = 0) {
+  for (let i = Math.max(0, from); i < lines.length; i += 1) if (opensTable(lines, i)) return i;
+  return -1;
+}
+function findTableEnd(lines, start) {
+  let end = Math.min(lines.length, start + 2);
+  while (end < lines.length && continuesTable(lines[end] ?? "")) end += 1;
+  return end;
+}
+
 exports.ACTION_BY_CLASS = ACTION_BY_CLASS;
 exports.AFTER_WINDOW = AFTER_WINDOW;
 exports.BacktickRuns = BacktickRuns;
@@ -7677,6 +7739,7 @@ exports.codeSpanText = codeSpanText;
 exports.collectReferencedKinds = collectReferencedKinds;
 exports.collectSchemaReferencedKinds = collectSchemaReferencedKinds;
 exports.compareWithExistingKindSchema = compareWithExistingKindSchema;
+exports.continuesTable = continuesTable;
 exports.convertAiSchemaToBlockFields = convertAiSchemaToBlockFields;
 exports.createFingerprinter = createFingerprinter;
 exports.createKindStreamParser = createKindStreamParser;
@@ -7700,6 +7763,8 @@ exports.fenceOpenerOf = fenceOpenerOf;
 exports.fenceParts = fenceParts;
 exports.fieldsToDbPayload = fieldsToDbPayload;
 exports.findCodeRanges = findCodeRanges;
+exports.findTableEnd = findTableEnd;
+exports.findTableStart = findTableStart;
 exports.fingerprintText = fingerprintText;
 exports.formatBlockLabel = formatBlockLabel;
 exports.getParseSession = getParseSession;
@@ -7713,11 +7778,13 @@ exports.isDirectiveClass = isDirectiveClass;
 exports.isDuplicateBlockSlug = isDuplicateBlockSlug;
 exports.isEmptyResidue = isEmptyResidue;
 exports.isEscapedBracketMath = isEscapedBracketMath;
+exports.isGfmDelimiterRow = isGfmDelimiterRow;
 exports.isIrEnvelopeCache = isIrEnvelopeCache;
 exports.isJsonAnyField = isJsonAnyField;
 exports.isKindDirective = isKindDirective;
 exports.isLegacyShell = isLegacyShell;
 exports.isMathPlaceholder = isMathPlaceholder;
+exports.isPipeLedRow = isPipeLedRow;
 exports.isProvisionalKind = isProvisionalKind;
 exports.isReservedDirectiveSlug = isReservedDirectiveSlug;
 exports.isScalarArrayType = isScalarArrayType;
@@ -7748,6 +7815,7 @@ exports.nounFamily = nounFamily;
 exports.nounLabel = nounLabel;
 exports.nounTitleColumn = nounTitleColumn;
 exports.openParseSession = openParseSession;
+exports.opensTable = opensTable;
 exports.pairDisplayMath = pairDisplayMath;
 exports.parseDirectiveSlug = parseDirectiveSlug;
 exports.parseFenceOpener = parseFenceOpener;
@@ -7765,6 +7833,7 @@ exports.rehydrateRunResult = rehydrateRunResult;
 exports.resetLegacyShellUses = resetLegacyShellUses;
 exports.resolvesInContent = resolvesInContent;
 exports.reuseEnvelopeIfCurrent = reuseEnvelopeIfCurrent;
+exports.rowCells = rowCells;
 exports.runKindDualGate = runKindDualGate;
 exports.runSchemaConversion = runSchemaConversion;
 exports.sanitizeInboundEnvelopeMetadata = sanitizeInboundEnvelopeMetadata;
@@ -7776,9 +7845,12 @@ exports.setJsonRootKeyLookup = setJsonRootKeyLookup;
 exports.singleDollarAfter = singleDollarAfter;
 exports.singleDollarMathEnd = singleDollarMathEnd;
 exports.spliceSave = spliceSave;
+exports.splitRowSegments = splitRowSegments;
 exports.splitsSurrogatePair = splitsSurrogatePair;
+exports.startsPipelessTable = startsPipelessTable;
 exports.storageToKindSchema = storageToKindSchema;
 exports.stripKindDeep = stripKindDeep;
+exports.tableStartsAt = tableStartsAt;
 exports.titleCaseToken = titleCaseToken;
 exports.toCodePointOffset = toCodePointOffset;
 exports.toUtf16Offset = toUtf16Offset;
@@ -7786,6 +7858,7 @@ exports.tokenizeSource = tokenizeSource;
 exports.trimFenceLine = trimFenceLine;
 exports.tryDecodeDirective = tryDecodeDirective;
 exports.tryDecodeDirectiveContent = tryDecodeDirectiveContent;
+exports.unescapeCellPipes = unescapeCellPipes;
 exports.validateBlockSchemaSavePlan = validateBlockSchemaSavePlan;
 exports.validateStructuralLeg = validateStructuralLeg;
 exports.withRootKind = withRootKind;