@ai-matrx/content-ir 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +119 -0
- package/dist/index.cjs +546 -98
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +539 -99
- package/dist/index.js.map +1 -1
- package/dist/source.cjs +546 -98
- package/dist/source.cjs.map +1 -1
- package/dist/source.d.cts +118 -14
- package/dist/source.d.ts +118 -14
- package/dist/source.js +539 -99
- package/dist/source.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -5169,20 +5169,84 @@ function normalizeFenceLanguage(language) {
|
|
|
5169
5169
|
return CODE_LANGUAGE_ALIASES[lower] ?? lower;
|
|
5170
5170
|
}
|
|
5171
5171
|
|
|
5172
|
+
// source/fence-nesting.ts
|
|
5173
|
+
var NESTING_FENCE_LANGUAGES = /* @__PURE__ */ new Set([
|
|
5174
|
+
"markdown",
|
|
5175
|
+
"md",
|
|
5176
|
+
"mdx"
|
|
5177
|
+
]);
|
|
5178
|
+
function fenceNestsInnerFences(language) {
|
|
5179
|
+
return !!language && NESTING_FENCE_LANGUAGES.has(language.toLowerCase());
|
|
5180
|
+
}
|
|
5181
|
+
var FENCE_WHITESPACE = " \n\r\f\v";
|
|
5182
|
+
function isFenceWhitespace(ch) {
|
|
5183
|
+
return ch !== void 0 && ch !== "" && FENCE_WHITESPACE.includes(ch);
|
|
5184
|
+
}
|
|
5185
|
+
function trimFenceLine(line) {
|
|
5186
|
+
let start = 0;
|
|
5187
|
+
let end = line.length;
|
|
5188
|
+
while (start < end && isFenceWhitespace(line[start])) start++;
|
|
5189
|
+
while (end > start && isFenceWhitespace(line[end - 1])) end--;
|
|
5190
|
+
return start === 0 && end === line.length ? line : line.slice(start, end);
|
|
5191
|
+
}
|
|
5192
|
+
function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
|
|
5193
|
+
let ticks = 0;
|
|
5194
|
+
while (ticks < trimmed.length && trimmed[ticks] === "`") ticks++;
|
|
5195
|
+
if (ticks < 3) return "content";
|
|
5196
|
+
const info = trimFenceLine(trimmed.slice(ticks));
|
|
5197
|
+
if (info === "") {
|
|
5198
|
+
if (ticks < openTicks) return "content";
|
|
5199
|
+
if (nests && nestedDepth > 0) return "close-nested";
|
|
5200
|
+
return "close-outer";
|
|
5201
|
+
}
|
|
5202
|
+
if (nests && ticks >= openTicks && !info.includes("`")) return "open-nested";
|
|
5203
|
+
return "content";
|
|
5204
|
+
}
|
|
5205
|
+
|
|
5172
5206
|
// source/math.ts
|
|
5173
5207
|
var TEX_SIGNAL = /\\[A-Za-z]+|[\\^_{}=<>+]/;
|
|
5174
5208
|
var LONE_VARIABLE = /^[A-Za-z](?:'|[0-9])?$/;
|
|
5209
|
+
var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
|
|
5210
|
+
var FUNCTION_NAMES = /* @__PURE__ */ new Set([
|
|
5211
|
+
"sin",
|
|
5212
|
+
"cos",
|
|
5213
|
+
"tan",
|
|
5214
|
+
"sec",
|
|
5215
|
+
"csc",
|
|
5216
|
+
"cot",
|
|
5217
|
+
"log",
|
|
5218
|
+
"ln",
|
|
5219
|
+
"exp",
|
|
5220
|
+
"lim",
|
|
5221
|
+
"max",
|
|
5222
|
+
"min",
|
|
5223
|
+
"det",
|
|
5224
|
+
"gcd",
|
|
5225
|
+
"mod",
|
|
5226
|
+
"arg",
|
|
5227
|
+
"sinh",
|
|
5228
|
+
"cosh",
|
|
5229
|
+
"tanh"
|
|
5230
|
+
]);
|
|
5231
|
+
function isMathShape(content) {
|
|
5232
|
+
if (!MATH_SHAPE_CHARS.test(content)) return false;
|
|
5233
|
+
const words = content.match(/[A-Za-z]+/g);
|
|
5234
|
+
if (!words) return false;
|
|
5235
|
+
if (!words.every((w) => w.length === 1 || FUNCTION_NAMES.has(w))) return false;
|
|
5236
|
+
return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
|
|
5237
|
+
}
|
|
5175
5238
|
function isSingleDollarMath(content, after) {
|
|
5176
5239
|
if (!content || content.length > 400) return false;
|
|
5177
5240
|
if (/^\s/.test(content) || /\s$/.test(content)) return false;
|
|
5178
5241
|
if (after !== void 0 && /[0-9]/.test(after)) return false;
|
|
5179
5242
|
if (LONE_VARIABLE.test(content)) return true;
|
|
5180
5243
|
if (/^[0-9]/.test(content)) return /\\[A-Za-z]+|[\\^_{}]/.test(content);
|
|
5181
|
-
return TEX_SIGNAL.test(content);
|
|
5244
|
+
return TEX_SIGNAL.test(content) || isMathShape(content);
|
|
5182
5245
|
}
|
|
5183
5246
|
function singleDollarMathEnd(text, open, limit = text.length) {
|
|
5247
|
+
const bound = Math.min(limit, open + 403);
|
|
5184
5248
|
let close = -1;
|
|
5185
|
-
for (let k = open + 1; k <
|
|
5249
|
+
for (let k = open + 1; k < bound; k += 1) {
|
|
5186
5250
|
const c = text[k];
|
|
5187
5251
|
if (c === "\n") break;
|
|
5188
5252
|
if (c === "\\") {
|
|
@@ -5199,11 +5263,69 @@ function singleDollarMathEnd(text, open, limit = text.length) {
|
|
|
5199
5263
|
return isSingleDollarMath(content, text[close + 1]) ? close + 1 : -1;
|
|
5200
5264
|
}
|
|
5201
5265
|
|
|
5266
|
+
// source/math-pairs.ts
|
|
5267
|
+
var MAX_MATH_SPAN = 600;
|
|
5268
|
+
var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(?:^|\n)[ \t]*[-*+][ \t]+|(?:^|\n)[ \t]*\d+[.)][ \t]/;
|
|
5269
|
+
var LATEX_COMMAND = /\\[a-zA-Z]/;
|
|
5270
|
+
var PROSE_WORD = /[A-Za-z]{3,}/g;
|
|
5271
|
+
var PROSE_WORD_LIMIT = 6;
|
|
5272
|
+
function looksLikeDisplayMath(inner) {
|
|
5273
|
+
const s = inner.trim();
|
|
5274
|
+
if (!s) return false;
|
|
5275
|
+
if (STRUCTURAL_MARKDOWN.test(s)) return false;
|
|
5276
|
+
if (s.length > MAX_MATH_SPAN) return false;
|
|
5277
|
+
if (LATEX_COMMAND.test(s)) {
|
|
5278
|
+
return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT * 3;
|
|
5279
|
+
}
|
|
5280
|
+
if (/\n[ \t]*\n/.test(s)) return false;
|
|
5281
|
+
return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT;
|
|
5282
|
+
}
|
|
5283
|
+
function codeRanges(text) {
|
|
5284
|
+
const ranges = [];
|
|
5285
|
+
for (const re of [/```[\s\S]*?(?:```|$)/g, /~~~[\s\S]*?(?:~~~|$)/g, /`[^`\n]*`/g]) {
|
|
5286
|
+
let m;
|
|
5287
|
+
while ((m = re.exec(text)) !== null) {
|
|
5288
|
+
ranges.push([m.index, m.index + m[0].length]);
|
|
5289
|
+
if (m[0].length === 0) re.lastIndex++;
|
|
5290
|
+
}
|
|
5291
|
+
}
|
|
5292
|
+
return ranges.sort((a, b) => a[0] - b[0]);
|
|
5293
|
+
}
|
|
5294
|
+
function pairDisplayMath(text) {
|
|
5295
|
+
const pairs = /* @__PURE__ */ new Map();
|
|
5296
|
+
if (!text.includes("$$")) return pairs;
|
|
5297
|
+
const ranges = codeRanges(text);
|
|
5298
|
+
const tokens = [];
|
|
5299
|
+
let r = 0;
|
|
5300
|
+
let maxEnd = -1;
|
|
5301
|
+
for (let i = text.indexOf("$$"); i !== -1 && i < text.length - 1; i = text.indexOf("$$", i)) {
|
|
5302
|
+
while (r < ranges.length && ranges[r][0] <= i) {
|
|
5303
|
+
maxEnd = Math.max(maxEnd, ranges[r][1]);
|
|
5304
|
+
r++;
|
|
5305
|
+
}
|
|
5306
|
+
if (maxEnd <= i) tokens.push(i);
|
|
5307
|
+
i += 2;
|
|
5308
|
+
}
|
|
5309
|
+
let j = 0;
|
|
5310
|
+
while (j < tokens.length) {
|
|
5311
|
+
const open = tokens[j];
|
|
5312
|
+
const close = tokens[j + 1];
|
|
5313
|
+
if (close === void 0) break;
|
|
5314
|
+
if (looksLikeDisplayMath(text.slice(open + 2, close))) {
|
|
5315
|
+
pairs.set(open, close);
|
|
5316
|
+
j += 2;
|
|
5317
|
+
continue;
|
|
5318
|
+
}
|
|
5319
|
+
j += 1;
|
|
5320
|
+
}
|
|
5321
|
+
return pairs;
|
|
5322
|
+
}
|
|
5323
|
+
|
|
5202
5324
|
// source/xml-tag.ts
|
|
5203
5325
|
var XML_NAME_START = /[A-Za-z_]/;
|
|
5204
5326
|
var XML_NAME_CHARACTER = /[\w.:-]/;
|
|
5205
5327
|
var WHITESPACE = /\s/;
|
|
5206
|
-
function readXmlTag(content, start) {
|
|
5328
|
+
function readXmlTag(content, start, find = (needle, from) => content.indexOf(needle, from)) {
|
|
5207
5329
|
let cursor = start + 1;
|
|
5208
5330
|
const isClosing = content[cursor] === "/";
|
|
5209
5331
|
if (isClosing) cursor++;
|
|
@@ -5256,8 +5378,9 @@ function readXmlTag(content, start) {
|
|
|
5256
5378
|
const quote = content[cursor];
|
|
5257
5379
|
if (quote !== '"' && quote !== "'") return null;
|
|
5258
5380
|
const valueStart = ++cursor;
|
|
5259
|
-
|
|
5260
|
-
if (
|
|
5381
|
+
const valueEnd = find(quote, valueStart);
|
|
5382
|
+
if (valueEnd === -1) return null;
|
|
5383
|
+
cursor = valueEnd;
|
|
5261
5384
|
attributes.push({ name, value: content.slice(valueStart, cursor) });
|
|
5262
5385
|
cursor++;
|
|
5263
5386
|
}
|
|
@@ -5354,6 +5477,10 @@ var XmlContainerTracker = class {
|
|
|
5354
5477
|
|
|
5355
5478
|
// source/tokenize.ts
|
|
5356
5479
|
var NOT_WHITESPACE = /\S/;
|
|
5480
|
+
var DIRECTIVE_OPEN = /^(:{2,})([A-Za-z][\w-]*)/;
|
|
5481
|
+
var MKDOCS_OPEN = /^(!!!|\?\?\?\+?)[ \t]+([A-Za-z][\w-]*)/;
|
|
5482
|
+
var CALLOUT_OPEN = /^>[ \t]*\[!([A-Za-z][\w-]*)\]([+-])?/;
|
|
5483
|
+
var FOOTNOTE_DEF = /^\[\^([^\]\s]+)\]:/;
|
|
5357
5484
|
function countStructuralBraces(source) {
|
|
5358
5485
|
let opens = 0;
|
|
5359
5486
|
let closes = 0;
|
|
@@ -5476,11 +5603,16 @@ function backtickRunLength(str2, pos) {
|
|
|
5476
5603
|
while (pos + n < str2.length && str2[pos + n] === "`") n++;
|
|
5477
5604
|
return n;
|
|
5478
5605
|
}
|
|
5606
|
+
function isSplitterTreeLine(line) {
|
|
5607
|
+
return TREE_GLYPHS.test(line) || /^[\s│|]*[├└+|][\s─\-]+/.test(line);
|
|
5608
|
+
}
|
|
5479
5609
|
var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
|
|
5480
5610
|
var Tokenizer = class {
|
|
5481
|
-
constructor(text) {
|
|
5611
|
+
constructor(text, streaming = false) {
|
|
5482
5612
|
this.text = text;
|
|
5613
|
+
this.streaming = streaming;
|
|
5483
5614
|
this.n = text.length;
|
|
5615
|
+
this.mathPairs = pairDisplayMath(text);
|
|
5484
5616
|
let start = 0;
|
|
5485
5617
|
for (let i = 0; i < text.length; i++) {
|
|
5486
5618
|
if (text.charCodeAt(i) === 10) {
|
|
@@ -5497,6 +5629,7 @@ var Tokenizer = class {
|
|
|
5497
5629
|
}
|
|
5498
5630
|
}
|
|
5499
5631
|
text;
|
|
5632
|
+
streaming;
|
|
5500
5633
|
n;
|
|
5501
5634
|
lineStarts = [];
|
|
5502
5635
|
contentEnds = [];
|
|
@@ -5506,6 +5639,38 @@ var Tokenizer = class {
|
|
|
5506
5639
|
/** Lazily built: for line i, first line j >= i where brace net from i drops to <= 0. */
|
|
5507
5640
|
braceStop = null;
|
|
5508
5641
|
braceNet = null;
|
|
5642
|
+
/** Every occurrence offset of each needle searched so far (overlapping, like indexOf). */
|
|
5643
|
+
occurrences = /* @__PURE__ */ new Map();
|
|
5644
|
+
/** `$$` opener → closer for the pairs the core renders as math. */
|
|
5645
|
+
mathPairs;
|
|
5646
|
+
finder = (needle, from) => this.next(needle, from);
|
|
5647
|
+
/**
|
|
5648
|
+
* `text.indexOf(needle, from)` in O(log k): the first query for a needle
|
|
5649
|
+
* indexes all of its occurrences once, so repeated searches for a closer
|
|
5650
|
+
* that never comes stay linear over the whole text.
|
|
5651
|
+
*/
|
|
5652
|
+
next(needle, from) {
|
|
5653
|
+
let list = this.occurrences.get(needle);
|
|
5654
|
+
if (!list) {
|
|
5655
|
+
list = [];
|
|
5656
|
+
for (let at = this.text.indexOf(needle); at !== -1; at = this.text.indexOf(needle, at + 1)) {
|
|
5657
|
+
list.push(at);
|
|
5658
|
+
}
|
|
5659
|
+
this.occurrences.set(needle, list);
|
|
5660
|
+
}
|
|
5661
|
+
let lo = 0;
|
|
5662
|
+
let hi = list.length;
|
|
5663
|
+
while (lo < hi) {
|
|
5664
|
+
const mid = lo + hi >>> 1;
|
|
5665
|
+
if (list[mid] < from) lo = mid + 1;
|
|
5666
|
+
else hi = mid;
|
|
5667
|
+
}
|
|
5668
|
+
return lo < list.length ? list[lo] : -1;
|
|
5669
|
+
}
|
|
5670
|
+
/** Streaming only: the final line, still waiting for its terminator. */
|
|
5671
|
+
isOpenTail(k) {
|
|
5672
|
+
return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
|
|
5673
|
+
}
|
|
5509
5674
|
get lineCount() {
|
|
5510
5675
|
return this.lineStarts.length;
|
|
5511
5676
|
}
|
|
@@ -5587,6 +5752,22 @@ var Tokenizer = class {
|
|
|
5587
5752
|
const fm = this.frontMatter(li, t);
|
|
5588
5753
|
if (fm) return fm;
|
|
5589
5754
|
}
|
|
5755
|
+
if (indent <= 3 && t.startsWith("::")) {
|
|
5756
|
+
const directive = this.directive(p, li, t);
|
|
5757
|
+
if (directive) return directive;
|
|
5758
|
+
}
|
|
5759
|
+
if (indent === 0 && (t.startsWith("!!!") || t.startsWith("???"))) {
|
|
5760
|
+
const admonition = this.mkdocsAdmonition(p, li, t);
|
|
5761
|
+
if (admonition) return admonition;
|
|
5762
|
+
}
|
|
5763
|
+
if (indent <= 3 && t.startsWith(">")) {
|
|
5764
|
+
const callout = this.callout(p, li, t);
|
|
5765
|
+
if (callout) return callout;
|
|
5766
|
+
}
|
|
5767
|
+
if (atParaStart && indent <= 3 && t.startsWith("[^")) {
|
|
5768
|
+
const footnote = this.footnoteDefinition(p, li, t);
|
|
5769
|
+
if (footnote) return footnote;
|
|
5770
|
+
}
|
|
5590
5771
|
if (t.startsWith("```")) return this.backtickFence(p, li, t);
|
|
5591
5772
|
if (indent <= 3 && t.startsWith("~~~")) {
|
|
5592
5773
|
const tilde = this.tildeFence(p, li, t);
|
|
@@ -5624,20 +5805,116 @@ var Tokenizer = class {
|
|
|
5624
5805
|
return { type, start, end, complete, meta };
|
|
5625
5806
|
}
|
|
5626
5807
|
frontMatter(li, t) {
|
|
5627
|
-
|
|
5808
|
+
const fence = this.lineText(li);
|
|
5809
|
+
if (fence !== "---" && fence !== "+++" || t !== fence) return null;
|
|
5810
|
+
const toml = fence === "+++";
|
|
5628
5811
|
let sawContent = false;
|
|
5629
5812
|
for (let k = li + 1; k < this.lineCount; k++) {
|
|
5630
5813
|
const line = this.lineText(k);
|
|
5631
|
-
if (line ===
|
|
5632
|
-
return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true) : null;
|
|
5814
|
+
if (line === fence || !toml && line === "...") {
|
|
5815
|
+
return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true, { format: toml ? "toml" : "yaml" }) : null;
|
|
5633
5816
|
}
|
|
5634
5817
|
if (!NOT_WHITESPACE.test(line)) continue;
|
|
5635
|
-
|
|
5818
|
+
const shaped = toml ? /^\s*(?:[A-Za-z0-9_."-]+\s*=|\[|#)/.test(line) : /^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s|#)/.test(line);
|
|
5819
|
+
if (!shaped) return null;
|
|
5636
5820
|
sawContent = true;
|
|
5637
5821
|
}
|
|
5638
5822
|
return null;
|
|
5639
5823
|
}
|
|
5640
|
-
/**
|
|
5824
|
+
/**
|
|
5825
|
+
* `:::name[label]{attrs}` … closing `:::` (a colon run at least as long as
|
|
5826
|
+
* the opener's, nesting counted), or a leaf `::name[…]{…}` line. The whole
|
|
5827
|
+
* container is ONE island: its grammar (fence lengths, labels, attributes,
|
|
5828
|
+
* nested tabs) is exactly what a visual editor would re-serialize wrongly.
|
|
5829
|
+
*/
|
|
5830
|
+
directive(p, li, t) {
|
|
5831
|
+
const open = DIRECTIVE_OPEN.exec(t);
|
|
5832
|
+
if (!open) return null;
|
|
5833
|
+
const colons = open[1].length;
|
|
5834
|
+
const name = open[2];
|
|
5835
|
+
if (colons === 2) {
|
|
5836
|
+
return this.island("directive", p, this.contentEnds[li], true, { name, leaf: true });
|
|
5837
|
+
}
|
|
5838
|
+
const stack = [colons];
|
|
5839
|
+
for (let k = li + 1; k < this.lineCount; k++) {
|
|
5840
|
+
const line = this.lineText(k).trim();
|
|
5841
|
+
const inner = DIRECTIVE_OPEN.exec(line);
|
|
5842
|
+
if (inner && inner[1].length >= 3) {
|
|
5843
|
+
stack.push(inner[1].length);
|
|
5844
|
+
continue;
|
|
5845
|
+
}
|
|
5846
|
+
const close = /^(:{3,})\s*$/.exec(line);
|
|
5847
|
+
if (close && close[1].length >= stack[stack.length - 1]) {
|
|
5848
|
+
stack.pop();
|
|
5849
|
+
if (stack.length === 0) return this.island("directive", p, this.contentEnds[k], true, { name });
|
|
5850
|
+
}
|
|
5851
|
+
}
|
|
5852
|
+
return this.island("directive", p, this.n, false, { name });
|
|
5853
|
+
}
|
|
5854
|
+
/** MkDocs `!!! type "Title"` / `??? type` / `???+ type` and its indented (or blank-separated indented) body. */
|
|
5855
|
+
mkdocsAdmonition(p, li, t) {
|
|
5856
|
+
const open = MKDOCS_OPEN.exec(t);
|
|
5857
|
+
if (!open) return null;
|
|
5858
|
+
let last = li;
|
|
5859
|
+
for (let k = li + 1; k < this.lineCount; k++) {
|
|
5860
|
+
const line = this.lineText(k);
|
|
5861
|
+
if (/^(?: {4}|\t)/.test(line) && NOT_WHITESPACE.test(line)) {
|
|
5862
|
+
last = k;
|
|
5863
|
+
continue;
|
|
5864
|
+
}
|
|
5865
|
+
if (!NOT_WHITESPACE.test(line)) continue;
|
|
5866
|
+
break;
|
|
5867
|
+
}
|
|
5868
|
+
return this.island("directive", p, this.contentEnds[last], true, {
|
|
5869
|
+
name: open[2].toLowerCase(),
|
|
5870
|
+
syntax: "mkdocs"
|
|
5871
|
+
});
|
|
5872
|
+
}
|
|
5873
|
+
/** A blockquote whose first line is `> [!TYPE]…` — every following `>` line belongs to it. */
|
|
5874
|
+
callout(p, li, t) {
|
|
5875
|
+
const open = CALLOUT_OPEN.exec(t);
|
|
5876
|
+
if (!open) return null;
|
|
5877
|
+
let last = li;
|
|
5878
|
+
for (let k = li + 1; k < this.lineCount; k++) {
|
|
5879
|
+
if (!/^ {0,3}>/.test(this.lineText(k))) break;
|
|
5880
|
+
last = k;
|
|
5881
|
+
}
|
|
5882
|
+
return this.island("callout", p, this.contentEnds[last], true, {
|
|
5883
|
+
name: open[1].toLowerCase(),
|
|
5884
|
+
...open[2] ? { fold: open[2] === "-" ? "closed" : "open" } : {}
|
|
5885
|
+
});
|
|
5886
|
+
}
|
|
5887
|
+
/** `[^id]: text` plus its continuation lines (indented, or lazy text before a blank line). */
|
|
5888
|
+
footnoteDefinition(p, li, t) {
|
|
5889
|
+
const open = FOOTNOTE_DEF.exec(t);
|
|
5890
|
+
if (!open) return null;
|
|
5891
|
+
let last = li;
|
|
5892
|
+
for (let k = li + 1; k < this.lineCount; k++) {
|
|
5893
|
+
const line = this.lineText(k);
|
|
5894
|
+
if (!NOT_WHITESPACE.test(line)) {
|
|
5895
|
+
let j = k + 1;
|
|
5896
|
+
while (j < this.lineCount && !NOT_WHITESPACE.test(this.lineText(j))) j++;
|
|
5897
|
+
if (j < this.lineCount && /^(?: {4}|\t)/.test(this.lineText(j))) {
|
|
5898
|
+
k = j - 1;
|
|
5899
|
+
continue;
|
|
5900
|
+
}
|
|
5901
|
+
break;
|
|
5902
|
+
}
|
|
5903
|
+
if (/^(?: {4}|\t)/.test(line)) {
|
|
5904
|
+
last = k;
|
|
5905
|
+
continue;
|
|
5906
|
+
}
|
|
5907
|
+
if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
|
|
5908
|
+
last = k;
|
|
5909
|
+
}
|
|
5910
|
+
return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
|
|
5911
|
+
}
|
|
5912
|
+
/**
|
|
5913
|
+
* Backtick fence — the renderer splitter's close rules: JSON-string aware for
|
|
5914
|
+
* ```json, and THE nested-fence rule (source/fence-nesting.ts) for markdown
|
|
5915
|
+
* fences, with the splitter's strict-CommonMark retry when a nested fence is
|
|
5916
|
+
* still open at the end of the text.
|
|
5917
|
+
*/
|
|
5641
5918
|
backtickFence(p, li, t) {
|
|
5642
5919
|
const openTicks = backtickRunLength(t, 0);
|
|
5643
5920
|
const info = t.slice(openTicks).trim();
|
|
@@ -5646,10 +5923,23 @@ var Tokenizer = class {
|
|
|
5646
5923
|
const normalized = normalizeFenceLanguage(lang);
|
|
5647
5924
|
const meta = { fence: "`", ticks: openTicks, lang };
|
|
5648
5925
|
if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
|
|
5926
|
+
const nesting = fenceNestsInnerFences(lang);
|
|
5927
|
+
let close = this.backtickFenceClose(li, openTicks, isJson, nesting);
|
|
5928
|
+
if (close === "retry-strict") close = this.backtickFenceClose(li, openTicks, isJson, false);
|
|
5929
|
+
if (typeof close === "number") {
|
|
5930
|
+
return this.fenceIsland(p, this.contentEnds[close], true, meta, li, close, isJson);
|
|
5931
|
+
}
|
|
5932
|
+
return this.fenceIsland(p, this.n, false, meta, li, this.lineCount, isJson);
|
|
5933
|
+
}
|
|
5934
|
+
/** The closing line of a backtick fence opened on line `li`, or null when it never closes. */
|
|
5935
|
+
backtickFenceClose(li, openTicks, isJson, nesting) {
|
|
5649
5936
|
let state = { inString: false, escaped: false };
|
|
5937
|
+
let nestedDepth = 0;
|
|
5938
|
+
let sawNested = false;
|
|
5650
5939
|
for (let k = li + 1; k < this.lineCount; k++) {
|
|
5940
|
+
if (this.isOpenTail(k)) continue;
|
|
5651
5941
|
const line = this.lineText(k);
|
|
5652
|
-
const trimmedLine = line
|
|
5942
|
+
const trimmedLine = trimFenceLine(line);
|
|
5653
5943
|
if (trimmedLine.startsWith("```")) {
|
|
5654
5944
|
const at2 = line.indexOf("```");
|
|
5655
5945
|
if (isJson) {
|
|
@@ -5659,11 +5949,13 @@ var Tokenizer = class {
|
|
|
5659
5949
|
continue;
|
|
5660
5950
|
}
|
|
5661
5951
|
}
|
|
5662
|
-
const
|
|
5663
|
-
|
|
5664
|
-
if (
|
|
5665
|
-
|
|
5952
|
+
const kind = classifyInnerFenceLine(trimmedLine, openTicks, nesting, nestedDepth);
|
|
5953
|
+
if (kind === "close-outer") return k;
|
|
5954
|
+
if (kind === "open-nested") {
|
|
5955
|
+
nestedDepth++;
|
|
5956
|
+
sawNested = true;
|
|
5666
5957
|
}
|
|
5958
|
+
if (kind === "close-nested") nestedDepth--;
|
|
5667
5959
|
if (isJson) state = jsonStringState(line, 0, line.length, state);
|
|
5668
5960
|
continue;
|
|
5669
5961
|
}
|
|
@@ -5677,16 +5969,14 @@ var Tokenizer = class {
|
|
|
5677
5969
|
}
|
|
5678
5970
|
}
|
|
5679
5971
|
const closeTicks = backtickRunLength(line, at);
|
|
5680
|
-
const after = line.slice(at + closeTicks)
|
|
5681
|
-
if (closeTicks >= openTicks && after === "")
|
|
5682
|
-
return this.fenceIsland(p, this.contentEnds[k], true, meta, li, k, isJson);
|
|
5683
|
-
}
|
|
5972
|
+
const after = trimFenceLine(line.slice(at + closeTicks));
|
|
5973
|
+
if (closeTicks >= openTicks && after === "" && nestedDepth === 0) return k;
|
|
5684
5974
|
if (isJson) state = jsonStringState(line, 0, line.length, state);
|
|
5685
5975
|
continue;
|
|
5686
5976
|
}
|
|
5687
5977
|
if (isJson) state = jsonStringState(line, 0, line.length, state);
|
|
5688
5978
|
}
|
|
5689
|
-
return
|
|
5979
|
+
return sawNested && nestedDepth > 0 ? "retry-strict" : null;
|
|
5690
5980
|
}
|
|
5691
5981
|
fenceIsland(p, end, complete, meta, openLine, closeLine, isJson) {
|
|
5692
5982
|
if (isJson && openLine + 1 < this.lineCount) {
|
|
@@ -5706,24 +5996,29 @@ var Tokenizer = class {
|
|
|
5706
5996
|
if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
|
|
5707
5997
|
const close = new RegExp(`^ {0,3}~{${ticks},}[ \\t]*$`);
|
|
5708
5998
|
for (let k = li + 1; k < this.lineCount; k++) {
|
|
5999
|
+
if (this.isOpenTail(k)) continue;
|
|
5709
6000
|
if (close.test(this.lineText(k))) {
|
|
5710
6001
|
return this.island("fence", p, this.contentEnds[k], true, meta);
|
|
5711
6002
|
}
|
|
5712
6003
|
}
|
|
5713
6004
|
return this.island("fence", p, this.n, false, meta);
|
|
5714
6005
|
}
|
|
6006
|
+
/**
|
|
6007
|
+
* A line-leading `$$` that OPENS a math pair (source/math-pairs.ts — the
|
|
6008
|
+
* core's rule). A closer, an unpaired `$$`, or a prose pair is never a block:
|
|
6009
|
+
* a lone `$$` renders as literal text and locks nothing.
|
|
6010
|
+
*/
|
|
5715
6011
|
mathBlock(p, tp) {
|
|
6012
|
+
const close = this.mathPairs.get(tp);
|
|
6013
|
+
if (close === void 0) return null;
|
|
6014
|
+
const end = close + 2;
|
|
5716
6015
|
const li = this.lineOf(tp);
|
|
5717
6016
|
const ce = this.contentEnds[li];
|
|
5718
|
-
|
|
5719
|
-
|
|
5720
|
-
|
|
5721
|
-
|
|
5722
|
-
|
|
5723
|
-
}
|
|
5724
|
-
const close = this.text.indexOf("$$", tp + 2);
|
|
5725
|
-
if (close === -1) return this.island("math_block", p, this.n, false, { delimiter: "$$" });
|
|
5726
|
-
return this.island("math_block", p, close + 2, true, { delimiter: "$$" });
|
|
6017
|
+
if (end <= ce && !this.isBlank(end, ce)) return null;
|
|
6018
|
+
const endLine = this.lineOf(close);
|
|
6019
|
+
const endCe = this.contentEnds[endLine];
|
|
6020
|
+
if (endLine !== li && !this.isBlank(end, endCe)) return null;
|
|
6021
|
+
return this.island("math_block", p, end, true, { delimiter: "$$" });
|
|
5727
6022
|
}
|
|
5728
6023
|
/** `<artifact …>`, `<decision …>`, … at the start of a line. */
|
|
5729
6024
|
attributeXml(p, tp, t) {
|
|
@@ -5740,7 +6035,7 @@ var Tokenizer = class {
|
|
|
5740
6035
|
attributeElement(name, p, tp, type) {
|
|
5741
6036
|
const li = this.lineOf(tp);
|
|
5742
6037
|
const ce = this.contentEnds[li];
|
|
5743
|
-
const tag = readXmlTag(this.text, tp);
|
|
6038
|
+
const tag = readXmlTag(this.text, tp, this.finder);
|
|
5744
6039
|
let openEnd;
|
|
5745
6040
|
if (tag && !tag.isClosing) {
|
|
5746
6041
|
if (tag.isSelfClosing) {
|
|
@@ -5748,13 +6043,13 @@ var Tokenizer = class {
|
|
|
5748
6043
|
}
|
|
5749
6044
|
openEnd = tp + tag.raw.length;
|
|
5750
6045
|
} else {
|
|
5751
|
-
const gt = this.
|
|
6046
|
+
const gt = this.next(">", tp);
|
|
5752
6047
|
if (gt === -1) return this.island(type, p, this.n, false, { tag: name });
|
|
5753
6048
|
if (gt >= ce) return null;
|
|
5754
6049
|
openEnd = gt + 1;
|
|
5755
6050
|
}
|
|
5756
6051
|
const closer = `</${name}>`;
|
|
5757
|
-
const close = this.
|
|
6052
|
+
const close = this.next(closer, openEnd);
|
|
5758
6053
|
if (close === -1) return this.island(type, p, this.n, false, { tag: name });
|
|
5759
6054
|
return this.island(type, p, close + closer.length, true, { tag: name });
|
|
5760
6055
|
}
|
|
@@ -5764,7 +6059,7 @@ var Tokenizer = class {
|
|
|
5764
6059
|
const opener = `<${name}>`;
|
|
5765
6060
|
if (!t.startsWith(opener)) continue;
|
|
5766
6061
|
const closer = `</${name}>`;
|
|
5767
|
-
const close = this.
|
|
6062
|
+
const close = this.next(closer, tp + opener.length);
|
|
5768
6063
|
if (close === -1) return this.island("xml_region", p, this.n, false, { tag: name });
|
|
5769
6064
|
return this.island("xml_region", p, close + closer.length, true, { tag: name });
|
|
5770
6065
|
}
|
|
@@ -5800,7 +6095,7 @@ var Tokenizer = class {
|
|
|
5800
6095
|
return null;
|
|
5801
6096
|
}
|
|
5802
6097
|
blockComment(p, tp, li) {
|
|
5803
|
-
const close = this.
|
|
6098
|
+
const close = this.next("-->", tp + 4);
|
|
5804
6099
|
if (close === -1) return this.island("html_comment", p, this.n, false);
|
|
5805
6100
|
const end = close + 3;
|
|
5806
6101
|
const raw = this.text.slice(tp, end);
|
|
@@ -5878,9 +6173,8 @@ var Tokenizer = class {
|
|
|
5878
6173
|
}
|
|
5879
6174
|
}
|
|
5880
6175
|
if (lastLine === null) {
|
|
5881
|
-
|
|
5882
|
-
|
|
5883
|
-
return this.island("json", p, this.n, false, jsonMeta(partial));
|
|
6176
|
+
if (isRecognizedJsonHead(this.text.slice(tp, tp + 512))) {
|
|
6177
|
+
return this.island("json", p, this.n, false, jsonMeta(this.text.slice(tp)));
|
|
5884
6178
|
}
|
|
5885
6179
|
return null;
|
|
5886
6180
|
}
|
|
@@ -5915,11 +6209,43 @@ var Tokenizer = class {
|
|
|
5915
6209
|
// ── prose ──────────────────────────────────────────────────────────────
|
|
5916
6210
|
paragraph(pos, li) {
|
|
5917
6211
|
let end = this.contentEnds[li];
|
|
6212
|
+
let breakLine = -1;
|
|
5918
6213
|
for (let k = li + 1; k < this.lineCount; k++) {
|
|
5919
6214
|
if (this.lineIsBlank(k)) break;
|
|
5920
|
-
if (this.detectBlock(this.lineStarts[k], k, false))
|
|
6215
|
+
if (this.detectBlock(this.lineStarts[k], k, false)) {
|
|
6216
|
+
breakLine = k;
|
|
6217
|
+
break;
|
|
6218
|
+
}
|
|
5921
6219
|
end = this.contentEnds[k];
|
|
5922
6220
|
}
|
|
6221
|
+
if (breakLine !== -1) {
|
|
6222
|
+
const breakIsland = this.detectBlock(this.lineStarts[breakLine], breakLine, false);
|
|
6223
|
+
if (breakIsland?.type === "tree") {
|
|
6224
|
+
let root = breakLine;
|
|
6225
|
+
for (let j = breakLine - 1; j >= li; j--) {
|
|
6226
|
+
const trimmed = this.lineText(j).trim();
|
|
6227
|
+
if (!trimmed || isSplitterTreeLine(trimmed) || /^#{1,6}\s/.test(trimmed)) break;
|
|
6228
|
+
root = j;
|
|
6229
|
+
}
|
|
6230
|
+
if (root < breakLine) {
|
|
6231
|
+
const treeStart = root === li ? pos : this.lineStarts[root];
|
|
6232
|
+
const tree = { ...breakIsland, start: treeStart };
|
|
6233
|
+
if (treeStart > pos) {
|
|
6234
|
+
const proseEnd = this.contentEnds[root - 1];
|
|
6235
|
+
const scan2 = this.scanInline(pos, proseEnd);
|
|
6236
|
+
if (!scan2.breakIsland) {
|
|
6237
|
+
this.push({ kind: "prose", start: pos, end: proseEnd, island: null, inlines: scan2.inlines });
|
|
6238
|
+
this.push({ kind: "gap", start: proseEnd, end: treeStart, island: null, inlines: [] });
|
|
6239
|
+
this.push({ kind: "island", start: treeStart, end: tree.end, island: tree, inlines: [] });
|
|
6240
|
+
return tree.end;
|
|
6241
|
+
}
|
|
6242
|
+
} else {
|
|
6243
|
+
this.push({ kind: "island", start: treeStart, end: tree.end, island: tree, inlines: [] });
|
|
6244
|
+
return tree.end;
|
|
6245
|
+
}
|
|
6246
|
+
}
|
|
6247
|
+
}
|
|
6248
|
+
}
|
|
5923
6249
|
const scan = this.scanInline(pos, end);
|
|
5924
6250
|
if (!scan.breakIsland) {
|
|
5925
6251
|
this.push({ kind: "prose", start: pos, end, island: null, inlines: scan.inlines });
|
|
@@ -5952,17 +6278,49 @@ var Tokenizer = class {
|
|
|
5952
6278
|
const text = this.text;
|
|
5953
6279
|
const inlines = [];
|
|
5954
6280
|
const hasKind = text.slice(ps, pe).includes("__kind");
|
|
6281
|
+
let kindBudget = 4 * (pe - ps) + 65536;
|
|
5955
6282
|
const stop = (island, orphan = false) => ({ inlines, breakIsland: island, orphan });
|
|
6283
|
+
const within = (needle, from) => {
|
|
6284
|
+
const at = this.next(needle, from);
|
|
6285
|
+
return at !== -1 && at + needle.length <= pe ? at : -1;
|
|
6286
|
+
};
|
|
6287
|
+
let runs = null;
|
|
6288
|
+
const codeSpanClose = (n, from) => {
|
|
6289
|
+
if (!runs) {
|
|
6290
|
+
runs = /* @__PURE__ */ new Map();
|
|
6291
|
+
for (let k = ps; k < pe; ) {
|
|
6292
|
+
if (text.charCodeAt(k) !== 96) {
|
|
6293
|
+
k++;
|
|
6294
|
+
continue;
|
|
6295
|
+
}
|
|
6296
|
+
const len = backtickRunLength(text, k);
|
|
6297
|
+
const list2 = runs.get(len);
|
|
6298
|
+
if (list2) list2.push(k);
|
|
6299
|
+
else runs.set(len, [k]);
|
|
6300
|
+
k += len;
|
|
6301
|
+
}
|
|
6302
|
+
}
|
|
6303
|
+
const list = runs.get(n);
|
|
6304
|
+
if (!list) return -1;
|
|
6305
|
+
let lo = 0;
|
|
6306
|
+
let hi = list.length;
|
|
6307
|
+
while (lo < hi) {
|
|
6308
|
+
const mid = lo + hi >>> 1;
|
|
6309
|
+
if (list[mid] < from) lo = mid + 1;
|
|
6310
|
+
else hi = mid;
|
|
6311
|
+
}
|
|
6312
|
+
const at = list[lo];
|
|
6313
|
+
return at !== void 0 && at + n <= pe ? at : -1;
|
|
6314
|
+
};
|
|
5956
6315
|
let i = ps;
|
|
5957
6316
|
while (i < pe) {
|
|
5958
6317
|
const ch = text[i];
|
|
5959
6318
|
if (ch === "\\") {
|
|
5960
|
-
const
|
|
5961
|
-
if (
|
|
5962
|
-
const
|
|
5963
|
-
|
|
5964
|
-
|
|
5965
|
-
inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: next === "(" ? "\\(" : "\\[" }));
|
|
6319
|
+
const nextCh = text[i + 1];
|
|
6320
|
+
if (nextCh === "(" || nextCh === "[") {
|
|
6321
|
+
const close = within(nextCh === "(" ? "\\)" : "\\]", i + 2);
|
|
6322
|
+
if (close !== -1) {
|
|
6323
|
+
inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: nextCh === "(" ? "\\(" : "\\[" }));
|
|
5966
6324
|
i = close + 2;
|
|
5967
6325
|
continue;
|
|
5968
6326
|
}
|
|
@@ -5972,33 +6330,20 @@ var Tokenizer = class {
|
|
|
5972
6330
|
}
|
|
5973
6331
|
if (ch === "`") {
|
|
5974
6332
|
const n = backtickRunLength(text, i);
|
|
5975
|
-
const
|
|
5976
|
-
let k = i + n;
|
|
5977
|
-
let close = -1;
|
|
5978
|
-
while (k < pe) {
|
|
5979
|
-
const idx = text.indexOf(run, k);
|
|
5980
|
-
if (idx === -1 || idx >= pe) break;
|
|
5981
|
-
if (text[idx + n] === "`") {
|
|
5982
|
-
let m = idx;
|
|
5983
|
-
while (text[m] === "`") m++;
|
|
5984
|
-
k = m;
|
|
5985
|
-
continue;
|
|
5986
|
-
}
|
|
5987
|
-
close = idx;
|
|
5988
|
-
break;
|
|
5989
|
-
}
|
|
6333
|
+
const close = codeSpanClose(n, i + n);
|
|
5990
6334
|
i = close === -1 ? i + n : close + n;
|
|
5991
6335
|
continue;
|
|
5992
6336
|
}
|
|
5993
6337
|
if (ch === "$") {
|
|
5994
6338
|
if (text[i + 1] === "$") {
|
|
5995
|
-
const close =
|
|
5996
|
-
if (close
|
|
5997
|
-
inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: "$$" }));
|
|
5998
|
-
i = close + 2;
|
|
5999
|
-
} else {
|
|
6339
|
+
const close = this.mathPairs.get(i);
|
|
6340
|
+
if (close === void 0) {
|
|
6000
6341
|
i += 2;
|
|
6342
|
+
continue;
|
|
6001
6343
|
}
|
|
6344
|
+
if (close + 2 > pe) return stop(this.island("math_block", i, close + 2, true, { delimiter: "$$" }));
|
|
6345
|
+
inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: "$$" }));
|
|
6346
|
+
i = close + 2;
|
|
6002
6347
|
continue;
|
|
6003
6348
|
}
|
|
6004
6349
|
const end = singleDollarMathEnd(text, i, pe);
|
|
@@ -6012,7 +6357,7 @@ var Tokenizer = class {
|
|
|
6012
6357
|
}
|
|
6013
6358
|
if (ch === "{") {
|
|
6014
6359
|
if (text[i + 1] === "{") {
|
|
6015
|
-
const close =
|
|
6360
|
+
const close = this.next("}}", i + 2);
|
|
6016
6361
|
if (close !== -1 && close + 2 <= pe && close - i <= 200 && !text.slice(i, close).includes("\n")) {
|
|
6017
6362
|
const raw = text.slice(i, close + 2);
|
|
6018
6363
|
const strict = STRICT_VARIABLE_RE.exec(raw);
|
|
@@ -6025,8 +6370,10 @@ var Tokenizer = class {
|
|
|
6025
6370
|
i += 2;
|
|
6026
6371
|
continue;
|
|
6027
6372
|
}
|
|
6028
|
-
if (hasKind) {
|
|
6029
|
-
const
|
|
6373
|
+
if (hasKind && kindBudget > 0 && /^\{\s*"/.test(text.slice(i, i + 64))) {
|
|
6374
|
+
const limit = Math.min(pe, i + 65536);
|
|
6375
|
+
const end = matchingJsonObjectEnd(text, i, limit);
|
|
6376
|
+
kindBudget -= (end ?? limit) - i;
|
|
6030
6377
|
if (end !== null) {
|
|
6031
6378
|
const kind = declaredKind(text.slice(i, end));
|
|
6032
6379
|
if (kind) {
|
|
@@ -6039,11 +6386,27 @@ var Tokenizer = class {
|
|
|
6039
6386
|
i++;
|
|
6040
6387
|
continue;
|
|
6041
6388
|
}
|
|
6389
|
+
if (ch === "[" && text[i + 1] === "[" || ch === "!" && text[i + 1] === "[" && text[i + 2] === "[") {
|
|
6390
|
+
const open = ch === "!" ? i + 3 : i + 2;
|
|
6391
|
+
const close = within("]]", open);
|
|
6392
|
+
const newline = this.next("\n", open);
|
|
6393
|
+
const inner = close === -1 ? "" : text.slice(open, close);
|
|
6394
|
+
if (close !== -1 && (newline === -1 || newline > close) && inner.trim() && !/[[\]]/.test(inner)) {
|
|
6395
|
+
const cut = inner.search(/\\?\|/);
|
|
6396
|
+
const target = (cut < 0 ? inner : inner.slice(0, cut)).trim();
|
|
6397
|
+
inlines.push(
|
|
6398
|
+
this.island("wikilink", i, close + 2, true, ch === "!" ? { target, embed: true } : { target })
|
|
6399
|
+
);
|
|
6400
|
+
i = close + 2;
|
|
6401
|
+
continue;
|
|
6402
|
+
}
|
|
6403
|
+
}
|
|
6042
6404
|
if (ch === "[") {
|
|
6043
6405
|
const media = MEDIA_REF_PREFIXES.find((prefix) => text.startsWith(prefix, i));
|
|
6044
6406
|
if (media) {
|
|
6045
|
-
const close =
|
|
6046
|
-
|
|
6407
|
+
const close = within("]", i + media.length);
|
|
6408
|
+
const newline = this.next("\n", i);
|
|
6409
|
+
if (close !== -1 && (newline === -1 || newline > close)) {
|
|
6047
6410
|
inlines.push(this.island("media_ref", i, close + 1, true, { media: media.slice(1, 6).toLowerCase() }));
|
|
6048
6411
|
i = close + 1;
|
|
6049
6412
|
continue;
|
|
@@ -6054,10 +6417,10 @@ var Tokenizer = class {
|
|
|
6054
6417
|
}
|
|
6055
6418
|
if (ch === "<") {
|
|
6056
6419
|
if (text.startsWith("<!--", i)) {
|
|
6057
|
-
const close =
|
|
6420
|
+
const close = this.next("-->", i + 4);
|
|
6058
6421
|
if (close === -1) return stop(this.island("html_comment", i, this.n, false));
|
|
6059
6422
|
const end = close + 3;
|
|
6060
|
-
const anchor = PINNED_ANCHOR_RE.exec(text.slice(i, end));
|
|
6423
|
+
const anchor = end - i <= 64 ? PINNED_ANCHOR_RE.exec(text.slice(i, end)) : null;
|
|
6061
6424
|
const type = anchor ? "anchor" : "html_comment";
|
|
6062
6425
|
const meta = anchor ? { id: anchor[1] } : {};
|
|
6063
6426
|
if (end > pe) return stop(this.island(type, i, end, true, meta));
|
|
@@ -6071,7 +6434,7 @@ var Tokenizer = class {
|
|
|
6071
6434
|
i += cite[0].length;
|
|
6072
6435
|
continue;
|
|
6073
6436
|
}
|
|
6074
|
-
const tag = readXmlTag(text, i);
|
|
6437
|
+
const tag = readXmlTag(text, i, this.finder);
|
|
6075
6438
|
if (tag && i + tag.raw.length <= pe) {
|
|
6076
6439
|
const name = tag.tagName;
|
|
6077
6440
|
const tagEnd = i + tag.raw.length;
|
|
@@ -6084,7 +6447,7 @@ var Tokenizer = class {
|
|
|
6084
6447
|
continue;
|
|
6085
6448
|
}
|
|
6086
6449
|
const closer = `</${name}>`;
|
|
6087
|
-
const close =
|
|
6450
|
+
const close = this.next(closer, tagEnd);
|
|
6088
6451
|
const blockType = isAttr ? "xml_attr" : "xml_region";
|
|
6089
6452
|
if (close === -1) return stop(this.island(blockType, i, this.n, false, { tag: name }));
|
|
6090
6453
|
const end = close + closer.length;
|
|
@@ -6108,7 +6471,7 @@ var Tokenizer = class {
|
|
|
6108
6471
|
const pending = ATTRIBUTE_XML_NAMES.find(
|
|
6109
6472
|
(name) => text.startsWith(`<${name}`, i) && /\s/.test(text[i + name.length + 1] ?? "")
|
|
6110
6473
|
);
|
|
6111
|
-
if (pending &&
|
|
6474
|
+
if (pending && this.next(">", i) === -1) {
|
|
6112
6475
|
return stop(this.island("xml_attr", i, this.n, false, { tag: pending }));
|
|
6113
6476
|
}
|
|
6114
6477
|
i++;
|
|
@@ -6119,8 +6482,8 @@ var Tokenizer = class {
|
|
|
6119
6482
|
return { inlines, breakIsland: null, orphan: false };
|
|
6120
6483
|
}
|
|
6121
6484
|
};
|
|
6122
|
-
function tokenizeSource(text) {
|
|
6123
|
-
const raw = new Tokenizer(text).run();
|
|
6485
|
+
function tokenizeSource(text, options = {}) {
|
|
6486
|
+
const raw = new Tokenizer(text, options.streaming ?? false).run();
|
|
6124
6487
|
const cp = buildCodePointIndex(text);
|
|
6125
6488
|
const inline = (island) => ({
|
|
6126
6489
|
islandType: island.type,
|
|
@@ -6169,7 +6532,7 @@ function listIslands(blocks) {
|
|
|
6169
6532
|
raw: block.raw
|
|
6170
6533
|
});
|
|
6171
6534
|
} else {
|
|
6172
|
-
|
|
6535
|
+
for (const inline of block.inlines) out.push(inline);
|
|
6173
6536
|
}
|
|
6174
6537
|
}
|
|
6175
6538
|
return out;
|
|
@@ -6199,9 +6562,44 @@ var SourceSpliceError = class extends Error {
|
|
|
6199
6562
|
function blockEdit(block, text) {
|
|
6200
6563
|
return { start: block.start, end: block.end, text };
|
|
6201
6564
|
}
|
|
6565
|
+
function islandEdit(island, text) {
|
|
6566
|
+
return { start: island.start, end: island.end, text, island: true };
|
|
6567
|
+
}
|
|
6202
6568
|
function blockRangeEdit(first, last, text) {
|
|
6203
6569
|
return { start: first.start, end: last.end, text };
|
|
6204
6570
|
}
|
|
6571
|
+
var CONTEXT_WINDOW = 64;
|
|
6572
|
+
function locateIsland(text, raw, from, expected, expectedFromEnd, oldText, oldStart, oldEnd) {
|
|
6573
|
+
const candidates = /* @__PURE__ */ new Set();
|
|
6574
|
+
if (expected >= from && text.startsWith(raw, expected)) candidates.add(expected);
|
|
6575
|
+
if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) candidates.add(expectedFromEnd);
|
|
6576
|
+
const after = text.indexOf(raw, Math.max(from, expected));
|
|
6577
|
+
if (after !== -1) candidates.add(after);
|
|
6578
|
+
const before = expected > from ? text.lastIndexOf(raw, expected) : -1;
|
|
6579
|
+
if (before >= from) candidates.add(before);
|
|
6580
|
+
if (candidates.size === 0) return text.indexOf(raw, from);
|
|
6581
|
+
if (candidates.size === 1) return candidates.values().next().value;
|
|
6582
|
+
let best = -1;
|
|
6583
|
+
let bestScore = -1;
|
|
6584
|
+
for (const at of candidates) {
|
|
6585
|
+
let score = 0;
|
|
6586
|
+
for (let k = 0; k < CONTEXT_WINDOW; k++) {
|
|
6587
|
+
const n = text[at + raw.length + k];
|
|
6588
|
+
if (n === void 0 || n !== oldText[oldEnd + k]) break;
|
|
6589
|
+
score++;
|
|
6590
|
+
}
|
|
6591
|
+
for (let k = 1; k <= CONTEXT_WINDOW; k++) {
|
|
6592
|
+
const n = text[at - k];
|
|
6593
|
+
if (n === void 0 || n !== oldText[oldStart - k]) break;
|
|
6594
|
+
score++;
|
|
6595
|
+
}
|
|
6596
|
+
if (score > bestScore || score === bestScore && at === expected) {
|
|
6597
|
+
best = at;
|
|
6598
|
+
bestScore = score;
|
|
6599
|
+
}
|
|
6600
|
+
}
|
|
6601
|
+
return best;
|
|
6602
|
+
}
|
|
6205
6603
|
function commonPrefix(a, b) {
|
|
6206
6604
|
const max = Math.min(a.length, b.length);
|
|
6207
6605
|
let i = 0;
|
|
@@ -6226,13 +6624,23 @@ function spliceSave(original, edits, options = {}) {
|
|
|
6226
6624
|
boundaries.add(block.start);
|
|
6227
6625
|
boundaries.add(block.end);
|
|
6228
6626
|
}
|
|
6627
|
+
const islands = listIslands(blocks);
|
|
6628
|
+
const islandAt = /* @__PURE__ */ new Map();
|
|
6629
|
+
for (const island of islands) islandAt.set(`${island.start}:${island.end}`, island);
|
|
6229
6630
|
const sorted = [...edits].sort((a, b) => a.start - b.start || a.end - b.end);
|
|
6230
6631
|
let previousEnd = -1;
|
|
6231
6632
|
for (const edit of sorted) {
|
|
6232
6633
|
if (edit.start < 0 || edit.end > original.length || edit.start > edit.end) {
|
|
6233
6634
|
throw new SourceSpliceError("out_of_range", `edit [${edit.start}, ${edit.end}) is outside the text`);
|
|
6234
6635
|
}
|
|
6235
|
-
if (
|
|
6636
|
+
if (edit.island) {
|
|
6637
|
+
if (!islandAt.has(`${edit.start}:${edit.end}`)) {
|
|
6638
|
+
throw new SourceSpliceError(
|
|
6639
|
+
"misaligned",
|
|
6640
|
+
`islandEdit [${edit.start}, ${edit.end}) does not name an island's exact span`
|
|
6641
|
+
);
|
|
6642
|
+
}
|
|
6643
|
+
} else if (!boundaries.has(edit.start) || !boundaries.has(edit.end)) {
|
|
6236
6644
|
throw new SourceSpliceError(
|
|
6237
6645
|
"misaligned",
|
|
6238
6646
|
`edit [${edit.start}, ${edit.end}) does not start and end on block boundaries`
|
|
@@ -6244,18 +6652,51 @@ function spliceSave(original, edits, options = {}) {
|
|
|
6244
6652
|
previousEnd = edit.end;
|
|
6245
6653
|
}
|
|
6246
6654
|
const effective = [];
|
|
6247
|
-
|
|
6248
|
-
|
|
6249
|
-
|
|
6250
|
-
|
|
6251
|
-
const
|
|
6655
|
+
const replacedIslands = /* @__PURE__ */ new Set();
|
|
6656
|
+
const pushSegment = (oldStart, oldEnd, replacement) => {
|
|
6657
|
+
const old = original.slice(oldStart, oldEnd);
|
|
6658
|
+
if (old === replacement) return;
|
|
6659
|
+
const prefix = commonPrefix(old, replacement);
|
|
6660
|
+
const suffix = commonSuffix(old, replacement, prefix);
|
|
6252
6661
|
effective.push({
|
|
6253
|
-
|
|
6254
|
-
|
|
6255
|
-
|
|
6256
|
-
oldEnd: edit.end - suffix,
|
|
6257
|
-
replacement: edit.text.slice(prefix, edit.text.length - suffix)
|
|
6662
|
+
oldStart: oldStart + prefix,
|
|
6663
|
+
oldEnd: oldEnd - suffix,
|
|
6664
|
+
replacement: replacement.slice(prefix, replacement.length - suffix)
|
|
6258
6665
|
});
|
|
6666
|
+
};
|
|
6667
|
+
for (const edit of sorted) {
|
|
6668
|
+
if (edit.island) {
|
|
6669
|
+
replacedIslands.add(`${edit.start}:${edit.end}`);
|
|
6670
|
+
pushSegment(edit.start, edit.end, edit.text);
|
|
6671
|
+
continue;
|
|
6672
|
+
}
|
|
6673
|
+
if (edit.text === original.slice(edit.start, edit.end)) continue;
|
|
6674
|
+
const inside = islands.filter((island) => island.start >= edit.start && island.end <= edit.end);
|
|
6675
|
+
let oldCursor2 = edit.start;
|
|
6676
|
+
let newCursor2 = 0;
|
|
6677
|
+
for (const island of inside) {
|
|
6678
|
+
const at = locateIsland(
|
|
6679
|
+
edit.text,
|
|
6680
|
+
island.raw,
|
|
6681
|
+
newCursor2,
|
|
6682
|
+
newCursor2 + (island.start - oldCursor2),
|
|
6683
|
+
edit.text.length - (edit.end - island.start),
|
|
6684
|
+
original.slice(edit.start, edit.end),
|
|
6685
|
+
island.start - edit.start,
|
|
6686
|
+
island.end - edit.start
|
|
6687
|
+
);
|
|
6688
|
+
if (at === -1) {
|
|
6689
|
+
const name = island.meta.tag ?? island.meta.name ?? island.meta.lang ?? "";
|
|
6690
|
+
throw new SourceSpliceError(
|
|
6691
|
+
"island_edit",
|
|
6692
|
+
`edit [${edit.start}, ${edit.end}) changes or removes the protected ${island.islandType}${name ? ` (${String(name)})` : ""} island at [${island.start}, ${island.end}); islands change only through islandEdit()`
|
|
6693
|
+
);
|
|
6694
|
+
}
|
|
6695
|
+
pushSegment(oldCursor2, island.start, edit.text.slice(newCursor2, at));
|
|
6696
|
+
oldCursor2 = island.end;
|
|
6697
|
+
newCursor2 = at + island.raw.length;
|
|
6698
|
+
}
|
|
6699
|
+
pushSegment(oldCursor2, edit.end, edit.text.slice(newCursor2));
|
|
6259
6700
|
}
|
|
6260
6701
|
if (effective.length === 0) {
|
|
6261
6702
|
return {
|
|
@@ -6275,9 +6716,10 @@ function spliceSave(original, edits, options = {}) {
|
|
|
6275
6716
|
const newStart = edit.oldStart + delta;
|
|
6276
6717
|
text += edit.replacement;
|
|
6277
6718
|
const newEnd = newStart + edit.replacement.length;
|
|
6719
|
+
const owner = sorted.find((e) => e.start <= edit.oldStart && e.end >= edit.oldEnd);
|
|
6278
6720
|
pending.push({
|
|
6279
|
-
blockStart: edit.
|
|
6280
|
-
blockEnd: edit.
|
|
6721
|
+
blockStart: owner?.start ?? edit.oldStart,
|
|
6722
|
+
blockEnd: owner?.end ?? edit.oldEnd,
|
|
6281
6723
|
oldStart: edit.oldStart,
|
|
6282
6724
|
oldEnd: edit.oldEnd,
|
|
6283
6725
|
newStart,
|
|
@@ -6315,11 +6757,8 @@ function spliceSave(original, edits, options = {}) {
|
|
|
6315
6757
|
else newIslandsByStart.set(island.start, [island.raw]);
|
|
6316
6758
|
}
|
|
6317
6759
|
const disturbed = [];
|
|
6318
|
-
for (const island of
|
|
6319
|
-
|
|
6320
|
-
(edit) => island.start < edit.blockEnd && island.end > edit.blockStart
|
|
6321
|
-
);
|
|
6322
|
-
if (touched) continue;
|
|
6760
|
+
for (const island of islands) {
|
|
6761
|
+
if (replacedIslands.has(`${island.start}:${island.end}`)) continue;
|
|
6323
6762
|
const mappedStart = mapPosition(changes, island.start, { assoc: 1 }).pos;
|
|
6324
6763
|
const candidates = newIslandsByStart.get(mappedStart);
|
|
6325
6764
|
if (!candidates || !candidates.includes(island.raw)) {
|
|
@@ -6337,10 +6776,11 @@ function spliceSave(original, edits, options = {}) {
|
|
|
6337
6776
|
bytesOutsideEditsIdentical,
|
|
6338
6777
|
disturbed
|
|
6339
6778
|
};
|
|
6340
|
-
if (options.requireIntegrity && !integrity.ok) {
|
|
6779
|
+
if ((options.requireIntegrity ?? true) && !integrity.ok) {
|
|
6780
|
+
const named = disturbed.slice(0, 5).map((island) => `${island.islandType}@${island.start}`).join(", ");
|
|
6341
6781
|
throw new SourceSpliceError(
|
|
6342
6782
|
"integrity",
|
|
6343
|
-
`the
|
|
6783
|
+
`integrity: the save disturbed ${disturbed.length} protected island(s) it did not replace${named ? ` (${named}${disturbed.length > 5 ? ", \u2026" : ""})` : ""}${bytesOutsideEditsIdentical ? "" : "; bytes outside the edits changed"}`
|
|
6344
6784
|
);
|
|
6345
6785
|
}
|
|
6346
6786
|
return { text, changed: true, changes, integrity, blocks: newBlocks };
|
|
@@ -6382,6 +6822,6 @@ function mapRange(changes, start, end, unit = "utf16") {
|
|
|
6382
6822
|
return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
|
|
6383
6823
|
}
|
|
6384
6824
|
|
|
6385
|
-
export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, parseDirectiveSlug, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
|
|
6825
|
+
export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
|
|
6386
6826
|
//# sourceMappingURL=index.js.map
|
|
6387
6827
|
//# sourceMappingURL=index.js.map
|