@ai-matrx/content-ir 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
7
7
  export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.cjs';
8
8
  export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.cjs';
9
9
  export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.cjs';
10
- export { BlockIslandType, CodePointIndex, DisturbedIsland, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource } from './source.cjs';
10
+ export { BlockIslandType, CodePointIndex, DisturbedIsland, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.cjs';
11
11
  import 'react';
package/dist/index.d.ts CHANGED
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
7
7
  export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.js';
8
8
  export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.js';
9
9
  export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.js';
10
- export { BlockIslandType, CodePointIndex, DisturbedIsland, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource } from './source.js';
10
+ export { BlockIslandType, CodePointIndex, DisturbedIsland, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.js';
11
11
  import 'react';
package/dist/index.js CHANGED
@@ -5178,11 +5178,22 @@ var NESTING_FENCE_LANGUAGES = /* @__PURE__ */ new Set([
5178
5178
  function fenceNestsInnerFences(language) {
5179
5179
  return !!language && NESTING_FENCE_LANGUAGES.has(language.toLowerCase());
5180
5180
  }
5181
+ var FENCE_WHITESPACE = " \n\r\f\v";
5182
+ function isFenceWhitespace(ch) {
5183
+ return ch !== void 0 && ch !== "" && FENCE_WHITESPACE.includes(ch);
5184
+ }
5185
+ function trimFenceLine(line) {
5186
+ let start = 0;
5187
+ let end = line.length;
5188
+ while (start < end && isFenceWhitespace(line[start])) start++;
5189
+ while (end > start && isFenceWhitespace(line[end - 1])) end--;
5190
+ return start === 0 && end === line.length ? line : line.slice(start, end);
5191
+ }
5181
5192
  function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
5182
5193
  let ticks = 0;
5183
5194
  while (ticks < trimmed.length && trimmed[ticks] === "`") ticks++;
5184
5195
  if (ticks < 3) return "content";
5185
- const info = trimmed.slice(ticks).trim();
5196
+ const info = trimFenceLine(trimmed.slice(ticks));
5186
5197
  if (info === "") {
5187
5198
  if (ticks < openTicks) return "content";
5188
5199
  if (nests && nestedDepth > 0) return "close-nested";
@@ -5195,13 +5206,42 @@ function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
5195
5206
  // source/math.ts
5196
5207
  var TEX_SIGNAL = /\\[A-Za-z]+|[\\^_{}=<>+]/;
5197
5208
  var LONE_VARIABLE = /^[A-Za-z](?:'|[0-9])?$/;
5209
+ var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
5210
+ var FUNCTION_NAMES = /* @__PURE__ */ new Set([
5211
+ "sin",
5212
+ "cos",
5213
+ "tan",
5214
+ "sec",
5215
+ "csc",
5216
+ "cot",
5217
+ "log",
5218
+ "ln",
5219
+ "exp",
5220
+ "lim",
5221
+ "max",
5222
+ "min",
5223
+ "det",
5224
+ "gcd",
5225
+ "mod",
5226
+ "arg",
5227
+ "sinh",
5228
+ "cosh",
5229
+ "tanh"
5230
+ ]);
5231
+ function isMathShape(content) {
5232
+ if (!MATH_SHAPE_CHARS.test(content)) return false;
5233
+ const words = content.match(/[A-Za-z]+/g);
5234
+ if (!words) return false;
5235
+ if (!words.every((w) => w.length === 1 || FUNCTION_NAMES.has(w))) return false;
5236
+ return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
5237
+ }
5198
5238
  function isSingleDollarMath(content, after) {
5199
5239
  if (!content || content.length > 400) return false;
5200
5240
  if (/^\s/.test(content) || /\s$/.test(content)) return false;
5201
5241
  if (after !== void 0 && /[0-9]/.test(after)) return false;
5202
5242
  if (LONE_VARIABLE.test(content)) return true;
5203
5243
  if (/^[0-9]/.test(content)) return /\\[A-Za-z]+|[\\^_{}]/.test(content);
5204
- return TEX_SIGNAL.test(content);
5244
+ return TEX_SIGNAL.test(content) || isMathShape(content);
5205
5245
  }
5206
5246
  function singleDollarMathEnd(text, open, limit = text.length) {
5207
5247
  const bound = Math.min(limit, open + 403);
@@ -5437,6 +5477,10 @@ var XmlContainerTracker = class {
5437
5477
 
5438
5478
  // source/tokenize.ts
5439
5479
  var NOT_WHITESPACE = /\S/;
5480
+ var DIRECTIVE_OPEN = /^(:{2,})([A-Za-z][\w-]*)/;
5481
+ var MKDOCS_OPEN = /^(!!!|\?\?\?\+?)[ \t]+([A-Za-z][\w-]*)/;
5482
+ var CALLOUT_OPEN = /^>[ \t]*\[!([A-Za-z][\w-]*)\]([+-])?/;
5483
+ var FOOTNOTE_DEF = /^\[\^([^\]\s]+)\]:/;
5440
5484
  function countStructuralBraces(source) {
5441
5485
  let opens = 0;
5442
5486
  let closes = 0;
@@ -5564,8 +5608,9 @@ function isSplitterTreeLine(line) {
5564
5608
  }
5565
5609
  var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
5566
5610
  var Tokenizer = class {
5567
- constructor(text) {
5611
+ constructor(text, streaming = false) {
5568
5612
  this.text = text;
5613
+ this.streaming = streaming;
5569
5614
  this.n = text.length;
5570
5615
  this.mathPairs = pairDisplayMath(text);
5571
5616
  let start = 0;
@@ -5584,6 +5629,7 @@ var Tokenizer = class {
5584
5629
  }
5585
5630
  }
5586
5631
  text;
5632
+ streaming;
5587
5633
  n;
5588
5634
  lineStarts = [];
5589
5635
  contentEnds = [];
@@ -5621,6 +5667,10 @@ var Tokenizer = class {
5621
5667
  }
5622
5668
  return lo < list.length ? list[lo] : -1;
5623
5669
  }
5670
+ /** Streaming only: the final line, still waiting for its terminator. */
5671
+ isOpenTail(k) {
5672
+ return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
5673
+ }
5624
5674
  get lineCount() {
5625
5675
  return this.lineStarts.length;
5626
5676
  }
@@ -5702,6 +5752,22 @@ var Tokenizer = class {
5702
5752
  const fm = this.frontMatter(li, t);
5703
5753
  if (fm) return fm;
5704
5754
  }
5755
+ if (indent <= 3 && t.startsWith("::")) {
5756
+ const directive = this.directive(p, li, t);
5757
+ if (directive) return directive;
5758
+ }
5759
+ if (indent === 0 && (t.startsWith("!!!") || t.startsWith("???"))) {
5760
+ const admonition = this.mkdocsAdmonition(p, li, t);
5761
+ if (admonition) return admonition;
5762
+ }
5763
+ if (indent <= 3 && t.startsWith(">")) {
5764
+ const callout = this.callout(p, li, t);
5765
+ if (callout) return callout;
5766
+ }
5767
+ if (atParaStart && indent <= 3 && t.startsWith("[^")) {
5768
+ const footnote = this.footnoteDefinition(p, li, t);
5769
+ if (footnote) return footnote;
5770
+ }
5705
5771
  if (t.startsWith("```")) return this.backtickFence(p, li, t);
5706
5772
  if (indent <= 3 && t.startsWith("~~~")) {
5707
5773
  const tilde = this.tildeFence(p, li, t);
@@ -5739,19 +5805,110 @@ var Tokenizer = class {
5739
5805
  return { type, start, end, complete, meta };
5740
5806
  }
5741
5807
  frontMatter(li, t) {
5742
- if (this.lineText(li) !== "---" || t !== "---") return null;
5808
+ const fence = this.lineText(li);
5809
+ if (fence !== "---" && fence !== "+++" || t !== fence) return null;
5810
+ const toml = fence === "+++";
5743
5811
  let sawContent = false;
5744
5812
  for (let k = li + 1; k < this.lineCount; k++) {
5745
5813
  const line = this.lineText(k);
5746
- if (line === "---" || line === "...") {
5747
- return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true) : null;
5814
+ if (line === fence || !toml && line === "...") {
5815
+ return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true, { format: toml ? "toml" : "yaml" }) : null;
5748
5816
  }
5749
5817
  if (!NOT_WHITESPACE.test(line)) continue;
5750
- if (!/^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s)/.test(line)) return null;
5818
+ const shaped = toml ? /^\s*(?:[A-Za-z0-9_."-]+\s*=|\[|#)/.test(line) : /^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s|#)/.test(line);
5819
+ if (!shaped) return null;
5751
5820
  sawContent = true;
5752
5821
  }
5753
5822
  return null;
5754
5823
  }
5824
+ /**
5825
+ * `:::name[label]{attrs}` … closing `:::` (a colon run at least as long as
5826
+ * the opener's, nesting counted), or a leaf `::name[…]{…}` line. The whole
5827
+ * container is ONE island: its grammar (fence lengths, labels, attributes,
5828
+ * nested tabs) is exactly what a visual editor would re-serialize wrongly.
5829
+ */
5830
+ directive(p, li, t) {
5831
+ const open = DIRECTIVE_OPEN.exec(t);
5832
+ if (!open) return null;
5833
+ const colons = open[1].length;
5834
+ const name = open[2];
5835
+ if (colons === 2) {
5836
+ return this.island("directive", p, this.contentEnds[li], true, { name, leaf: true });
5837
+ }
5838
+ const stack = [colons];
5839
+ for (let k = li + 1; k < this.lineCount; k++) {
5840
+ const line = this.lineText(k).trim();
5841
+ const inner = DIRECTIVE_OPEN.exec(line);
5842
+ if (inner && inner[1].length >= 3) {
5843
+ stack.push(inner[1].length);
5844
+ continue;
5845
+ }
5846
+ const close = /^(:{3,})\s*$/.exec(line);
5847
+ if (close && close[1].length >= stack[stack.length - 1]) {
5848
+ stack.pop();
5849
+ if (stack.length === 0) return this.island("directive", p, this.contentEnds[k], true, { name });
5850
+ }
5851
+ }
5852
+ return this.island("directive", p, this.n, false, { name });
5853
+ }
5854
+ /** MkDocs `!!! type "Title"` / `??? type` / `???+ type` and its indented (or blank-separated indented) body. */
5855
+ mkdocsAdmonition(p, li, t) {
5856
+ const open = MKDOCS_OPEN.exec(t);
5857
+ if (!open) return null;
5858
+ let last = li;
5859
+ for (let k = li + 1; k < this.lineCount; k++) {
5860
+ const line = this.lineText(k);
5861
+ if (/^(?: {4}|\t)/.test(line) && NOT_WHITESPACE.test(line)) {
5862
+ last = k;
5863
+ continue;
5864
+ }
5865
+ if (!NOT_WHITESPACE.test(line)) continue;
5866
+ break;
5867
+ }
5868
+ return this.island("directive", p, this.contentEnds[last], true, {
5869
+ name: open[2].toLowerCase(),
5870
+ syntax: "mkdocs"
5871
+ });
5872
+ }
5873
+ /** A blockquote whose first line is `> [!TYPE]…` — every following `>` line belongs to it. */
5874
+ callout(p, li, t) {
5875
+ const open = CALLOUT_OPEN.exec(t);
5876
+ if (!open) return null;
5877
+ let last = li;
5878
+ for (let k = li + 1; k < this.lineCount; k++) {
5879
+ if (!/^ {0,3}>/.test(this.lineText(k))) break;
5880
+ last = k;
5881
+ }
5882
+ return this.island("callout", p, this.contentEnds[last], true, {
5883
+ name: open[1].toLowerCase(),
5884
+ ...open[2] ? { fold: open[2] === "-" ? "closed" : "open" } : {}
5885
+ });
5886
+ }
5887
+ /** `[^id]: text` plus its continuation lines (indented, or lazy text before a blank line). */
5888
+ footnoteDefinition(p, li, t) {
5889
+ const open = FOOTNOTE_DEF.exec(t);
5890
+ if (!open) return null;
5891
+ let last = li;
5892
+ for (let k = li + 1; k < this.lineCount; k++) {
5893
+ const line = this.lineText(k);
5894
+ if (!NOT_WHITESPACE.test(line)) {
5895
+ let j = k + 1;
5896
+ while (j < this.lineCount && !NOT_WHITESPACE.test(this.lineText(j))) j++;
5897
+ if (j < this.lineCount && /^(?: {4}|\t)/.test(this.lineText(j))) {
5898
+ k = j - 1;
5899
+ continue;
5900
+ }
5901
+ break;
5902
+ }
5903
+ if (/^(?: {4}|\t)/.test(line)) {
5904
+ last = k;
5905
+ continue;
5906
+ }
5907
+ if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
5908
+ last = k;
5909
+ }
5910
+ return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
5911
+ }
5755
5912
  /**
5756
5913
  * Backtick fence — the renderer splitter's close rules: JSON-string aware for
5757
5914
  * ```json, and THE nested-fence rule (source/fence-nesting.ts) for markdown
@@ -5780,8 +5937,9 @@ var Tokenizer = class {
5780
5937
  let nestedDepth = 0;
5781
5938
  let sawNested = false;
5782
5939
  for (let k = li + 1; k < this.lineCount; k++) {
5940
+ if (this.isOpenTail(k)) continue;
5783
5941
  const line = this.lineText(k);
5784
- const trimmedLine = line.trim();
5942
+ const trimmedLine = trimFenceLine(line);
5785
5943
  if (trimmedLine.startsWith("```")) {
5786
5944
  const at2 = line.indexOf("```");
5787
5945
  if (isJson) {
@@ -5811,7 +5969,7 @@ var Tokenizer = class {
5811
5969
  }
5812
5970
  }
5813
5971
  const closeTicks = backtickRunLength(line, at);
5814
- const after = line.slice(at + closeTicks).trim();
5972
+ const after = trimFenceLine(line.slice(at + closeTicks));
5815
5973
  if (closeTicks >= openTicks && after === "" && nestedDepth === 0) return k;
5816
5974
  if (isJson) state = jsonStringState(line, 0, line.length, state);
5817
5975
  continue;
@@ -5838,6 +5996,7 @@ var Tokenizer = class {
5838
5996
  if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
5839
5997
  const close = new RegExp(`^ {0,3}~{${ticks},}[ \\t]*$`);
5840
5998
  for (let k = li + 1; k < this.lineCount; k++) {
5999
+ if (this.isOpenTail(k)) continue;
5841
6000
  if (close.test(this.lineText(k))) {
5842
6001
  return this.island("fence", p, this.contentEnds[k], true, meta);
5843
6002
  }
@@ -6227,6 +6386,21 @@ var Tokenizer = class {
6227
6386
  i++;
6228
6387
  continue;
6229
6388
  }
6389
+ if (ch === "[" && text[i + 1] === "[" || ch === "!" && text[i + 1] === "[" && text[i + 2] === "[") {
6390
+ const open = ch === "!" ? i + 3 : i + 2;
6391
+ const close = within("]]", open);
6392
+ const newline = this.next("\n", open);
6393
+ const inner = close === -1 ? "" : text.slice(open, close);
6394
+ if (close !== -1 && (newline === -1 || newline > close) && inner.trim() && !/[[\]]/.test(inner)) {
6395
+ const cut = inner.search(/\\?\|/);
6396
+ const target = (cut < 0 ? inner : inner.slice(0, cut)).trim();
6397
+ inlines.push(
6398
+ this.island("wikilink", i, close + 2, true, ch === "!" ? { target, embed: true } : { target })
6399
+ );
6400
+ i = close + 2;
6401
+ continue;
6402
+ }
6403
+ }
6230
6404
  if (ch === "[") {
6231
6405
  const media = MEDIA_REF_PREFIXES.find((prefix) => text.startsWith(prefix, i));
6232
6406
  if (media) {
@@ -6308,8 +6482,8 @@ var Tokenizer = class {
6308
6482
  return { inlines, breakIsland: null, orphan: false };
6309
6483
  }
6310
6484
  };
6311
- function tokenizeSource(text) {
6312
- const raw = new Tokenizer(text).run();
6485
+ function tokenizeSource(text, options = {}) {
6486
+ const raw = new Tokenizer(text, options.streaming ?? false).run();
6313
6487
  const cp = buildCodePointIndex(text);
6314
6488
  const inline = (island) => ({
6315
6489
  islandType: island.type,
@@ -6358,7 +6532,7 @@ function listIslands(blocks) {
6358
6532
  raw: block.raw
6359
6533
  });
6360
6534
  } else {
6361
- out.push(...block.inlines);
6535
+ for (const inline of block.inlines) out.push(inline);
6362
6536
  }
6363
6537
  }
6364
6538
  return out;
@@ -6394,15 +6568,37 @@ function islandEdit(island, text) {
6394
6568
  function blockRangeEdit(first, last, text) {
6395
6569
  return { start: first.start, end: last.end, text };
6396
6570
  }
6397
- function locateIsland(text, raw, from, expected, expectedFromEnd) {
6398
- if (expected >= from && text.startsWith(raw, expected)) return expected;
6399
- if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) return expectedFromEnd;
6571
+ var CONTEXT_WINDOW = 64;
6572
+ function locateIsland(text, raw, from, expected, expectedFromEnd, oldText, oldStart, oldEnd) {
6573
+ const candidates = /* @__PURE__ */ new Set();
6574
+ if (expected >= from && text.startsWith(raw, expected)) candidates.add(expected);
6575
+ if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) candidates.add(expectedFromEnd);
6400
6576
  const after = text.indexOf(raw, Math.max(from, expected));
6577
+ if (after !== -1) candidates.add(after);
6401
6578
  const before = expected > from ? text.lastIndexOf(raw, expected) : -1;
6402
- const beforeOk = before >= from ? before : -1;
6403
- if (after === -1) return beforeOk;
6404
- if (beforeOk === -1) return after;
6405
- return expected - beforeOk <= after - expected ? beforeOk : after;
6579
+ if (before >= from) candidates.add(before);
6580
+ if (candidates.size === 0) return text.indexOf(raw, from);
6581
+ if (candidates.size === 1) return candidates.values().next().value;
6582
+ let best = -1;
6583
+ let bestScore = -1;
6584
+ for (const at of candidates) {
6585
+ let score = 0;
6586
+ for (let k = 0; k < CONTEXT_WINDOW; k++) {
6587
+ const n = text[at + raw.length + k];
6588
+ if (n === void 0 || n !== oldText[oldEnd + k]) break;
6589
+ score++;
6590
+ }
6591
+ for (let k = 1; k <= CONTEXT_WINDOW; k++) {
6592
+ const n = text[at - k];
6593
+ if (n === void 0 || n !== oldText[oldStart - k]) break;
6594
+ score++;
6595
+ }
6596
+ if (score > bestScore || score === bestScore && at === expected) {
6597
+ best = at;
6598
+ bestScore = score;
6599
+ }
6600
+ }
6601
+ return best;
6406
6602
  }
6407
6603
  function commonPrefix(a, b) {
6408
6604
  const max = Math.min(a.length, b.length);
@@ -6479,7 +6675,16 @@ function spliceSave(original, edits, options = {}) {
6479
6675
  let oldCursor2 = edit.start;
6480
6676
  let newCursor2 = 0;
6481
6677
  for (const island of inside) {
6482
- const at = locateIsland(edit.text, island.raw, newCursor2, newCursor2 + (island.start - oldCursor2), edit.text.length - (edit.end - island.start));
6678
+ const at = locateIsland(
6679
+ edit.text,
6680
+ island.raw,
6681
+ newCursor2,
6682
+ newCursor2 + (island.start - oldCursor2),
6683
+ edit.text.length - (edit.end - island.start),
6684
+ original.slice(edit.start, edit.end),
6685
+ island.start - edit.start,
6686
+ island.end - edit.start
6687
+ );
6483
6688
  if (at === -1) {
6484
6689
  const name = island.meta.tag ?? island.meta.name ?? island.meta.lang ?? "";
6485
6690
  throw new SourceSpliceError(
@@ -6617,6 +6822,6 @@ function mapRange(changes, start, end, unit = "utf16") {
6617
6822
  return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
6618
6823
  }
6619
6824
 
6620
- export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
6825
+ export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
6621
6826
  //# sourceMappingURL=index.js.map
6622
6827
  //# sourceMappingURL=index.js.map