@ai-matrx/content-ir 0.13.0 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
7
7
  export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.cjs';
8
8
  export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.cjs';
9
9
  export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.cjs';
10
- export { BlockIslandType, CodePointIndex, DisturbedIsland, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource } from './source.cjs';
10
+ export { BlockIslandType, CodePointIndex, DisturbedIsland, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.cjs';
11
11
  import 'react';
package/dist/index.d.ts CHANGED
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
7
7
  export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.js';
8
8
  export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.js';
9
9
  export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.js';
10
- export { BlockIslandType, CodePointIndex, DisturbedIsland, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource } from './source.js';
10
+ export { BlockIslandType, CodePointIndex, DisturbedIsland, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.js';
11
11
  import 'react';
package/dist/index.js CHANGED
@@ -5178,11 +5178,22 @@ var NESTING_FENCE_LANGUAGES = /* @__PURE__ */ new Set([
5178
5178
  function fenceNestsInnerFences(language) {
5179
5179
  return !!language && NESTING_FENCE_LANGUAGES.has(language.toLowerCase());
5180
5180
  }
5181
+ var FENCE_WHITESPACE = " \n\r\f\v";
5182
+ function isFenceWhitespace(ch) {
5183
+ return ch !== void 0 && ch !== "" && FENCE_WHITESPACE.includes(ch);
5184
+ }
5185
+ function trimFenceLine(line) {
5186
+ let start = 0;
5187
+ let end = line.length;
5188
+ while (start < end && isFenceWhitespace(line[start])) start++;
5189
+ while (end > start && isFenceWhitespace(line[end - 1])) end--;
5190
+ return start === 0 && end === line.length ? line : line.slice(start, end);
5191
+ }
5181
5192
  function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
5182
5193
  let ticks = 0;
5183
5194
  while (ticks < trimmed.length && trimmed[ticks] === "`") ticks++;
5184
5195
  if (ticks < 3) return "content";
5185
- const info = trimmed.slice(ticks).trim();
5196
+ const info = trimFenceLine(trimmed.slice(ticks));
5186
5197
  if (info === "") {
5187
5198
  if (ticks < openTicks) return "content";
5188
5199
  if (nests && nestedDepth > 0) return "close-nested";
@@ -5195,13 +5206,43 @@ function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
5195
5206
  // source/math.ts
5196
5207
  var TEX_SIGNAL = /\\[A-Za-z]+|[\\^_{}=<>+]/;
5197
5208
  var LONE_VARIABLE = /^[A-Za-z](?:'|[0-9])?$/;
5209
+ var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
5210
+ var FUNCTION_NAMES = /* @__PURE__ */ new Set([
5211
+ "sin",
5212
+ "cos",
5213
+ "tan",
5214
+ "sec",
5215
+ "csc",
5216
+ "cot",
5217
+ "log",
5218
+ "ln",
5219
+ "exp",
5220
+ "lim",
5221
+ "max",
5222
+ "min",
5223
+ "det",
5224
+ "gcd",
5225
+ "mod",
5226
+ "arg",
5227
+ "sinh",
5228
+ "cosh",
5229
+ "tanh"
5230
+ ]);
5231
+ function isMathShape(content) {
5232
+ if (!MATH_SHAPE_CHARS.test(content)) return false;
5233
+ const words = content.match(/[A-Za-z]+/g);
5234
+ if (!words) return false;
5235
+ if (!words.every((w) => w.length === 1 || FUNCTION_NAMES.has(w))) return false;
5236
+ if (/[A-Za-z]'+[A-Za-z]/.test(content) || /(?:^|[\s(,])[A-Za-z]\.(?:[A-Za-z]\.|\s|$)/.test(content)) return false;
5237
+ return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
5238
+ }
5198
5239
  function isSingleDollarMath(content, after) {
5199
5240
  if (!content || content.length > 400) return false;
5200
5241
  if (/^\s/.test(content) || /\s$/.test(content)) return false;
5201
5242
  if (after !== void 0 && /[0-9]/.test(after)) return false;
5202
5243
  if (LONE_VARIABLE.test(content)) return true;
5203
5244
  if (/^[0-9]/.test(content)) return /\\[A-Za-z]+|[\\^_{}]/.test(content);
5204
- return TEX_SIGNAL.test(content);
5245
+ return TEX_SIGNAL.test(content) || isMathShape(content);
5205
5246
  }
5206
5247
  function singleDollarMathEnd(text, open, limit = text.length) {
5207
5248
  const bound = Math.min(limit, open + 403);
@@ -5437,6 +5478,10 @@ var XmlContainerTracker = class {
5437
5478
 
5438
5479
  // source/tokenize.ts
5439
5480
  var NOT_WHITESPACE = /\S/;
5481
+ var DIRECTIVE_OPEN = /^(:{2,})([A-Za-z][\w-]*)/;
5482
+ var MKDOCS_OPEN = /^(!!!|\?\?\?\+?)[ \t]+([A-Za-z][\w-]*)/;
5483
+ var CALLOUT_OPEN = /^>[ \t]*\[!([A-Za-z][\w-]*)\]([+-])?/;
5484
+ var FOOTNOTE_DEF = /^\[\^([^\]\s]+)\]:/;
5440
5485
  function countStructuralBraces(source) {
5441
5486
  let opens = 0;
5442
5487
  let closes = 0;
@@ -5564,8 +5609,9 @@ function isSplitterTreeLine(line) {
5564
5609
  }
5565
5610
  var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
5566
5611
  var Tokenizer = class {
5567
- constructor(text) {
5612
+ constructor(text, streaming = false) {
5568
5613
  this.text = text;
5614
+ this.streaming = streaming;
5569
5615
  this.n = text.length;
5570
5616
  this.mathPairs = pairDisplayMath(text);
5571
5617
  let start = 0;
@@ -5584,6 +5630,7 @@ var Tokenizer = class {
5584
5630
  }
5585
5631
  }
5586
5632
  text;
5633
+ streaming;
5587
5634
  n;
5588
5635
  lineStarts = [];
5589
5636
  contentEnds = [];
@@ -5621,6 +5668,10 @@ var Tokenizer = class {
5621
5668
  }
5622
5669
  return lo < list.length ? list[lo] : -1;
5623
5670
  }
5671
+ /** Streaming only: the final line, still waiting for its terminator. */
5672
+ isOpenTail(k) {
5673
+ return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
5674
+ }
5624
5675
  get lineCount() {
5625
5676
  return this.lineStarts.length;
5626
5677
  }
@@ -5702,6 +5753,22 @@ var Tokenizer = class {
5702
5753
  const fm = this.frontMatter(li, t);
5703
5754
  if (fm) return fm;
5704
5755
  }
5756
+ if (indent <= 3 && t.startsWith("::")) {
5757
+ const directive = this.directive(p, li, t);
5758
+ if (directive) return directive;
5759
+ }
5760
+ if (indent === 0 && (t.startsWith("!!!") || t.startsWith("???"))) {
5761
+ const admonition = this.mkdocsAdmonition(p, li, t);
5762
+ if (admonition) return admonition;
5763
+ }
5764
+ if (indent <= 3 && t.startsWith(">")) {
5765
+ const callout = this.callout(p, li, t);
5766
+ if (callout) return callout;
5767
+ }
5768
+ if (atParaStart && indent <= 3 && t.startsWith("[^")) {
5769
+ const footnote = this.footnoteDefinition(p, li, t);
5770
+ if (footnote) return footnote;
5771
+ }
5705
5772
  if (t.startsWith("```")) return this.backtickFence(p, li, t);
5706
5773
  if (indent <= 3 && t.startsWith("~~~")) {
5707
5774
  const tilde = this.tildeFence(p, li, t);
@@ -5739,19 +5806,110 @@ var Tokenizer = class {
5739
5806
  return { type, start, end, complete, meta };
5740
5807
  }
5741
5808
  frontMatter(li, t) {
5742
- if (this.lineText(li) !== "---" || t !== "---") return null;
5809
+ const fence = this.lineText(li);
5810
+ if (fence !== "---" && fence !== "+++" || t !== fence) return null;
5811
+ const toml = fence === "+++";
5743
5812
  let sawContent = false;
5744
5813
  for (let k = li + 1; k < this.lineCount; k++) {
5745
5814
  const line = this.lineText(k);
5746
- if (line === "---" || line === "...") {
5747
- return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true) : null;
5815
+ if (line === fence || !toml && line === "...") {
5816
+ return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true, { format: toml ? "toml" : "yaml" }) : null;
5748
5817
  }
5749
5818
  if (!NOT_WHITESPACE.test(line)) continue;
5750
- if (!/^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s)/.test(line)) return null;
5819
+ const shaped = toml ? /^\s*(?:[A-Za-z0-9_."-]+\s*=|\[|#)/.test(line) : /^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s|#)/.test(line);
5820
+ if (!shaped) return null;
5751
5821
  sawContent = true;
5752
5822
  }
5753
5823
  return null;
5754
5824
  }
5825
+ /**
5826
+ * `:::name[label]{attrs}` … closing `:::` (a colon run at least as long as
5827
+ * the opener's, nesting counted), or a leaf `::name[…]{…}` line. The whole
5828
+ * container is ONE island: its grammar (fence lengths, labels, attributes,
5829
+ * nested tabs) is exactly what a visual editor would re-serialize wrongly.
5830
+ */
5831
+ directive(p, li, t) {
5832
+ const open = DIRECTIVE_OPEN.exec(t);
5833
+ if (!open) return null;
5834
+ const colons = open[1].length;
5835
+ const name = open[2];
5836
+ if (colons === 2) {
5837
+ return this.island("directive", p, this.contentEnds[li], true, { name, leaf: true });
5838
+ }
5839
+ const stack = [colons];
5840
+ for (let k = li + 1; k < this.lineCount; k++) {
5841
+ const line = this.lineText(k).trim();
5842
+ const inner = DIRECTIVE_OPEN.exec(line);
5843
+ if (inner && inner[1].length >= 3) {
5844
+ stack.push(inner[1].length);
5845
+ continue;
5846
+ }
5847
+ const close = /^(:{3,})\s*$/.exec(line);
5848
+ if (close && close[1].length >= stack[stack.length - 1]) {
5849
+ stack.pop();
5850
+ if (stack.length === 0) return this.island("directive", p, this.contentEnds[k], true, { name });
5851
+ }
5852
+ }
5853
+ return this.island("directive", p, this.n, false, { name });
5854
+ }
5855
+ /** MkDocs `!!! type "Title"` / `??? type` / `???+ type` and its indented (or blank-separated indented) body. */
5856
+ mkdocsAdmonition(p, li, t) {
5857
+ const open = MKDOCS_OPEN.exec(t);
5858
+ if (!open) return null;
5859
+ let last = li;
5860
+ for (let k = li + 1; k < this.lineCount; k++) {
5861
+ const line = this.lineText(k);
5862
+ if (/^(?: {4}|\t)/.test(line) && NOT_WHITESPACE.test(line)) {
5863
+ last = k;
5864
+ continue;
5865
+ }
5866
+ if (!NOT_WHITESPACE.test(line)) continue;
5867
+ break;
5868
+ }
5869
+ return this.island("directive", p, this.contentEnds[last], true, {
5870
+ name: open[2].toLowerCase(),
5871
+ syntax: "mkdocs"
5872
+ });
5873
+ }
5874
+ /** A blockquote whose first line is `> [!TYPE]…` — every following `>` line belongs to it. */
5875
+ callout(p, li, t) {
5876
+ const open = CALLOUT_OPEN.exec(t);
5877
+ if (!open) return null;
5878
+ let last = li;
5879
+ for (let k = li + 1; k < this.lineCount; k++) {
5880
+ if (!/^ {0,3}>/.test(this.lineText(k))) break;
5881
+ last = k;
5882
+ }
5883
+ return this.island("callout", p, this.contentEnds[last], true, {
5884
+ name: open[1].toLowerCase(),
5885
+ ...open[2] ? { fold: open[2] === "-" ? "closed" : "open" } : {}
5886
+ });
5887
+ }
5888
+ /** `[^id]: text` plus its continuation lines (indented, or lazy text before a blank line). */
5889
+ footnoteDefinition(p, li, t) {
5890
+ const open = FOOTNOTE_DEF.exec(t);
5891
+ if (!open) return null;
5892
+ let last = li;
5893
+ for (let k = li + 1; k < this.lineCount; k++) {
5894
+ const line = this.lineText(k);
5895
+ if (!NOT_WHITESPACE.test(line)) {
5896
+ let j = k + 1;
5897
+ while (j < this.lineCount && !NOT_WHITESPACE.test(this.lineText(j))) j++;
5898
+ if (j < this.lineCount && /^(?: {4}|\t)/.test(this.lineText(j))) {
5899
+ k = j - 1;
5900
+ continue;
5901
+ }
5902
+ break;
5903
+ }
5904
+ if (/^(?: {4}|\t)/.test(line)) {
5905
+ last = k;
5906
+ continue;
5907
+ }
5908
+ if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
5909
+ last = k;
5910
+ }
5911
+ return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
5912
+ }
5755
5913
  /**
5756
5914
  * Backtick fence — the renderer splitter's close rules: JSON-string aware for
5757
5915
  * ```json, and THE nested-fence rule (source/fence-nesting.ts) for markdown
@@ -5780,8 +5938,9 @@ var Tokenizer = class {
5780
5938
  let nestedDepth = 0;
5781
5939
  let sawNested = false;
5782
5940
  for (let k = li + 1; k < this.lineCount; k++) {
5941
+ if (this.isOpenTail(k)) continue;
5783
5942
  const line = this.lineText(k);
5784
- const trimmedLine = line.trim();
5943
+ const trimmedLine = trimFenceLine(line);
5785
5944
  if (trimmedLine.startsWith("```")) {
5786
5945
  const at2 = line.indexOf("```");
5787
5946
  if (isJson) {
@@ -5811,7 +5970,7 @@ var Tokenizer = class {
5811
5970
  }
5812
5971
  }
5813
5972
  const closeTicks = backtickRunLength(line, at);
5814
- const after = line.slice(at + closeTicks).trim();
5973
+ const after = trimFenceLine(line.slice(at + closeTicks));
5815
5974
  if (closeTicks >= openTicks && after === "" && nestedDepth === 0) return k;
5816
5975
  if (isJson) state = jsonStringState(line, 0, line.length, state);
5817
5976
  continue;
@@ -5838,6 +5997,7 @@ var Tokenizer = class {
5838
5997
  if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
5839
5998
  const close = new RegExp(`^ {0,3}~{${ticks},}[ \\t]*$`);
5840
5999
  for (let k = li + 1; k < this.lineCount; k++) {
6000
+ if (this.isOpenTail(k)) continue;
5841
6001
  if (close.test(this.lineText(k))) {
5842
6002
  return this.island("fence", p, this.contentEnds[k], true, meta);
5843
6003
  }
@@ -6227,6 +6387,21 @@ var Tokenizer = class {
6227
6387
  i++;
6228
6388
  continue;
6229
6389
  }
6390
+ if (ch === "[" && text[i + 1] === "[" || ch === "!" && text[i + 1] === "[" && text[i + 2] === "[") {
6391
+ const open = ch === "!" ? i + 3 : i + 2;
6392
+ const close = within("]]", open);
6393
+ const newline = this.next("\n", open);
6394
+ const inner = close === -1 ? "" : text.slice(open, close);
6395
+ if (close !== -1 && (newline === -1 || newline > close) && inner.trim() && !/[[\]]/.test(inner)) {
6396
+ const cut = inner.search(/\\?\|/);
6397
+ const target = (cut < 0 ? inner : inner.slice(0, cut)).trim();
6398
+ inlines.push(
6399
+ this.island("wikilink", i, close + 2, true, ch === "!" ? { target, embed: true } : { target })
6400
+ );
6401
+ i = close + 2;
6402
+ continue;
6403
+ }
6404
+ }
6230
6405
  if (ch === "[") {
6231
6406
  const media = MEDIA_REF_PREFIXES.find((prefix) => text.startsWith(prefix, i));
6232
6407
  if (media) {
@@ -6308,8 +6483,8 @@ var Tokenizer = class {
6308
6483
  return { inlines, breakIsland: null, orphan: false };
6309
6484
  }
6310
6485
  };
6311
- function tokenizeSource(text) {
6312
- const raw = new Tokenizer(text).run();
6486
+ function tokenizeSource(text, options = {}) {
6487
+ const raw = new Tokenizer(text, options.streaming ?? false).run();
6313
6488
  const cp = buildCodePointIndex(text);
6314
6489
  const inline = (island) => ({
6315
6490
  islandType: island.type,
@@ -6358,7 +6533,7 @@ function listIslands(blocks) {
6358
6533
  raw: block.raw
6359
6534
  });
6360
6535
  } else {
6361
- out.push(...block.inlines);
6536
+ for (const inline of block.inlines) out.push(inline);
6362
6537
  }
6363
6538
  }
6364
6539
  return out;
@@ -6394,15 +6569,37 @@ function islandEdit(island, text) {
6394
6569
  function blockRangeEdit(first, last, text) {
6395
6570
  return { start: first.start, end: last.end, text };
6396
6571
  }
6397
- function locateIsland(text, raw, from, expected, expectedFromEnd) {
6398
- if (expected >= from && text.startsWith(raw, expected)) return expected;
6399
- if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) return expectedFromEnd;
6572
+ var CONTEXT_WINDOW = 64;
6573
+ function locateIsland(text, raw, from, expected, expectedFromEnd, oldText, oldStart, oldEnd) {
6574
+ const candidates = /* @__PURE__ */ new Set();
6575
+ if (expected >= from && text.startsWith(raw, expected)) candidates.add(expected);
6576
+ if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) candidates.add(expectedFromEnd);
6400
6577
  const after = text.indexOf(raw, Math.max(from, expected));
6578
+ if (after !== -1) candidates.add(after);
6401
6579
  const before = expected > from ? text.lastIndexOf(raw, expected) : -1;
6402
- const beforeOk = before >= from ? before : -1;
6403
- if (after === -1) return beforeOk;
6404
- if (beforeOk === -1) return after;
6405
- return expected - beforeOk <= after - expected ? beforeOk : after;
6580
+ if (before >= from) candidates.add(before);
6581
+ if (candidates.size === 0) return text.indexOf(raw, from);
6582
+ if (candidates.size === 1) return candidates.values().next().value;
6583
+ let best = -1;
6584
+ let bestScore = -1;
6585
+ for (const at of candidates) {
6586
+ let score = 0;
6587
+ for (let k = 0; k < CONTEXT_WINDOW; k++) {
6588
+ const n = text[at + raw.length + k];
6589
+ if (n === void 0 || n !== oldText[oldEnd + k]) break;
6590
+ score++;
6591
+ }
6592
+ for (let k = 1; k <= CONTEXT_WINDOW; k++) {
6593
+ const n = text[at - k];
6594
+ if (n === void 0 || n !== oldText[oldStart - k]) break;
6595
+ score++;
6596
+ }
6597
+ if (score > bestScore || score === bestScore && at === expected) {
6598
+ best = at;
6599
+ bestScore = score;
6600
+ }
6601
+ }
6602
+ return best;
6406
6603
  }
6407
6604
  function commonPrefix(a, b) {
6408
6605
  const max = Math.min(a.length, b.length);
@@ -6479,7 +6676,16 @@ function spliceSave(original, edits, options = {}) {
6479
6676
  let oldCursor2 = edit.start;
6480
6677
  let newCursor2 = 0;
6481
6678
  for (const island of inside) {
6482
- const at = locateIsland(edit.text, island.raw, newCursor2, newCursor2 + (island.start - oldCursor2), edit.text.length - (edit.end - island.start));
6679
+ const at = locateIsland(
6680
+ edit.text,
6681
+ island.raw,
6682
+ newCursor2,
6683
+ newCursor2 + (island.start - oldCursor2),
6684
+ edit.text.length - (edit.end - island.start),
6685
+ original.slice(edit.start, edit.end),
6686
+ island.start - edit.start,
6687
+ island.end - edit.start
6688
+ );
6483
6689
  if (at === -1) {
6484
6690
  const name = island.meta.tag ?? island.meta.name ?? island.meta.lang ?? "";
6485
6691
  throw new SourceSpliceError(
@@ -6617,6 +6823,6 @@ function mapRange(changes, start, end, unit = "utf16") {
6617
6823
  return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
6618
6824
  }
6619
6825
 
6620
- export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
6826
+ export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
6621
6827
  //# sourceMappingURL=index.js.map
6622
6828
  //# sourceMappingURL=index.js.map