@ai-matrx/content-ir 0.15.1 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
7
7
  export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.cjs';
8
8
  export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.cjs';
9
9
  export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.cjs';
10
- export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.cjs';
10
+ export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.cjs';
11
11
  import 'react';
package/dist/index.d.ts CHANGED
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
7
7
  export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.js';
8
8
  export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.js';
9
9
  export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.js';
10
- export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.js';
10
+ export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.js';
11
11
  import 'react';
package/dist/index.js CHANGED
@@ -5315,6 +5315,50 @@ function codeRanges(text) {
5315
5315
  }
5316
5316
  return ranges.sort((a, b) => a[0] - b[0]);
5317
5317
  }
5318
+ function pendingDisplayMathOpener(text) {
5319
+ if (!text.includes("$$")) return -1;
5320
+ const ranges = codeRanges(text);
5321
+ const tokens = [];
5322
+ let r = 0;
5323
+ let maxEnd = -1;
5324
+ for (let i = text.indexOf("$$"); i !== -1 && i < text.length - 1; i = text.indexOf("$$", i)) {
5325
+ while (r < ranges.length && ranges[r][0] <= i) {
5326
+ maxEnd = Math.max(maxEnd, ranges[r][1]);
5327
+ r++;
5328
+ }
5329
+ if (maxEnd <= i) tokens.push(i);
5330
+ i += 2;
5331
+ }
5332
+ const paired = /* @__PURE__ */ new Set();
5333
+ let j = 0;
5334
+ while (j < tokens.length) {
5335
+ const open = tokens[j];
5336
+ const close = tokens[j + 1];
5337
+ if (close === void 0) break;
5338
+ if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
5339
+ paired.add(open);
5340
+ paired.add(close);
5341
+ j += 2;
5342
+ continue;
5343
+ }
5344
+ j += 1;
5345
+ }
5346
+ let trail = 0;
5347
+ while (trail < text.length && /\s/.test(text[text.length - 1 - trail])) trail++;
5348
+ for (const token of tokens) {
5349
+ if (paired.has(token)) continue;
5350
+ let lead = 0;
5351
+ while (lead <= MAX_MATH_SPAN && /\s/.test(text[token + 2 + lead] ?? "x")) lead++;
5352
+ if (text.length - (token + 2) - lead - trail > MAX_MATH_SPAN) continue;
5353
+ const inner = text.slice(token + 2);
5354
+ const s = inner.trim();
5355
+ if (STRUCTURAL_MARKDOWN.test(s) || s.length > MAX_MATH_SPAN) continue;
5356
+ if (!opensFlowBlock(text, token) && /\n[ \t]*\n/.test(inner)) continue;
5357
+ if ((s.match(PROSE_WORD) ?? []).length >= PROSE_WORD_LIMIT * 3) continue;
5358
+ return token;
5359
+ }
5360
+ return -1;
5361
+ }
5318
5362
  function pairDisplayMath(text) {
5319
5363
  const pairs = /* @__PURE__ */ new Map();
5320
5364
  if (!text.includes("$$")) return pairs;
@@ -5663,11 +5707,15 @@ function isSplitterTreeLine(line) {
5663
5707
  }
5664
5708
  var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
5665
5709
  var Tokenizer = class {
5666
- constructor(text, streaming = false) {
5710
+ constructor(text, streaming = false, meter = void 0) {
5667
5711
  this.text = text;
5668
5712
  this.streaming = streaming;
5713
+ this.meter = meter;
5669
5714
  this.n = text.length;
5715
+ this.tick(text.length);
5670
5716
  this.mathPairs = pairDisplayMath(text);
5717
+ this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
5718
+ if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
5671
5719
  let start = 0;
5672
5720
  for (let i = 0; i < text.length; i++) {
5673
5721
  if (text.charCodeAt(i) === 10) {
@@ -5678,13 +5726,15 @@ var Tokenizer = class {
5678
5726
  }
5679
5727
  }
5680
5728
  if (start < text.length || text.length === 0) {
5729
+ this.pendingCR = streaming && text.length > start && text.charCodeAt(text.length - 1) === 13;
5681
5730
  this.lineStarts.push(start);
5682
- this.contentEnds.push(text.length);
5731
+ this.contentEnds.push(this.pendingCR ? text.length - 1 : text.length);
5683
5732
  this.lineEnds.push(text.length);
5684
5733
  }
5685
5734
  }
5686
5735
  text;
5687
5736
  streaming;
5737
+ meter;
5688
5738
  n;
5689
5739
  lineStarts = [];
5690
5740
  contentEnds = [];
@@ -5699,6 +5749,23 @@ var Tokenizer = class {
5699
5749
  /** `$$` opener → closer for the pairs the core renders as math. */
5700
5750
  mathPairs;
5701
5751
  finder = (needle, from) => this.next(needle, from);
5752
+ /** Streaming: where provisional text starts (a pending `$$`), or +∞. */
5753
+ provisionalFrom = Number.POSITIVE_INFINITY;
5754
+ /** Streaming: the text ends in a lone CR whose LF has not arrived yet. */
5755
+ pendingCR = false;
5756
+ /**
5757
+ * Streaming: an open last line that begins with `{` is undecided — each new
5758
+ * byte can balance or unbalance its braces and flip the whole line between
5759
+ * prose and a JSON island — so it is provisional until it ends (G5).
5760
+ */
5761
+ markOpenJsonTail() {
5762
+ const last = this.lineStarts.length - 1;
5763
+ if (!this.isOpenTail(last)) return;
5764
+ const start = this.lineStarts[last];
5765
+ if (this.lineText(last).trimStart().startsWith("{")) {
5766
+ this.provisionalFrom = Math.min(this.provisionalFrom, start);
5767
+ }
5768
+ }
5702
5769
  /**
5703
5770
  * `text.indexOf(needle, from)` in O(log k): the first query for a needle
5704
5771
  * indexes all of its occurrences once, so repeated searches for a closer
@@ -5722,9 +5789,21 @@ var Tokenizer = class {
5722
5789
  }
5723
5790
  return lo < list.length ? list[lo] : -1;
5724
5791
  }
5792
+ /** Charge `n` visited characters to the diagnostic meter (complexity tests). */
5793
+ tick(n) {
5794
+ if (this.meter) this.meter.work += n;
5795
+ }
5796
+ /** Streaming only: a paragraph ending at `pe` may still be continued by a line not yet arrived. */
5797
+ paragraphMayGrow(pe) {
5798
+ if (!this.streaming) return false;
5799
+ const rest = this.n - pe;
5800
+ if (rest === 0) return true;
5801
+ const tail = this.text.slice(pe);
5802
+ return tail === "\n" || tail === "\r\n" || tail === "\r";
5803
+ }
5725
5804
  /** Streaming only: the final line, still waiting for its terminator. */
5726
5805
  isOpenTail(k) {
5727
- return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
5806
+ return this.streaming && k === this.lineStarts.length - 1 && (this.lineEnds[k] === this.contentEnds[k] || this.pendingCR);
5728
5807
  }
5729
5808
  get lineCount() {
5730
5809
  return this.lineStarts.length;
@@ -5749,6 +5828,7 @@ var Tokenizer = class {
5749
5828
  return this.text.slice(this.lineStarts[li], this.contentEnds[li]);
5750
5829
  }
5751
5830
  run() {
5831
+ if (this.streaming) this.markOpenJsonTail();
5752
5832
  let pos = 0;
5753
5833
  while (pos < this.n) {
5754
5834
  const li = this.lineOf(pos);
@@ -5757,6 +5837,10 @@ var Tokenizer = class {
5757
5837
  pos = this.gap(pos, li);
5758
5838
  continue;
5759
5839
  }
5840
+ if (pos >= this.provisionalFrom) {
5841
+ this.push({ kind: "prose", start: pos, end: this.n, island: null, inlines: [] });
5842
+ break;
5843
+ }
5760
5844
  const island = this.detectBlock(pos, li, true);
5761
5845
  if (island) {
5762
5846
  this.push({ kind: "island", start: island.start, end: island.end, island, inlines: [] });
@@ -5807,7 +5891,8 @@ var Tokenizer = class {
5807
5891
  const fm = this.frontMatter(li, t);
5808
5892
  if (fm) return fm;
5809
5893
  }
5810
- if (indent <= 3 && t.startsWith("::")) {
5894
+ const listNested = indent > 0 && this.inListContinuation(li);
5895
+ if (indent <= 3 && !listNested && t.startsWith("::")) {
5811
5896
  const directive = this.directive(p, li, t);
5812
5897
  if (directive) return directive;
5813
5898
  }
@@ -5815,7 +5900,7 @@ var Tokenizer = class {
5815
5900
  const admonition = this.mkdocsAdmonition(p, li, t);
5816
5901
  if (admonition) return admonition;
5817
5902
  }
5818
- if (indent <= 3 && t.startsWith(">")) {
5903
+ if (indent <= 3 && !listNested && t.startsWith(">")) {
5819
5904
  const callout = this.callout(p, li, t);
5820
5905
  if (callout) return callout;
5821
5906
  }
@@ -5856,6 +5941,17 @@ var Tokenizer = class {
5856
5941
  }
5857
5942
  return null;
5858
5943
  }
5944
+ /** True when line `li` continues a list item (the nearest non-blank line above is a list item or indented text under one). */
5945
+ inListContinuation(li) {
5946
+ for (let k = li - 1; k >= 0; k--) {
5947
+ const line = this.lineText(k);
5948
+ if (!NOT_WHITESPACE.test(line)) continue;
5949
+ if (/^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]/.test(line)) return true;
5950
+ if (/^[ \t]/.test(line)) continue;
5951
+ return false;
5952
+ }
5953
+ return false;
5954
+ }
5859
5955
  island(type, start, end, complete, meta = {}) {
5860
5956
  return { type, start, end, complete, meta };
5861
5957
  }
@@ -5960,6 +6056,7 @@ var Tokenizer = class {
5960
6056
  continue;
5961
6057
  }
5962
6058
  if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
6059
+ if (this.isOpenTail(k)) break;
5963
6060
  last = k;
5964
6061
  }
5965
6062
  return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
@@ -6121,6 +6218,7 @@ var Tokenizer = class {
6121
6218
  const li = this.lineOf(tp);
6122
6219
  const ce = this.contentEnds[li];
6123
6220
  const tag = readXmlTag(this.text, tp, this.finder);
6221
+ this.tick(tag ? tag.raw.length : 1);
6124
6222
  let openEnd;
6125
6223
  if (tag && !tag.isClosing) {
6126
6224
  if (tag.isSelfClosing) {
@@ -6205,7 +6303,7 @@ var Tokenizer = class {
6205
6303
  return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
6206
6304
  }
6207
6305
  const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
6208
- const typeSeven = atParaStart && t.trimEnd() === tag.raw;
6306
+ const typeSeven = atParaStart && t.trimEnd() === tag.raw && !this.isOpenTail(li);
6209
6307
  if (!typeSix && !typeSeven) return null;
6210
6308
  let end = this.contentEnds[li];
6211
6309
  for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
@@ -6297,6 +6395,10 @@ var Tokenizer = class {
6297
6395
  let breakLine = -1;
6298
6396
  for (let k = li + 1; k < this.lineCount; k++) {
6299
6397
  if (this.lineIsBlank(k)) break;
6398
+ if (this.lineStarts[k] >= this.provisionalFrom) {
6399
+ end = this.n;
6400
+ break;
6401
+ }
6300
6402
  if (this.detectBlock(this.lineStarts[k], k, false)) {
6301
6403
  breakLine = k;
6302
6404
  break;
@@ -6362,6 +6464,7 @@ var Tokenizer = class {
6362
6464
  scanInline(ps, pe) {
6363
6465
  const text = this.text;
6364
6466
  const inlines = [];
6467
+ this.tick(pe - ps);
6365
6468
  const hasKind = text.slice(ps, pe).includes("__kind");
6366
6469
  let kindBudget = 4 * (pe - ps) + 65536;
6367
6470
  const stop = (island, orphan = false) => ({ inlines, breakIsland: island, orphan });
@@ -6372,6 +6475,7 @@ var Tokenizer = class {
6372
6475
  let runs = null;
6373
6476
  const codeSpanClose = (n, from) => {
6374
6477
  if (!runs) {
6478
+ this.tick(pe - ps);
6375
6479
  runs = /* @__PURE__ */ new Map();
6376
6480
  for (let k = ps; k < pe; ) {
6377
6481
  if (text.charCodeAt(k) !== 96) {
@@ -6398,7 +6502,8 @@ var Tokenizer = class {
6398
6502
  return at !== void 0 && at + n <= pe ? at : -1;
6399
6503
  };
6400
6504
  let i = ps;
6401
- while (i < pe) {
6505
+ const scanEnd = Math.min(pe, this.provisionalFrom);
6506
+ while (i < scanEnd) {
6402
6507
  const ch = text[i];
6403
6508
  if (ch === "\\") {
6404
6509
  const nextCh = text[i + 1];
@@ -6416,6 +6521,9 @@ var Tokenizer = class {
6416
6521
  if (ch === "`") {
6417
6522
  const n = backtickRunLength(text, i);
6418
6523
  const close = codeSpanClose(n, i + n);
6524
+ if (close === -1 && this.streaming && this.paragraphMayGrow(pe)) {
6525
+ return { inlines, breakIsland: null, orphan: false };
6526
+ }
6419
6527
  i = close === -1 ? i + n : close + n;
6420
6528
  continue;
6421
6529
  }
@@ -6432,6 +6540,7 @@ var Tokenizer = class {
6432
6540
  continue;
6433
6541
  }
6434
6542
  const end = singleDollarMathEnd(text, i, pe);
6543
+ this.tick(end > 0 ? end - i : Math.min(403, pe - i));
6435
6544
  if (end > 0 && !(this.streaming && end === this.n)) {
6436
6545
  inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
6437
6546
  i = end;
@@ -6459,6 +6568,7 @@ var Tokenizer = class {
6459
6568
  const limit = Math.min(pe, i + 65536);
6460
6569
  const end = matchingJsonObjectEnd(text, i, limit);
6461
6570
  kindBudget -= (end ?? limit) - i;
6571
+ this.tick((end ?? limit) - i);
6462
6572
  if (end !== null) {
6463
6573
  const kind = declaredKind(text.slice(i, end));
6464
6574
  if (kind) {
@@ -6520,6 +6630,7 @@ var Tokenizer = class {
6520
6630
  continue;
6521
6631
  }
6522
6632
  const tag = readXmlTag(text, i, this.finder);
6633
+ this.tick(tag ? tag.raw.length : 1);
6523
6634
  if (tag && i + tag.raw.length <= pe) {
6524
6635
  const name = tag.tagName;
6525
6636
  const tagEnd = i + tag.raw.length;
@@ -6568,7 +6679,7 @@ var Tokenizer = class {
6568
6679
  }
6569
6680
  };
6570
6681
  function tokenizeSource(text, options = {}) {
6571
- const raw = new Tokenizer(text, options.streaming ?? false).run();
6682
+ const raw = new Tokenizer(text, options.streaming ?? false, options.meter).run();
6572
6683
  const cp = buildCodePointIndex(text);
6573
6684
  const inline = (island) => ({
6574
6685
  islandType: island.type,
@@ -6921,6 +7032,6 @@ function mapRange(changes, start, end, unit = "utf16") {
6921
7032
  return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
6922
7033
  }
6923
7034
 
6924
- export { ACTION_BY_CLASS, CALLOUT_NAMES, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, parseFenceOpener, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
7035
+ export { ACTION_BY_CLASS, CALLOUT_NAMES, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, parseFenceOpener, pendingDisplayMathOpener, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
6925
7036
  //# sourceMappingURL=index.js.map
6926
7037
  //# sourceMappingURL=index.js.map