@ai-matrx/content-ir 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
7
7
  export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.cjs';
8
8
  export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.cjs';
9
9
  export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.cjs';
10
- export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.cjs';
10
+ export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.cjs';
11
11
  import 'react';
package/dist/index.d.ts CHANGED
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
7
7
  export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.js';
8
8
  export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.js';
9
9
  export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.js';
10
- export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.js';
10
+ export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.js';
11
11
  import 'react';
package/dist/index.js CHANGED
@@ -5288,15 +5288,20 @@ var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(
5288
5288
  var LATEX_COMMAND = /\\[a-zA-Z]/;
5289
5289
  var PROSE_WORD = /[A-Za-z]{3,}/g;
5290
5290
  var PROSE_WORD_LIMIT = 6;
5291
- function looksLikeDisplayMath(inner) {
5291
+ function opensFlowBlock(text, open) {
5292
+ for (let k = open - 1; k >= 0 && text[k] !== "\n"; k--) if (text[k] !== " " && text[k] !== " ") return false;
5293
+ for (let k = open + 2; k < text.length && text[k] !== "\n"; k++) if (text[k] !== " " && text[k] !== " " && text[k] !== "\r") return false;
5294
+ return true;
5295
+ }
5296
+ function looksLikeDisplayMath(inner, flow = false) {
5292
5297
  const s = inner.trim();
5293
5298
  if (!s) return false;
5294
5299
  if (STRUCTURAL_MARKDOWN.test(s)) return false;
5295
5300
  if (s.length > MAX_MATH_SPAN) return false;
5301
+ if (!flow && /\n[ \t]*\n/.test(inner)) return false;
5296
5302
  if (LATEX_COMMAND.test(s)) {
5297
5303
  return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT * 3;
5298
5304
  }
5299
- if (/\n[ \t]*\n/.test(s)) return false;
5300
5305
  return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT;
5301
5306
  }
5302
5307
  function codeRanges(text) {
@@ -5310,6 +5315,50 @@ function codeRanges(text) {
5310
5315
  }
5311
5316
  return ranges.sort((a, b) => a[0] - b[0]);
5312
5317
  }
5318
+ function pendingDisplayMathOpener(text) {
5319
+ if (!text.includes("$$")) return -1;
5320
+ const ranges = codeRanges(text);
5321
+ const tokens = [];
5322
+ let r = 0;
5323
+ let maxEnd = -1;
5324
+ for (let i = text.indexOf("$$"); i !== -1 && i < text.length - 1; i = text.indexOf("$$", i)) {
5325
+ while (r < ranges.length && ranges[r][0] <= i) {
5326
+ maxEnd = Math.max(maxEnd, ranges[r][1]);
5327
+ r++;
5328
+ }
5329
+ if (maxEnd <= i) tokens.push(i);
5330
+ i += 2;
5331
+ }
5332
+ const paired = /* @__PURE__ */ new Set();
5333
+ let j = 0;
5334
+ while (j < tokens.length) {
5335
+ const open = tokens[j];
5336
+ const close = tokens[j + 1];
5337
+ if (close === void 0) break;
5338
+ if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
5339
+ paired.add(open);
5340
+ paired.add(close);
5341
+ j += 2;
5342
+ continue;
5343
+ }
5344
+ j += 1;
5345
+ }
5346
+ let trail = 0;
5347
+ while (trail < text.length && /\s/.test(text[text.length - 1 - trail])) trail++;
5348
+ for (const token of tokens) {
5349
+ if (paired.has(token)) continue;
5350
+ let lead = 0;
5351
+ while (lead <= MAX_MATH_SPAN && /\s/.test(text[token + 2 + lead] ?? "x")) lead++;
5352
+ if (text.length - (token + 2) - lead - trail > MAX_MATH_SPAN) continue;
5353
+ const inner = text.slice(token + 2);
5354
+ const s = inner.trim();
5355
+ if (STRUCTURAL_MARKDOWN.test(s) || s.length > MAX_MATH_SPAN) continue;
5356
+ if (!opensFlowBlock(text, token) && /\n[ \t]*\n/.test(inner)) continue;
5357
+ if ((s.match(PROSE_WORD) ?? []).length >= PROSE_WORD_LIMIT * 3) continue;
5358
+ return token;
5359
+ }
5360
+ return -1;
5361
+ }
5313
5362
  function pairDisplayMath(text) {
5314
5363
  const pairs = /* @__PURE__ */ new Map();
5315
5364
  if (!text.includes("$$")) return pairs;
@@ -5330,7 +5379,7 @@ function pairDisplayMath(text) {
5330
5379
  const open = tokens[j];
5331
5380
  const close = tokens[j + 1];
5332
5381
  if (close === void 0) break;
5333
- if (looksLikeDisplayMath(text.slice(open + 2, close))) {
5382
+ if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
5334
5383
  pairs.set(open, close);
5335
5384
  j += 2;
5336
5385
  continue;
@@ -5658,11 +5707,15 @@ function isSplitterTreeLine(line) {
5658
5707
  }
5659
5708
  var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
5660
5709
  var Tokenizer = class {
5661
- constructor(text, streaming = false) {
5710
+ constructor(text, streaming = false, meter = void 0) {
5662
5711
  this.text = text;
5663
5712
  this.streaming = streaming;
5713
+ this.meter = meter;
5664
5714
  this.n = text.length;
5715
+ this.tick(text.length);
5665
5716
  this.mathPairs = pairDisplayMath(text);
5717
+ this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
5718
+ if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
5666
5719
  let start = 0;
5667
5720
  for (let i = 0; i < text.length; i++) {
5668
5721
  if (text.charCodeAt(i) === 10) {
@@ -5673,13 +5726,15 @@ var Tokenizer = class {
5673
5726
  }
5674
5727
  }
5675
5728
  if (start < text.length || text.length === 0) {
5729
+ this.pendingCR = streaming && text.length > start && text.charCodeAt(text.length - 1) === 13;
5676
5730
  this.lineStarts.push(start);
5677
- this.contentEnds.push(text.length);
5731
+ this.contentEnds.push(this.pendingCR ? text.length - 1 : text.length);
5678
5732
  this.lineEnds.push(text.length);
5679
5733
  }
5680
5734
  }
5681
5735
  text;
5682
5736
  streaming;
5737
+ meter;
5683
5738
  n;
5684
5739
  lineStarts = [];
5685
5740
  contentEnds = [];
@@ -5694,6 +5749,23 @@ var Tokenizer = class {
5694
5749
  /** `$$` opener → closer for the pairs the core renders as math. */
5695
5750
  mathPairs;
5696
5751
  finder = (needle, from) => this.next(needle, from);
5752
+ /** Streaming: where provisional text starts (a pending `$$`), or +∞. */
5753
+ provisionalFrom = Number.POSITIVE_INFINITY;
5754
+ /** Streaming: the text ends in a lone CR whose LF has not arrived yet. */
5755
+ pendingCR = false;
5756
+ /**
5757
+ * Streaming: an open last line that begins with `{` is undecided — each new
5758
+ * byte can balance or unbalance its braces and flip the whole line between
5759
+ * prose and a JSON island — so it is provisional until it ends (G5).
5760
+ */
5761
+ markOpenJsonTail() {
5762
+ const last = this.lineStarts.length - 1;
5763
+ if (!this.isOpenTail(last)) return;
5764
+ const start = this.lineStarts[last];
5765
+ if (this.lineText(last).trimStart().startsWith("{")) {
5766
+ this.provisionalFrom = Math.min(this.provisionalFrom, start);
5767
+ }
5768
+ }
5697
5769
  /**
5698
5770
  * `text.indexOf(needle, from)` in O(log k): the first query for a needle
5699
5771
  * indexes all of its occurrences once, so repeated searches for a closer
@@ -5717,9 +5789,21 @@ var Tokenizer = class {
5717
5789
  }
5718
5790
  return lo < list.length ? list[lo] : -1;
5719
5791
  }
5792
+ /** Charge `n` visited characters to the diagnostic meter (complexity tests). */
5793
+ tick(n) {
5794
+ if (this.meter) this.meter.work += n;
5795
+ }
5796
+ /** Streaming only: a paragraph ending at `pe` may still be continued by a line not yet arrived. */
5797
+ paragraphMayGrow(pe) {
5798
+ if (!this.streaming) return false;
5799
+ const rest = this.n - pe;
5800
+ if (rest === 0) return true;
5801
+ const tail = this.text.slice(pe);
5802
+ return tail === "\n" || tail === "\r\n" || tail === "\r";
5803
+ }
5720
5804
  /** Streaming only: the final line, still waiting for its terminator. */
5721
5805
  isOpenTail(k) {
5722
- return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
5806
+ return this.streaming && k === this.lineStarts.length - 1 && (this.lineEnds[k] === this.contentEnds[k] || this.pendingCR);
5723
5807
  }
5724
5808
  get lineCount() {
5725
5809
  return this.lineStarts.length;
@@ -5744,6 +5828,7 @@ var Tokenizer = class {
5744
5828
  return this.text.slice(this.lineStarts[li], this.contentEnds[li]);
5745
5829
  }
5746
5830
  run() {
5831
+ if (this.streaming) this.markOpenJsonTail();
5747
5832
  let pos = 0;
5748
5833
  while (pos < this.n) {
5749
5834
  const li = this.lineOf(pos);
@@ -5752,6 +5837,10 @@ var Tokenizer = class {
5752
5837
  pos = this.gap(pos, li);
5753
5838
  continue;
5754
5839
  }
5840
+ if (pos >= this.provisionalFrom) {
5841
+ this.push({ kind: "prose", start: pos, end: this.n, island: null, inlines: [] });
5842
+ break;
5843
+ }
5755
5844
  const island = this.detectBlock(pos, li, true);
5756
5845
  if (island) {
5757
5846
  this.push({ kind: "island", start: island.start, end: island.end, island, inlines: [] });
@@ -5955,6 +6044,7 @@ var Tokenizer = class {
5955
6044
  continue;
5956
6045
  }
5957
6046
  if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
6047
+ if (this.isOpenTail(k)) break;
5958
6048
  last = k;
5959
6049
  }
5960
6050
  return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
@@ -6116,6 +6206,7 @@ var Tokenizer = class {
6116
6206
  const li = this.lineOf(tp);
6117
6207
  const ce = this.contentEnds[li];
6118
6208
  const tag = readXmlTag(this.text, tp, this.finder);
6209
+ this.tick(tag ? tag.raw.length : 1);
6119
6210
  let openEnd;
6120
6211
  if (tag && !tag.isClosing) {
6121
6212
  if (tag.isSelfClosing) {
@@ -6200,7 +6291,7 @@ var Tokenizer = class {
6200
6291
  return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
6201
6292
  }
6202
6293
  const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
6203
- const typeSeven = atParaStart && t.trimEnd() === tag.raw;
6294
+ const typeSeven = atParaStart && t.trimEnd() === tag.raw && !this.isOpenTail(li);
6204
6295
  if (!typeSix && !typeSeven) return null;
6205
6296
  let end = this.contentEnds[li];
6206
6297
  for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
@@ -6292,6 +6383,10 @@ var Tokenizer = class {
6292
6383
  let breakLine = -1;
6293
6384
  for (let k = li + 1; k < this.lineCount; k++) {
6294
6385
  if (this.lineIsBlank(k)) break;
6386
+ if (this.lineStarts[k] >= this.provisionalFrom) {
6387
+ end = this.n;
6388
+ break;
6389
+ }
6295
6390
  if (this.detectBlock(this.lineStarts[k], k, false)) {
6296
6391
  breakLine = k;
6297
6392
  break;
@@ -6357,6 +6452,7 @@ var Tokenizer = class {
6357
6452
  scanInline(ps, pe) {
6358
6453
  const text = this.text;
6359
6454
  const inlines = [];
6455
+ this.tick(pe - ps);
6360
6456
  const hasKind = text.slice(ps, pe).includes("__kind");
6361
6457
  let kindBudget = 4 * (pe - ps) + 65536;
6362
6458
  const stop = (island, orphan = false) => ({ inlines, breakIsland: island, orphan });
@@ -6367,6 +6463,7 @@ var Tokenizer = class {
6367
6463
  let runs = null;
6368
6464
  const codeSpanClose = (n, from) => {
6369
6465
  if (!runs) {
6466
+ this.tick(pe - ps);
6370
6467
  runs = /* @__PURE__ */ new Map();
6371
6468
  for (let k = ps; k < pe; ) {
6372
6469
  if (text.charCodeAt(k) !== 96) {
@@ -6393,7 +6490,8 @@ var Tokenizer = class {
6393
6490
  return at !== void 0 && at + n <= pe ? at : -1;
6394
6491
  };
6395
6492
  let i = ps;
6396
- while (i < pe) {
6493
+ const scanEnd = Math.min(pe, this.provisionalFrom);
6494
+ while (i < scanEnd) {
6397
6495
  const ch = text[i];
6398
6496
  if (ch === "\\") {
6399
6497
  const nextCh = text[i + 1];
@@ -6411,6 +6509,9 @@ var Tokenizer = class {
6411
6509
  if (ch === "`") {
6412
6510
  const n = backtickRunLength(text, i);
6413
6511
  const close = codeSpanClose(n, i + n);
6512
+ if (close === -1 && this.streaming && this.paragraphMayGrow(pe)) {
6513
+ return { inlines, breakIsland: null, orphan: false };
6514
+ }
6414
6515
  i = close === -1 ? i + n : close + n;
6415
6516
  continue;
6416
6517
  }
@@ -6427,6 +6528,7 @@ var Tokenizer = class {
6427
6528
  continue;
6428
6529
  }
6429
6530
  const end = singleDollarMathEnd(text, i, pe);
6531
+ this.tick(end > 0 ? end - i : Math.min(403, pe - i));
6430
6532
  if (end > 0 && !(this.streaming && end === this.n)) {
6431
6533
  inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
6432
6534
  i = end;
@@ -6454,6 +6556,7 @@ var Tokenizer = class {
6454
6556
  const limit = Math.min(pe, i + 65536);
6455
6557
  const end = matchingJsonObjectEnd(text, i, limit);
6456
6558
  kindBudget -= (end ?? limit) - i;
6559
+ this.tick((end ?? limit) - i);
6457
6560
  if (end !== null) {
6458
6561
  const kind = declaredKind(text.slice(i, end));
6459
6562
  if (kind) {
@@ -6515,6 +6618,7 @@ var Tokenizer = class {
6515
6618
  continue;
6516
6619
  }
6517
6620
  const tag = readXmlTag(text, i, this.finder);
6621
+ this.tick(tag ? tag.raw.length : 1);
6518
6622
  if (tag && i + tag.raw.length <= pe) {
6519
6623
  const name = tag.tagName;
6520
6624
  const tagEnd = i + tag.raw.length;
@@ -6563,7 +6667,7 @@ var Tokenizer = class {
6563
6667
  }
6564
6668
  };
6565
6669
  function tokenizeSource(text, options = {}) {
6566
- const raw = new Tokenizer(text, options.streaming ?? false).run();
6670
+ const raw = new Tokenizer(text, options.streaming ?? false, options.meter).run();
6567
6671
  const cp = buildCodePointIndex(text);
6568
6672
  const inline = (island) => ({
6569
6673
  islandType: island.type,
@@ -6916,6 +7020,6 @@ function mapRange(changes, start, end, unit = "utf16") {
6916
7020
  return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
6917
7021
  }
6918
7022
 
6919
- export { ACTION_BY_CLASS, CALLOUT_NAMES, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, parseFenceOpener, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
7023
+ export { ACTION_BY_CLASS, CALLOUT_NAMES, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, parseFenceOpener, pendingDisplayMathOpener, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
6920
7024
  //# sourceMappingURL=index.js.map
6921
7025
  //# sourceMappingURL=index.js.map