@ai-matrx/content-ir 0.15.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/dist/index.cjs +106 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +106 -7
- package/dist/index.js.map +1 -1
- package/dist/source.cjs +106 -6
- package/dist/source.cjs.map +1 -1
- package/dist/source.d.cts +19 -1
- package/dist/source.d.ts +19 -1
- package/dist/source.js +106 -7
- package/dist/source.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
|
|
|
7
7
|
export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.cjs';
|
|
8
8
|
export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.cjs';
|
|
9
9
|
export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.cjs';
|
|
10
|
-
export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.cjs';
|
|
10
|
+
export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.cjs';
|
|
11
11
|
import 'react';
|
package/dist/index.d.ts
CHANGED
|
@@ -7,5 +7,5 @@ export { DualGateDefinition, DualGateInput, DualGateResolvedComponent, DualGateR
|
|
|
7
7
|
export { ArrayItemKindBinding, BlockSchemaDraft, ConversionProblem, DroppedMetadata, FieldComparison, ItemKindRefCheck, JsonSchemaNode, KindJsonSchemaExport, KindJsonSchemaOptions, KindSchemaFromJsonResult, SavePlanEntry, SavePlanValidation, SchemaConversionResult, buildAgentSchemaWithRenderBlockSupport, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, fieldsToDbPayload, injectKindIntoObjectSchema, isDuplicateBlockSlug, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, normalizeAiSchemaInput, runSchemaConversion, validateBlockSchemaSavePlan } from './convert.js';
|
|
8
8
|
export { AnyPartialKindEvent, IR_PARTIAL_KEY, KindVerdict, NODE_OUTCOME_KIND, NodeOutcomeWrapper, PartialKindEvent, PartialKindNode, PartialKindState, RUN_RESULT_KIND, RetractedKindEvent, RunResultWrapper, SupersededKindEvent, advancePartialKind, emitPayloadFence, emitPayloadJson, isProvisionalKind, isTerminalKindEvent, kindVerdictOf, makePartialKindStalenessGate, readNodeOutcomeValue, readOutputRef, readPartialKindEvent, readRunResultValue, rehydrateNodeOutcome, rehydrateRunResult, sanitizeInboundPartialKindMetadata, withRootKind } from './wire.js';
|
|
9
9
|
export { ACTION_BY_CLASS, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DecodedDirective, DirectiveCapability, DirectiveClass, DirectiveDecodeError, DirectiveDisplay, DirectiveItemKindLookup, DirectiveItemsEdge, DirectiveNounCatalog, DirectiveNounEntry, DirectiveSlug, IN_CONTENT_CLASSES, ItemFact, KindDirectiveShell, RESERVED_PREFIX, SIDE_EFFECT_CLASSES, SLUG_PREFIX, asKindInstance, buildDirectiveSlug, buildKindDirective, capabilityOf, decodeDirective, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, executesAtOutputRoot, isDirectiveClass, isKindDirective, isLegacyShell, isReservedDirectiveSlug, itemFacts, itemSubtitle, itemTitle, legacyShellUses, looksLikeDirectiveHead, nounFamily, nounLabel, nounTitleColumn, parseDirectiveSlug, resetLegacyShellUses, resolvesInContent, titleCaseToken, tryDecodeDirective, tryDecodeDirectiveContent } from './directives.js';
|
|
10
|
-
export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.js';
|
|
10
|
+
export { BlockIslandType, CALLOUT_NAMES, CodePointIndex, DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, InlineIslandType, InnerFenceLine, IslandMeta, IslandType, MapOptions, MappedPosition, MappedRange, NESTING_FENCE_LANGUAGES, ReadXmlTag, SourceBlock, SourceBlockKind, SourceChange, SourceEdit, SourceIsland, SourceSpan, SourceSpliceError, SourceSpliceErrorCode, SpliceIntegrity, SpliceOptions, SpliceResult, TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine } from './source.js';
|
|
11
11
|
import 'react';
|
package/dist/index.js
CHANGED
|
@@ -5315,6 +5315,50 @@ function codeRanges(text) {
|
|
|
5315
5315
|
}
|
|
5316
5316
|
return ranges.sort((a, b) => a[0] - b[0]);
|
|
5317
5317
|
}
|
|
5318
|
+
function pendingDisplayMathOpener(text) {
|
|
5319
|
+
if (!text.includes("$$")) return -1;
|
|
5320
|
+
const ranges = codeRanges(text);
|
|
5321
|
+
const tokens = [];
|
|
5322
|
+
let r = 0;
|
|
5323
|
+
let maxEnd = -1;
|
|
5324
|
+
for (let i = text.indexOf("$$"); i !== -1 && i < text.length - 1; i = text.indexOf("$$", i)) {
|
|
5325
|
+
while (r < ranges.length && ranges[r][0] <= i) {
|
|
5326
|
+
maxEnd = Math.max(maxEnd, ranges[r][1]);
|
|
5327
|
+
r++;
|
|
5328
|
+
}
|
|
5329
|
+
if (maxEnd <= i) tokens.push(i);
|
|
5330
|
+
i += 2;
|
|
5331
|
+
}
|
|
5332
|
+
const paired = /* @__PURE__ */ new Set();
|
|
5333
|
+
let j = 0;
|
|
5334
|
+
while (j < tokens.length) {
|
|
5335
|
+
const open = tokens[j];
|
|
5336
|
+
const close = tokens[j + 1];
|
|
5337
|
+
if (close === void 0) break;
|
|
5338
|
+
if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
|
|
5339
|
+
paired.add(open);
|
|
5340
|
+
paired.add(close);
|
|
5341
|
+
j += 2;
|
|
5342
|
+
continue;
|
|
5343
|
+
}
|
|
5344
|
+
j += 1;
|
|
5345
|
+
}
|
|
5346
|
+
let trail = 0;
|
|
5347
|
+
while (trail < text.length && /\s/.test(text[text.length - 1 - trail])) trail++;
|
|
5348
|
+
for (const token of tokens) {
|
|
5349
|
+
if (paired.has(token)) continue;
|
|
5350
|
+
let lead = 0;
|
|
5351
|
+
while (lead <= MAX_MATH_SPAN && /\s/.test(text[token + 2 + lead] ?? "x")) lead++;
|
|
5352
|
+
if (text.length - (token + 2) - lead - trail > MAX_MATH_SPAN) continue;
|
|
5353
|
+
const inner = text.slice(token + 2);
|
|
5354
|
+
const s = inner.trim();
|
|
5355
|
+
if (STRUCTURAL_MARKDOWN.test(s) || s.length > MAX_MATH_SPAN) continue;
|
|
5356
|
+
if (!opensFlowBlock(text, token) && /\n[ \t]*\n/.test(inner)) continue;
|
|
5357
|
+
if ((s.match(PROSE_WORD) ?? []).length >= PROSE_WORD_LIMIT * 3) continue;
|
|
5358
|
+
return token;
|
|
5359
|
+
}
|
|
5360
|
+
return -1;
|
|
5361
|
+
}
|
|
5318
5362
|
function pairDisplayMath(text) {
|
|
5319
5363
|
const pairs = /* @__PURE__ */ new Map();
|
|
5320
5364
|
if (!text.includes("$$")) return pairs;
|
|
@@ -5663,11 +5707,15 @@ function isSplitterTreeLine(line) {
|
|
|
5663
5707
|
}
|
|
5664
5708
|
var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
|
|
5665
5709
|
var Tokenizer = class {
|
|
5666
|
-
constructor(text, streaming = false) {
|
|
5710
|
+
constructor(text, streaming = false, meter = void 0) {
|
|
5667
5711
|
this.text = text;
|
|
5668
5712
|
this.streaming = streaming;
|
|
5713
|
+
this.meter = meter;
|
|
5669
5714
|
this.n = text.length;
|
|
5715
|
+
this.tick(text.length);
|
|
5670
5716
|
this.mathPairs = pairDisplayMath(text);
|
|
5717
|
+
this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
|
|
5718
|
+
if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
|
|
5671
5719
|
let start = 0;
|
|
5672
5720
|
for (let i = 0; i < text.length; i++) {
|
|
5673
5721
|
if (text.charCodeAt(i) === 10) {
|
|
@@ -5678,13 +5726,15 @@ var Tokenizer = class {
|
|
|
5678
5726
|
}
|
|
5679
5727
|
}
|
|
5680
5728
|
if (start < text.length || text.length === 0) {
|
|
5729
|
+
this.pendingCR = streaming && text.length > start && text.charCodeAt(text.length - 1) === 13;
|
|
5681
5730
|
this.lineStarts.push(start);
|
|
5682
|
-
this.contentEnds.push(text.length);
|
|
5731
|
+
this.contentEnds.push(this.pendingCR ? text.length - 1 : text.length);
|
|
5683
5732
|
this.lineEnds.push(text.length);
|
|
5684
5733
|
}
|
|
5685
5734
|
}
|
|
5686
5735
|
text;
|
|
5687
5736
|
streaming;
|
|
5737
|
+
meter;
|
|
5688
5738
|
n;
|
|
5689
5739
|
lineStarts = [];
|
|
5690
5740
|
contentEnds = [];
|
|
@@ -5699,6 +5749,23 @@ var Tokenizer = class {
|
|
|
5699
5749
|
/** `$$` opener → closer for the pairs the core renders as math. */
|
|
5700
5750
|
mathPairs;
|
|
5701
5751
|
finder = (needle, from) => this.next(needle, from);
|
|
5752
|
+
/** Streaming: where provisional text starts (a pending `$$`), or +∞. */
|
|
5753
|
+
provisionalFrom = Number.POSITIVE_INFINITY;
|
|
5754
|
+
/** Streaming: the text ends in a lone CR whose LF has not arrived yet. */
|
|
5755
|
+
pendingCR = false;
|
|
5756
|
+
/**
|
|
5757
|
+
* Streaming: an open last line that begins with `{` is undecided — each new
|
|
5758
|
+
* byte can balance or unbalance its braces and flip the whole line between
|
|
5759
|
+
* prose and a JSON island — so it is provisional until it ends (G5).
|
|
5760
|
+
*/
|
|
5761
|
+
markOpenJsonTail() {
|
|
5762
|
+
const last = this.lineStarts.length - 1;
|
|
5763
|
+
if (!this.isOpenTail(last)) return;
|
|
5764
|
+
const start = this.lineStarts[last];
|
|
5765
|
+
if (this.lineText(last).trimStart().startsWith("{")) {
|
|
5766
|
+
this.provisionalFrom = Math.min(this.provisionalFrom, start);
|
|
5767
|
+
}
|
|
5768
|
+
}
|
|
5702
5769
|
/**
|
|
5703
5770
|
* `text.indexOf(needle, from)` in O(log k): the first query for a needle
|
|
5704
5771
|
* indexes all of its occurrences once, so repeated searches for a closer
|
|
@@ -5722,9 +5789,21 @@ var Tokenizer = class {
|
|
|
5722
5789
|
}
|
|
5723
5790
|
return lo < list.length ? list[lo] : -1;
|
|
5724
5791
|
}
|
|
5792
|
+
/** Charge `n` visited characters to the diagnostic meter (complexity tests). */
|
|
5793
|
+
tick(n) {
|
|
5794
|
+
if (this.meter) this.meter.work += n;
|
|
5795
|
+
}
|
|
5796
|
+
/** Streaming only: a paragraph ending at `pe` may still be continued by a line not yet arrived. */
|
|
5797
|
+
paragraphMayGrow(pe) {
|
|
5798
|
+
if (!this.streaming) return false;
|
|
5799
|
+
const rest = this.n - pe;
|
|
5800
|
+
if (rest === 0) return true;
|
|
5801
|
+
const tail = this.text.slice(pe);
|
|
5802
|
+
return tail === "\n" || tail === "\r\n" || tail === "\r";
|
|
5803
|
+
}
|
|
5725
5804
|
/** Streaming only: the final line, still waiting for its terminator. */
|
|
5726
5805
|
isOpenTail(k) {
|
|
5727
|
-
return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
|
|
5806
|
+
return this.streaming && k === this.lineStarts.length - 1 && (this.lineEnds[k] === this.contentEnds[k] || this.pendingCR);
|
|
5728
5807
|
}
|
|
5729
5808
|
get lineCount() {
|
|
5730
5809
|
return this.lineStarts.length;
|
|
@@ -5749,6 +5828,7 @@ var Tokenizer = class {
|
|
|
5749
5828
|
return this.text.slice(this.lineStarts[li], this.contentEnds[li]);
|
|
5750
5829
|
}
|
|
5751
5830
|
run() {
|
|
5831
|
+
if (this.streaming) this.markOpenJsonTail();
|
|
5752
5832
|
let pos = 0;
|
|
5753
5833
|
while (pos < this.n) {
|
|
5754
5834
|
const li = this.lineOf(pos);
|
|
@@ -5757,6 +5837,10 @@ var Tokenizer = class {
|
|
|
5757
5837
|
pos = this.gap(pos, li);
|
|
5758
5838
|
continue;
|
|
5759
5839
|
}
|
|
5840
|
+
if (pos >= this.provisionalFrom) {
|
|
5841
|
+
this.push({ kind: "prose", start: pos, end: this.n, island: null, inlines: [] });
|
|
5842
|
+
break;
|
|
5843
|
+
}
|
|
5760
5844
|
const island = this.detectBlock(pos, li, true);
|
|
5761
5845
|
if (island) {
|
|
5762
5846
|
this.push({ kind: "island", start: island.start, end: island.end, island, inlines: [] });
|
|
@@ -5960,6 +6044,7 @@ var Tokenizer = class {
|
|
|
5960
6044
|
continue;
|
|
5961
6045
|
}
|
|
5962
6046
|
if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
|
|
6047
|
+
if (this.isOpenTail(k)) break;
|
|
5963
6048
|
last = k;
|
|
5964
6049
|
}
|
|
5965
6050
|
return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
|
|
@@ -6121,6 +6206,7 @@ var Tokenizer = class {
|
|
|
6121
6206
|
const li = this.lineOf(tp);
|
|
6122
6207
|
const ce = this.contentEnds[li];
|
|
6123
6208
|
const tag = readXmlTag(this.text, tp, this.finder);
|
|
6209
|
+
this.tick(tag ? tag.raw.length : 1);
|
|
6124
6210
|
let openEnd;
|
|
6125
6211
|
if (tag && !tag.isClosing) {
|
|
6126
6212
|
if (tag.isSelfClosing) {
|
|
@@ -6205,7 +6291,7 @@ var Tokenizer = class {
|
|
|
6205
6291
|
return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
|
|
6206
6292
|
}
|
|
6207
6293
|
const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
|
|
6208
|
-
const typeSeven = atParaStart && t.trimEnd() === tag.raw;
|
|
6294
|
+
const typeSeven = atParaStart && t.trimEnd() === tag.raw && !this.isOpenTail(li);
|
|
6209
6295
|
if (!typeSix && !typeSeven) return null;
|
|
6210
6296
|
let end = this.contentEnds[li];
|
|
6211
6297
|
for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
|
|
@@ -6297,6 +6383,10 @@ var Tokenizer = class {
|
|
|
6297
6383
|
let breakLine = -1;
|
|
6298
6384
|
for (let k = li + 1; k < this.lineCount; k++) {
|
|
6299
6385
|
if (this.lineIsBlank(k)) break;
|
|
6386
|
+
if (this.lineStarts[k] >= this.provisionalFrom) {
|
|
6387
|
+
end = this.n;
|
|
6388
|
+
break;
|
|
6389
|
+
}
|
|
6300
6390
|
if (this.detectBlock(this.lineStarts[k], k, false)) {
|
|
6301
6391
|
breakLine = k;
|
|
6302
6392
|
break;
|
|
@@ -6362,6 +6452,7 @@ var Tokenizer = class {
|
|
|
6362
6452
|
scanInline(ps, pe) {
|
|
6363
6453
|
const text = this.text;
|
|
6364
6454
|
const inlines = [];
|
|
6455
|
+
this.tick(pe - ps);
|
|
6365
6456
|
const hasKind = text.slice(ps, pe).includes("__kind");
|
|
6366
6457
|
let kindBudget = 4 * (pe - ps) + 65536;
|
|
6367
6458
|
const stop = (island, orphan = false) => ({ inlines, breakIsland: island, orphan });
|
|
@@ -6372,6 +6463,7 @@ var Tokenizer = class {
|
|
|
6372
6463
|
let runs = null;
|
|
6373
6464
|
const codeSpanClose = (n, from) => {
|
|
6374
6465
|
if (!runs) {
|
|
6466
|
+
this.tick(pe - ps);
|
|
6375
6467
|
runs = /* @__PURE__ */ new Map();
|
|
6376
6468
|
for (let k = ps; k < pe; ) {
|
|
6377
6469
|
if (text.charCodeAt(k) !== 96) {
|
|
@@ -6398,7 +6490,8 @@ var Tokenizer = class {
|
|
|
6398
6490
|
return at !== void 0 && at + n <= pe ? at : -1;
|
|
6399
6491
|
};
|
|
6400
6492
|
let i = ps;
|
|
6401
|
-
|
|
6493
|
+
const scanEnd = Math.min(pe, this.provisionalFrom);
|
|
6494
|
+
while (i < scanEnd) {
|
|
6402
6495
|
const ch = text[i];
|
|
6403
6496
|
if (ch === "\\") {
|
|
6404
6497
|
const nextCh = text[i + 1];
|
|
@@ -6416,6 +6509,9 @@ var Tokenizer = class {
|
|
|
6416
6509
|
if (ch === "`") {
|
|
6417
6510
|
const n = backtickRunLength(text, i);
|
|
6418
6511
|
const close = codeSpanClose(n, i + n);
|
|
6512
|
+
if (close === -1 && this.streaming && this.paragraphMayGrow(pe)) {
|
|
6513
|
+
return { inlines, breakIsland: null, orphan: false };
|
|
6514
|
+
}
|
|
6419
6515
|
i = close === -1 ? i + n : close + n;
|
|
6420
6516
|
continue;
|
|
6421
6517
|
}
|
|
@@ -6432,6 +6528,7 @@ var Tokenizer = class {
|
|
|
6432
6528
|
continue;
|
|
6433
6529
|
}
|
|
6434
6530
|
const end = singleDollarMathEnd(text, i, pe);
|
|
6531
|
+
this.tick(end > 0 ? end - i : Math.min(403, pe - i));
|
|
6435
6532
|
if (end > 0 && !(this.streaming && end === this.n)) {
|
|
6436
6533
|
inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
|
|
6437
6534
|
i = end;
|
|
@@ -6459,6 +6556,7 @@ var Tokenizer = class {
|
|
|
6459
6556
|
const limit = Math.min(pe, i + 65536);
|
|
6460
6557
|
const end = matchingJsonObjectEnd(text, i, limit);
|
|
6461
6558
|
kindBudget -= (end ?? limit) - i;
|
|
6559
|
+
this.tick((end ?? limit) - i);
|
|
6462
6560
|
if (end !== null) {
|
|
6463
6561
|
const kind = declaredKind(text.slice(i, end));
|
|
6464
6562
|
if (kind) {
|
|
@@ -6520,6 +6618,7 @@ var Tokenizer = class {
|
|
|
6520
6618
|
continue;
|
|
6521
6619
|
}
|
|
6522
6620
|
const tag = readXmlTag(text, i, this.finder);
|
|
6621
|
+
this.tick(tag ? tag.raw.length : 1);
|
|
6523
6622
|
if (tag && i + tag.raw.length <= pe) {
|
|
6524
6623
|
const name = tag.tagName;
|
|
6525
6624
|
const tagEnd = i + tag.raw.length;
|
|
@@ -6568,7 +6667,7 @@ var Tokenizer = class {
|
|
|
6568
6667
|
}
|
|
6569
6668
|
};
|
|
6570
6669
|
function tokenizeSource(text, options = {}) {
|
|
6571
|
-
const raw = new Tokenizer(text, options.streaming ?? false).run();
|
|
6670
|
+
const raw = new Tokenizer(text, options.streaming ?? false, options.meter).run();
|
|
6572
6671
|
const cp = buildCodePointIndex(text);
|
|
6573
6672
|
const inline = (island) => ({
|
|
6574
6673
|
islandType: island.type,
|
|
@@ -6921,6 +7020,6 @@ function mapRange(changes, start, end, unit = "utf16") {
|
|
|
6921
7020
|
return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
|
|
6922
7021
|
}
|
|
6923
7022
|
|
|
6924
|
-
export { ACTION_BY_CLASS, CALLOUT_NAMES, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, parseFenceOpener, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
|
|
7023
|
+
export { ACTION_BY_CLASS, CALLOUT_NAMES, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, parseFenceOpener, pendingDisplayMathOpener, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
|
|
6925
7024
|
//# sourceMappingURL=index.js.map
|
|
6926
7025
|
//# sourceMappingURL=index.js.map
|