@ai-matrx/content-ir 0.18.4 → 0.18.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -1
- package/README.md +3 -1
- package/dist/index.cjs +211 -29
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +210 -30
- package/dist/index.js.map +1 -1
- package/dist/source.cjs +211 -29
- package/dist/source.cjs.map +1 -1
- package/dist/source.d.cts +24 -2
- package/dist/source.d.ts +24 -2
- package/dist/source.js +210 -30
- package/dist/source.js.map +1 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,12 +1,72 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.18.6
|
|
4
|
+
|
|
5
|
+
Automatic changed-only republish (docs/metadata drift since the last tag — see
|
|
6
|
+
`git diff npm/content-ir-core/v0.18.5..npm/content-ir-core/v0.18.6 -- apps/shared/content-ir-core`).
|
|
7
|
+
No source changes intended and no consumer action required.
|
|
8
|
+
|
|
9
|
+
## 0.19.0
|
|
10
|
+
|
|
11
|
+
**Carries: RC-B3 fix round (verify-RC-B3 § 0.18.3) and RC-B3r R3 — builder:
|
|
12
|
+
the RC-B3 standard lane.** (aidream 1174edb640 — RC-B10, the print lane —
|
|
13
|
+
shipped in 0.18.3, and 094594d3a1 — the escaped-bracket rule, "Unreleased"
|
|
14
|
+
below — shipped in 0.18.4; both under generic "no source changes" notes.)
|
|
15
|
+
|
|
16
|
+
**`/source` streaming: settled means settled** (F3/F4). Found with the
|
|
17
|
+
verifier's generator (realistic model fragments, every prefix, with and
|
|
18
|
+
without a final newline), now permanent in `__tests__/source-attack.test.ts`
|
|
19
|
+
(`LLM-fragment documents…`; 12,000 documents in `SOURCE_ATTACK=long`) plus 27
|
|
20
|
+
pinned reproducers. `tokenizeSource(prefix, { streaming: true })` now also
|
|
21
|
+
holds back, as prose or as an open island:
|
|
22
|
+
- lines a box-drawing tree still forming at the end may claim as its root
|
|
23
|
+
label (the renderer's walk-back), the open last line included;
|
|
24
|
+
- a paragraph cut short by a block that may still dissolve (a `$$` pair
|
|
25
|
+
closing on the open line, an unbalanced JSON head, any opener still
|
|
26
|
+
arriving on the open line) — and a footnote before one;
|
|
27
|
+
- everything inline on the open last line, and a block island ending there;
|
|
28
|
+
- a nested-fence closer found only by the strict-CommonMark retry, a tilde
|
|
29
|
+
fence whose inner block may still grow past it, braces that balance on
|
|
30
|
+
the open last line;
|
|
31
|
+
- a code span that a blank-so-far or `>`-only open line may still close
|
|
32
|
+
(`pendingCodeSpanStart`, `source/code-ranges.ts`).
|
|
33
|
+
The two lone-HTML-tag holdbacks (M-N6/N7) collapsed into the open-tail block
|
|
34
|
+
rule. Each holdback turns the attack suite red on its own when removed
|
|
35
|
+
(planted mutations on a scratch copy). The H2 exception is unchanged.
|
|
36
|
+
|
|
37
|
+
**`/source` one-shot: a mid-line reasoning mention that never closes is prose**
|
|
38
|
+
(RC-B3r R3). `grep -c '<thinking>' log.txt > count.txt` no longer opens an
|
|
39
|
+
`xml_region` to the end of the text (the renderer shows prose). While a stream
|
|
40
|
+
is still arriving it stays an open region (its closer may yet come).
|
|
41
|
+
|
|
42
|
+
**`isSingleDollarMath` refuses shell and template variable chains** (F5):
|
|
43
|
+
`$USER$HOSTNAME`, `A=$A$B`, `df$col$sub`, `$name.$ext`, `PS1='$USER@$HOST'`,
|
|
44
|
+
`${a}${b}` stay text; `$n$th`, `$x_i$s` stay math. Measured read-only with
|
|
45
|
+
matrx-frontend `scripts/measure-single-dollar-rule.ts` over 85,325 assistant
|
|
46
|
+
messages and 4,317 notes: 0 spans gained, 3 lost (2 shell-variable-chain, 1
|
|
47
|
+
template-literal), each an intended false positive; notes unchanged. New
|
|
48
|
+
export `singleDollarAfter(text, close)` — the `after` argument the rule now
|
|
49
|
+
reads (the following name, `\n` marking the end of text). Shared vectors grew
|
|
50
|
+
(`__tests__/fixtures/single-dollar-vectors.json`, including the trailing-space
|
|
51
|
+
guard case `echo $HOME $PATH`).
|
|
52
|
+
- Consumer action: callers of `isSingleDollarMath` pass
|
|
53
|
+
`singleDollarAfter(text, close)` instead of `text[close + 1]` (a one-character
|
|
54
|
+
`after` still works; it just cannot see a variable chain).
|
|
55
|
+
|
|
56
|
+
## 0.18.5
|
|
57
|
+
|
|
58
|
+
Automatic republish. Despite the generated note, this version DID carry source
|
|
59
|
+
changes: the first half of the 0.19.0 work above (aidream edb3ec222d streaming
|
|
60
|
+
holds, 3a1dae805c + 9a6c8db0b2 variable-chain guard, 60d3f61803
|
|
61
|
+
`singleDollarAfter`). 0.19.0 supersedes it.
|
|
62
|
+
|
|
3
63
|
## 0.18.4
|
|
4
64
|
|
|
5
65
|
Automatic changed-only republish (docs/metadata drift since the last tag — see
|
|
6
66
|
`git diff npm/content-ir-core/v0.18.3..npm/content-ir-core/v0.18.4 -- apps/shared/content-ir-core`).
|
|
7
67
|
No source changes intended and no consumer action required.
|
|
8
68
|
|
|
9
|
-
##
|
|
69
|
+
## Shipped in 0.18.4 (listed here before the release)
|
|
10
70
|
|
|
11
71
|
**`/source`: THE escaped-bracket rule** (verify-RC-B4 R3-2). `\[ … \]` is a
|
|
12
72
|
display-math island only when its content is real TeX — a command, `^ { } =
|
|
@@ -891,3 +951,4 @@ unchanged. Two things are worth doing at your next working session:
|
|
|
891
951
|
- Promoted the existing verified Content IR twin into an independently installable public package.
|
|
892
952
|
- Added ESM and declaration builds, package linting, packed-artifact analysis, and an isolated consumer canary.
|
|
893
953
|
- Preserved all parser, session, normalization, schema conversion, and structural-validation behavior with the existing shared fixtures and fingerprint vectors.
|
|
954
|
+
|
package/README.md
CHANGED
|
@@ -48,7 +48,9 @@ The package has no browser, React runtime, application store, database client, r
|
|
|
48
48
|
`tokenizeSource(text)` reads stored text; `tokenizeSource(prefix, { streaming: true })`
|
|
49
49
|
reads a message still arriving. In streaming mode anything the next bytes could
|
|
50
50
|
still change is held back as plain prose (an open last line, an unclosed code
|
|
51
|
-
span, a `$$` that can still pair, a footnote whose continuation is undecided,
|
|
51
|
+
span, a `$$` that can still pair, a footnote whose continuation is undecided,
|
|
52
|
+
lines a box-drawing tree still forming below may claim as its root label, a
|
|
53
|
+
paragraph cut short by a block that may still dissolve, …),
|
|
52
54
|
so **an island reported `complete` is the island the finished text has** — with
|
|
53
55
|
ONE documented exception:
|
|
54
56
|
|
package/dist/index.cjs
CHANGED
|
@@ -5296,6 +5296,7 @@ var FenceReader = class {
|
|
|
5296
5296
|
function closeFence(src, open, opener) {
|
|
5297
5297
|
const first = readFence(src, open, new FenceReader(opener));
|
|
5298
5298
|
if (first.closed || !first.reader.sawNested || first.reader.nestedDepth <= 0) return { line: first.line, closed: first.closed };
|
|
5299
|
+
if (src.streaming) return { line: first.line, closed: false };
|
|
5299
5300
|
const strict = readFence(src, open, new FenceReader(opener, { nesting: false }));
|
|
5300
5301
|
return { line: strict.line, closed: strict.closed };
|
|
5301
5302
|
}
|
|
@@ -5559,6 +5560,8 @@ function fenceLineKinds(text) {
|
|
|
5559
5560
|
function pendingCodeSpanStart(text) {
|
|
5560
5561
|
const n = text.length;
|
|
5561
5562
|
if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
|
|
5563
|
+
const lastLine = text.slice(text.lastIndexOf("\n") + 1);
|
|
5564
|
+
if (lastLine && /^(?: {0,3}>[ \t]?)*[ \t]*$/.test(lastLine)) return pendingCodeSpanStart(`${text}x`);
|
|
5562
5565
|
const s = scan(text);
|
|
5563
5566
|
const para = s.pendingPara;
|
|
5564
5567
|
if (s.endsInFence || para === -1) return -1;
|
|
@@ -5705,13 +5708,26 @@ var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
|
|
|
5705
5708
|
function isSingleDollarMath(content, after) {
|
|
5706
5709
|
if (!content || content.length > 400) return false;
|
|
5707
5710
|
if (/^\s/.test(content) || /\s$/.test(content)) return false;
|
|
5708
|
-
if (after !== void 0 &&
|
|
5711
|
+
if (after !== void 0 && /^[0-9]/.test(after)) return false;
|
|
5712
|
+
if (isVariableChain(content, after)) return false;
|
|
5709
5713
|
if (/(?:^|[^\\])%/.test(content)) return false;
|
|
5710
5714
|
if (/^[)\]}"/]/.test(content) || /[(\[{":;,/]$/.test(content)) return false;
|
|
5711
|
-
if (/^[0-9]/.test(content) && after !== void 0 &&
|
|
5715
|
+
if (/^[0-9]/.test(content) && after !== void 0 && /^[A-Za-z]/.test(after)) return false;
|
|
5712
5716
|
if (isMathPlaceholder(content)) return false;
|
|
5713
5717
|
return TEX_SIGNAL.test(content) || !readsAsProse(content);
|
|
5714
5718
|
}
|
|
5719
|
+
var IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
5720
|
+
var MATH_SUFFIX = /^(?:s|es|th|st|nd|rd)$/i;
|
|
5721
|
+
function isVariableChain(content, after) {
|
|
5722
|
+
if (/^\{[A-Za-z_][\w.]*\}$/.test(content)) return true;
|
|
5723
|
+
if (!after) return false;
|
|
5724
|
+
if (after[0] === "{") return IDENTIFIER.test(content);
|
|
5725
|
+
const run = /^[A-Za-z0-9_]+/.exec(after)?.[0];
|
|
5726
|
+
if (!run) return false;
|
|
5727
|
+
if (after.length === 1) return false;
|
|
5728
|
+
if (/^[A-Za-z_][A-Za-z0-9_]*[.@-]$/.test(content)) return true;
|
|
5729
|
+
return IDENTIFIER.test(content) && !MATH_SUFFIX.test(run);
|
|
5730
|
+
}
|
|
5715
5731
|
var FUNCTION_NAMES = /* @__PURE__ */ new Set([
|
|
5716
5732
|
"sin",
|
|
5717
5733
|
"cos",
|
|
@@ -5755,6 +5771,12 @@ function isMathShape(content) {
|
|
|
5755
5771
|
if (/[A-Za-z]'+[A-Za-z]/.test(content) || /(?:^|[\s(,])[A-Za-z]\.(?:[A-Za-z]\.|\s|$)/.test(content)) return false;
|
|
5756
5772
|
return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
|
|
5757
5773
|
}
|
|
5774
|
+
var AFTER_WINDOW = 32;
|
|
5775
|
+
function singleDollarAfter(text, close) {
|
|
5776
|
+
const rest = text.slice(close + 1, close + 1 + AFTER_WINDOW);
|
|
5777
|
+
return rest.length < AFTER_WINDOW ? `${rest}
|
|
5778
|
+
` : rest;
|
|
5779
|
+
}
|
|
5758
5780
|
function singleDollarMathEnd(text, open, limit = text.length) {
|
|
5759
5781
|
const bound = Math.min(limit, open + 403);
|
|
5760
5782
|
let close = -1;
|
|
@@ -5772,7 +5794,7 @@ function singleDollarMathEnd(text, open, limit = text.length) {
|
|
|
5772
5794
|
}
|
|
5773
5795
|
if (close === -1 || text[close + 1] === "$") return -1;
|
|
5774
5796
|
const content = text.slice(open + 1, close);
|
|
5775
|
-
return isSingleDollarMath(content, text
|
|
5797
|
+
return isSingleDollarMath(content, singleDollarAfter(text, close)) ? close + 1 : -1;
|
|
5776
5798
|
}
|
|
5777
5799
|
var BRACKET_TEX_SIGNAL = /\\[A-Za-z]+|[\^{}=<>+]|_\{|(?:^|[^A-Za-z0-9])[A-Za-z]_[A-Za-z0-9](?![A-Za-z0-9])/;
|
|
5778
5800
|
function isEscapedBracketMath(content, before) {
|
|
@@ -6143,18 +6165,14 @@ var Tokenizer = class {
|
|
|
6143
6165
|
/**
|
|
6144
6166
|
* Streaming: an open last line that begins with `{` is undecided — each new
|
|
6145
6167
|
* byte can balance or unbalance its braces and flip the whole line between
|
|
6146
|
-
* prose and a JSON island —
|
|
6147
|
-
* HTML tag
|
|
6168
|
+
* prose and a JSON island — so it is provisional until it ends (G5).
|
|
6169
|
+
* (A lone HTML tag on that line is held by {@link holdUndecidedTail}.)
|
|
6148
6170
|
*/
|
|
6149
6171
|
markOpenJsonTail() {
|
|
6150
6172
|
const last = this.lineStarts.length - 1;
|
|
6151
6173
|
if (!this.isOpenTail(last)) return;
|
|
6152
|
-
|
|
6153
|
-
|
|
6154
|
-
const oneTag = t.startsWith("<") ? readXmlTag(t, 0) : null;
|
|
6155
|
-
const loneTag = !!oneTag && oneTag.raw === t && ALLOWED_RAW_HTML_TAGS.has(oneTag.tagName.toLowerCase());
|
|
6156
|
-
if (t.startsWith("{") || loneTag) {
|
|
6157
|
-
this.provisionalFrom = Math.min(this.provisionalFrom, start);
|
|
6174
|
+
if (this.lineText(last).trim().startsWith("{")) {
|
|
6175
|
+
this.provisionalFrom = Math.min(this.provisionalFrom, this.lineStarts[last]);
|
|
6158
6176
|
}
|
|
6159
6177
|
}
|
|
6160
6178
|
/**
|
|
@@ -6187,11 +6205,25 @@ var Tokenizer = class {
|
|
|
6187
6205
|
/** Streaming only: a paragraph ending at `pe` may still be continued by a line not yet arrived. */
|
|
6188
6206
|
paragraphMayGrow(pe) {
|
|
6189
6207
|
if (!this.streaming) return false;
|
|
6208
|
+
if (this.undecidedBreaks.has(pe)) return true;
|
|
6190
6209
|
const rest = this.n - pe;
|
|
6191
6210
|
if (rest === 0) return true;
|
|
6192
6211
|
const tail = this.text.slice(pe);
|
|
6193
|
-
return
|
|
6212
|
+
return /^(?:\r\n|\r|\n)[ \t]*$/.test(tail);
|
|
6194
6213
|
}
|
|
6214
|
+
/**
|
|
6215
|
+
* Streaming: an open block that may still turn back into prose — a `$$`
|
|
6216
|
+
* pair whose closer is on the open last line (prose after it makes it
|
|
6217
|
+
* inline), a JSON head whose braces have not balanced (it may never parse),
|
|
6218
|
+
* or any opener that is itself still arriving on the open last line.
|
|
6219
|
+
*/
|
|
6220
|
+
mayDissolve(block) {
|
|
6221
|
+
if (!this.streaming || block.complete) return false;
|
|
6222
|
+
if (block.type === "math_block" || block.type === "json") return true;
|
|
6223
|
+
return this.isOpenTail(this.lineOf(block.start));
|
|
6224
|
+
}
|
|
6225
|
+
/** Streaming: paragraph ends where an undecided, dissolvable block cut them short. */
|
|
6226
|
+
undecidedBreaks = /* @__PURE__ */ new Set();
|
|
6195
6227
|
/** Streaming only: the final line, still waiting for its terminator. */
|
|
6196
6228
|
isOpenTail(k) {
|
|
6197
6229
|
return this.streaming && k === this.lineStarts.length - 1 && (this.lineEnds[k] === this.contentEnds[k] || this.pendingCR);
|
|
@@ -6209,6 +6241,12 @@ var Tokenizer = class {
|
|
|
6209
6241
|
}
|
|
6210
6242
|
return lo;
|
|
6211
6243
|
}
|
|
6244
|
+
/** How many whitespace characters open the text at `from` (up to `to`). */
|
|
6245
|
+
leadingSpace(from, to) {
|
|
6246
|
+
let k = from;
|
|
6247
|
+
while (k < to && /\s/.test(this.text[k])) k++;
|
|
6248
|
+
return k - from;
|
|
6249
|
+
}
|
|
6212
6250
|
isBlank(from, to) {
|
|
6213
6251
|
return !NOT_WHITESPACE.test(this.text.slice(from, to));
|
|
6214
6252
|
}
|
|
@@ -6233,6 +6271,10 @@ var Tokenizer = class {
|
|
|
6233
6271
|
break;
|
|
6234
6272
|
}
|
|
6235
6273
|
const island = this.detectBlock(pos, li, true);
|
|
6274
|
+
if (island && pos + this.leadingSpace(pos, this.contentEnds[li]) >= this.provisionalFrom) {
|
|
6275
|
+
this.push({ kind: "prose", start: pos, end: this.n, island: null, inlines: [] });
|
|
6276
|
+
break;
|
|
6277
|
+
}
|
|
6236
6278
|
if (island) {
|
|
6237
6279
|
this.push({ kind: "island", start: island.start, end: island.end, island, inlines: [] });
|
|
6238
6280
|
pos = island.end;
|
|
@@ -6240,8 +6282,122 @@ var Tokenizer = class {
|
|
|
6240
6282
|
}
|
|
6241
6283
|
pos = this.paragraph(pos, li);
|
|
6242
6284
|
}
|
|
6285
|
+
if (this.streaming) this.holdUndecidedTail();
|
|
6243
6286
|
return this.blocks;
|
|
6244
6287
|
}
|
|
6288
|
+
/**
|
|
6289
|
+
* Streaming: settle nothing the next bytes can still change (verify-RC-B3 F3).
|
|
6290
|
+
*
|
|
6291
|
+
* 1. A block island that ends on the open last line is undecided — the next
|
|
6292
|
+
* bytes on that line can re-read it or dissolve it (`<!--@a:k1-->` +
|
|
6293
|
+
* ` more` is an inline anchor, `<br>` + `x` is prose). It runs open to
|
|
6294
|
+
* the end of the text.
|
|
6295
|
+
* 2. A box-drawing tree still forming at the end of the text claims, as its
|
|
6296
|
+
* root label, the lines directly above its first connector (the
|
|
6297
|
+
* renderer's `findTreeBlockStart`), and the open last line itself may
|
|
6298
|
+
* become one of its connector lines — so inlines on those lines are held
|
|
6299
|
+
* back as prose until a blank line, a heading or a tree line decides.
|
|
6300
|
+
* This also holds everything a closer still able to arrive on the open
|
|
6301
|
+
* last line could change (`$x$` + `5` is currency, G4; `{{v}}` + `}`).
|
|
6302
|
+
*/
|
|
6303
|
+
holdUndecidedTail() {
|
|
6304
|
+
const last = this.lineStarts.length - 1;
|
|
6305
|
+
if (last < 0) return;
|
|
6306
|
+
if (this.isOpenTail(last)) {
|
|
6307
|
+
const lineStart = this.lineStarts[last];
|
|
6308
|
+
for (const block2 of this.blocks) {
|
|
6309
|
+
if (block2.kind === "prose" && block2.end > lineStart) {
|
|
6310
|
+
block2.inlines = block2.inlines.filter((inline) => inline.end <= lineStart);
|
|
6311
|
+
}
|
|
6312
|
+
}
|
|
6313
|
+
let b = this.blocks.length - 1;
|
|
6314
|
+
while (b >= 0 && this.blocks[b].kind === "gap") b--;
|
|
6315
|
+
const block = this.blocks[b];
|
|
6316
|
+
if (block?.island?.complete && block.island.end > this.lineStarts[last]) {
|
|
6317
|
+
block.island.complete = false;
|
|
6318
|
+
block.island.end = this.n;
|
|
6319
|
+
block.end = this.n;
|
|
6320
|
+
this.blocks.length = b + 1;
|
|
6321
|
+
}
|
|
6322
|
+
}
|
|
6323
|
+
const from = this.treeRootHoldFrom();
|
|
6324
|
+
for (let b = this.blocks.length - 1; b >= 0; b--) {
|
|
6325
|
+
const block = this.blocks[b];
|
|
6326
|
+
if (block.end <= from) break;
|
|
6327
|
+
if (block.kind === "prose") block.inlines = block.inlines.filter((inline) => inline.end <= from);
|
|
6328
|
+
if (block.island) this.blocks[b] = { kind: "prose", start: block.start, end: block.end, island: null, inlines: [] };
|
|
6329
|
+
}
|
|
6330
|
+
}
|
|
6331
|
+
/**
|
|
6332
|
+
* Streaming: where a tree that may still form at the end of the text would
|
|
6333
|
+
* start claiming lines (`n` when none can). Only paragraph lines can be
|
|
6334
|
+
* claimed — a line that starts inside an island (a fence body, an XML
|
|
6335
|
+
* region…) is never a tree line nor a root label, so it stops the walk.
|
|
6336
|
+
*/
|
|
6337
|
+
treeRootHoldFrom() {
|
|
6338
|
+
const claimFrom = (k2) => {
|
|
6339
|
+
const lineStart = this.lineStarts[k2];
|
|
6340
|
+
const lineEnd = this.contentEnds[k2];
|
|
6341
|
+
let pos = lineStart;
|
|
6342
|
+
for (; ; ) {
|
|
6343
|
+
const block = this.blockAt(pos);
|
|
6344
|
+
if (!block || block.kind !== "island" || !block.island) return pos;
|
|
6345
|
+
if (block.island.fromParagraph === true || this.mayDissolve(block.island)) return pos;
|
|
6346
|
+
if (block.start < lineStart) {
|
|
6347
|
+
const opened = this.lineOf(block.start);
|
|
6348
|
+
if (this.lineStarts[opened] !== block.start && claimFrom(opened) !== -1) return pos;
|
|
6349
|
+
if (block.end >= lineEnd) return -1;
|
|
6350
|
+
pos = block.end;
|
|
6351
|
+
continue;
|
|
6352
|
+
}
|
|
6353
|
+
if (block.end >= lineEnd) return -1;
|
|
6354
|
+
pos = block.end;
|
|
6355
|
+
}
|
|
6356
|
+
};
|
|
6357
|
+
let k = this.lineCount - 1;
|
|
6358
|
+
let start = this.lineCount;
|
|
6359
|
+
let startPos = this.n;
|
|
6360
|
+
if (k >= 0 && this.isOpenTail(k)) {
|
|
6361
|
+
const at = claimFrom(k);
|
|
6362
|
+
if (at === -1) return this.n;
|
|
6363
|
+
start = k;
|
|
6364
|
+
startPos = at;
|
|
6365
|
+
k--;
|
|
6366
|
+
}
|
|
6367
|
+
for (; k >= 0; k--) {
|
|
6368
|
+
this.tick(1);
|
|
6369
|
+
const at = claimFrom(k);
|
|
6370
|
+
if (at === -1) break;
|
|
6371
|
+
if (this.lineIsBlank(k)) continue;
|
|
6372
|
+
if (!TREE_GLYPHS.test(this.lineText(k))) break;
|
|
6373
|
+
start = k;
|
|
6374
|
+
startPos = at;
|
|
6375
|
+
}
|
|
6376
|
+
let rootPos = startPos;
|
|
6377
|
+
for (let j = start - 1; j >= 0; j--) {
|
|
6378
|
+
const at = claimFrom(j);
|
|
6379
|
+
if (at === -1) break;
|
|
6380
|
+
const trimmed = this.lineText(j).trim();
|
|
6381
|
+
this.tick(trimmed.length + 1);
|
|
6382
|
+
if (!trimmed || isSplitterTreeLine(trimmed) || /^#{1,6}\s/.test(trimmed)) break;
|
|
6383
|
+
rootPos = at;
|
|
6384
|
+
if (at > this.lineStarts[j]) break;
|
|
6385
|
+
}
|
|
6386
|
+
return rootPos;
|
|
6387
|
+
}
|
|
6388
|
+
/** The block (so far) containing offset `pos`, or null. */
|
|
6389
|
+
blockAt(pos) {
|
|
6390
|
+
let lo = 0;
|
|
6391
|
+
let hi = this.blocks.length - 1;
|
|
6392
|
+
while (lo <= hi) {
|
|
6393
|
+
const mid = lo + hi >>> 1;
|
|
6394
|
+
const block = this.blocks[mid];
|
|
6395
|
+
if (pos < block.start) hi = mid - 1;
|
|
6396
|
+
else if (pos >= block.end) lo = mid + 1;
|
|
6397
|
+
else return block;
|
|
6398
|
+
}
|
|
6399
|
+
return null;
|
|
6400
|
+
}
|
|
6245
6401
|
push(block) {
|
|
6246
6402
|
if (block.end <= block.start) return;
|
|
6247
6403
|
if (block.kind === "prose" && this.isBlank(block.start, block.end)) {
|
|
@@ -6445,7 +6601,10 @@ var Tokenizer = class {
|
|
|
6445
6601
|
continue;
|
|
6446
6602
|
}
|
|
6447
6603
|
if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
|
|
6448
|
-
if (FOOTNOTE_DEF.test(line.trimStart())
|
|
6604
|
+
if (FOOTNOTE_DEF.test(line.trimStart())) break;
|
|
6605
|
+
const block = this.detectBlock(this.lineStarts[k], k, false);
|
|
6606
|
+
if (block && this.mayDissolve(block)) return this.heldFootnote(p, open[1]);
|
|
6607
|
+
if (block) break;
|
|
6449
6608
|
last = k;
|
|
6450
6609
|
}
|
|
6451
6610
|
const end = this.contentEnds[last];
|
|
@@ -6461,7 +6620,8 @@ var Tokenizer = class {
|
|
|
6461
6620
|
return {
|
|
6462
6621
|
count: this.lineCount,
|
|
6463
6622
|
line: (k) => this.lineText(k),
|
|
6464
|
-
openTail: (k) => this.isOpenTail(k)
|
|
6623
|
+
openTail: (k) => this.isOpenTail(k),
|
|
6624
|
+
streaming: this.streaming
|
|
6465
6625
|
};
|
|
6466
6626
|
}
|
|
6467
6627
|
/**
|
|
@@ -6521,6 +6681,10 @@ var Tokenizer = class {
|
|
|
6521
6681
|
if (fenceOpenerOf(this.lineText(j))?.char === "~") continue;
|
|
6522
6682
|
const inner = this.detectBlockUncached(start, j, false);
|
|
6523
6683
|
if (!inner) continue;
|
|
6684
|
+
if (this.streaming && inner.end >= tildeEnd && this.paragraphMayGrow(inner.end)) {
|
|
6685
|
+
this.overflowComplete = false;
|
|
6686
|
+
return this.n;
|
|
6687
|
+
}
|
|
6524
6688
|
if (inner.end > end) {
|
|
6525
6689
|
end = inner.end;
|
|
6526
6690
|
this.overflowComplete = inner.complete;
|
|
@@ -6649,12 +6813,7 @@ var Tokenizer = class {
|
|
|
6649
6813
|
return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
|
|
6650
6814
|
}
|
|
6651
6815
|
const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
|
|
6652
|
-
const
|
|
6653
|
-
if (loneTag && this.isOpenTail(li)) {
|
|
6654
|
-
this.provisionalFrom = Math.min(this.provisionalFrom, p);
|
|
6655
|
-
return null;
|
|
6656
|
-
}
|
|
6657
|
-
const typeSeven = loneTag;
|
|
6816
|
+
const typeSeven = atParaStart && t.trimEnd() === tag.raw;
|
|
6658
6817
|
if (!typeSix && !typeSeven) return null;
|
|
6659
6818
|
let end = this.contentEnds[li];
|
|
6660
6819
|
for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
|
|
@@ -6712,6 +6871,13 @@ var Tokenizer = class {
|
|
|
6712
6871
|
}
|
|
6713
6872
|
return null;
|
|
6714
6873
|
}
|
|
6874
|
+
if (this.isOpenTail(lastLine)) {
|
|
6875
|
+
if (isRecognizedJsonHead(this.text.slice(tp, tp + 512))) {
|
|
6876
|
+
return this.island("json", p, this.n, false, jsonMeta(this.text.slice(tp)));
|
|
6877
|
+
}
|
|
6878
|
+
this.provisionalFrom = Math.min(this.provisionalFrom, p);
|
|
6879
|
+
return null;
|
|
6880
|
+
}
|
|
6715
6881
|
if (net !== 0) return null;
|
|
6716
6882
|
const end = this.contentEnds[lastLine];
|
|
6717
6883
|
const content = this.text.slice(tp, end).trim();
|
|
@@ -6746,8 +6912,10 @@ var Tokenizer = class {
|
|
|
6746
6912
|
let breakLine = -1;
|
|
6747
6913
|
for (let k = li + 1; k < this.lineCount; k++) {
|
|
6748
6914
|
if (this.lineIsBlank(k)) break;
|
|
6749
|
-
|
|
6915
|
+
const block = this.detectBlock(this.lineStarts[k], k, false);
|
|
6916
|
+
if (block) {
|
|
6750
6917
|
breakLine = k;
|
|
6918
|
+
if (this.mayDissolve(block)) this.undecidedBreaks.add(end);
|
|
6751
6919
|
break;
|
|
6752
6920
|
}
|
|
6753
6921
|
end = this.contentEnds[k];
|
|
@@ -6785,7 +6953,7 @@ var Tokenizer = class {
|
|
|
6785
6953
|
this.push({ kind: "prose", start: pos, end, island: null, inlines: scan2.inlines });
|
|
6786
6954
|
return end;
|
|
6787
6955
|
}
|
|
6788
|
-
const br = scan2.breakIsland;
|
|
6956
|
+
const br = { ...scan2.breakIsland, fromParagraph: true };
|
|
6789
6957
|
if (scan2.orphan) {
|
|
6790
6958
|
let start = pos;
|
|
6791
6959
|
while (this.blocks.length) {
|
|
@@ -6808,6 +6976,15 @@ var Tokenizer = class {
|
|
|
6808
6976
|
this.push({ kind: "island", start: br.start, end: br.end, island: br, inlines: [] });
|
|
6809
6977
|
return br.end;
|
|
6810
6978
|
}
|
|
6979
|
+
/**
|
|
6980
|
+
* An island that opens inside a paragraph and closes past its end becomes a
|
|
6981
|
+
* block. Streaming: while the paragraph may still grow over its closer, the
|
|
6982
|
+
* island may yet be inline — undecided, so it runs open to the end (F3).
|
|
6983
|
+
*/
|
|
6984
|
+
spill(type, start, end, pe, meta) {
|
|
6985
|
+
if (this.paragraphMayGrow(pe)) return this.island(type, start, this.n, false, meta);
|
|
6986
|
+
return this.island(type, start, end, true, meta);
|
|
6987
|
+
}
|
|
6811
6988
|
scanInline(ps, pe) {
|
|
6812
6989
|
const text = this.text;
|
|
6813
6990
|
const inlines = [];
|
|
@@ -6842,6 +7019,9 @@ var Tokenizer = class {
|
|
|
6842
7019
|
i = close + 2;
|
|
6843
7020
|
continue;
|
|
6844
7021
|
}
|
|
7022
|
+
if (close === -1 && this.streaming && this.paragraphMayGrow(pe)) {
|
|
7023
|
+
return { inlines, breakIsland: null, orphan: false };
|
|
7024
|
+
}
|
|
6845
7025
|
}
|
|
6846
7026
|
i += 2;
|
|
6847
7027
|
continue;
|
|
@@ -6862,17 +7042,13 @@ var Tokenizer = class {
|
|
|
6862
7042
|
i += 2;
|
|
6863
7043
|
continue;
|
|
6864
7044
|
}
|
|
6865
|
-
if (close + 2 > pe) return stop(this.
|
|
7045
|
+
if (close + 2 > pe) return stop(this.spill("math_block", i, close + 2, pe, { delimiter: "$$" }));
|
|
6866
7046
|
inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: "$$" }));
|
|
6867
7047
|
i = close + 2;
|
|
6868
7048
|
continue;
|
|
6869
7049
|
}
|
|
6870
7050
|
const end = singleDollarMathEnd(text, i, pe);
|
|
6871
7051
|
this.tick(end > 0 ? end - i : Math.min(403, pe - i));
|
|
6872
|
-
if (end > 0 && this.streaming && end === this.n) {
|
|
6873
|
-
this.provisionalFrom = Math.min(this.provisionalFrom, i);
|
|
6874
|
-
return { inlines, breakIsland: null, orphan: false };
|
|
6875
|
-
}
|
|
6876
7052
|
if (end > 0) {
|
|
6877
7053
|
inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
|
|
6878
7054
|
i = end;
|
|
@@ -6950,7 +7126,7 @@ var Tokenizer = class {
|
|
|
6950
7126
|
const anchor = end - i <= 64 ? PINNED_ANCHOR_RE.exec(text.slice(i, end)) : null;
|
|
6951
7127
|
const type = anchor ? "anchor" : "html_comment";
|
|
6952
7128
|
const meta = anchor ? { id: anchor[1] } : {};
|
|
6953
|
-
if (end > pe) return stop(this.
|
|
7129
|
+
if (end > pe) return stop(this.spill(type, i, end, pe, meta));
|
|
6954
7130
|
inlines.push(this.island(type, i, end, true, meta));
|
|
6955
7131
|
i = end;
|
|
6956
7132
|
continue;
|
|
@@ -6977,9 +7153,13 @@ var Tokenizer = class {
|
|
|
6977
7153
|
const closer = `</${name}>`;
|
|
6978
7154
|
const close = this.next(closer, tagEnd);
|
|
6979
7155
|
const blockType = isAttr ? "xml_attr" : "xml_region";
|
|
7156
|
+
if (close === -1 && isThinking && !this.streaming) {
|
|
7157
|
+
i = tagEnd;
|
|
7158
|
+
continue;
|
|
7159
|
+
}
|
|
6980
7160
|
if (close === -1) return stop(this.island(blockType, i, this.n, false, { tag: name }));
|
|
6981
7161
|
const end = close + closer.length;
|
|
6982
|
-
if (end > pe) return stop(this.
|
|
7162
|
+
if (end > pe) return stop(this.spill(blockType, i, end, pe, { tag: name }));
|
|
6983
7163
|
inlines.push(this.island("xml_inline", i, end, true, { tag: name }));
|
|
6984
7164
|
i = end;
|
|
6985
7165
|
continue;
|
|
@@ -7365,6 +7545,7 @@ function mapRange(changes, start, end, unit = "utf16") {
|
|
|
7365
7545
|
}
|
|
7366
7546
|
|
|
7367
7547
|
exports.ACTION_BY_CLASS = ACTION_BY_CLASS;
|
|
7548
|
+
exports.AFTER_WINDOW = AFTER_WINDOW;
|
|
7368
7549
|
exports.BacktickRuns = BacktickRuns;
|
|
7369
7550
|
exports.CALLOUT_NAMES = CALLOUT_NAMES;
|
|
7370
7551
|
exports.CAPABILITY_BY_CLASS = CAPABILITY_BY_CLASS;
|
|
@@ -7508,6 +7689,7 @@ exports.scalarArrayItemType = scalarArrayItemType;
|
|
|
7508
7689
|
exports.schemaLayoutMode = schemaLayoutMode;
|
|
7509
7690
|
exports.schemaStructureDepth = schemaStructureDepth;
|
|
7510
7691
|
exports.setJsonRootKeyLookup = setJsonRootKeyLookup;
|
|
7692
|
+
exports.singleDollarAfter = singleDollarAfter;
|
|
7511
7693
|
exports.singleDollarMathEnd = singleDollarMathEnd;
|
|
7512
7694
|
exports.spliceSave = spliceSave;
|
|
7513
7695
|
exports.splitsSurrogatePair = splitsSurrogatePair;
|