@copilotkit/markdown 0.0.1 → 2.0.0-rc.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +23 -2
- package/dist/index.cjs +2023 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +361 -0
- package/dist/index.d.cts.map +1 -0
- package/dist/index.d.mts +361 -0
- package/dist/index.d.mts.map +1 -0
- package/dist/index.mjs +2018 -0
- package/dist/index.mjs.map +1 -0
- package/dist/react/index.cjs +538 -0
- package/dist/react/index.cjs.map +1 -0
- package/dist/react/index.d.cts +190 -0
- package/dist/react/index.d.cts.map +1 -0
- package/dist/react/index.d.mts +190 -0
- package/dist/react/index.d.mts.map +1 -0
- package/dist/react/index.mjs +535 -0
- package/dist/react/index.mjs.map +1 -0
- package/dist/react-native/index.cjs +330 -0
- package/dist/react-native/index.cjs.map +1 -0
- package/dist/react-native/index.d.cts +74 -0
- package/dist/react-native/index.d.cts.map +1 -0
- package/dist/react-native/index.d.mts +74 -0
- package/dist/react-native/index.d.mts.map +1 -0
- package/dist/react-native/index.mjs +300 -0
- package/dist/react-native/index.mjs.map +1 -0
- package/dist/vue/index.cjs +256 -0
- package/dist/vue/index.cjs.map +1 -0
- package/dist/vue/index.d.cts +116 -0
- package/dist/vue/index.d.cts.map +1 -0
- package/dist/vue/index.d.mts +116 -0
- package/dist/vue/index.d.mts.map +1 -0
- package/dist/vue/index.mjs +255 -0
- package/dist/vue/index.mjs.map +1 -0
- package/package.json +75 -2
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,2018 @@
|
|
|
1
|
+
//#region src/parser/helpers.ts
|
|
2
|
+
/**
|
|
3
|
+
* Default parser options for streaming markdown.
|
|
4
|
+
*/
|
|
5
|
+
const DEFAULT_OPTIONS = {
|
|
6
|
+
segmenter: true,
|
|
7
|
+
enableTables: true,
|
|
8
|
+
enableAutolinks: true
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Merges user-provided parser options with defaults.
|
|
12
|
+
*
|
|
13
|
+
* @param options - Optional partial parser options.
|
|
14
|
+
* @returns Fully populated parser options.
|
|
15
|
+
*/
|
|
16
|
+
function normalizeOptions(options) {
|
|
17
|
+
return {
|
|
18
|
+
segmenter: options?.segmenter ?? DEFAULT_OPTIONS.segmenter,
|
|
19
|
+
enableTables: options?.enableTables ?? DEFAULT_OPTIONS.enableTables,
|
|
20
|
+
enableAutolinks: options?.enableAutolinks ?? DEFAULT_OPTIONS.enableAutolinks
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Normalizes streaming input chunks by handling trailing carriage-return boundaries.
|
|
25
|
+
*
|
|
26
|
+
* @param chunk - Incoming raw chunk.
|
|
27
|
+
* @param hadPendingCR - Whether the previous chunk ended with `\\r`.
|
|
28
|
+
* @returns Normalized text and whether this chunk now ends with a pending `\\r`.
|
|
29
|
+
*/
|
|
30
|
+
function normalizeChunk(chunk, hadPendingCR) {
|
|
31
|
+
const input = (hadPendingCR ? "\r" : "") + chunk;
|
|
32
|
+
const pendingCarriageReturn = input.endsWith("\r");
|
|
33
|
+
return {
|
|
34
|
+
text: (pendingCarriageReturn ? input.slice(0, -1) : input).replace(/\r\n?/g, "\n"),
|
|
35
|
+
pendingCarriageReturn
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Splits normalized source text into lines with source offsets.
|
|
40
|
+
*
|
|
41
|
+
* @param source - Normalized source text.
|
|
42
|
+
* @returns Line descriptors preserving absolute ranges.
|
|
43
|
+
*/
|
|
44
|
+
function splitLines(source) {
|
|
45
|
+
const lines = [];
|
|
46
|
+
let start = 0;
|
|
47
|
+
for (let i = 0; i < source.length; i += 1) {
|
|
48
|
+
if (source[i] !== "\n") continue;
|
|
49
|
+
lines.push({
|
|
50
|
+
text: source.slice(start, i),
|
|
51
|
+
start,
|
|
52
|
+
end: i + 1,
|
|
53
|
+
hasNewline: true
|
|
54
|
+
});
|
|
55
|
+
start = i + 1;
|
|
56
|
+
}
|
|
57
|
+
if (start < source.length || source.length === 0) lines.push({
|
|
58
|
+
text: source.slice(start),
|
|
59
|
+
start,
|
|
60
|
+
end: source.length,
|
|
61
|
+
hasNewline: false
|
|
62
|
+
});
|
|
63
|
+
return lines;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Returns the trailing unterminated line text.
|
|
67
|
+
*
|
|
68
|
+
* @param source - Full source text.
|
|
69
|
+
* @returns Last line content after the final newline.
|
|
70
|
+
*/
|
|
71
|
+
function getLineBuffer(source) {
|
|
72
|
+
const newline = source.lastIndexOf("\n");
|
|
73
|
+
return newline < 0 ? source : source.slice(newline + 1);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Converts an absolute source index to 1-based line/column.
|
|
77
|
+
*
|
|
78
|
+
* @param source - Source text.
|
|
79
|
+
* @param index - Absolute index.
|
|
80
|
+
* @returns 1-based line and column coordinates.
|
|
81
|
+
*/
|
|
82
|
+
function toLineColumn(source, index) {
|
|
83
|
+
let line = 1;
|
|
84
|
+
let column = 1;
|
|
85
|
+
for (let i = 0; i < index; i += 1) if (source[i] === "\n") {
|
|
86
|
+
line += 1;
|
|
87
|
+
column = 1;
|
|
88
|
+
} else column += 1;
|
|
89
|
+
return {
|
|
90
|
+
line,
|
|
91
|
+
column
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Returns true for blank lines.
|
|
96
|
+
*/
|
|
97
|
+
function isBlank(value) {
|
|
98
|
+
return /^\s*$/.test(value);
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Returns true when a line is a citation definition.
|
|
102
|
+
*/
|
|
103
|
+
function isCitationDefinitionLine(value) {
|
|
104
|
+
return /^\s{0,3}\[\^([^\]\s]+)\]:\s*(.*)$/.test(value);
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Returns true when a line is an unfinished citation-definition prefix candidate.
|
|
108
|
+
*
|
|
109
|
+
* This enables optimistic streaming behavior for definition lines that begin with
|
|
110
|
+
* `[^` but are not yet complete enough to classify as either:
|
|
111
|
+
* 1) a valid citation definition, or
|
|
112
|
+
* 2) plain paragraph text.
|
|
113
|
+
*
|
|
114
|
+
* @param value - Raw line text.
|
|
115
|
+
* @param hasNewline - Whether the source line is newline-terminated.
|
|
116
|
+
* @param isComplete - Whether the parser has been finalized.
|
|
117
|
+
*/
|
|
118
|
+
function isCitationDefinitionPrefixCandidate(value, hasNewline, isComplete) {
|
|
119
|
+
if (hasNewline || isComplete || isCitationDefinitionLine(value)) return false;
|
|
120
|
+
return /^\s{0,3}\[\^[^\]\s]*$/.test(value) || /^\s{0,3}\[\^[^\]\s]+\]$/.test(value);
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Matches an opening fenced code block marker.
|
|
124
|
+
*/
|
|
125
|
+
function matchFenceOpen(value) {
|
|
126
|
+
const trimmed = value.trimStart();
|
|
127
|
+
const backtick = /^(`{3,})(.*)$/.exec(trimmed);
|
|
128
|
+
if (backtick) return {
|
|
129
|
+
marker: "```",
|
|
130
|
+
length: backtick[1].length,
|
|
131
|
+
info: backtick[2].trim()
|
|
132
|
+
};
|
|
133
|
+
const tilde = /^(~{3,})(.*)$/.exec(trimmed);
|
|
134
|
+
if (tilde) return {
|
|
135
|
+
marker: "~~~",
|
|
136
|
+
length: tilde[1].length,
|
|
137
|
+
info: tilde[2].trim()
|
|
138
|
+
};
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Matches a valid fenced code block closing line.
|
|
143
|
+
*/
|
|
144
|
+
function matchFenceClose(value, marker, length) {
|
|
145
|
+
if (!(marker === "```" ? /^\s*`{3,}\s*$/ : /^\s*~{3,}\s*$/).test(value)) return false;
|
|
146
|
+
return value.trim().length >= length;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Matches ATX heading syntax and extracts heading metadata.
|
|
150
|
+
*/
|
|
151
|
+
function matchAtxHeading(value) {
|
|
152
|
+
const match = /^\s{0,3}(#{1,6})[ \t]+(.+?)\s*#*\s*$/.exec(value);
|
|
153
|
+
if (!match) return null;
|
|
154
|
+
return {
|
|
155
|
+
level: match[1].length,
|
|
156
|
+
text: match[2],
|
|
157
|
+
contentOffset: value.indexOf(match[2])
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Returns true for setext heading underline lines.
|
|
162
|
+
*/
|
|
163
|
+
function isSetextUnderline(value) {
|
|
164
|
+
return /^\s{0,3}(=+|-+)\s*$/.test(value);
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Returns true for thematic break syntax.
|
|
168
|
+
*/
|
|
169
|
+
function isThematicBreak(value) {
|
|
170
|
+
return /^\s{0,3}((\*\s*){3,}|(-\s*){3,}|(_\s*){3,})$/.test(value);
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Matches ordered and unordered list-item markers.
|
|
174
|
+
*/
|
|
175
|
+
function matchListItem(value) {
|
|
176
|
+
const unordered = /^(\s{0,3})([-+*])(\s+)(.*)$/.exec(value);
|
|
177
|
+
if (unordered) {
|
|
178
|
+
const indent = unordered[1].length;
|
|
179
|
+
const contentIndent = indent + unordered[2].length + unordered[3].length;
|
|
180
|
+
return {
|
|
181
|
+
ordered: false,
|
|
182
|
+
start: 1,
|
|
183
|
+
content: unordered[4],
|
|
184
|
+
indent,
|
|
185
|
+
contentIndent
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
const ordered = /^(\s{0,3})(\d{1,9})([.)])(\s+)(.*)$/.exec(value);
|
|
189
|
+
if (ordered) {
|
|
190
|
+
const indent = ordered[1].length;
|
|
191
|
+
const contentIndent = indent + (ordered[2].length + ordered[3].length) + ordered[4].length;
|
|
192
|
+
return {
|
|
193
|
+
ordered: true,
|
|
194
|
+
start: Number(ordered[2]),
|
|
195
|
+
content: ordered[5],
|
|
196
|
+
indent,
|
|
197
|
+
contentIndent
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Returns true for blockquote marker lines.
|
|
204
|
+
*/
|
|
205
|
+
function isBlockquoteLine(value) {
|
|
206
|
+
return /^\s{0,3}>\s?/.test(value);
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Returns true when two lines form a GFM table header + divider pair.
|
|
210
|
+
*/
|
|
211
|
+
function isPipeTableHeader(first, second) {
|
|
212
|
+
return looksLikeTableRow(first.text) && isTableDividerRow(second.text);
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Returns true when a line appears to be a table row.
|
|
216
|
+
*/
|
|
217
|
+
function looksLikeTableRow(value) {
|
|
218
|
+
return value.includes("|") && /\S/.test(value.replace(/\|/g, ""));
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Returns true for GFM table divider rows.
|
|
222
|
+
*/
|
|
223
|
+
function isTableDividerRow(value) {
|
|
224
|
+
const cells = splitTableCells(value);
|
|
225
|
+
if (cells.length === 0) return false;
|
|
226
|
+
return cells.every((cell) => /^:?-{3,}:?$/.test(cell.trim()));
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Splits a pipe-table row into trimmed cell values.
|
|
230
|
+
*/
|
|
231
|
+
function splitTableCells(value) {
|
|
232
|
+
const trimmed = value.trim();
|
|
233
|
+
const body = trimmed.startsWith("|") ? trimmed.slice(1) : trimmed;
|
|
234
|
+
return (body.endsWith("|") ? body.slice(0, -1) : body).split("|").map((cell) => cell.trim());
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Converts table divider cells into alignment metadata.
|
|
238
|
+
*/
|
|
239
|
+
function parseTableAlignment(cells) {
|
|
240
|
+
return cells.map((cell) => {
|
|
241
|
+
const trimmed = cell.trim();
|
|
242
|
+
const left = trimmed.startsWith(":");
|
|
243
|
+
const right = trimmed.endsWith(":");
|
|
244
|
+
if (left && right) return "center";
|
|
245
|
+
if (left) return "left";
|
|
246
|
+
if (right) return "right";
|
|
247
|
+
return "none";
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Determines whether a line starts a new block construct.
|
|
252
|
+
*/
|
|
253
|
+
function startsNewBlock(lines, index, enableTables) {
|
|
254
|
+
const line = lines[index].text;
|
|
255
|
+
return !!matchFenceOpen(line) || !!matchAtxHeading(line) || isThematicBreak(line) || !!matchListItem(line) || isBlockquoteLine(line) || enableTables && index + 1 < lines.length && isPipeTableHeader(lines[index], lines[index + 1]);
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Returns true when a character is escapable in CommonMark inline syntax.
|
|
259
|
+
*/
|
|
260
|
+
function isEscapable(ch) {
|
|
261
|
+
return /[!"#$%&'()*+,\-./:;<=>?@[\\\]^_`{|}~]/.test(ch);
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Finds the matching closing delimiter for a nested delimiter pair.
|
|
265
|
+
*/
|
|
266
|
+
function findClosing(input, openAt, openChar, closeChar) {
|
|
267
|
+
let depth = 0;
|
|
268
|
+
for (let i = openAt; i < input.length; i += 1) {
|
|
269
|
+
const ch = input[i];
|
|
270
|
+
if (ch === "\\") {
|
|
271
|
+
i += 1;
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
if (ch === openChar) {
|
|
275
|
+
depth += 1;
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
if (ch === closeChar) {
|
|
279
|
+
depth -= 1;
|
|
280
|
+
if (depth === 0) return i;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return -1;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Parses a markdown link destination and optional title.
|
|
287
|
+
*/
|
|
288
|
+
function parseLinkDestination(input) {
|
|
289
|
+
const trimmed = input.trim();
|
|
290
|
+
if (trimmed.startsWith("<")) {
|
|
291
|
+
const close = trimmed.indexOf(">");
|
|
292
|
+
if (close > 0) {
|
|
293
|
+
const url = trimmed.slice(1, close);
|
|
294
|
+
const title = parseLinkTitle(trimmed.slice(close + 1).trim());
|
|
295
|
+
return title ? {
|
|
296
|
+
url,
|
|
297
|
+
title
|
|
298
|
+
} : { url };
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
const firstSpace = trimmed.search(/\s/);
|
|
302
|
+
if (firstSpace < 0) return { url: trimmed };
|
|
303
|
+
const url = trimmed.slice(0, firstSpace);
|
|
304
|
+
const title = parseLinkTitle(trimmed.slice(firstSpace).trim());
|
|
305
|
+
return title ? {
|
|
306
|
+
url,
|
|
307
|
+
title
|
|
308
|
+
} : { url };
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* Parses CommonMark-style link titles (`\"...\"`, `'...'`, `( ... )`).
|
|
312
|
+
*/
|
|
313
|
+
function parseLinkTitle(input) {
|
|
314
|
+
if (!input) return;
|
|
315
|
+
if (input.startsWith("\"") && input.endsWith("\"") || input.startsWith("'") && input.endsWith("'")) return input.slice(1, -1);
|
|
316
|
+
if (input.startsWith("(") && input.endsWith(")")) return input.slice(1, -1);
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Returns true for URL-shaped text supported by autolink detection.
|
|
320
|
+
*/
|
|
321
|
+
function isUrl(value) {
|
|
322
|
+
return /^(https?:\/\/|www\.)\S+$/.test(value);
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Returns true for email-shaped text supported by autolink detection.
|
|
326
|
+
*/
|
|
327
|
+
function isEmail(value) {
|
|
328
|
+
return /^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/.test(value);
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Trims punctuation suffixes from autolink candidates.
|
|
332
|
+
*/
|
|
333
|
+
function trimAutolinkTrailingPunctuation(value) {
|
|
334
|
+
let end = value.length;
|
|
335
|
+
while (end > 0 && /[.,;:!?]/.test(value[end - 1])) end -= 1;
|
|
336
|
+
let candidate = value.slice(0, end);
|
|
337
|
+
if (candidate.endsWith(")")) {
|
|
338
|
+
const open = (candidate.match(/\(/g) ?? []).length;
|
|
339
|
+
if ((candidate.match(/\)/g) ?? []).length > open) candidate = candidate.slice(0, -1);
|
|
340
|
+
}
|
|
341
|
+
return candidate;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
//#endregion
|
|
345
|
+
//#region src/parser/citations.ts
|
|
346
|
+
/**
|
|
347
|
+
* Extracts citation definitions from source text.
|
|
348
|
+
*
|
|
349
|
+
* @param source - Full normalized markdown source.
|
|
350
|
+
* @param warnings - Mutable warning collection for parse diagnostics.
|
|
351
|
+
* @returns Citation state with collected definitions and empty reference order.
|
|
352
|
+
*/
|
|
353
|
+
function parseCitationDefinitions(source) {
|
|
354
|
+
const definitions = {};
|
|
355
|
+
const warnings = [];
|
|
356
|
+
const lines = source.split("\n");
|
|
357
|
+
let offset = 0;
|
|
358
|
+
for (const line of lines) {
|
|
359
|
+
const match = /^\s{0,3}\[\^([^\]\s]+)\]:\s*(.*)$/.exec(line);
|
|
360
|
+
if (!match) {
|
|
361
|
+
offset += line.length + 1;
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
const id = match[1];
|
|
365
|
+
if (definitions[id]) {
|
|
366
|
+
warnings.push({
|
|
367
|
+
code: "invalid_citation_definition",
|
|
368
|
+
at: offset
|
|
369
|
+
});
|
|
370
|
+
offset += line.length + 1;
|
|
371
|
+
continue;
|
|
372
|
+
}
|
|
373
|
+
const extracted = extractTrailingUrl(match[2].trim());
|
|
374
|
+
definitions[id] = {
|
|
375
|
+
id,
|
|
376
|
+
text: extracted.text,
|
|
377
|
+
...extracted.url ? { url: extracted.url } : {}
|
|
378
|
+
};
|
|
379
|
+
offset += line.length + 1;
|
|
380
|
+
}
|
|
381
|
+
return {
|
|
382
|
+
citations: {
|
|
383
|
+
order: [],
|
|
384
|
+
numbers: {},
|
|
385
|
+
definitions
|
|
386
|
+
},
|
|
387
|
+
warnings
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Assigns or returns a stable citation number by first-seen inline reference order.
|
|
392
|
+
*
|
|
393
|
+
* @param citations - Citation state that tracks numbering.
|
|
394
|
+
* @param idRef - Citation identifier (without `[^` and `]` markers).
|
|
395
|
+
* @returns Stable numeric citation number.
|
|
396
|
+
*/
|
|
397
|
+
function assignCitationNumber(citations, idRef) {
|
|
398
|
+
const existing = citations.numbers[idRef];
|
|
399
|
+
if (typeof existing === "number") return {
|
|
400
|
+
citations,
|
|
401
|
+
number: existing
|
|
402
|
+
};
|
|
403
|
+
const next = citations.order.length + 1;
|
|
404
|
+
return {
|
|
405
|
+
citations: {
|
|
406
|
+
...citations,
|
|
407
|
+
order: [...citations.order, idRef],
|
|
408
|
+
numbers: {
|
|
409
|
+
...citations.numbers,
|
|
410
|
+
[idRef]: next
|
|
411
|
+
}
|
|
412
|
+
},
|
|
413
|
+
number: next
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
function extractTrailingUrl(input) {
|
|
417
|
+
if (!input) return { text: "" };
|
|
418
|
+
const match = /(?:\s|^)(https?:\/\/\S+|www\.\S+)$/.exec(input);
|
|
419
|
+
if (!match) return { text: input };
|
|
420
|
+
return {
|
|
421
|
+
text: input.slice(0, match.index).trimEnd(),
|
|
422
|
+
url: match[1]
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
//#endregion
|
|
427
|
+
//#region src/parser/segments.ts
|
|
428
|
+
/**
|
|
429
|
+
* Computes parse-time text segments for animation and incremental rendering.
|
|
430
|
+
*
|
|
431
|
+
* @param text - Raw text content for a single text node.
|
|
432
|
+
* @param absoluteStart - Absolute source index where `text` begins.
|
|
433
|
+
* @param context - Parse context containing segmenter options and warning state.
|
|
434
|
+
* @returns Segment list or empty list when segmentation is disabled/unavailable.
|
|
435
|
+
*/
|
|
436
|
+
function createSegments(text, absoluteStart, options) {
|
|
437
|
+
if (options.segmenter === false) return {
|
|
438
|
+
segments: [],
|
|
439
|
+
hasWarnedSegmenterUnavailable: options.hasWarnedSegmenterUnavailable
|
|
440
|
+
};
|
|
441
|
+
const segmenterCtor = typeof Intl === "undefined" ? void 0 : Intl.Segmenter;
|
|
442
|
+
if (!segmenterCtor) {
|
|
443
|
+
if (!options.hasWarnedSegmenterUnavailable) return {
|
|
444
|
+
segments: [],
|
|
445
|
+
warning: {
|
|
446
|
+
code: "segmenter_unavailable",
|
|
447
|
+
at: absoluteStart
|
|
448
|
+
},
|
|
449
|
+
hasWarnedSegmenterUnavailable: true
|
|
450
|
+
};
|
|
451
|
+
return {
|
|
452
|
+
segments: [],
|
|
453
|
+
hasWarnedSegmenterUnavailable: true
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
const granularity = options.segmenter === true ? "word" : options.segmenter.granularity ?? "word";
|
|
457
|
+
const segmenter = new segmenterCtor(options.segmenter === true ? void 0 : options.segmenter.locale, { granularity });
|
|
458
|
+
const segments = [];
|
|
459
|
+
for (const segment of segmenter.segment(text)) {
|
|
460
|
+
const segmentText = segment.segment;
|
|
461
|
+
const start = absoluteStart + segment.index;
|
|
462
|
+
segments.push({
|
|
463
|
+
text: segmentText,
|
|
464
|
+
start,
|
|
465
|
+
end: start + segmentText.length,
|
|
466
|
+
kind: granularity,
|
|
467
|
+
isWhitespace: /^\s+$/.test(segmentText)
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
return {
|
|
471
|
+
segments: clusterPunctuationSegments(segments),
|
|
472
|
+
hasWarnedSegmenterUnavailable: options.hasWarnedSegmenterUnavailable
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
function clusterPunctuationSegments(segments) {
|
|
476
|
+
const withClosingAttached = segments.reduce((acc, segment) => {
|
|
477
|
+
const previous = acc[acc.length - 1];
|
|
478
|
+
if (previous && !previous.isWhitespace && isClosingPunctuationOnly(segment.text)) {
|
|
479
|
+
acc[acc.length - 1] = {
|
|
480
|
+
...previous,
|
|
481
|
+
text: `${previous.text}${segment.text}`,
|
|
482
|
+
end: segment.end
|
|
483
|
+
};
|
|
484
|
+
return acc;
|
|
485
|
+
}
|
|
486
|
+
return [...acc, segment];
|
|
487
|
+
}, []);
|
|
488
|
+
const clustered = [];
|
|
489
|
+
let i = 0;
|
|
490
|
+
while (i < withClosingAttached.length) {
|
|
491
|
+
const current = withClosingAttached[i];
|
|
492
|
+
const next = i + 1 < withClosingAttached.length ? withClosingAttached[i + 1] : null;
|
|
493
|
+
if (next && !next.isWhitespace && isOpeningPunctuationOnly(current.text)) {
|
|
494
|
+
clustered.push({
|
|
495
|
+
...next,
|
|
496
|
+
text: `${current.text}${next.text}`,
|
|
497
|
+
start: current.start
|
|
498
|
+
});
|
|
499
|
+
i += 2;
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
clustered.push(current);
|
|
503
|
+
i += 1;
|
|
504
|
+
}
|
|
505
|
+
return clustered;
|
|
506
|
+
}
|
|
507
|
+
function isClosingPunctuationOnly(value) {
|
|
508
|
+
return isOnlyFromSet(value, CLOSING_PUNCTUATION$1);
|
|
509
|
+
}
|
|
510
|
+
function isOpeningPunctuationOnly(value) {
|
|
511
|
+
return isOnlyFromSet(value, OPENING_PUNCTUATION);
|
|
512
|
+
}
|
|
513
|
+
function isOnlyFromSet(value, charset) {
|
|
514
|
+
return value.length > 0 && [...value].every((character) => charset.has(character));
|
|
515
|
+
}
|
|
516
|
+
const CLOSING_PUNCTUATION$1 = new Set([
|
|
517
|
+
"]",
|
|
518
|
+
",",
|
|
519
|
+
".",
|
|
520
|
+
"!",
|
|
521
|
+
"?",
|
|
522
|
+
";",
|
|
523
|
+
":",
|
|
524
|
+
"%",
|
|
525
|
+
")",
|
|
526
|
+
"}",
|
|
527
|
+
">",
|
|
528
|
+
"\"",
|
|
529
|
+
"'",
|
|
530
|
+
"、",
|
|
531
|
+
"。",
|
|
532
|
+
",",
|
|
533
|
+
".",
|
|
534
|
+
"!",
|
|
535
|
+
"?",
|
|
536
|
+
":",
|
|
537
|
+
";",
|
|
538
|
+
"%",
|
|
539
|
+
")",
|
|
540
|
+
"]",
|
|
541
|
+
"}",
|
|
542
|
+
"〉",
|
|
543
|
+
"》",
|
|
544
|
+
"」",
|
|
545
|
+
"』",
|
|
546
|
+
"】",
|
|
547
|
+
"〕",
|
|
548
|
+
"〗"
|
|
549
|
+
]);
|
|
550
|
+
const OPENING_PUNCTUATION = new Set([
|
|
551
|
+
"(",
|
|
552
|
+
"{",
|
|
553
|
+
"[",
|
|
554
|
+
"\"",
|
|
555
|
+
"'",
|
|
556
|
+
"〈",
|
|
557
|
+
"《",
|
|
558
|
+
"「",
|
|
559
|
+
"『",
|
|
560
|
+
"【",
|
|
561
|
+
"〔",
|
|
562
|
+
"〖",
|
|
563
|
+
"(",
|
|
564
|
+
"[",
|
|
565
|
+
"{"
|
|
566
|
+
]);
|
|
567
|
+
|
|
568
|
+
//#endregion
|
|
569
|
+
//#region src/parser/inline-parser.ts
|
|
570
|
+
/**
|
|
571
|
+
* Parses inline markdown content into inline draft nodes.
|
|
572
|
+
*
|
|
573
|
+
* @param text - Inline source text.
|
|
574
|
+
* @param absoluteStart - Absolute source offset for the first character in `text`.
|
|
575
|
+
* @param path - Stable path prefix for emitted nodes.
|
|
576
|
+
* @param context - Immutable parse context shared across block/inline parsing.
|
|
577
|
+
* @returns Inline draft nodes and updated immutable parser context.
|
|
578
|
+
*/
|
|
579
|
+
function parseInline(text, absoluteStart, path, context) {
|
|
580
|
+
const nodes = [];
|
|
581
|
+
let i = 0;
|
|
582
|
+
let textBuffer = "";
|
|
583
|
+
let textBufferStart = -1;
|
|
584
|
+
let citations = context.citations;
|
|
585
|
+
let warnings = context.warnings;
|
|
586
|
+
let hasWarnedSegmenterUnavailable = context.hasWarnedSegmenterUnavailable;
|
|
587
|
+
const flushTextBuffer = () => {
|
|
588
|
+
if (textBuffer.length === 0 || textBufferStart < 0) return;
|
|
589
|
+
const appendResult = appendText(nodes, textBuffer, textBufferStart, path, context.options.segmenter, hasWarnedSegmenterUnavailable);
|
|
590
|
+
warnings = appendResult.warnings.length > 0 ? [...warnings, ...appendResult.warnings] : warnings;
|
|
591
|
+
hasWarnedSegmenterUnavailable = appendResult.hasWarnedSegmenterUnavailable;
|
|
592
|
+
textBuffer = "";
|
|
593
|
+
textBufferStart = -1;
|
|
594
|
+
};
|
|
595
|
+
const appendBufferedText = (value, start) => {
|
|
596
|
+
if (value.length === 0) return;
|
|
597
|
+
if (textBuffer.length === 0) textBufferStart = start;
|
|
598
|
+
textBuffer += value;
|
|
599
|
+
};
|
|
600
|
+
while (i < text.length) {
|
|
601
|
+
const current = text[i];
|
|
602
|
+
if (current === "\n") {
|
|
603
|
+
const hardByBackslash = countTrailing(text, i - 1, "\\") % 2 === 1;
|
|
604
|
+
const hardBySpaces = i > 1 && text.slice(i - 2, i) === " ";
|
|
605
|
+
const hard = hardByBackslash || hardBySpaces;
|
|
606
|
+
if (hardByBackslash && textBuffer.endsWith("\\")) textBuffer = textBuffer.slice(0, -1);
|
|
607
|
+
flushTextBuffer();
|
|
608
|
+
nodes.push({
|
|
609
|
+
path: `${path}.${nodes.length}`,
|
|
610
|
+
type: hard ? "hard-break" : "soft-break",
|
|
611
|
+
range: {
|
|
612
|
+
start: absoluteStart + i,
|
|
613
|
+
end: absoluteStart + i + 1
|
|
614
|
+
},
|
|
615
|
+
closed: true,
|
|
616
|
+
props: {},
|
|
617
|
+
children: []
|
|
618
|
+
});
|
|
619
|
+
i += 1;
|
|
620
|
+
continue;
|
|
621
|
+
}
|
|
622
|
+
if (current === "\\" && i + 1 < text.length && isEscapable(text[i + 1])) {
|
|
623
|
+
appendBufferedText(text[i + 1], absoluteStart + i);
|
|
624
|
+
i += 2;
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
627
|
+
const citation = parseCitationInline(text, i, absoluteStart, path, citations, nodes.length, context.isComplete);
|
|
628
|
+
if (citation) {
|
|
629
|
+
flushTextBuffer();
|
|
630
|
+
nodes.push(citation.node);
|
|
631
|
+
i = citation.next;
|
|
632
|
+
citations = citation.citations;
|
|
633
|
+
continue;
|
|
634
|
+
}
|
|
635
|
+
const image = parseImageInline(text, i, absoluteStart, path, nodes.length);
|
|
636
|
+
if (image) {
|
|
637
|
+
flushTextBuffer();
|
|
638
|
+
nodes.push(image.node);
|
|
639
|
+
i = image.next;
|
|
640
|
+
continue;
|
|
641
|
+
}
|
|
642
|
+
const link = parseLinkInline(text, i, absoluteStart, path, nodes.length, context, citations, warnings, hasWarnedSegmenterUnavailable);
|
|
643
|
+
if (link) {
|
|
644
|
+
flushTextBuffer();
|
|
645
|
+
nodes.push(link.node);
|
|
646
|
+
i = link.next;
|
|
647
|
+
citations = link.citations;
|
|
648
|
+
warnings = link.warnings;
|
|
649
|
+
hasWarnedSegmenterUnavailable = link.hasWarnedSegmenterUnavailable;
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
const auto = parseAutolinkInline(text, i, absoluteStart, path, context.options.enableAutolinks, nodes.length);
|
|
653
|
+
if (auto) {
|
|
654
|
+
flushTextBuffer();
|
|
655
|
+
nodes.push(auto.node);
|
|
656
|
+
i = auto.next;
|
|
657
|
+
continue;
|
|
658
|
+
}
|
|
659
|
+
const code = parseInlineCode(text, i, absoluteStart, path, nodes.length);
|
|
660
|
+
if (code) {
|
|
661
|
+
flushTextBuffer();
|
|
662
|
+
nodes.push(code.node);
|
|
663
|
+
i = code.next;
|
|
664
|
+
continue;
|
|
665
|
+
}
|
|
666
|
+
const strong = parseDelimitedInline(text, i, "**", "strong", absoluteStart, path, nodes.length, context, citations, warnings, hasWarnedSegmenterUnavailable) ?? parseDelimitedInline(text, i, "__", "strong", absoluteStart, path, nodes.length, context, citations, warnings, hasWarnedSegmenterUnavailable);
|
|
667
|
+
if (strong) {
|
|
668
|
+
flushTextBuffer();
|
|
669
|
+
nodes.push(strong.node);
|
|
670
|
+
i = strong.next;
|
|
671
|
+
citations = strong.citations;
|
|
672
|
+
warnings = strong.warnings;
|
|
673
|
+
hasWarnedSegmenterUnavailable = strong.hasWarnedSegmenterUnavailable;
|
|
674
|
+
continue;
|
|
675
|
+
}
|
|
676
|
+
const strike = parseDelimitedInline(text, i, "~~", "strikethrough", absoluteStart, path, nodes.length, context, citations, warnings, hasWarnedSegmenterUnavailable);
|
|
677
|
+
if (strike) {
|
|
678
|
+
flushTextBuffer();
|
|
679
|
+
nodes.push(strike.node);
|
|
680
|
+
i = strike.next;
|
|
681
|
+
citations = strike.citations;
|
|
682
|
+
warnings = strike.warnings;
|
|
683
|
+
hasWarnedSegmenterUnavailable = strike.hasWarnedSegmenterUnavailable;
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
const em = parseDelimitedInline(text, i, "*", "em", absoluteStart, path, nodes.length, context, citations, warnings, hasWarnedSegmenterUnavailable) ?? parseDelimitedInline(text, i, "_", "em", absoluteStart, path, nodes.length, context, citations, warnings, hasWarnedSegmenterUnavailable);
|
|
687
|
+
if (em) {
|
|
688
|
+
flushTextBuffer();
|
|
689
|
+
nodes.push(em.node);
|
|
690
|
+
i = em.next;
|
|
691
|
+
citations = em.citations;
|
|
692
|
+
warnings = em.warnings;
|
|
693
|
+
hasWarnedSegmenterUnavailable = em.hasWarnedSegmenterUnavailable;
|
|
694
|
+
continue;
|
|
695
|
+
}
|
|
696
|
+
appendBufferedText(text[i], absoluteStart + i);
|
|
697
|
+
i += 1;
|
|
698
|
+
}
|
|
699
|
+
flushTextBuffer();
|
|
700
|
+
return {
|
|
701
|
+
value: annotateNoBreakBeforeSegments(nodes),
|
|
702
|
+
citations,
|
|
703
|
+
warnings,
|
|
704
|
+
hasWarnedSegmenterUnavailable
|
|
705
|
+
};
|
|
706
|
+
}
|
|
707
|
+
function countTrailing(text, index, value) {
|
|
708
|
+
let run = 0;
|
|
709
|
+
let i = index;
|
|
710
|
+
while (i >= 0 && text[i] === value) {
|
|
711
|
+
run += 1;
|
|
712
|
+
i -= 1;
|
|
713
|
+
}
|
|
714
|
+
return run;
|
|
715
|
+
}
|
|
716
|
+
function annotateNoBreakBeforeSegments(nodes) {
|
|
717
|
+
let changed = false;
|
|
718
|
+
const next = nodes.map((node, index) => {
|
|
719
|
+
if (node.type !== "text") return node;
|
|
720
|
+
const segments = node.props["segments"];
|
|
721
|
+
if (!Array.isArray(segments) || segments.length === 0) return node;
|
|
722
|
+
const typedSegments = segments;
|
|
723
|
+
const firstSegment = typedSegments[0];
|
|
724
|
+
if (!firstSegment) return node;
|
|
725
|
+
const previous = index > 0 ? nodes[index - 1] : null;
|
|
726
|
+
const shouldMark = startsWithClosingPunctuation(firstSegment.text) && shouldPreventBreakBeforeNode(previous);
|
|
727
|
+
if (shouldMark === (firstSegment.noBreakBefore === true)) return node;
|
|
728
|
+
const nextSegments = [shouldMark ? {
|
|
729
|
+
...firstSegment,
|
|
730
|
+
noBreakBefore: true
|
|
731
|
+
} : {
|
|
732
|
+
...firstSegment,
|
|
733
|
+
noBreakBefore: void 0
|
|
734
|
+
}, ...typedSegments.slice(1)];
|
|
735
|
+
changed = true;
|
|
736
|
+
return {
|
|
737
|
+
...node,
|
|
738
|
+
props: {
|
|
739
|
+
...node.props,
|
|
740
|
+
segments: nextSegments
|
|
741
|
+
}
|
|
742
|
+
};
|
|
743
|
+
});
|
|
744
|
+
return changed ? next : nodes;
|
|
745
|
+
}
|
|
746
|
+
function shouldPreventBreakBeforeNode(node) {
|
|
747
|
+
if (!node) return false;
|
|
748
|
+
if (node.type === "soft-break" || node.type === "hard-break") return false;
|
|
749
|
+
if (node.type === "text") {
|
|
750
|
+
const text = String(node.props["text"] ?? "");
|
|
751
|
+
if (!text || /\s$/u.test(text)) return false;
|
|
752
|
+
}
|
|
753
|
+
return true;
|
|
754
|
+
}
|
|
755
|
+
function startsWithClosingPunctuation(value) {
|
|
756
|
+
if (!value) return false;
|
|
757
|
+
const [first] = [...value];
|
|
758
|
+
return first != null && CLOSING_PUNCTUATION.has(first);
|
|
759
|
+
}
|
|
760
|
+
const CLOSING_PUNCTUATION = new Set([
|
|
761
|
+
"]",
|
|
762
|
+
",",
|
|
763
|
+
".",
|
|
764
|
+
"!",
|
|
765
|
+
"?",
|
|
766
|
+
";",
|
|
767
|
+
":",
|
|
768
|
+
"%",
|
|
769
|
+
")",
|
|
770
|
+
"}",
|
|
771
|
+
">",
|
|
772
|
+
"\"",
|
|
773
|
+
"'",
|
|
774
|
+
"、",
|
|
775
|
+
"。",
|
|
776
|
+
",",
|
|
777
|
+
".",
|
|
778
|
+
"!",
|
|
779
|
+
"?",
|
|
780
|
+
":",
|
|
781
|
+
";",
|
|
782
|
+
"%",
|
|
783
|
+
")",
|
|
784
|
+
"]",
|
|
785
|
+
"}",
|
|
786
|
+
"〉",
|
|
787
|
+
"》",
|
|
788
|
+
"」",
|
|
789
|
+
"』",
|
|
790
|
+
"】",
|
|
791
|
+
"〕",
|
|
792
|
+
"〗"
|
|
793
|
+
]);
|
|
794
|
+
function appendText(nodes, text, start, path, segmenter, hasWarnedSegmenterUnavailable) {
|
|
795
|
+
if (!text) return {
|
|
796
|
+
warnings: [],
|
|
797
|
+
hasWarnedSegmenterUnavailable
|
|
798
|
+
};
|
|
799
|
+
const segmentation = createSegments(text, start, {
|
|
800
|
+
segmenter,
|
|
801
|
+
hasWarnedSegmenterUnavailable
|
|
802
|
+
});
|
|
803
|
+
nodes.push({
|
|
804
|
+
path: `${path}.${nodes.length}`,
|
|
805
|
+
type: "text",
|
|
806
|
+
range: {
|
|
807
|
+
start,
|
|
808
|
+
end: start + text.length
|
|
809
|
+
},
|
|
810
|
+
closed: true,
|
|
811
|
+
props: {
|
|
812
|
+
text,
|
|
813
|
+
segments: segmentation.segments
|
|
814
|
+
},
|
|
815
|
+
children: []
|
|
816
|
+
});
|
|
817
|
+
return {
|
|
818
|
+
warnings: segmentation.warning ? [segmentation.warning] : [],
|
|
819
|
+
hasWarnedSegmenterUnavailable: segmentation.hasWarnedSegmenterUnavailable
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
function parseCitationInline(text, at, start, path, citations, index, isComplete) {
|
|
823
|
+
if (!text.startsWith("[^", at)) return null;
|
|
824
|
+
const close = text.indexOf("]", at + 2);
|
|
825
|
+
if (close >= 0) {
|
|
826
|
+
const idRef = text.slice(at + 2, close).trim();
|
|
827
|
+
if (!idRef) return null;
|
|
828
|
+
const numbered = assignCitationNumber(citations, idRef);
|
|
829
|
+
return {
|
|
830
|
+
node: {
|
|
831
|
+
path: `${path}.${index}`,
|
|
832
|
+
type: "citation",
|
|
833
|
+
range: {
|
|
834
|
+
start: start + at,
|
|
835
|
+
end: start + close + 1
|
|
836
|
+
},
|
|
837
|
+
closed: true,
|
|
838
|
+
props: {
|
|
839
|
+
idRef,
|
|
840
|
+
number: numbered.number
|
|
841
|
+
},
|
|
842
|
+
children: []
|
|
843
|
+
},
|
|
844
|
+
next: close + 1,
|
|
845
|
+
citations: numbered.citations
|
|
846
|
+
};
|
|
847
|
+
}
|
|
848
|
+
if (isComplete) return null;
|
|
849
|
+
const partial = text.slice(at);
|
|
850
|
+
if (!/^\[\^[^\]\s]+$/u.test(partial)) return null;
|
|
851
|
+
const partialIdRef = partial.slice(2);
|
|
852
|
+
return {
|
|
853
|
+
node: {
|
|
854
|
+
path: `${path}.${index}`,
|
|
855
|
+
type: "citation",
|
|
856
|
+
range: {
|
|
857
|
+
start: start + at,
|
|
858
|
+
end: start + text.length
|
|
859
|
+
},
|
|
860
|
+
closed: false,
|
|
861
|
+
props: { idRef: partialIdRef },
|
|
862
|
+
children: []
|
|
863
|
+
},
|
|
864
|
+
next: text.length,
|
|
865
|
+
citations
|
|
866
|
+
};
|
|
867
|
+
}
|
|
868
|
+
function parseImageInline(text, at, start, path, index) {
|
|
869
|
+
if (!text.startsWith("![", at)) return null;
|
|
870
|
+
const labelEnd = findClosing(text, at + 1, "[", "]");
|
|
871
|
+
if (labelEnd < 0 || text[labelEnd + 1] !== "(") return null;
|
|
872
|
+
const destEnd = findClosing(text, labelEnd + 1, "(", ")");
|
|
873
|
+
if (destEnd < 0) return null;
|
|
874
|
+
const alt = text.slice(at + 2, labelEnd);
|
|
875
|
+
const parsed = parseLinkDestination(text.slice(labelEnd + 2, destEnd));
|
|
876
|
+
return {
|
|
877
|
+
node: {
|
|
878
|
+
path: `${path}.${index}`,
|
|
879
|
+
type: "image",
|
|
880
|
+
range: {
|
|
881
|
+
start: start + at,
|
|
882
|
+
end: start + destEnd + 1
|
|
883
|
+
},
|
|
884
|
+
closed: true,
|
|
885
|
+
props: {
|
|
886
|
+
url: parsed.url,
|
|
887
|
+
alt,
|
|
888
|
+
...parsed.title ? { title: parsed.title } : {}
|
|
889
|
+
},
|
|
890
|
+
children: []
|
|
891
|
+
},
|
|
892
|
+
next: destEnd + 1
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
function parseLinkInline(text, at, start, path, index, context, citations, warnings, hasWarnedSegmenterUnavailable) {
|
|
896
|
+
if (text[at] !== "[" || text.startsWith("[^", at)) return null;
|
|
897
|
+
const labelEnd = findClosing(text, at, "[", "]");
|
|
898
|
+
if (labelEnd < 0 || text[labelEnd + 1] !== "(") return null;
|
|
899
|
+
const destEnd = findClosing(text, labelEnd + 1, "(", ")");
|
|
900
|
+
if (destEnd < 0) return null;
|
|
901
|
+
const label = text.slice(at + 1, labelEnd);
|
|
902
|
+
const parsed = parseLinkDestination(text.slice(labelEnd + 2, destEnd));
|
|
903
|
+
const childResult = parseInline(label, start + at + 1, `${path}.${index}`, {
|
|
904
|
+
...context,
|
|
905
|
+
citations,
|
|
906
|
+
warnings,
|
|
907
|
+
hasWarnedSegmenterUnavailable
|
|
908
|
+
});
|
|
909
|
+
return {
|
|
910
|
+
node: {
|
|
911
|
+
path: `${path}.${index}`,
|
|
912
|
+
type: "link",
|
|
913
|
+
range: {
|
|
914
|
+
start: start + at,
|
|
915
|
+
end: start + destEnd + 1
|
|
916
|
+
},
|
|
917
|
+
closed: true,
|
|
918
|
+
props: {
|
|
919
|
+
url: parsed.url,
|
|
920
|
+
...parsed.title ? { title: parsed.title } : {}
|
|
921
|
+
},
|
|
922
|
+
children: childResult.value
|
|
923
|
+
},
|
|
924
|
+
next: destEnd + 1,
|
|
925
|
+
citations: childResult.citations,
|
|
926
|
+
warnings: childResult.warnings,
|
|
927
|
+
hasWarnedSegmenterUnavailable: childResult.hasWarnedSegmenterUnavailable
|
|
928
|
+
};
|
|
929
|
+
}
|
|
930
|
+
function parseAutolinkInline(text, at, start, path, enabled, index) {
|
|
931
|
+
if (!enabled) return null;
|
|
932
|
+
if (text[at] === "<") {
|
|
933
|
+
const close = text.indexOf(">", at + 1);
|
|
934
|
+
if (close > at + 1) {
|
|
935
|
+
const value = text.slice(at + 1, close).trim();
|
|
936
|
+
if (isUrl(value) || isEmail(value)) {
|
|
937
|
+
const url = isEmail(value) ? `mailto:${value}` : value;
|
|
938
|
+
return {
|
|
939
|
+
node: {
|
|
940
|
+
path: `${path}.${index}`,
|
|
941
|
+
type: "autolink",
|
|
942
|
+
range: {
|
|
943
|
+
start: start + at,
|
|
944
|
+
end: start + close + 1
|
|
945
|
+
},
|
|
946
|
+
closed: true,
|
|
947
|
+
props: {
|
|
948
|
+
url,
|
|
949
|
+
text: value
|
|
950
|
+
},
|
|
951
|
+
children: []
|
|
952
|
+
},
|
|
953
|
+
next: close + 1
|
|
954
|
+
};
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
if (!(at === 0 || /\s|[([{"'`]/.test(text[at - 1]))) return null;
|
|
959
|
+
const match = /^(https?:\/\/\S+|www\.\S+|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,})/.exec(text.slice(at));
|
|
960
|
+
if (!match) return null;
|
|
961
|
+
const raw = trimAutolinkTrailingPunctuation(match[1]);
|
|
962
|
+
if (!raw) return null;
|
|
963
|
+
const url = isEmail(raw) ? `mailto:${raw}` : raw.startsWith("www.") ? `https://${raw}` : raw;
|
|
964
|
+
return {
|
|
965
|
+
node: {
|
|
966
|
+
path: `${path}.${index}`,
|
|
967
|
+
type: "autolink",
|
|
968
|
+
range: {
|
|
969
|
+
start: start + at,
|
|
970
|
+
end: start + at + raw.length
|
|
971
|
+
},
|
|
972
|
+
closed: true,
|
|
973
|
+
props: {
|
|
974
|
+
url,
|
|
975
|
+
text: raw
|
|
976
|
+
},
|
|
977
|
+
children: []
|
|
978
|
+
},
|
|
979
|
+
next: at + raw.length
|
|
980
|
+
};
|
|
981
|
+
}
|
|
982
|
+
function parseInlineCode(text, at, start, path, index) {
|
|
983
|
+
if (text[at] !== "`") return null;
|
|
984
|
+
let len = 1;
|
|
985
|
+
while (text[at + len] === "`") len += 1;
|
|
986
|
+
const delimiter = "`".repeat(len);
|
|
987
|
+
const close = text.indexOf(delimiter, at + len);
|
|
988
|
+
if (close < 0) return {
|
|
989
|
+
node: {
|
|
990
|
+
path: `${path}.${index}`,
|
|
991
|
+
type: "inline-code",
|
|
992
|
+
range: {
|
|
993
|
+
start: start + at,
|
|
994
|
+
end: start + text.length
|
|
995
|
+
},
|
|
996
|
+
closed: false,
|
|
997
|
+
props: { text: text.slice(at + len) },
|
|
998
|
+
children: []
|
|
999
|
+
},
|
|
1000
|
+
next: text.length
|
|
1001
|
+
};
|
|
1002
|
+
return {
|
|
1003
|
+
node: {
|
|
1004
|
+
path: `${path}.${index}`,
|
|
1005
|
+
type: "inline-code",
|
|
1006
|
+
range: {
|
|
1007
|
+
start: start + at,
|
|
1008
|
+
end: start + close + len
|
|
1009
|
+
},
|
|
1010
|
+
closed: true,
|
|
1011
|
+
props: { text: text.slice(at + len, close) },
|
|
1012
|
+
children: []
|
|
1013
|
+
},
|
|
1014
|
+
next: close + len
|
|
1015
|
+
};
|
|
1016
|
+
}
|
|
1017
|
+
function parseDelimitedInline(text, at, delimiter, type, start, path, index, context, citations, warnings, hasWarnedSegmenterUnavailable) {
|
|
1018
|
+
if (!text.startsWith(delimiter, at)) return null;
|
|
1019
|
+
const close = text.indexOf(delimiter, at + delimiter.length);
|
|
1020
|
+
if (close < 0) {
|
|
1021
|
+
const children = parseInline(text.slice(at + delimiter.length), start + at + delimiter.length, `${path}.${index}`, {
|
|
1022
|
+
...context,
|
|
1023
|
+
citations,
|
|
1024
|
+
warnings,
|
|
1025
|
+
hasWarnedSegmenterUnavailable
|
|
1026
|
+
});
|
|
1027
|
+
return {
|
|
1028
|
+
node: {
|
|
1029
|
+
path: `${path}.${index}`,
|
|
1030
|
+
type,
|
|
1031
|
+
range: {
|
|
1032
|
+
start: start + at,
|
|
1033
|
+
end: start + text.length
|
|
1034
|
+
},
|
|
1035
|
+
closed: false,
|
|
1036
|
+
props: {},
|
|
1037
|
+
children: children.value
|
|
1038
|
+
},
|
|
1039
|
+
next: text.length,
|
|
1040
|
+
citations: children.citations,
|
|
1041
|
+
warnings: children.warnings,
|
|
1042
|
+
hasWarnedSegmenterUnavailable: children.hasWarnedSegmenterUnavailable
|
|
1043
|
+
};
|
|
1044
|
+
}
|
|
1045
|
+
const children = parseInline(text.slice(at + delimiter.length, close), start + at + delimiter.length, `${path}.${index}`, {
|
|
1046
|
+
...context,
|
|
1047
|
+
citations,
|
|
1048
|
+
warnings,
|
|
1049
|
+
hasWarnedSegmenterUnavailable
|
|
1050
|
+
});
|
|
1051
|
+
return {
|
|
1052
|
+
node: {
|
|
1053
|
+
path: `${path}.${index}`,
|
|
1054
|
+
type,
|
|
1055
|
+
range: {
|
|
1056
|
+
start: start + at,
|
|
1057
|
+
end: start + close + delimiter.length
|
|
1058
|
+
},
|
|
1059
|
+
closed: true,
|
|
1060
|
+
props: {},
|
|
1061
|
+
children: children.value
|
|
1062
|
+
},
|
|
1063
|
+
next: close + delimiter.length,
|
|
1064
|
+
citations: children.citations,
|
|
1065
|
+
warnings: children.warnings,
|
|
1066
|
+
hasWarnedSegmenterUnavailable: children.hasWarnedSegmenterUnavailable
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
//#endregion
|
|
1071
|
+
//#region src/parser/block-parser.ts
|
|
1072
|
+
/**
|
|
1073
|
+
* Parses a contiguous range of source lines into block-level draft nodes.
|
|
1074
|
+
*
|
|
1075
|
+
* @param lines - Line-split source with offsets.
|
|
1076
|
+
* @param from - Inclusive start line index.
|
|
1077
|
+
* @param to - Exclusive end line index.
|
|
1078
|
+
* @param path - Stable path prefix for emitted nodes.
|
|
1079
|
+
* @param context - Shared parse context carrying options, warnings, and citations.
|
|
1080
|
+
* @param isRoot - Whether this parse call is for the document root.
|
|
1081
|
+
* @returns Root draft node plus updated immutable parse context.
|
|
1082
|
+
*/
|
|
1083
|
+
function parseBlocks(lines, from, to, path, context, isRoot) {
|
|
1084
|
+
let root = {
|
|
1085
|
+
path,
|
|
1086
|
+
type: "document",
|
|
1087
|
+
range: {
|
|
1088
|
+
start: from < to ? lines[from].start : 0,
|
|
1089
|
+
end: from < to ? lines[to - 1].end : 0
|
|
1090
|
+
},
|
|
1091
|
+
closed: false,
|
|
1092
|
+
props: {},
|
|
1093
|
+
children: []
|
|
1094
|
+
};
|
|
1095
|
+
let i = from;
|
|
1096
|
+
let citations = context.citations;
|
|
1097
|
+
let warnings = context.warnings;
|
|
1098
|
+
let hasWarnedSegmenterUnavailable = context.hasWarnedSegmenterUnavailable;
|
|
1099
|
+
while (i < to) {
|
|
1100
|
+
const line = lines[i];
|
|
1101
|
+
const isCitationDefinition = isCitationDefinitionLine(line.text);
|
|
1102
|
+
const isCitationDefinitionPrefix = isCitationDefinitionPrefixCandidate(line.text, line.hasNewline, context.isComplete);
|
|
1103
|
+
if (isBlank(line.text) || isCitationDefinition || isCitationDefinitionPrefix) {
|
|
1104
|
+
i += 1;
|
|
1105
|
+
continue;
|
|
1106
|
+
}
|
|
1107
|
+
const fenced = matchFenceOpen(line.text);
|
|
1108
|
+
if (fenced) {
|
|
1109
|
+
const nodePath = `${path}.${root.children.length}`;
|
|
1110
|
+
const parsed = parseCodeFence(lines, i, to, nodePath, context, fenced);
|
|
1111
|
+
root.children = [...root.children, parsed.node];
|
|
1112
|
+
i = parsed.next;
|
|
1113
|
+
continue;
|
|
1114
|
+
}
|
|
1115
|
+
if (isBlockquoteLine(line.text)) {
|
|
1116
|
+
const nodePath = `${path}.${root.children.length}`;
|
|
1117
|
+
const parsed = parseBlockquote(lines, i, to, nodePath, {
|
|
1118
|
+
...context,
|
|
1119
|
+
citations,
|
|
1120
|
+
warnings,
|
|
1121
|
+
hasWarnedSegmenterUnavailable
|
|
1122
|
+
});
|
|
1123
|
+
root.children = [...root.children, parsed.value];
|
|
1124
|
+
i = parsed.next;
|
|
1125
|
+
citations = parsed.citations;
|
|
1126
|
+
warnings = parsed.warnings;
|
|
1127
|
+
hasWarnedSegmenterUnavailable = parsed.hasWarnedSegmenterUnavailable;
|
|
1128
|
+
continue;
|
|
1129
|
+
}
|
|
1130
|
+
const listMarker = matchListItem(line.text);
|
|
1131
|
+
if (listMarker) {
|
|
1132
|
+
const nodePath = `${path}.${root.children.length}`;
|
|
1133
|
+
const parsed = parseList(lines, i, to, nodePath, {
|
|
1134
|
+
...context,
|
|
1135
|
+
citations,
|
|
1136
|
+
warnings,
|
|
1137
|
+
hasWarnedSegmenterUnavailable
|
|
1138
|
+
}, listMarker.ordered);
|
|
1139
|
+
root.children = [...root.children, parsed.value];
|
|
1140
|
+
i = parsed.next;
|
|
1141
|
+
citations = parsed.citations;
|
|
1142
|
+
warnings = parsed.warnings;
|
|
1143
|
+
hasWarnedSegmenterUnavailable = parsed.hasWarnedSegmenterUnavailable;
|
|
1144
|
+
continue;
|
|
1145
|
+
}
|
|
1146
|
+
const heading = matchAtxHeading(line.text);
|
|
1147
|
+
if (heading) {
|
|
1148
|
+
const nodePath = `${path}.${root.children.length}`;
|
|
1149
|
+
const children = parseInline(heading.text, line.start + heading.contentOffset, nodePath, {
|
|
1150
|
+
...context,
|
|
1151
|
+
citations,
|
|
1152
|
+
warnings,
|
|
1153
|
+
hasWarnedSegmenterUnavailable
|
|
1154
|
+
});
|
|
1155
|
+
root.children = [...root.children, {
|
|
1156
|
+
path: nodePath,
|
|
1157
|
+
type: "heading",
|
|
1158
|
+
range: {
|
|
1159
|
+
start: line.start,
|
|
1160
|
+
end: line.end
|
|
1161
|
+
},
|
|
1162
|
+
closed: context.isComplete || line.hasNewline,
|
|
1163
|
+
props: { level: heading.level },
|
|
1164
|
+
children: children.value
|
|
1165
|
+
}];
|
|
1166
|
+
citations = children.citations;
|
|
1167
|
+
warnings = children.warnings;
|
|
1168
|
+
hasWarnedSegmenterUnavailable = children.hasWarnedSegmenterUnavailable;
|
|
1169
|
+
i += 1;
|
|
1170
|
+
continue;
|
|
1171
|
+
}
|
|
1172
|
+
if (i + 1 < to && isSetextUnderline(lines[i + 1].text) && !isBlank(line.text)) {
|
|
1173
|
+
const nodePath = `${path}.${root.children.length}`;
|
|
1174
|
+
const level = lines[i + 1].text.trim().startsWith("=") ? 1 : 2;
|
|
1175
|
+
const children = parseInline(line.text, line.start, nodePath, {
|
|
1176
|
+
...context,
|
|
1177
|
+
citations,
|
|
1178
|
+
warnings,
|
|
1179
|
+
hasWarnedSegmenterUnavailable
|
|
1180
|
+
});
|
|
1181
|
+
root.children = [...root.children, {
|
|
1182
|
+
path: nodePath,
|
|
1183
|
+
type: "heading",
|
|
1184
|
+
range: {
|
|
1185
|
+
start: line.start,
|
|
1186
|
+
end: lines[i + 1].end
|
|
1187
|
+
},
|
|
1188
|
+
closed: context.isComplete || lines[i + 1].hasNewline,
|
|
1189
|
+
props: { level },
|
|
1190
|
+
children: children.value
|
|
1191
|
+
}];
|
|
1192
|
+
citations = children.citations;
|
|
1193
|
+
warnings = children.warnings;
|
|
1194
|
+
hasWarnedSegmenterUnavailable = children.hasWarnedSegmenterUnavailable;
|
|
1195
|
+
i += 2;
|
|
1196
|
+
continue;
|
|
1197
|
+
}
|
|
1198
|
+
if (isThematicBreak(line.text)) {
|
|
1199
|
+
root.children = [...root.children, {
|
|
1200
|
+
path: `${path}.${root.children.length}`,
|
|
1201
|
+
type: "thematic-break",
|
|
1202
|
+
range: {
|
|
1203
|
+
start: line.start,
|
|
1204
|
+
end: line.end
|
|
1205
|
+
},
|
|
1206
|
+
closed: true,
|
|
1207
|
+
props: {},
|
|
1208
|
+
children: []
|
|
1209
|
+
}];
|
|
1210
|
+
i += 1;
|
|
1211
|
+
continue;
|
|
1212
|
+
}
|
|
1213
|
+
if (context.options.enableTables && i + 1 < to && isPipeTableHeader(lines[i], lines[i + 1])) {
|
|
1214
|
+
const nodePath = `${path}.${root.children.length}`;
|
|
1215
|
+
const parsed = parsePipeTable(lines, i, to, nodePath, {
|
|
1216
|
+
...context,
|
|
1217
|
+
citations,
|
|
1218
|
+
warnings,
|
|
1219
|
+
hasWarnedSegmenterUnavailable
|
|
1220
|
+
});
|
|
1221
|
+
root.children = [...root.children, parsed.value];
|
|
1222
|
+
i = parsed.next;
|
|
1223
|
+
citations = parsed.citations;
|
|
1224
|
+
warnings = parsed.warnings;
|
|
1225
|
+
hasWarnedSegmenterUnavailable = parsed.hasWarnedSegmenterUnavailable;
|
|
1226
|
+
continue;
|
|
1227
|
+
}
|
|
1228
|
+
const nodePath = `${path}.${root.children.length}`;
|
|
1229
|
+
const parsed = parseParagraph(lines, i, to, nodePath, {
|
|
1230
|
+
...context,
|
|
1231
|
+
citations,
|
|
1232
|
+
warnings,
|
|
1233
|
+
hasWarnedSegmenterUnavailable
|
|
1234
|
+
});
|
|
1235
|
+
root.children = [...root.children, parsed.value];
|
|
1236
|
+
i = parsed.next;
|
|
1237
|
+
citations = parsed.citations;
|
|
1238
|
+
warnings = parsed.warnings;
|
|
1239
|
+
hasWarnedSegmenterUnavailable = parsed.hasWarnedSegmenterUnavailable;
|
|
1240
|
+
}
|
|
1241
|
+
root = {
|
|
1242
|
+
...root,
|
|
1243
|
+
closed: !isRoot || context.isComplete || to > from && lines[to - 1].hasNewline && root.children.every((child) => child.closed)
|
|
1244
|
+
};
|
|
1245
|
+
return {
|
|
1246
|
+
value: root,
|
|
1247
|
+
citations,
|
|
1248
|
+
warnings,
|
|
1249
|
+
hasWarnedSegmenterUnavailable
|
|
1250
|
+
};
|
|
1251
|
+
}
|
|
1252
|
+
function parseCodeFence(lines, from, to, path, context, fenceOpen) {
|
|
1253
|
+
let i = from + 1;
|
|
1254
|
+
let closeLine = -1;
|
|
1255
|
+
while (i < to) {
|
|
1256
|
+
if (matchFenceClose(lines[i].text, fenceOpen.marker, fenceOpen.length)) {
|
|
1257
|
+
closeLine = i;
|
|
1258
|
+
break;
|
|
1259
|
+
}
|
|
1260
|
+
i += 1;
|
|
1261
|
+
}
|
|
1262
|
+
const endLine = closeLine >= 0 ? closeLine : to - 1;
|
|
1263
|
+
const text = lines.slice(from + 1, closeLine >= 0 ? closeLine : to).map((line) => line.text).join("\n");
|
|
1264
|
+
const closed = closeLine >= 0 || context.isComplete;
|
|
1265
|
+
return {
|
|
1266
|
+
node: {
|
|
1267
|
+
path,
|
|
1268
|
+
type: "code-block",
|
|
1269
|
+
range: {
|
|
1270
|
+
start: lines[from].start,
|
|
1271
|
+
end: lines[endLine].end
|
|
1272
|
+
},
|
|
1273
|
+
closed,
|
|
1274
|
+
props: {
|
|
1275
|
+
fence: fenceOpen.marker,
|
|
1276
|
+
...fenceOpen.info ? { info: fenceOpen.info.split(/\s+/)[0] } : {},
|
|
1277
|
+
...fenceOpen.info.includes(" ") ? { meta: fenceOpen.info.slice(fenceOpen.info.indexOf(" ") + 1) } : {},
|
|
1278
|
+
text
|
|
1279
|
+
},
|
|
1280
|
+
children: []
|
|
1281
|
+
},
|
|
1282
|
+
next: closeLine >= 0 ? closeLine + 1 : to
|
|
1283
|
+
};
|
|
1284
|
+
}
|
|
1285
|
+
function parseBlockquote(lines, from, to, path, context) {
|
|
1286
|
+
const picked = [];
|
|
1287
|
+
let i = from;
|
|
1288
|
+
while (i < to) {
|
|
1289
|
+
const line = lines[i];
|
|
1290
|
+
if (isBlank(line.text) && picked.length > 0) {
|
|
1291
|
+
picked.push(line);
|
|
1292
|
+
i += 1;
|
|
1293
|
+
continue;
|
|
1294
|
+
}
|
|
1295
|
+
if (isBlockquoteLine(line.text) || !isBlank(line.text) && picked.length > 0 && !startsNewBlock(lines, i, context.options.enableTables)) {
|
|
1296
|
+
picked.push(line);
|
|
1297
|
+
i += 1;
|
|
1298
|
+
continue;
|
|
1299
|
+
}
|
|
1300
|
+
break;
|
|
1301
|
+
}
|
|
1302
|
+
const stripped = picked.map((line) => {
|
|
1303
|
+
const marker = /^(\s{0,3}>\s?)/.exec(line.text);
|
|
1304
|
+
const markerWidth = marker?.[1].length ?? 0;
|
|
1305
|
+
const text = marker ? line.text.slice(markerWidth) : line.text;
|
|
1306
|
+
const start = line.start + markerWidth;
|
|
1307
|
+
return {
|
|
1308
|
+
text,
|
|
1309
|
+
start,
|
|
1310
|
+
end: start + text.length,
|
|
1311
|
+
hasNewline: line.hasNewline
|
|
1312
|
+
};
|
|
1313
|
+
});
|
|
1314
|
+
const children = parseBlocks(stripped, 0, stripped.length, path, context, false);
|
|
1315
|
+
return {
|
|
1316
|
+
value: {
|
|
1317
|
+
path,
|
|
1318
|
+
type: "blockquote",
|
|
1319
|
+
range: {
|
|
1320
|
+
start: picked[0].start,
|
|
1321
|
+
end: picked[picked.length - 1].end
|
|
1322
|
+
},
|
|
1323
|
+
closed: context.isComplete || picked[picked.length - 1].hasNewline,
|
|
1324
|
+
props: {},
|
|
1325
|
+
children: children.value.children
|
|
1326
|
+
},
|
|
1327
|
+
next: i,
|
|
1328
|
+
citations: children.citations,
|
|
1329
|
+
warnings: children.warnings,
|
|
1330
|
+
hasWarnedSegmenterUnavailable: children.hasWarnedSegmenterUnavailable
|
|
1331
|
+
};
|
|
1332
|
+
}
|
|
1333
|
+
function parseList(lines, from, to, path, context, ordered) {
|
|
1334
|
+
const items = [];
|
|
1335
|
+
let i = from;
|
|
1336
|
+
let sawBlankBetweenItems = false;
|
|
1337
|
+
let listStartNumber = null;
|
|
1338
|
+
let citations = context.citations;
|
|
1339
|
+
let warnings = context.warnings;
|
|
1340
|
+
let hasWarnedSegmenterUnavailable = context.hasWarnedSegmenterUnavailable;
|
|
1341
|
+
while (i < to) {
|
|
1342
|
+
const marker = matchListItem(lines[i].text);
|
|
1343
|
+
if (!marker || marker.ordered !== ordered) break;
|
|
1344
|
+
if (ordered && listStartNumber === null) listStartNumber = marker.start;
|
|
1345
|
+
const contentLines = [{
|
|
1346
|
+
...lines[i],
|
|
1347
|
+
text: marker.content
|
|
1348
|
+
}];
|
|
1349
|
+
const contentIndent = marker.contentIndent;
|
|
1350
|
+
i += 1;
|
|
1351
|
+
let hadBlank = false;
|
|
1352
|
+
while (i < to) {
|
|
1353
|
+
const next = lines[i];
|
|
1354
|
+
if (matchListItem(next.text)) break;
|
|
1355
|
+
if (isBlank(next.text)) {
|
|
1356
|
+
hadBlank = true;
|
|
1357
|
+
contentLines.push(next);
|
|
1358
|
+
i += 1;
|
|
1359
|
+
if (i < to) {
|
|
1360
|
+
if (matchListItem(lines[i].text)) {
|
|
1361
|
+
sawBlankBetweenItems = true;
|
|
1362
|
+
break;
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
continue;
|
|
1366
|
+
}
|
|
1367
|
+
if ((/^\s*/.exec(next.text)?.[0].length ?? 0) < contentIndent) break;
|
|
1368
|
+
const continuation = next.text.slice(contentIndent);
|
|
1369
|
+
contentLines.push({
|
|
1370
|
+
...next,
|
|
1371
|
+
text: continuation
|
|
1372
|
+
});
|
|
1373
|
+
i += 1;
|
|
1374
|
+
}
|
|
1375
|
+
const paragraphText = contentLines.map((line) => line.text).join("\n").trimEnd();
|
|
1376
|
+
const itemPath = `${path}.${items.length}`;
|
|
1377
|
+
const parsedInline = parseInline(paragraphText, contentLines[0].start, `${itemPath}.0`, {
|
|
1378
|
+
...context,
|
|
1379
|
+
citations,
|
|
1380
|
+
warnings,
|
|
1381
|
+
hasWarnedSegmenterUnavailable
|
|
1382
|
+
});
|
|
1383
|
+
const paragraph = {
|
|
1384
|
+
path: `${itemPath}.0`,
|
|
1385
|
+
type: "paragraph",
|
|
1386
|
+
range: {
|
|
1387
|
+
start: contentLines[0].start,
|
|
1388
|
+
end: contentLines[contentLines.length - 1].end
|
|
1389
|
+
},
|
|
1390
|
+
closed: context.isComplete || contentLines[contentLines.length - 1].hasNewline,
|
|
1391
|
+
props: {},
|
|
1392
|
+
children: parsedInline.value
|
|
1393
|
+
};
|
|
1394
|
+
items.push({
|
|
1395
|
+
path: itemPath,
|
|
1396
|
+
type: "list-item",
|
|
1397
|
+
range: {
|
|
1398
|
+
start: contentLines[0].start,
|
|
1399
|
+
end: contentLines[contentLines.length - 1].end
|
|
1400
|
+
},
|
|
1401
|
+
closed: context.isComplete || contentLines[contentLines.length - 1].hasNewline,
|
|
1402
|
+
props: {},
|
|
1403
|
+
children: [paragraph]
|
|
1404
|
+
});
|
|
1405
|
+
citations = parsedInline.citations;
|
|
1406
|
+
warnings = parsedInline.warnings;
|
|
1407
|
+
hasWarnedSegmenterUnavailable = parsedInline.hasWarnedSegmenterUnavailable;
|
|
1408
|
+
if (hadBlank) sawBlankBetweenItems = true;
|
|
1409
|
+
}
|
|
1410
|
+
return {
|
|
1411
|
+
value: {
|
|
1412
|
+
path,
|
|
1413
|
+
type: "list",
|
|
1414
|
+
range: {
|
|
1415
|
+
start: lines[from].start,
|
|
1416
|
+
end: lines[Math.max(from, i - 1)].end
|
|
1417
|
+
},
|
|
1418
|
+
closed: context.isComplete || lines[Math.max(from, i - 1)].hasNewline,
|
|
1419
|
+
props: {
|
|
1420
|
+
ordered,
|
|
1421
|
+
start: ordered ? listStartNumber ?? 1 : null,
|
|
1422
|
+
tight: !sawBlankBetweenItems
|
|
1423
|
+
},
|
|
1424
|
+
children: items
|
|
1425
|
+
},
|
|
1426
|
+
next: i,
|
|
1427
|
+
citations,
|
|
1428
|
+
warnings,
|
|
1429
|
+
hasWarnedSegmenterUnavailable
|
|
1430
|
+
};
|
|
1431
|
+
}
|
|
1432
|
+
function parsePipeTable(lines, from, to, path, context) {
|
|
1433
|
+
const header = splitTableCells(lines[from].text);
|
|
1434
|
+
const align = parseTableAlignment(splitTableCells(lines[from + 1].text));
|
|
1435
|
+
const rows = [];
|
|
1436
|
+
let i = from + 2;
|
|
1437
|
+
while (i < to && looksLikeTableRow(lines[i].text)) {
|
|
1438
|
+
rows.push(splitTableCells(lines[i].text));
|
|
1439
|
+
i += 1;
|
|
1440
|
+
}
|
|
1441
|
+
let citations = context.citations;
|
|
1442
|
+
let warnings = context.warnings;
|
|
1443
|
+
let hasWarnedSegmenterUnavailable = context.hasWarnedSegmenterUnavailable;
|
|
1444
|
+
const headerChildren = [];
|
|
1445
|
+
for (let cellIndex = 0; cellIndex < header.length; cellIndex += 1) {
|
|
1446
|
+
const parsed = parseInline(header[cellIndex], lines[from].start, `${path}.0.${cellIndex}`, {
|
|
1447
|
+
...context,
|
|
1448
|
+
citations,
|
|
1449
|
+
warnings,
|
|
1450
|
+
hasWarnedSegmenterUnavailable
|
|
1451
|
+
});
|
|
1452
|
+
headerChildren.push({
|
|
1453
|
+
path: `${path}.0.${cellIndex}`,
|
|
1454
|
+
type: "table-cell",
|
|
1455
|
+
range: {
|
|
1456
|
+
start: lines[from].start,
|
|
1457
|
+
end: lines[from].end
|
|
1458
|
+
},
|
|
1459
|
+
closed: true,
|
|
1460
|
+
props: {},
|
|
1461
|
+
children: parsed.value
|
|
1462
|
+
});
|
|
1463
|
+
citations = parsed.citations;
|
|
1464
|
+
warnings = parsed.warnings;
|
|
1465
|
+
hasWarnedSegmenterUnavailable = parsed.hasWarnedSegmenterUnavailable;
|
|
1466
|
+
}
|
|
1467
|
+
const headerRow = {
|
|
1468
|
+
path: `${path}.0`,
|
|
1469
|
+
type: "table-row",
|
|
1470
|
+
range: {
|
|
1471
|
+
start: lines[from].start,
|
|
1472
|
+
end: lines[from].end
|
|
1473
|
+
},
|
|
1474
|
+
closed: true,
|
|
1475
|
+
props: { isHeader: true },
|
|
1476
|
+
children: headerChildren
|
|
1477
|
+
};
|
|
1478
|
+
const bodyRows = [];
|
|
1479
|
+
for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) {
|
|
1480
|
+
const row = rows[rowIndex];
|
|
1481
|
+
const cellNodes = [];
|
|
1482
|
+
for (let cellIndex = 0; cellIndex < row.length; cellIndex += 1) {
|
|
1483
|
+
const parsed = parseInline(row[cellIndex], lines[from + 2 + rowIndex].start, `${path}.${rowIndex + 1}.${cellIndex}`, {
|
|
1484
|
+
...context,
|
|
1485
|
+
citations,
|
|
1486
|
+
warnings,
|
|
1487
|
+
hasWarnedSegmenterUnavailable
|
|
1488
|
+
});
|
|
1489
|
+
cellNodes.push({
|
|
1490
|
+
path: `${path}.${rowIndex + 1}.${cellIndex}`,
|
|
1491
|
+
type: "table-cell",
|
|
1492
|
+
range: {
|
|
1493
|
+
start: lines[from + 2 + rowIndex].start,
|
|
1494
|
+
end: lines[from + 2 + rowIndex].end
|
|
1495
|
+
},
|
|
1496
|
+
closed: true,
|
|
1497
|
+
props: {},
|
|
1498
|
+
children: parsed.value
|
|
1499
|
+
});
|
|
1500
|
+
citations = parsed.citations;
|
|
1501
|
+
warnings = parsed.warnings;
|
|
1502
|
+
hasWarnedSegmenterUnavailable = parsed.hasWarnedSegmenterUnavailable;
|
|
1503
|
+
}
|
|
1504
|
+
bodyRows.push({
|
|
1505
|
+
path: `${path}.${rowIndex + 1}`,
|
|
1506
|
+
type: "table-row",
|
|
1507
|
+
range: {
|
|
1508
|
+
start: lines[from + 2 + rowIndex].start,
|
|
1509
|
+
end: lines[from + 2 + rowIndex].end
|
|
1510
|
+
},
|
|
1511
|
+
closed: true,
|
|
1512
|
+
props: { isHeader: false },
|
|
1513
|
+
children: cellNodes
|
|
1514
|
+
});
|
|
1515
|
+
}
|
|
1516
|
+
return {
|
|
1517
|
+
value: {
|
|
1518
|
+
path,
|
|
1519
|
+
type: "table",
|
|
1520
|
+
range: {
|
|
1521
|
+
start: lines[from].start,
|
|
1522
|
+
end: lines[i - 1].end
|
|
1523
|
+
},
|
|
1524
|
+
closed: context.isComplete || lines[i - 1].hasNewline,
|
|
1525
|
+
props: { align },
|
|
1526
|
+
children: [headerRow, ...bodyRows]
|
|
1527
|
+
},
|
|
1528
|
+
next: i,
|
|
1529
|
+
citations,
|
|
1530
|
+
warnings,
|
|
1531
|
+
hasWarnedSegmenterUnavailable
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1534
|
+
function parseParagraph(lines, from, to, path, context) {
|
|
1535
|
+
const picked = [];
|
|
1536
|
+
let i = from;
|
|
1537
|
+
while (i < to) {
|
|
1538
|
+
if (isBlank(lines[i].text)) break;
|
|
1539
|
+
if (picked.length > 0 && startsNewBlock(lines, i, context.options.enableTables)) break;
|
|
1540
|
+
if (isCitationDefinitionLine(lines[i].text) || isCitationDefinitionPrefixCandidate(lines[i].text, lines[i].hasNewline, context.isComplete)) break;
|
|
1541
|
+
picked.push(lines[i]);
|
|
1542
|
+
i += 1;
|
|
1543
|
+
}
|
|
1544
|
+
const children = parseInline(picked.map((line) => line.text).join("\n"), picked[0].start, path, context);
|
|
1545
|
+
return {
|
|
1546
|
+
value: {
|
|
1547
|
+
path,
|
|
1548
|
+
type: "paragraph",
|
|
1549
|
+
range: {
|
|
1550
|
+
start: picked[0].start,
|
|
1551
|
+
end: picked[picked.length - 1].end
|
|
1552
|
+
},
|
|
1553
|
+
closed: context.isComplete || picked[picked.length - 1].hasNewline,
|
|
1554
|
+
props: {},
|
|
1555
|
+
children: children.value
|
|
1556
|
+
},
|
|
1557
|
+
next: i,
|
|
1558
|
+
citations: children.citations,
|
|
1559
|
+
warnings: children.warnings,
|
|
1560
|
+
hasWarnedSegmenterUnavailable: children.hasWarnedSegmenterUnavailable
|
|
1561
|
+
};
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
//#endregion
|
|
1565
|
+
//#region src/parser/ops.ts
|
|
1566
|
+
/**
|
|
1567
|
+
* Flattens a draft tree into reducer operations.
|
|
1568
|
+
*
|
|
1569
|
+
* @param root - Draft AST root.
|
|
1570
|
+
* @returns Ordered operations for reducer application.
|
|
1571
|
+
*/
|
|
1572
|
+
function createAstOpsFromDraft(root) {
|
|
1573
|
+
const ops = [];
|
|
1574
|
+
function visit(node, parentPath) {
|
|
1575
|
+
const childPaths = node.children.map((child, index) => `${node.path}.${index}`);
|
|
1576
|
+
ops.push({
|
|
1577
|
+
kind: "upsert-node",
|
|
1578
|
+
path: node.path,
|
|
1579
|
+
parentPath,
|
|
1580
|
+
type: node.type,
|
|
1581
|
+
range: node.range,
|
|
1582
|
+
closed: node.closed,
|
|
1583
|
+
props: node.props,
|
|
1584
|
+
childPaths
|
|
1585
|
+
});
|
|
1586
|
+
for (let index = 0; index < node.children.length; index += 1) {
|
|
1587
|
+
const child = node.children[index];
|
|
1588
|
+
visit({
|
|
1589
|
+
...child,
|
|
1590
|
+
path: `${node.path}.${index}`
|
|
1591
|
+
}, node.path);
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
visit(root, null);
|
|
1595
|
+
return ops;
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Applies AST ops with identity reuse for unchanged nodes.
|
|
1599
|
+
*
|
|
1600
|
+
* @param input - Previous parser state fragments and ops to apply.
|
|
1601
|
+
* @returns Next materialized state fragments.
|
|
1602
|
+
*/
|
|
1603
|
+
function applyAstOps(input) {
|
|
1604
|
+
const previousNodeById = new Map(input.previousNodes.map((node) => [node.id, node]));
|
|
1605
|
+
const nodeOps = input.ops.filter((op) => op.kind === "upsert-node");
|
|
1606
|
+
const pathToId = {};
|
|
1607
|
+
let nextId = input.nextId;
|
|
1608
|
+
for (const op of nodeOps) {
|
|
1609
|
+
const id = input.previousPathToId[op.path] ?? nextId++;
|
|
1610
|
+
pathToId[op.path] = id;
|
|
1611
|
+
}
|
|
1612
|
+
const nodesById = /* @__PURE__ */ new Map();
|
|
1613
|
+
for (const op of nodeOps) {
|
|
1614
|
+
const id = pathToId[op.path];
|
|
1615
|
+
const materialized = opToAstNode(op, id, op.parentPath ? pathToId[op.parentPath] : null, op.childPaths.map((childPath) => pathToId[childPath]));
|
|
1616
|
+
const previous = previousNodeById.get(id);
|
|
1617
|
+
const reconciled = previous?.type === "text" && materialized.type === "text" ? reconcileTextNode(previous, materialized) : materialized;
|
|
1618
|
+
nodesById.set(id, previous && astNodeEquals(previous, reconciled) ? previous : reconciled);
|
|
1619
|
+
}
|
|
1620
|
+
const rootPath = nodeOps[0]?.path;
|
|
1621
|
+
if (!rootPath) throw new Error("StreamingMarkdown op stream did not include a root node");
|
|
1622
|
+
const warnings = input.ops.find((op) => op.kind === "set-warnings")?.warnings ?? [];
|
|
1623
|
+
const citations = input.ops.find((op) => op.kind === "set-citations")?.citations ?? {
|
|
1624
|
+
order: [],
|
|
1625
|
+
numbers: {},
|
|
1626
|
+
definitions: {}
|
|
1627
|
+
};
|
|
1628
|
+
const hasWarnedSegmenterUnavailable = input.ops.find((op) => op.kind === "set-segmenter-warning-state")?.hasWarnedSegmenterUnavailable ?? false;
|
|
1629
|
+
return {
|
|
1630
|
+
nodes: Array.from(nodesById.values()).sort((a, b) => a.id - b.id),
|
|
1631
|
+
rootId: pathToId[rootPath],
|
|
1632
|
+
nextId,
|
|
1633
|
+
pathToId,
|
|
1634
|
+
warnings,
|
|
1635
|
+
citations,
|
|
1636
|
+
hasWarnedSegmenterUnavailable
|
|
1637
|
+
};
|
|
1638
|
+
}
|
|
1639
|
+
function opToAstNode(op, id, parentId, children) {
|
|
1640
|
+
const base = {
|
|
1641
|
+
id,
|
|
1642
|
+
type: op.type,
|
|
1643
|
+
parentId,
|
|
1644
|
+
closed: op.closed,
|
|
1645
|
+
range: op.range
|
|
1646
|
+
};
|
|
1647
|
+
switch (op.type) {
|
|
1648
|
+
case "document": return {
|
|
1649
|
+
...base,
|
|
1650
|
+
type: "document",
|
|
1651
|
+
children
|
|
1652
|
+
};
|
|
1653
|
+
case "paragraph": return {
|
|
1654
|
+
...base,
|
|
1655
|
+
type: "paragraph",
|
|
1656
|
+
children
|
|
1657
|
+
};
|
|
1658
|
+
case "heading": return {
|
|
1659
|
+
...base,
|
|
1660
|
+
type: "heading",
|
|
1661
|
+
level: op.props["level"],
|
|
1662
|
+
children
|
|
1663
|
+
};
|
|
1664
|
+
case "blockquote": return {
|
|
1665
|
+
...base,
|
|
1666
|
+
type: "blockquote",
|
|
1667
|
+
children
|
|
1668
|
+
};
|
|
1669
|
+
case "list": return {
|
|
1670
|
+
...base,
|
|
1671
|
+
type: "list",
|
|
1672
|
+
ordered: op.props["ordered"],
|
|
1673
|
+
start: op.props["start"],
|
|
1674
|
+
tight: op.props["tight"],
|
|
1675
|
+
children
|
|
1676
|
+
};
|
|
1677
|
+
case "list-item": return {
|
|
1678
|
+
...base,
|
|
1679
|
+
type: "list-item",
|
|
1680
|
+
children
|
|
1681
|
+
};
|
|
1682
|
+
case "code-block": return {
|
|
1683
|
+
...base,
|
|
1684
|
+
type: "code-block",
|
|
1685
|
+
fence: op.props["fence"],
|
|
1686
|
+
info: op.props["info"],
|
|
1687
|
+
meta: op.props["meta"],
|
|
1688
|
+
text: op.props["text"]
|
|
1689
|
+
};
|
|
1690
|
+
case "table": return {
|
|
1691
|
+
...base,
|
|
1692
|
+
type: "table",
|
|
1693
|
+
align: op.props["align"],
|
|
1694
|
+
children
|
|
1695
|
+
};
|
|
1696
|
+
case "table-row": return {
|
|
1697
|
+
...base,
|
|
1698
|
+
type: "table-row",
|
|
1699
|
+
isHeader: op.props["isHeader"],
|
|
1700
|
+
children
|
|
1701
|
+
};
|
|
1702
|
+
case "table-cell": return {
|
|
1703
|
+
...base,
|
|
1704
|
+
type: "table-cell",
|
|
1705
|
+
children
|
|
1706
|
+
};
|
|
1707
|
+
case "thematic-break": return {
|
|
1708
|
+
...base,
|
|
1709
|
+
type: "thematic-break"
|
|
1710
|
+
};
|
|
1711
|
+
case "text": return {
|
|
1712
|
+
...base,
|
|
1713
|
+
type: "text",
|
|
1714
|
+
text: op.props["text"],
|
|
1715
|
+
segments: op.props["segments"]
|
|
1716
|
+
};
|
|
1717
|
+
case "em": return {
|
|
1718
|
+
...base,
|
|
1719
|
+
type: "em",
|
|
1720
|
+
children
|
|
1721
|
+
};
|
|
1722
|
+
case "strong": return {
|
|
1723
|
+
...base,
|
|
1724
|
+
type: "strong",
|
|
1725
|
+
children
|
|
1726
|
+
};
|
|
1727
|
+
case "strikethrough": return {
|
|
1728
|
+
...base,
|
|
1729
|
+
type: "strikethrough",
|
|
1730
|
+
children
|
|
1731
|
+
};
|
|
1732
|
+
case "inline-code": return {
|
|
1733
|
+
...base,
|
|
1734
|
+
type: "inline-code",
|
|
1735
|
+
text: op.props["text"]
|
|
1736
|
+
};
|
|
1737
|
+
case "soft-break": return {
|
|
1738
|
+
...base,
|
|
1739
|
+
type: "soft-break"
|
|
1740
|
+
};
|
|
1741
|
+
case "hard-break": return {
|
|
1742
|
+
...base,
|
|
1743
|
+
type: "hard-break"
|
|
1744
|
+
};
|
|
1745
|
+
case "link": return {
|
|
1746
|
+
...base,
|
|
1747
|
+
type: "link",
|
|
1748
|
+
url: op.props["url"],
|
|
1749
|
+
title: op.props["title"],
|
|
1750
|
+
children
|
|
1751
|
+
};
|
|
1752
|
+
case "image": return {
|
|
1753
|
+
...base,
|
|
1754
|
+
type: "image",
|
|
1755
|
+
url: op.props["url"],
|
|
1756
|
+
title: op.props["title"],
|
|
1757
|
+
alt: op.props["alt"]
|
|
1758
|
+
};
|
|
1759
|
+
case "autolink": return {
|
|
1760
|
+
...base,
|
|
1761
|
+
type: "autolink",
|
|
1762
|
+
url: op.props["url"],
|
|
1763
|
+
text: op.props["text"]
|
|
1764
|
+
};
|
|
1765
|
+
default: return {
|
|
1766
|
+
...base,
|
|
1767
|
+
type: "citation",
|
|
1768
|
+
idRef: op.props["idRef"],
|
|
1769
|
+
number: op.props["number"]
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
function astNodeEquals(a, b) {
|
|
1774
|
+
return JSON.stringify(a) === JSON.stringify(b);
|
|
1775
|
+
}
|
|
1776
|
+
function reconcileTextNode(previous, next) {
|
|
1777
|
+
if (next.segments.length === 0 || previous.segments.length === 0) return next;
|
|
1778
|
+
const previousBySignature = groupSegmentsBySignature(previous.segments);
|
|
1779
|
+
let reusedAnySegment = false;
|
|
1780
|
+
const segments = next.segments.map((segment) => {
|
|
1781
|
+
const signature = getSegmentSignature(segment);
|
|
1782
|
+
const bucket = previousBySignature.get(signature);
|
|
1783
|
+
if (!bucket?.length) return segment;
|
|
1784
|
+
const [reused, ...rest] = bucket;
|
|
1785
|
+
if (rest.length) previousBySignature.set(signature, rest);
|
|
1786
|
+
else previousBySignature.delete(signature);
|
|
1787
|
+
reusedAnySegment = true;
|
|
1788
|
+
return reused;
|
|
1789
|
+
});
|
|
1790
|
+
if (areSegmentArraysIdentical(previous.segments, segments)) return {
|
|
1791
|
+
...next,
|
|
1792
|
+
segments: previous.segments
|
|
1793
|
+
};
|
|
1794
|
+
if (!reusedAnySegment) return next;
|
|
1795
|
+
return {
|
|
1796
|
+
...next,
|
|
1797
|
+
segments
|
|
1798
|
+
};
|
|
1799
|
+
}
|
|
1800
|
+
/**
|
|
1801
|
+
* Returns true when both arrays contain the same segment object references in order.
|
|
1802
|
+
*/
|
|
1803
|
+
function areSegmentArraysIdentical(a, b) {
|
|
1804
|
+
if (a.length !== b.length) return false;
|
|
1805
|
+
return a.every((segment, index) => segment === b[index]);
|
|
1806
|
+
}
|
|
1807
|
+
/**
|
|
1808
|
+
* Produces a stable signature for matching segments across immutable parse iterations.
|
|
1809
|
+
*/
|
|
1810
|
+
function getSegmentSignature(segment) {
|
|
1811
|
+
return `${segment.start}:${segment.end}:${segment.text}:${segment.kind}:${String(segment.isWhitespace)}:${String(segment.noBreakBefore === true)}`;
|
|
1812
|
+
}
|
|
1813
|
+
/**
|
|
1814
|
+
* Groups previous segments by signature to support one-to-one identity reuse.
|
|
1815
|
+
*/
|
|
1816
|
+
function groupSegmentsBySignature(segments) {
|
|
1817
|
+
const bySignature = /* @__PURE__ */ new Map();
|
|
1818
|
+
for (const segment of segments) {
|
|
1819
|
+
const signature = getSegmentSignature(segment);
|
|
1820
|
+
const existing = bySignature.get(signature) ?? [];
|
|
1821
|
+
bySignature.set(signature, [...existing, segment]);
|
|
1822
|
+
}
|
|
1823
|
+
return bySignature;
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
//#endregion
|
|
1827
|
+
//#region src/parser/state.ts
|
|
1828
|
+
/**
|
|
1829
|
+
* Creates a new streaming markdown parser state.
|
|
1830
|
+
*
|
|
1831
|
+
* @param options - Optional parser option overrides.
|
|
1832
|
+
* @returns New immutable parser state.
|
|
1833
|
+
* @public
|
|
1834
|
+
*/
|
|
1835
|
+
function createStreamingMarkdownParserState(options) {
|
|
1836
|
+
return {
|
|
1837
|
+
nextId: 2,
|
|
1838
|
+
nodes: [{
|
|
1839
|
+
id: 1,
|
|
1840
|
+
type: "document",
|
|
1841
|
+
parentId: null,
|
|
1842
|
+
closed: false,
|
|
1843
|
+
range: {
|
|
1844
|
+
start: 0,
|
|
1845
|
+
end: 0
|
|
1846
|
+
},
|
|
1847
|
+
children: []
|
|
1848
|
+
}],
|
|
1849
|
+
rootId: 1,
|
|
1850
|
+
stack: [1],
|
|
1851
|
+
mode: "block",
|
|
1852
|
+
warnings: [],
|
|
1853
|
+
citations: {
|
|
1854
|
+
order: [],
|
|
1855
|
+
numbers: {},
|
|
1856
|
+
definitions: {}
|
|
1857
|
+
},
|
|
1858
|
+
lineBuffer: "",
|
|
1859
|
+
isComplete: false,
|
|
1860
|
+
index: 0,
|
|
1861
|
+
line: 1,
|
|
1862
|
+
column: 1,
|
|
1863
|
+
options: normalizeOptions(options),
|
|
1864
|
+
source: "",
|
|
1865
|
+
pathToId: { "0": 1 },
|
|
1866
|
+
pendingCarriageReturn: false,
|
|
1867
|
+
hasWarnedSegmenterUnavailable: false
|
|
1868
|
+
};
|
|
1869
|
+
}
|
|
1870
|
+
/**
|
|
1871
|
+
* Parses a streaming chunk and returns the next immutable parser state.
|
|
1872
|
+
*
|
|
1873
|
+
* @param state - Current parser state.
|
|
1874
|
+
* @param chunk - Incoming text chunk.
|
|
1875
|
+
* @returns Updated parser state.
|
|
1876
|
+
* @public
|
|
1877
|
+
*/
|
|
1878
|
+
function parseStreamingMarkdownChunk(state, chunk) {
|
|
1879
|
+
const normalized = normalizeChunk(chunk, state.pendingCarriageReturn);
|
|
1880
|
+
return rebuildState(state, {
|
|
1881
|
+
source: state.source + normalized.text,
|
|
1882
|
+
isComplete: false,
|
|
1883
|
+
pendingCarriageReturn: normalized.pendingCarriageReturn
|
|
1884
|
+
});
|
|
1885
|
+
}
|
|
1886
|
+
/**
|
|
1887
|
+
* Finalizes parsing by closing any provisional nodes and flushing trailing CR.
|
|
1888
|
+
*
|
|
1889
|
+
* @param state - Current parser state.
|
|
1890
|
+
* @returns Final parser state.
|
|
1891
|
+
* @public
|
|
1892
|
+
*/
|
|
1893
|
+
function finalizeStreamingMarkdown(state) {
|
|
1894
|
+
if (state.isComplete) return state;
|
|
1895
|
+
return rebuildState(state, {
|
|
1896
|
+
source: state.source + (state.pendingCarriageReturn ? "\n" : ""),
|
|
1897
|
+
isComplete: true,
|
|
1898
|
+
pendingCarriageReturn: false
|
|
1899
|
+
});
|
|
1900
|
+
}
|
|
1901
|
+
function rebuildState(prev, input) {
|
|
1902
|
+
const definitions = parseCitationDefinitions(input.source);
|
|
1903
|
+
const parseContext = {
|
|
1904
|
+
options: prev.options,
|
|
1905
|
+
warnings: definitions.warnings,
|
|
1906
|
+
citations: definitions.citations,
|
|
1907
|
+
isComplete: input.isComplete,
|
|
1908
|
+
hasWarnedSegmenterUnavailable: prev.hasWarnedSegmenterUnavailable
|
|
1909
|
+
};
|
|
1910
|
+
const lines = splitLines(input.source);
|
|
1911
|
+
const parsedBlocks = parseBlocks(lines, 0, lines.length, "0", parseContext, true);
|
|
1912
|
+
const ops = [
|
|
1913
|
+
{
|
|
1914
|
+
kind: "set-warnings",
|
|
1915
|
+
warnings: parsedBlocks.warnings
|
|
1916
|
+
},
|
|
1917
|
+
{
|
|
1918
|
+
kind: "set-citations",
|
|
1919
|
+
citations: parsedBlocks.citations
|
|
1920
|
+
},
|
|
1921
|
+
{
|
|
1922
|
+
kind: "set-segmenter-warning-state",
|
|
1923
|
+
hasWarnedSegmenterUnavailable: parsedBlocks.hasWarnedSegmenterUnavailable
|
|
1924
|
+
},
|
|
1925
|
+
...createAstOpsFromDraft(parsedBlocks.value)
|
|
1926
|
+
];
|
|
1927
|
+
const reduced = applyAstOps({
|
|
1928
|
+
previousNodes: prev.nodes,
|
|
1929
|
+
previousPathToId: prev.pathToId,
|
|
1930
|
+
nextId: prev.nextId,
|
|
1931
|
+
ops
|
|
1932
|
+
});
|
|
1933
|
+
const lineBuffer = getLineBuffer(input.source);
|
|
1934
|
+
const { line, column } = toLineColumn(input.source, input.source.length);
|
|
1935
|
+
return {
|
|
1936
|
+
...prev,
|
|
1937
|
+
nextId: reduced.nextId,
|
|
1938
|
+
nodes: reduced.nodes,
|
|
1939
|
+
rootId: reduced.rootId,
|
|
1940
|
+
stack: computeOpenStack(reduced.nodes, reduced.rootId),
|
|
1941
|
+
mode: inferMode(reduced.nodes, reduced.rootId),
|
|
1942
|
+
warnings: reduced.warnings,
|
|
1943
|
+
citations: reduced.citations,
|
|
1944
|
+
lineBuffer,
|
|
1945
|
+
isComplete: input.isComplete,
|
|
1946
|
+
index: input.source.length,
|
|
1947
|
+
line,
|
|
1948
|
+
column,
|
|
1949
|
+
source: input.source,
|
|
1950
|
+
pathToId: reduced.pathToId,
|
|
1951
|
+
pendingCarriageReturn: input.pendingCarriageReturn,
|
|
1952
|
+
hasWarnedSegmenterUnavailable: reduced.hasWarnedSegmenterUnavailable
|
|
1953
|
+
};
|
|
1954
|
+
}
|
|
1955
|
+
function computeOpenStack(nodes, rootId) {
|
|
1956
|
+
const nodeById = new Map(nodes.map((node) => [node.id, node]));
|
|
1957
|
+
const stack = [];
|
|
1958
|
+
let cursor = rootId;
|
|
1959
|
+
while (cursor != null) {
|
|
1960
|
+
const node = nodeById.get(cursor);
|
|
1961
|
+
if (!node || node.closed) break;
|
|
1962
|
+
stack.push(cursor);
|
|
1963
|
+
if (!("children" in node) || node.children.length === 0) break;
|
|
1964
|
+
const lastChild = node.children[node.children.length - 1];
|
|
1965
|
+
cursor = typeof lastChild === "number" ? lastChild : null;
|
|
1966
|
+
}
|
|
1967
|
+
return stack;
|
|
1968
|
+
}
|
|
1969
|
+
function inferMode(nodes, rootId) {
|
|
1970
|
+
if (rootId == null) return "block";
|
|
1971
|
+
const root = nodes.find((node) => node.id === rootId);
|
|
1972
|
+
if (!root || !("children" in root) || root.children.length === 0) return "block";
|
|
1973
|
+
const lastChild = nodes.find((node) => node.id === root.children[root.children.length - 1]);
|
|
1974
|
+
if (!lastChild || lastChild.closed) return "block";
|
|
1975
|
+
if (lastChild.type === "paragraph") return "paragraph";
|
|
1976
|
+
if (lastChild.type === "heading") return "heading";
|
|
1977
|
+
if (lastChild.type === "blockquote") return "blockquote";
|
|
1978
|
+
if (lastChild.type === "list-item") return "list-item";
|
|
1979
|
+
if (lastChild.type === "code-block") return "code-fence";
|
|
1980
|
+
if (lastChild.type === "table") return "table";
|
|
1981
|
+
return "block";
|
|
1982
|
+
}
|
|
1983
|
+
|
|
1984
|
+
//#endregion
|
|
1985
|
+
//#region src/dev-warning.ts
|
|
1986
|
+
let warned = false;
|
|
1987
|
+
const MIGRATION_GUIDE_URL = "https://docs.copilotkit.ai/migrate/markdown-renderer#migrate-with-your-ai-assistant";
|
|
1988
|
+
const HIGHLIGHTABLE_CODE = /(^|\n)[ \t]*(```|~~~)[ \t]*[A-Za-z][\w+#-]*/;
|
|
1989
|
+
const BLOCK_MATH = /\$\$[\s\S]+?\$\$/;
|
|
1990
|
+
const BRACKET_MATH = /\\\[[\s\S]+?\\\]/;
|
|
1991
|
+
function isProduction() {
|
|
1992
|
+
return typeof process !== "undefined" && process.env != null && process.env.NODE_ENV === "production";
|
|
1993
|
+
}
|
|
1994
|
+
/**
|
|
1995
|
+
* Dev-only, once-per-session warning emitted when CopilotKit's built-in default
|
|
1996
|
+
* markdown renderer encounters markdown it won't render richly — math
|
|
1997
|
+
* (`$$…$$` / `\[…\]`) or a language-tagged fenced code block (rendered as plain,
|
|
1998
|
+
* un-highlighted code). Helps people upgrading from the bundled Streamdown
|
|
1999
|
+
* default notice the behavioral change and points them at the migration guide.
|
|
2000
|
+
*
|
|
2001
|
+
* No-op in production and after the first warning. Only the built-in default
|
|
2002
|
+
* renderers call this — a custom `markdownRenderer` never triggers it.
|
|
2003
|
+
*
|
|
2004
|
+
* @internal
|
|
2005
|
+
*/
|
|
2006
|
+
function warnUnsupportedRichSyntaxOnce(content) {
|
|
2007
|
+
if (warned || !content || isProduction()) return;
|
|
2008
|
+
const hasMath = BLOCK_MATH.test(content) || BRACKET_MATH.test(content);
|
|
2009
|
+
const hasHighlightableCode = HIGHLIGHTABLE_CODE.test(content);
|
|
2010
|
+
if (!hasMath && !hasHighlightableCode) return;
|
|
2011
|
+
warned = true;
|
|
2012
|
+
const [feature, missing] = hasMath ? ["math", "math typesetting"] : ["a syntax-highlighted code block", "syntax highlighting"];
|
|
2013
|
+
console.warn(`[CopilotKit] The built-in markdown renderer rendered ${feature} without ${missing}. It supports GFM, tables, autolinks, and citations, but not math, syntax highlighting, or diagrams. To restore those, plug in a custom markdownRenderer (e.g. react-markdown or streamdown) — or paste the migration prompt into your AI assistant: ${MIGRATION_GUIDE_URL}`);
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
//#endregion
|
|
2017
|
+
export { createStreamingMarkdownParserState, finalizeStreamingMarkdown, parseStreamingMarkdownChunk, warnUnsupportedRichSyntaxOnce };
|
|
2018
|
+
//# sourceMappingURL=index.mjs.map
|