mdld-parse 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -3
- package/package.json +18 -2
- package/src/highlight.js +362 -0
- package/src/parse.js +120 -100
- package/src/render.js +337 -465
- package/src/tokenizers.js +594 -0
- package/src/utils.js +2 -2
|
@@ -0,0 +1,594 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Character-based Tokenizers for MDLD
|
|
3
|
+
*
|
|
4
|
+
* Replaces regex patterns with direct character scanning for better performance.
|
|
5
|
+
* Each tokenizer follows the EBNF grammar exactly.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
// ============================================================================
|
|
9
|
+
// BLOCK-LEVEL TOKENIZERS (Line-based)
|
|
10
|
+
// ============================================================================
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Detect code fence: 3+ backticks or tildes at line start
|
|
14
|
+
* EBNF: backticks = "`", "`", "`", { "`" } | tildes = "~", "~", "~", { "~" }
|
|
15
|
+
*/
|
|
16
|
+
export function detectFence(line) {
|
|
17
|
+
if (line.length < 3) return null;
|
|
18
|
+
|
|
19
|
+
const char = line[0];
|
|
20
|
+
if (char !== '`' && char !== '~') return null;
|
|
21
|
+
|
|
22
|
+
// Count consecutive fence characters
|
|
23
|
+
let count = 1;
|
|
24
|
+
while (count < line.length && line[count] === char) {
|
|
25
|
+
count++;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
if (count < 3) return null;
|
|
29
|
+
|
|
30
|
+
// Extract info string (everything after fence chars)
|
|
31
|
+
const rest = line.slice(count).trimStart();
|
|
32
|
+
const infoString = rest;
|
|
33
|
+
|
|
34
|
+
// Parse info string for language and attributes
|
|
35
|
+
const langMatch = infoString.match(/^([^\s{]+)/);
|
|
36
|
+
const lang = langMatch ? langMatch[1] : '';
|
|
37
|
+
|
|
38
|
+
// Find attributes in info string
|
|
39
|
+
const attrsMatch = infoString.match(/\{([^}]+)\}/);
|
|
40
|
+
const attrs = attrsMatch ? attrsMatch[1] : null;
|
|
41
|
+
|
|
42
|
+
return {
|
|
43
|
+
fenceChar: char,
|
|
44
|
+
fenceLength: count,
|
|
45
|
+
lang,
|
|
46
|
+
attrs,
|
|
47
|
+
infoString
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Detect prefix declaration: [prefix] <IRI>
|
|
53
|
+
* EBNF: contextDecl = "[", contextKey, "]", whitespace, "<", contextIri, ">"
|
|
54
|
+
*/
|
|
55
|
+
export function detectPrefix(line) {
|
|
56
|
+
if (line[0] !== '[') return null;
|
|
57
|
+
|
|
58
|
+
// Find closing bracket
|
|
59
|
+
const closeBracket = line.indexOf(']', 1);
|
|
60
|
+
if (closeBracket === -1) return null;
|
|
61
|
+
|
|
62
|
+
const prefix = line.slice(1, closeBracket).trim();
|
|
63
|
+
if (!prefix) return null;
|
|
64
|
+
|
|
65
|
+
// Find < after bracket (skip whitespace)
|
|
66
|
+
let pos = closeBracket + 1;
|
|
67
|
+
while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
|
|
68
|
+
pos++;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (pos >= line.length || line[pos] !== '<') return null;
|
|
72
|
+
|
|
73
|
+
// Find closing >
|
|
74
|
+
const closeAngle = line.indexOf('>', pos + 1);
|
|
75
|
+
if (closeAngle === -1) return null;
|
|
76
|
+
|
|
77
|
+
const iri = line.slice(pos + 1, closeAngle).trim();
|
|
78
|
+
if (!iri) return null;
|
|
79
|
+
|
|
80
|
+
return { prefix, iri };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Detect heading: 1-6 # characters followed by space and text
|
|
85
|
+
* EBNF: heading = "#", { "#" }, whitespace, text
|
|
86
|
+
*/
|
|
87
|
+
export function detectHeading(line) {
|
|
88
|
+
if (line[0] !== '#') return null;
|
|
89
|
+
|
|
90
|
+
// Count # characters (max 6)
|
|
91
|
+
let depth = 1;
|
|
92
|
+
while (depth < line.length && depth < 6 && line[depth] === '#') {
|
|
93
|
+
depth++;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Must be followed by whitespace
|
|
97
|
+
if (depth >= line.length || (line[depth] !== ' ' && line[depth] !== '\t')) {
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Extract content (skip # and whitespace)
|
|
102
|
+
let contentStart = depth;
|
|
103
|
+
while (contentStart < line.length && (line[contentStart] === ' ' || line[contentStart] === '\t')) {
|
|
104
|
+
contentStart++;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Check for trailing attributes {=...} or {...}
|
|
108
|
+
const trimmed = line.slice(contentStart);
|
|
109
|
+
const attrsMatch = trimmed.match(/\s*\{([^}]+)\}\s*$/);
|
|
110
|
+
|
|
111
|
+
let content = trimmed;
|
|
112
|
+
let attrs = null;
|
|
113
|
+
|
|
114
|
+
if (attrsMatch) {
|
|
115
|
+
attrs = attrsMatch[1];
|
|
116
|
+
content = trimmed.slice(0, -attrsMatch[0].length).trim();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return { depth, content, attrs };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Detect list item: -, *, +, or digits followed by .
|
|
124
|
+
* EBNF: listMarker = "-" | "*" | "+" | digit, { digit }, "."
|
|
125
|
+
*/
|
|
126
|
+
export function detectList(line) {
|
|
127
|
+
// Skip leading whitespace for indentation detection
|
|
128
|
+
let pos = 0;
|
|
129
|
+
while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
|
|
130
|
+
pos++;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const indent = pos;
|
|
134
|
+
if (pos >= line.length) return null;
|
|
135
|
+
|
|
136
|
+
const char = line[pos];
|
|
137
|
+
let marker;
|
|
138
|
+
let contentStart = pos + 1;
|
|
139
|
+
|
|
140
|
+
if (char === '-' || char === '*' || char === '+') {
|
|
141
|
+
marker = char;
|
|
142
|
+
} else if (char >= '0' && char <= '9') {
|
|
143
|
+
// Ordered list: digits followed by .
|
|
144
|
+
let numEnd = pos + 1;
|
|
145
|
+
while (numEnd < line.length && line[numEnd] >= '0' && line[numEnd] <= '9') {
|
|
146
|
+
numEnd++;
|
|
147
|
+
}
|
|
148
|
+
if (numEnd >= line.length || line[numEnd] !== '.') return null;
|
|
149
|
+
marker = line.slice(pos, numEnd + 1);
|
|
150
|
+
contentStart = numEnd + 1;
|
|
151
|
+
} else {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Must be followed by whitespace
|
|
156
|
+
if (contentStart >= line.length || (line[contentStart] !== ' ' && line[contentStart] !== '\t')) {
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Skip whitespace after marker
|
|
161
|
+
while (contentStart < line.length && (line[contentStart] === ' ' || line[contentStart] === '\t')) {
|
|
162
|
+
contentStart++;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Extract content and check for trailing attributes
|
|
166
|
+
let content = line.slice(contentStart);
|
|
167
|
+
let attrs = null;
|
|
168
|
+
|
|
169
|
+
const attrsMatch = content.match(/\s*\{([^}]+)\}\s*$/);
|
|
170
|
+
if (attrsMatch) {
|
|
171
|
+
attrs = attrsMatch[1];
|
|
172
|
+
content = content.slice(0, -attrsMatch[0].length).trim();
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return { indent, marker, content, attrs };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Detect blockquote: > followed by space and text
|
|
180
|
+
* EBNF: blockquote = ">", whitespace, text
|
|
181
|
+
*/
|
|
182
|
+
export function detectBlockquote(line) {
|
|
183
|
+
if (line[0] !== '>') return null;
|
|
184
|
+
|
|
185
|
+
// Must be followed by whitespace or end of line
|
|
186
|
+
if (line.length > 1 && line[1] !== ' ' && line[1] !== '\t') {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// Extract content
|
|
191
|
+
let contentStart = 1;
|
|
192
|
+
while (contentStart < line.length && (line[contentStart] === ' ' || line[contentStart] === '\t')) {
|
|
193
|
+
contentStart++;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
let content = line.slice(contentStart);
|
|
197
|
+
let attrs = null;
|
|
198
|
+
|
|
199
|
+
// Check for trailing attributes
|
|
200
|
+
const attrsMatch = content.match(/\s*\{([^}]+)\}\s*$/);
|
|
201
|
+
if (attrsMatch) {
|
|
202
|
+
attrs = attrsMatch[1];
|
|
203
|
+
content = content.slice(0, -attrsMatch[0].length).trim();
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
return { content, attrs };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Detect standalone subject: {=...} with optional whitespace
|
|
211
|
+
* EBNF: standaloneSubject = whitespace*, "{", "=", ( iriRef | "#", fragment ), "}", whitespace*
|
|
212
|
+
*/
|
|
213
|
+
export function detectStandaloneSubject(line) {
|
|
214
|
+
let pos = 0;
|
|
215
|
+
|
|
216
|
+
// Skip leading whitespace
|
|
217
|
+
while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
|
|
218
|
+
pos++;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
if (pos >= line.length || line[pos] !== '{') return null;
|
|
222
|
+
pos++;
|
|
223
|
+
|
|
224
|
+
if (pos >= line.length || line[pos] !== '=') return null;
|
|
225
|
+
pos++;
|
|
226
|
+
|
|
227
|
+
// Extract content until }
|
|
228
|
+
const contentStart = pos;
|
|
229
|
+
let braceCount = 1;
|
|
230
|
+
|
|
231
|
+
while (pos < line.length && braceCount > 0) {
|
|
232
|
+
if (line[pos] === '{') braceCount++;
|
|
233
|
+
if (line[pos] === '}') braceCount--;
|
|
234
|
+
if (braceCount > 0) pos++;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (braceCount > 0) return null; // No closing brace
|
|
238
|
+
|
|
239
|
+
const content = line.slice(contentStart, pos).trim();
|
|
240
|
+
|
|
241
|
+
// Skip trailing whitespace
|
|
242
|
+
pos++;
|
|
243
|
+
while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
|
|
244
|
+
pos++;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// Must consume entire line (after optional trailing whitespace)
|
|
248
|
+
if (pos < line.length) return null;
|
|
249
|
+
|
|
250
|
+
return { content };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// ============================================================================
|
|
254
|
+
// INLINE CARRIER TOKENIZERS (Character-based within block content)
|
|
255
|
+
// ============================================================================
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Find matching closing bracket, accounting for nesting
|
|
259
|
+
*/
|
|
260
|
+
export function findMatchingBracket(text, startPos, openChar = '[', closeChar = ']') {
|
|
261
|
+
let depth = 1;
|
|
262
|
+
let pos = startPos + 1;
|
|
263
|
+
|
|
264
|
+
while (pos < text.length && depth > 0) {
|
|
265
|
+
if (text[pos] === openChar) depth++;
|
|
266
|
+
if (text[pos] === closeChar) depth--;
|
|
267
|
+
if (depth > 0) pos++;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
return depth === 0 ? pos : null;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Extract attributes block after a position: {...}
|
|
275
|
+
*/
|
|
276
|
+
export function extractAttributes(text, startPos) {
|
|
277
|
+
let pos = startPos;
|
|
278
|
+
|
|
279
|
+
// Skip whitespace
|
|
280
|
+
while (pos < text.length && (text[pos] === ' ' || text[pos] === '\t')) {
|
|
281
|
+
pos++;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
if (pos >= text.length || text[pos] !== '{') return null;
|
|
285
|
+
|
|
286
|
+
// Find closing brace (simple version - no nesting for attribute blocks)
|
|
287
|
+
const closeBrace = text.indexOf('}', pos + 1);
|
|
288
|
+
if (closeBrace === -1) return null;
|
|
289
|
+
|
|
290
|
+
const attrs = text.slice(pos + 1, closeBrace);
|
|
291
|
+
const endPos = closeBrace + 1;
|
|
292
|
+
|
|
293
|
+
return { attrs, endPos };
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Detect URL in angle brackets: <URL>
|
|
298
|
+
* Returns null if not a valid URL per EBNF scheme rules
|
|
299
|
+
*/
|
|
300
|
+
export function detectAngleBracketUrl(text, startPos) {
|
|
301
|
+
if (text[startPos] !== '<') return null;
|
|
302
|
+
|
|
303
|
+
const closeAngle = text.indexOf('>', startPos + 1);
|
|
304
|
+
if (closeAngle === -1) return null;
|
|
305
|
+
|
|
306
|
+
const url = text.slice(startPos + 1, closeAngle).trim();
|
|
307
|
+
|
|
308
|
+
// Validate URL scheme (letter followed by letters/digits/+/ - /.)
|
|
309
|
+
const schemeMatch = url.match(/^[a-zA-Z][a-zA-Z0-9+\-.]*:/);
|
|
310
|
+
if (!schemeMatch) return null;
|
|
311
|
+
|
|
312
|
+
return { url, endPos: closeAngle + 1 };
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Detect emphasis/strong spans: *text* or _text_
|
|
317
|
+
* EBNF: emphasisSpan = "*", text, "*" | "_", text, "_"
|
|
318
|
+
* strongSpan = "**", text, "**" | "__", text, "__"
|
|
319
|
+
*/
|
|
320
|
+
export function detectEmphasis(text, startPos) {
|
|
321
|
+
const char = text[startPos];
|
|
322
|
+
if (char !== '*' && char !== '_') return null;
|
|
323
|
+
|
|
324
|
+
// Count delimiter characters
|
|
325
|
+
let delimCount = 1;
|
|
326
|
+
while (startPos + delimCount < text.length &&
|
|
327
|
+
text[startPos + delimCount] === char) {
|
|
328
|
+
delimCount++;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
if (delimCount > 2) return null; // Only 1 or 2 allowed
|
|
332
|
+
|
|
333
|
+
const type = delimCount === 1 ? 'emphasis' : 'strong';
|
|
334
|
+
const closer = char.repeat(delimCount);
|
|
335
|
+
|
|
336
|
+
// Find closing delimiter
|
|
337
|
+
const contentStart = startPos + delimCount;
|
|
338
|
+
let pos = contentStart;
|
|
339
|
+
|
|
340
|
+
while (pos < text.length) {
|
|
341
|
+
// Check for closing delimiter
|
|
342
|
+
if (text.slice(pos, pos + delimCount) === closer) {
|
|
343
|
+
// Must not be followed by same char (would be longer delimiter)
|
|
344
|
+
if (pos + delimCount < text.length && text[pos + delimCount] === char) {
|
|
345
|
+
pos++;
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
const content = text.slice(contentStart, pos);
|
|
350
|
+
const endPos = pos + delimCount;
|
|
351
|
+
|
|
352
|
+
// Check for attributes after
|
|
353
|
+
const attrsResult = extractAttributes(text, endPos);
|
|
354
|
+
const finalPos = attrsResult ? attrsResult.endPos : endPos;
|
|
355
|
+
|
|
356
|
+
return {
|
|
357
|
+
type,
|
|
358
|
+
content,
|
|
359
|
+
attrs: attrsResult?.attrs || null,
|
|
360
|
+
endPos: finalPos,
|
|
361
|
+
contentEnd: endPos
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
pos++;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
return null; // No closing delimiter found
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Detect code span: `text` or ``text``
|
|
372
|
+
* EBNF: codeSpan = "`", text, "`" | "``", text, "``"
|
|
373
|
+
*/
|
|
374
|
+
export function detectCodeSpan(text, startPos) {
|
|
375
|
+
if (text[startPos] !== '`') return null;
|
|
376
|
+
|
|
377
|
+
// Count opening backticks
|
|
378
|
+
let backtickCount = 1;
|
|
379
|
+
while (startPos + backtickCount < text.length &&
|
|
380
|
+
text[startPos + backtickCount] === '`') {
|
|
381
|
+
backtickCount++;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
if (backtickCount > 2) return null; // Only 1 or 2 allowed in current grammar
|
|
385
|
+
|
|
386
|
+
const opener = '`'.repeat(backtickCount);
|
|
387
|
+
const contentStart = startPos + backtickCount;
|
|
388
|
+
|
|
389
|
+
// Find closing backticks
|
|
390
|
+
let pos = contentStart;
|
|
391
|
+
|
|
392
|
+
while (pos < text.length) {
|
|
393
|
+
if (text.slice(pos, pos + backtickCount) === opener) {
|
|
394
|
+
const content = text.slice(contentStart, pos);
|
|
395
|
+
const endPos = pos + backtickCount;
|
|
396
|
+
|
|
397
|
+
// Check for attributes after
|
|
398
|
+
const attrsResult = extractAttributes(text, endPos);
|
|
399
|
+
const finalPos = attrsResult ? attrsResult.endPos : endPos;
|
|
400
|
+
|
|
401
|
+
return {
|
|
402
|
+
type: 'code',
|
|
403
|
+
content,
|
|
404
|
+
attrs: attrsResult?.attrs || null,
|
|
405
|
+
endPos: finalPos,
|
|
406
|
+
contentEnd: endPos
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
pos++;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
return null;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Detect bracket link: [text](url) or [text]{...} or [text] alone
|
|
417
|
+
* EBNF: linkSpan = "[", text, "](", text, ")" | "[", text, "]"
|
|
418
|
+
*/
|
|
419
|
+
export function detectBracketLink(text, startPos) {
|
|
420
|
+
if (text[startPos] !== '[') return null;
|
|
421
|
+
|
|
422
|
+
const bracketEnd = findMatchingBracket(text, startPos, '[', ']');
|
|
423
|
+
if (!bracketEnd) return null;
|
|
424
|
+
|
|
425
|
+
const linkText = text.slice(startPos + 1, bracketEnd);
|
|
426
|
+
let pos = bracketEnd + 1;
|
|
427
|
+
let url = null;
|
|
428
|
+
|
|
429
|
+
// Check for (url) after ]
|
|
430
|
+
if (pos < text.length && text[pos] === '(') {
|
|
431
|
+
const parenEnd = text.indexOf(')', pos + 1);
|
|
432
|
+
if (parenEnd !== -1) {
|
|
433
|
+
url = text.slice(pos + 1, parenEnd);
|
|
434
|
+
pos = parenEnd + 1;
|
|
435
|
+
}
|
|
436
|
+
} else if (pos < text.length && text[pos] === '<') {
|
|
437
|
+
// Check for <url> after ]
|
|
438
|
+
const closeAngle = text.indexOf('>', pos + 1);
|
|
439
|
+
if (closeAngle !== -1) {
|
|
440
|
+
const angleUrl = text.slice(pos + 1, closeAngle).trim();
|
|
441
|
+
// Validate URL scheme
|
|
442
|
+
const schemeMatch = angleUrl.match(/^[a-zA-Z][a-zA-Z0-9+\-.]*:/);
|
|
443
|
+
if (schemeMatch) {
|
|
444
|
+
url = angleUrl;
|
|
445
|
+
pos = closeAngle + 1;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
// Check for attributes
|
|
451
|
+
const attrsResult = extractAttributes(text, pos);
|
|
452
|
+
const finalPos = attrsResult ? attrsResult.endPos : pos;
|
|
453
|
+
|
|
454
|
+
// Determine carrier type: 'link' if URL present, 'span' otherwise
|
|
455
|
+
const carrierType = url ? 'link' : 'span';
|
|
456
|
+
|
|
457
|
+
return {
|
|
458
|
+
type: carrierType,
|
|
459
|
+
text: linkText,
|
|
460
|
+
url,
|
|
461
|
+
attrs: attrsResult?.attrs || null,
|
|
462
|
+
endPos: finalPos,
|
|
463
|
+
bracketEnd: bracketEnd + 1
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
// ============================================================================
|
|
468
|
+
// RANGE CALCULATION HELPERS
|
|
469
|
+
// ============================================================================
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* Calculate absolute ranges for a token given base offset
|
|
473
|
+
*/
|
|
474
|
+
export function calcRanges(token, baseOffset) {
|
|
475
|
+
return {
|
|
476
|
+
range: [baseOffset + token.start, baseOffset + token.end],
|
|
477
|
+
valueRange: token.valueStart !== undefined
|
|
478
|
+
? [baseOffset + token.valueStart, baseOffset + token.valueEnd]
|
|
479
|
+
: null,
|
|
480
|
+
attrsRange: token.attrsStart !== undefined
|
|
481
|
+
? [baseOffset + token.attrsStart, baseOffset + token.attrsEnd]
|
|
482
|
+
: null
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
// ============================================================================
|
|
487
|
+
// COMPLETE INLINE CARRIER SCANNER (replaces extractInlineCarriers)
|
|
488
|
+
// ============================================================================
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* Scan text for all inline carriers using character-based detection
|
|
492
|
+
* Memory-optimized version - minimizes object allocations
|
|
493
|
+
*/
|
|
494
|
+
export function scanInlineCarriers(text, baseOffset = 0) {
|
|
495
|
+
const carriers = [];
|
|
496
|
+
const len = text.length;
|
|
497
|
+
let pos = 0;
|
|
498
|
+
|
|
499
|
+
while (pos < len) {
|
|
500
|
+
const char = text[pos];
|
|
501
|
+
let detected = null;
|
|
502
|
+
|
|
503
|
+
// Fast path: only check carrier start characters
|
|
504
|
+
switch (char) {
|
|
505
|
+
case '<':
|
|
506
|
+
detected = detectAngleBracketUrl(text, pos);
|
|
507
|
+
if (detected) {
|
|
508
|
+
const attrsResult = extractAttributes(text, detected.endPos);
|
|
509
|
+
if (attrsResult) {
|
|
510
|
+
detected.attrs = attrsResult.attrs;
|
|
511
|
+
detected.endPos = attrsResult.endPos;
|
|
512
|
+
}
|
|
513
|
+
detected.type = 'link';
|
|
514
|
+
detected.text = detected.url;
|
|
515
|
+
}
|
|
516
|
+
break;
|
|
517
|
+
|
|
518
|
+
case '[':
|
|
519
|
+
detected = detectBracketLink(text, pos);
|
|
520
|
+
break;
|
|
521
|
+
|
|
522
|
+
case '*':
|
|
523
|
+
case '_':
|
|
524
|
+
detected = detectEmphasis(text, pos);
|
|
525
|
+
break;
|
|
526
|
+
|
|
527
|
+
case '`':
|
|
528
|
+
detected = detectCodeSpan(text, pos);
|
|
529
|
+
break;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
if (!detected) {
|
|
533
|
+
pos++;
|
|
534
|
+
continue;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// Skip annotation-only patterns
|
|
538
|
+
const url = detected.url;
|
|
539
|
+
const type = detected.type;
|
|
540
|
+
const attrs = detected.attrs;
|
|
541
|
+
|
|
542
|
+
if (url?.startsWith('=') || (type === 'link' && !attrs && !url)) {
|
|
543
|
+
pos = detected.endPos;
|
|
544
|
+
continue;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// Calculate ranges once
|
|
548
|
+
const absStart = baseOffset + pos;
|
|
549
|
+
const absEnd = baseOffset + detected.endPos;
|
|
550
|
+
const contentEndOffset = detected.contentEnd || detected.bracketEnd || detected.endPos;
|
|
551
|
+
const absContentEnd = baseOffset + contentEndOffset;
|
|
552
|
+
|
|
553
|
+
// Build carrier object with minimal properties
|
|
554
|
+
const carrier = {
|
|
555
|
+
type: type,
|
|
556
|
+
text: detected.content !== undefined ? detected.content : (detected.text !== undefined ? detected.text : url),
|
|
557
|
+
range: [absStart, absEnd],
|
|
558
|
+
valueRange: [absStart + (type === 'url' ? 1 : 0), absContentEnd - (type === 'url' ? 1 : 0)],
|
|
559
|
+
attrs: attrs,
|
|
560
|
+
url: url,
|
|
561
|
+
pos: detected.endPos
|
|
562
|
+
};
|
|
563
|
+
|
|
564
|
+
// Only add attrsRange if attrs exist (saves memory for carriers without attrs)
|
|
565
|
+
if (attrs) {
|
|
566
|
+
carrier.attrsRange = [absContentEnd, absEnd];
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
carriers.push(carrier);
|
|
570
|
+
pos = detected.endPos;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
return carriers;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
// ============================================================================
|
|
577
|
+
// TESTING EXPORTS (for validation)
|
|
578
|
+
// ============================================================================
|
|
579
|
+
|
|
580
|
+
export const TEST = {
|
|
581
|
+
detectFence,
|
|
582
|
+
detectPrefix,
|
|
583
|
+
detectHeading,
|
|
584
|
+
detectList,
|
|
585
|
+
detectBlockquote,
|
|
586
|
+
detectStandaloneSubject,
|
|
587
|
+
detectAngleBracketUrl,
|
|
588
|
+
detectEmphasis,
|
|
589
|
+
detectCodeSpan,
|
|
590
|
+
detectBracketLink,
|
|
591
|
+
extractAttributes,
|
|
592
|
+
findMatchingBracket,
|
|
593
|
+
scanInlineCarriers
|
|
594
|
+
};
|
package/src/utils.js
CHANGED
|
@@ -293,8 +293,8 @@ export function shortenIRI(iri, ctx) {
|
|
|
293
293
|
|
|
294
294
|
// Token pattern definitions for semantic block parsing
|
|
295
295
|
const TOKEN_PATTERNS = {
|
|
296
|
-
'=#': { kind: 'fragment', extract: t => t.substring(2) },
|
|
297
|
-
'+#': { kind: 'softFragment', extract: t => t.substring(2) },
|
|
296
|
+
'=#': { kind: 'fragment', extract: t => t.substring(2).replace('}', '') },
|
|
297
|
+
'+#': { kind: 'softFragment', extract: t => t.substring(2).replace('}', '') },
|
|
298
298
|
'+': { kind: 'object', extract: t => t.substring(1) },
|
|
299
299
|
'^^': { kind: 'datatype', extract: t => t.substring(2) },
|
|
300
300
|
'@': { kind: 'language', extract: t => t.substring(1) },
|