mdld-parse 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -1
- package/package.json +10 -4
- package/src/highlight.js +362 -0
- package/src/parse.js +53 -107
- package/src/tokenizers.js +594 -0
package/README.md
CHANGED
|
@@ -62,6 +62,39 @@ console.log(result.quads);
|
|
|
62
62
|
|
|
63
63
|
MD-LD allows you to author RDF graphs directly in Markdown using explicit `{...}` annotations:
|
|
64
64
|
|
|
65
|
+
## ⚡ Performance & Architecture
|
|
66
|
+
|
|
67
|
+
MD-LD v0.10.0 features a **character-based tokenization system** for optimal performance:
|
|
68
|
+
|
|
69
|
+
- **20-28% faster parsing** than regex-based approaches
|
|
70
|
+
- **Memory-efficient** with ~640 bytes per quad retained
|
|
71
|
+
- **Streaming-friendly** with O(n) linear time complexity
|
|
72
|
+
- **Character-based detection** replaces complex regex patterns
|
|
73
|
+
- **Unified tokenizer architecture** in `src/tokenizers.js`
|
|
74
|
+
|
|
75
|
+
### Tokenizer Architecture
|
|
76
|
+
|
|
77
|
+
The parser uses specialized character-based tokenizers:
|
|
78
|
+
|
|
79
|
+
```javascript
|
|
80
|
+
// Block-level tokenizers
|
|
81
|
+
detectFence() // ```code blocks
|
|
82
|
+
detectPrefix() // [prefix] <iri>
|
|
83
|
+
detectHeading() // # Headings
|
|
84
|
+
detectList() // - List items
|
|
85
|
+
detectBlockquote() // > Blockquotes
|
|
86
|
+
detectStandaloneSubject() // {=subject}
|
|
87
|
+
|
|
88
|
+
// Inline carrier scanner
|
|
89
|
+
scanInlineCarriers() // [text], **bold**, `code`, <URL>
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
This design provides:
|
|
93
|
+
- **Better maintainability** - Easier to debug and extend
|
|
94
|
+
- **Improved error handling** - More precise edge case detection
|
|
95
|
+
- **Cleaner code structure** - No complex regex patterns
|
|
96
|
+
- **Full backward compatibility** - All 127 tests passing
|
|
97
|
+
|
|
65
98
|
```markdown
|
|
66
99
|
[my] <tag:alice@example.com,2026:>
|
|
67
100
|
# 2024-07-18 {=my:journal-2024-07-18 .my:Event my:date ^^xsd:date}
|
|
@@ -380,7 +413,7 @@ Quads are compatible with:
|
|
|
380
413
|
The parser includes comprehensive tests covering all spec requirements:
|
|
381
414
|
|
|
382
415
|
```bash
|
|
383
|
-
|
|
416
|
+
pnpm test
|
|
384
417
|
```
|
|
385
418
|
|
|
386
419
|
Tests validate:
|
package/package.json
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mdld-parse",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.2",
|
|
4
4
|
"description": "A standards-compliant parser for **MD-LD (Markdown-Linked Data)** — a human-friendly RDF authoring format that extends Markdown with semantic annotations.",
|
|
5
5
|
"type": "module",
|
|
6
|
+
"packageManager": "pnpm@10.6.5",
|
|
6
7
|
"main": "index.js",
|
|
7
8
|
"exports": {
|
|
8
9
|
".": "./src/index.js"
|
|
@@ -14,7 +15,7 @@
|
|
|
14
15
|
"scripts": {
|
|
15
16
|
"test": "node tests/index.js",
|
|
16
17
|
"dev": "vite dev",
|
|
17
|
-
"build": "vite build && vite build --config vite.config.lib.js && cp -r docs/ dist/docs/ && cp -r examples/ dist/examples/ && cp -r spec/ dist/spec/"
|
|
18
|
+
"build": "vite build && vite build --config vite.config.lib.js && cp -r docs/ dist/docs/ && cp -r examples/ dist/examples/ && cp -r spec/ dist/spec/ && cp -r README.md dist/README.md"
|
|
18
19
|
},
|
|
19
20
|
"keywords": [
|
|
20
21
|
"mdld",
|
|
@@ -43,10 +44,15 @@
|
|
|
43
44
|
"devDependencies": {
|
|
44
45
|
"@unocss/extractor-pug": "^66.6.8",
|
|
45
46
|
"@vitejs/plugin-vue": "^6.0.6",
|
|
47
|
+
"@vueuse/core": "^14.3.0",
|
|
48
|
+
"markdown-it": "^14.1.1",
|
|
49
|
+
"n3": "^2.0.3",
|
|
50
|
+
"shiki": "^4.0.2",
|
|
46
51
|
"unocss": "^66.6.8",
|
|
47
52
|
"vite": "^8.0.10",
|
|
48
53
|
"vite-plugin-singlefile": "^2.3.3",
|
|
49
|
-
"
|
|
50
|
-
"
|
|
54
|
+
"vscode-textmate": "^9.3.2",
|
|
55
|
+
"vue": "^3.6.0-beta.10",
|
|
56
|
+
"codejar": "^4.3.0"
|
|
51
57
|
}
|
|
52
58
|
}
|
package/src/highlight.js
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MD-LD Syntax Highlighter
|
|
3
|
+
*
|
|
4
|
+
* A performant, token-based syntax highlighter for MD-LD (Markdown Linked Data).
|
|
5
|
+
* Uses simple string operations rather than complex grammars for reliability
|
|
6
|
+
* and maintainability.
|
|
7
|
+
*
|
|
8
|
+
* Features:
|
|
9
|
+
* - Token-based parsing with string operations
|
|
10
|
+
* - Markdown formatting support with semantic distinction
|
|
11
|
+
* - Lookahead detection for legal value carriers
|
|
12
|
+
* - Orange semantic markers vs default markdown spans
|
|
13
|
+
* - Annotation, value carrier, and heading highlighting
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
// HTML escaping function
|
|
17
|
+
export function escapeHtml(s) {
|
|
18
|
+
return String(s)
|
|
19
|
+
.replaceAll('&', '&')
|
|
20
|
+
.replaceAll('<', '<')
|
|
21
|
+
.replaceAll('>', '>')
|
|
22
|
+
.replaceAll('"', '"')
|
|
23
|
+
.replaceAll("'", ''');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Token color scheme
|
|
27
|
+
export const TOKEN_COLORS = {
|
|
28
|
+
text: '#6b7280', // Grayscale - main text
|
|
29
|
+
marker: '#f97316', // Orange - .?!^^@
|
|
30
|
+
retraction: '#dc2626', // Red - whole predicate with -
|
|
31
|
+
value: '#f97316', // Yellow - values in []
|
|
32
|
+
annotation: '#4b5563' // Darker gray - annotation braces
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
// Check if markdown formatting has following annotation
|
|
36
|
+
function hasFollowingAnnotation(code, endPos) {
|
|
37
|
+
let i = endPos;
|
|
38
|
+
while (i < code.length && /\s/.test(code[i])) {
|
|
39
|
+
i++;
|
|
40
|
+
}
|
|
41
|
+
return i < code.length && code[i] === '{';
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Process markdown formatting in text while preserving original syntax
|
|
45
|
+
export function processMarkdownFormatting(text, hasFollowingAnnotation = false) {
|
|
46
|
+
let result = escapeHtml(text);
|
|
47
|
+
|
|
48
|
+
if (hasFollowingAnnotation) {
|
|
49
|
+
// Orange for legal value carriers (with following annotation)
|
|
50
|
+
result = result.replace(/\*\*(.*?)\*\*/g, `<strong style="color: ${TOKEN_COLORS.marker}">$1</strong>`);
|
|
51
|
+
result = result.replace(/__(.*?)__/g, `<strong style="color: ${TOKEN_COLORS.marker}">$1</strong>`);
|
|
52
|
+
result = result.replace(/\*(.*?)\*/g, `<em style="color: ${TOKEN_COLORS.marker}">$1</em>`);
|
|
53
|
+
result = result.replace(/_(.*?)_/g, `<em style="color: ${TOKEN_COLORS.marker}">$1</em>`);
|
|
54
|
+
result = result.replace(/`(.*?)`/g, `<code style="color: ${TOKEN_COLORS.marker}">$1</code>`);
|
|
55
|
+
} else {
|
|
56
|
+
// Default browser styles for regular markdown spans
|
|
57
|
+
result = result.replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>');
|
|
58
|
+
result = result.replace(/__(.*?)__/g, '<strong>$1</strong>');
|
|
59
|
+
result = result.replace(/\*(.*?)\*/g, '<em>$1</em>');
|
|
60
|
+
result = result.replace(/_(.*?)_/g, '<em>$1</em>');
|
|
61
|
+
result = result.replace(/`(.*?)`/g, '<code>$1</code>');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return result;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Process markdown formatting at specific position for non-bracket text
|
|
68
|
+
function processMarkdownFormattingAtPosition(code, startPos) {
|
|
69
|
+
const char = code[startPos];
|
|
70
|
+
|
|
71
|
+
// Check for **bold**
|
|
72
|
+
if (char === '*' && startPos + 1 < code.length && code[startPos + 1] === '*') {
|
|
73
|
+
const endPos = code.indexOf('**', startPos + 2);
|
|
74
|
+
if (endPos !== -1) {
|
|
75
|
+
const content = code.slice(startPos + 2, endPos);
|
|
76
|
+
const hasAnnotation = hasFollowingAnnotation(code, endPos + 2);
|
|
77
|
+
const color = hasAnnotation ? ` style="color: ${TOKEN_COLORS.marker}"` : '';
|
|
78
|
+
return {
|
|
79
|
+
html: `<strong${color}>**${escapeHtml(content)}**</strong>`,
|
|
80
|
+
nextIndex: endPos + 2
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Check for __bold__
|
|
86
|
+
if (char === '_' && startPos + 1 < code.length && code[startPos + 1] === '_') {
|
|
87
|
+
const endPos = code.indexOf('__', startPos + 2);
|
|
88
|
+
if (endPos !== -1) {
|
|
89
|
+
const content = code.slice(startPos + 2, endPos);
|
|
90
|
+
const hasAnnotation = hasFollowingAnnotation(code, endPos + 2);
|
|
91
|
+
const color = hasAnnotation ? ` style="color: ${TOKEN_COLORS.marker}"` : '';
|
|
92
|
+
return {
|
|
93
|
+
html: `<strong${color}>__${escapeHtml(content)}__</strong>`,
|
|
94
|
+
nextIndex: endPos + 2
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Check for *italic*
|
|
100
|
+
if (char === '*') {
|
|
101
|
+
const endPos = code.indexOf('*', startPos + 1);
|
|
102
|
+
if (endPos !== -1) {
|
|
103
|
+
const content = code.slice(startPos + 1, endPos);
|
|
104
|
+
const hasAnnotation = hasFollowingAnnotation(code, endPos + 1);
|
|
105
|
+
const color = hasAnnotation ? ` style="color: ${TOKEN_COLORS.marker}"` : '';
|
|
106
|
+
return {
|
|
107
|
+
html: `<em${color}>*${escapeHtml(content)}*</em>`,
|
|
108
|
+
nextIndex: endPos + 1
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Check for _italic_
|
|
114
|
+
if (char === '_') {
|
|
115
|
+
const endPos = code.indexOf('_', startPos + 1);
|
|
116
|
+
if (endPos !== -1) {
|
|
117
|
+
const content = code.slice(startPos + 1, endPos);
|
|
118
|
+
const hasAnnotation = hasFollowingAnnotation(code, endPos + 1);
|
|
119
|
+
const color = hasAnnotation ? ` style="color: ${TOKEN_COLORS.marker}"` : '';
|
|
120
|
+
return {
|
|
121
|
+
html: `<em${color}>_${escapeHtml(content)}_</em>`,
|
|
122
|
+
nextIndex: endPos + 1
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Check for `code`
|
|
128
|
+
if (char === '`') {
|
|
129
|
+
const endPos = code.indexOf('`', startPos + 1);
|
|
130
|
+
if (endPos !== -1) {
|
|
131
|
+
const content = code.slice(startPos + 1, endPos);
|
|
132
|
+
const hasAnnotation = hasFollowingAnnotation(code, endPos + 1);
|
|
133
|
+
const color = hasAnnotation ? ` style="color: ${TOKEN_COLORS.marker}"` : '';
|
|
134
|
+
return {
|
|
135
|
+
html: `<code${color}>\`${escapeHtml(content)}\`</code>`,
|
|
136
|
+
nextIndex: endPos + 1
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Parse an annotation token { ... }
|
|
145
|
+
function parseAnnotation(content) {
|
|
146
|
+
const parts = content.trim().split(/\s+/);
|
|
147
|
+
let result = '';
|
|
148
|
+
let nextTermIsRetracted = false;
|
|
149
|
+
|
|
150
|
+
for (let i = 0; i < parts.length; i++) {
|
|
151
|
+
const part = parts[i];
|
|
152
|
+
if (!part) continue;
|
|
153
|
+
|
|
154
|
+
// Check for subject markers (= or +)
|
|
155
|
+
if (part.startsWith('=') || part.startsWith('+')) {
|
|
156
|
+
const marker = part[0]; // = or +
|
|
157
|
+
const rest = part.slice(1); // the rest of the subject
|
|
158
|
+
result += `<span style="color: ${TOKEN_COLORS.marker}">${escapeHtml(marker)}</span><span style="color: ${TOKEN_COLORS.text}">${escapeHtml(rest)}</span> `;
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Check for retraction marker (-)
|
|
163
|
+
if (part === '-') {
|
|
164
|
+
nextTermIsRetracted = true;
|
|
165
|
+
result += `<span style="color: ${TOKEN_COLORS.retraction}">${escapeHtml(part)}</span> `;
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// Handle case where - and predicate are combined (e.g., -predicate)
|
|
170
|
+
if (part.startsWith('-') && part.length > 1) {
|
|
171
|
+
result += `<span style="color: ${TOKEN_COLORS.retraction}">${escapeHtml(part)}</span> `;
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Apply retraction color to the immediate next term only
|
|
176
|
+
const isRetracted = nextTermIsRetracted;
|
|
177
|
+
nextTermIsRetracted = false; // Reset after one term
|
|
178
|
+
|
|
179
|
+
// Check for object markers (? or !)
|
|
180
|
+
if (part.startsWith('?') || part.startsWith('!')) {
|
|
181
|
+
const marker = part[0]; // ? or !
|
|
182
|
+
const rest = part.slice(1); // the CURIE or rest of the term
|
|
183
|
+
const color = isRetracted ? TOKEN_COLORS.retraction : TOKEN_COLORS.marker;
|
|
184
|
+
const textColor = isRetracted ? TOKEN_COLORS.retraction : TOKEN_COLORS.text;
|
|
185
|
+
result += `<span style="color: ${color}">${escapeHtml(marker)}</span><span style="color: ${textColor}">${escapeHtml(rest)}</span> `;
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// Check for datatype markers (^^, @)
|
|
190
|
+
if (part.startsWith('^^')) {
|
|
191
|
+
const marker = '^^'; // the datatype marker
|
|
192
|
+
const rest = part.slice(2); // the datatype
|
|
193
|
+
const color = isRetracted ? TOKEN_COLORS.retraction : TOKEN_COLORS.marker;
|
|
194
|
+
const textColor = isRetracted ? TOKEN_COLORS.retraction : TOKEN_COLORS.text;
|
|
195
|
+
result += `<span style="color: ${color}">${escapeHtml(marker)}</span><span style="color: ${textColor}">${escapeHtml(rest)}</span> `;
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
if (part.startsWith('@')) {
|
|
200
|
+
const marker = '@'; // the language marker
|
|
201
|
+
const rest = part.slice(1); // the language code
|
|
202
|
+
const color = isRetracted ? TOKEN_COLORS.retraction : TOKEN_COLORS.marker;
|
|
203
|
+
const textColor = isRetracted ? TOKEN_COLORS.retraction : TOKEN_COLORS.text;
|
|
204
|
+
result += `<span style="color: ${color}">${escapeHtml(marker)}</span><span style="color: ${textColor}">${escapeHtml(rest)}</span> `;
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Check for type marker (.)
|
|
209
|
+
if (part.startsWith('.')) {
|
|
210
|
+
const marker = '.'; // the dot
|
|
211
|
+
const rest = part.slice(1); // the class name
|
|
212
|
+
const color = isRetracted ? TOKEN_COLORS.retraction : TOKEN_COLORS.marker;
|
|
213
|
+
const textColor = isRetracted ? TOKEN_COLORS.retraction : TOKEN_COLORS.text;
|
|
214
|
+
result += `<span style="color: ${color}">${escapeHtml(marker)}</span><span style="color: ${textColor}">${escapeHtml(rest)}</span> `;
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Regular CURIE or IRI
|
|
219
|
+
const color = isRetracted ? TOKEN_COLORS.retraction : TOKEN_COLORS.text;
|
|
220
|
+
result += `<span style="color: ${color}">${escapeHtml(part)}</span> `;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
return result.trim();
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Main MDLD syntax highlighting function
|
|
227
|
+
export function highlightMDLD(code) {
|
|
228
|
+
let result = '';
|
|
229
|
+
let i = 0;
|
|
230
|
+
|
|
231
|
+
while (i < code.length) {
|
|
232
|
+
const char = code[i];
|
|
233
|
+
|
|
234
|
+
// Handle annotation blocks { ... }
|
|
235
|
+
if (char === '{') {
|
|
236
|
+
const endIdx = code.indexOf('}', i);
|
|
237
|
+
if (endIdx === -1) {
|
|
238
|
+
// No closing brace, treat as regular text
|
|
239
|
+
result += escapeHtml(char);
|
|
240
|
+
i++;
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const content = code.slice(i + 1, endIdx);
|
|
245
|
+
const parsed = parseAnnotation(content);
|
|
246
|
+
|
|
247
|
+
result += `<span style="color: ${TOKEN_COLORS.annotation}; opacity: 0.75">{</span>`;
|
|
248
|
+
result += parsed;
|
|
249
|
+
result += `<span style="color: ${TOKEN_COLORS.annotation}; opacity: 0.75">}</span>`;
|
|
250
|
+
|
|
251
|
+
i = endIdx + 1;
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// Handle value carriers [ ... ] and markdown formatting
|
|
256
|
+
if (char === '[') {
|
|
257
|
+
const endIdx = code.indexOf(']', i);
|
|
258
|
+
if (endIdx === -1) {
|
|
259
|
+
// No closing bracket, treat as regular text
|
|
260
|
+
result += escapeHtml(char);
|
|
261
|
+
i++;
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const content = code.slice(i + 1, endIdx);
|
|
266
|
+
const hasAnnotation = hasFollowingAnnotation(code, endIdx + 1);
|
|
267
|
+
const processedContent = processMarkdownFormatting(content, hasAnnotation);
|
|
268
|
+
result += `<span style="color: ${TOKEN_COLORS.value}; opacity: 0.85">[${processedContent}]</span>`;
|
|
269
|
+
|
|
270
|
+
i = endIdx + 1;
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// Handle markdown formatting outside brackets
|
|
275
|
+
if (char === '*' || char === '_' || char === '`') {
|
|
276
|
+
const processed = processMarkdownFormattingAtPosition(code, i);
|
|
277
|
+
if (processed) {
|
|
278
|
+
result += processed.html;
|
|
279
|
+
i = processed.nextIndex;
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// Handle prefix declarations < ... >
|
|
285
|
+
if (char === '<') {
|
|
286
|
+
const endIdx = code.indexOf('>', i);
|
|
287
|
+
if (endIdx === -1) {
|
|
288
|
+
result += escapeHtml(char);
|
|
289
|
+
i++;
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
const content = code.slice(i + 1, endIdx);
|
|
294
|
+
result += `<span style="color: ${TOKEN_COLORS.text}; opacity: 0.6"><${escapeHtml(content)}></span>`;
|
|
295
|
+
|
|
296
|
+
i = endIdx + 1;
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Handle markdown headers
|
|
301
|
+
if (char === '#') {
|
|
302
|
+
let hashCount = 0;
|
|
303
|
+
let j = i;
|
|
304
|
+
while (j < code.length && code[j] === '#') {
|
|
305
|
+
hashCount++;
|
|
306
|
+
j++;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
if (hashCount <= 6 && (j >= code.length || code[j] === ' ' || code[j] === '\t')) {
|
|
310
|
+
// Find end of line
|
|
311
|
+
const lineEnd = code.indexOf('\n', j);
|
|
312
|
+
const endIdx = lineEnd === -1 ? code.length : lineEnd;
|
|
313
|
+
const headerText = code.slice(j, endIdx).trim();
|
|
314
|
+
|
|
315
|
+
// Process heading text for annotations
|
|
316
|
+
let processedHeaderText = '';
|
|
317
|
+
let lastIndex = 0;
|
|
318
|
+
|
|
319
|
+
// Find and process annotations in heading text
|
|
320
|
+
let annotationStart = headerText.indexOf('{');
|
|
321
|
+
while (annotationStart !== -1) {
|
|
322
|
+
const annotationEnd = headerText.indexOf('}', annotationStart);
|
|
323
|
+
if (annotationEnd === -1) break;
|
|
324
|
+
|
|
325
|
+
// Add text before annotation (escaped)
|
|
326
|
+
const beforeText = headerText.slice(lastIndex, annotationStart);
|
|
327
|
+
processedHeaderText += escapeHtml(beforeText);
|
|
328
|
+
|
|
329
|
+
// Process annotation content
|
|
330
|
+
const annotationContent = headerText.slice(annotationStart + 1, annotationEnd);
|
|
331
|
+
const parsedAnnotation = parseAnnotation(annotationContent);
|
|
332
|
+
|
|
333
|
+
// Add annotation HTML
|
|
334
|
+
processedHeaderText += `<span style="color: ${TOKEN_COLORS.annotation}; opacity: 0.75">{</span>` +
|
|
335
|
+
parsedAnnotation +
|
|
336
|
+
`<span style="color: ${TOKEN_COLORS.annotation}; opacity: 0.75">}</span>`;
|
|
337
|
+
|
|
338
|
+
lastIndex = annotationEnd + 1;
|
|
339
|
+
annotationStart = headerText.indexOf('{', lastIndex);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// Add remaining text after last annotation (escaped)
|
|
343
|
+
processedHeaderText += escapeHtml(headerText.slice(lastIndex));
|
|
344
|
+
|
|
345
|
+
// Create HTML heading that preserves the original markdown syntax
|
|
346
|
+
const headingTag = `h${hashCount}`;
|
|
347
|
+
const headingStyle = `margin: 0.5em 0; font-weight: 600;`;
|
|
348
|
+
const hashes = '#'.repeat(hashCount);
|
|
349
|
+
result += `<${headingTag} style="${headingStyle}"><span style="color: ${TOKEN_COLORS.marker}; opacity: 0.8">${hashes}</span> ${processedHeaderText}</${headingTag}>`;
|
|
350
|
+
|
|
351
|
+
i = endIdx;
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Regular text
|
|
357
|
+
result += escapeHtml(char);
|
|
358
|
+
i++;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
return result;
|
|
362
|
+
}
|
package/src/parse.js
CHANGED
|
@@ -7,16 +7,16 @@ import {
|
|
|
7
7
|
} from './utils.js';
|
|
8
8
|
import {
|
|
9
9
|
DEFAULT_CONTEXT,
|
|
10
|
-
URL_REGEX,
|
|
11
|
-
FENCE_REGEX,
|
|
12
|
-
PREFIX_REGEX,
|
|
13
|
-
HEADING_REGEX,
|
|
14
|
-
UNORDERED_LIST_REGEX,
|
|
15
|
-
BLOCKQUOTE_REGEX,
|
|
16
|
-
STANDALONE_SUBJECT_REGEX,
|
|
17
|
-
CARRIER_PATTERN_ARRAY,
|
|
18
|
-
|
|
19
10
|
} from './constants.js';
|
|
11
|
+
import {
|
|
12
|
+
detectStandaloneSubject,
|
|
13
|
+
detectPrefix,
|
|
14
|
+
detectHeading,
|
|
15
|
+
detectList,
|
|
16
|
+
detectBlockquote,
|
|
17
|
+
detectFence,
|
|
18
|
+
scanInlineCarriers
|
|
19
|
+
} from './tokenizers.js';
|
|
20
20
|
import {
|
|
21
21
|
|
|
22
22
|
getFenceClosePattern,
|
|
@@ -144,31 +144,31 @@ function scanTokens(text) {
|
|
|
144
144
|
let pos = 0;
|
|
145
145
|
let codeBlock = null;
|
|
146
146
|
const PROCESSORS = [
|
|
147
|
-
{ type: 'fence', test: line =>
|
|
147
|
+
{ type: 'fence', test: line => detectFence(line.trim()), process: handleFence },
|
|
148
148
|
{ type: 'content', test: () => codeBlock, process: line => codeBlock.content.push(line) },
|
|
149
|
-
{ type: 'prefix', test: line =>
|
|
150
|
-
{ type: 'standalone', test: line =>
|
|
151
|
-
{ type: 'heading', test: line =>
|
|
152
|
-
{ type: 'list', test: line =>
|
|
153
|
-
{ type: 'blockquote', test: line =>
|
|
149
|
+
{ type: 'prefix', test: line => detectPrefix(line), process: handlePrefix },
|
|
150
|
+
{ type: 'standalone', test: line => detectStandaloneSubject(line), process: handleStandaloneSubject },
|
|
151
|
+
{ type: 'heading', test: line => detectHeading(line), process: handleHeading },
|
|
152
|
+
{ type: 'list', test: line => detectList(line), process: handleList },
|
|
153
|
+
{ type: 'blockquote', test: line => detectBlockquote(line), process: handleBlockquote },
|
|
154
154
|
{ type: 'para', test: line => line.trim(), process: handlePara }
|
|
155
155
|
];
|
|
156
156
|
|
|
157
157
|
function handleFence(line, lineStart, pos) {
|
|
158
158
|
const trimmedLine = line.trim();
|
|
159
159
|
if (!codeBlock) {
|
|
160
|
-
const
|
|
161
|
-
if (!
|
|
160
|
+
const fenceResult = detectFence(trimmedLine);
|
|
161
|
+
if (!fenceResult) return false;
|
|
162
162
|
|
|
163
|
-
const
|
|
163
|
+
const attrsText = fenceResult.attrs;
|
|
164
164
|
const attrsStartInLine = attrsText ? line.indexOf(attrsText) : -1;
|
|
165
165
|
const contentStart = lineStart + line.length + 1;
|
|
166
166
|
|
|
167
167
|
codeBlock = {
|
|
168
|
-
fence:
|
|
168
|
+
fence: fenceResult.fenceChar.repeat(fenceResult.fenceLength),
|
|
169
169
|
start: lineStart,
|
|
170
170
|
content: [],
|
|
171
|
-
lang,
|
|
171
|
+
lang: fenceResult.lang,
|
|
172
172
|
attrs: attrsText,
|
|
173
173
|
attrsRange: attrsText && attrsStartInLine >= 0 ? [lineStart + attrsStartInLine, lineStart + attrsStartInLine + attrsText.length] : null,
|
|
174
174
|
valueRangeStart: contentStart
|
|
@@ -214,47 +214,50 @@ function scanTokens(text) {
|
|
|
214
214
|
}
|
|
215
215
|
|
|
216
216
|
function handlePrefix(line, lineStart, pos) {
|
|
217
|
-
const
|
|
218
|
-
tokens.push({ type: 'prefix', prefix:
|
|
217
|
+
const result = detectPrefix(line);
|
|
218
|
+
tokens.push({ type: 'prefix', prefix: result.prefix, iri: result.iri });
|
|
219
219
|
// Prefixes are typically stripped from clean MD
|
|
220
220
|
return true;
|
|
221
221
|
}
|
|
222
222
|
|
|
223
223
|
function handleHeading(line, lineStart, pos) {
|
|
224
|
-
const
|
|
225
|
-
const attrs =
|
|
226
|
-
const afterHashes =
|
|
227
|
-
const rangeInfo = calcRangeInfo(line, attrs, lineStart, afterHashes,
|
|
228
|
-
tokens.push(createToken('heading', [lineStart, pos - 1],
|
|
229
|
-
rangeInfo.attrsRange, rangeInfo.valueRange, { depth:
|
|
224
|
+
const result = detectHeading(line);
|
|
225
|
+
const attrs = result.attrs;
|
|
226
|
+
const afterHashes = result.depth;
|
|
227
|
+
const rangeInfo = calcRangeInfo(line, attrs, lineStart, afterHashes, result.content.length);
|
|
228
|
+
tokens.push(createToken('heading', [lineStart, pos - 1], result.content, attrs,
|
|
229
|
+
rangeInfo.attrsRange, rangeInfo.valueRange, { depth: result.depth }));
|
|
230
230
|
|
|
231
231
|
// Add stripped heading to mdLines
|
|
232
|
-
const cleanHeading = `${
|
|
232
|
+
const cleanHeading = `${'#'.repeat(result.depth)} ${result.content}`;
|
|
233
233
|
mdLines.push(cleanHeading);
|
|
234
234
|
return true;
|
|
235
235
|
}
|
|
236
236
|
|
|
237
237
|
function handleList(line, lineStart, pos) {
|
|
238
|
-
const
|
|
238
|
+
const result = detectList(line);
|
|
239
|
+
// Create match array format expected by createListToken
|
|
240
|
+
const indentStr = ' '.repeat(result.indent);
|
|
241
|
+
const match = [line, indentStr, result.marker, result.content, result.attrs];
|
|
239
242
|
tokens.push(createListToken('list', line, lineStart, pos, match));
|
|
240
243
|
|
|
241
244
|
// Add stripped list item to mdLines
|
|
242
|
-
const cleanList = `${
|
|
245
|
+
const cleanList = `${indentStr}${result.marker} ${result.content}`;
|
|
243
246
|
mdLines.push(cleanList);
|
|
244
247
|
return true;
|
|
245
248
|
}
|
|
246
249
|
|
|
247
250
|
function handleBlockquote(line, lineStart, pos) {
|
|
248
|
-
const
|
|
249
|
-
const attrs =
|
|
251
|
+
const result = detectBlockquote(line);
|
|
252
|
+
const attrs = result.attrs;
|
|
250
253
|
const valueStartInLine = line.startsWith('> ') ? 2 : line.indexOf('>') + 1;
|
|
251
|
-
const valueEndInLine = valueStartInLine +
|
|
252
|
-
tokens.push(createToken('blockquote', [lineStart, pos - 1],
|
|
254
|
+
const valueEndInLine = valueStartInLine + result.content.length;
|
|
255
|
+
tokens.push(createToken('blockquote', [lineStart, pos - 1], result.content, attrs,
|
|
253
256
|
calcAttrsRange(line, attrs, lineStart),
|
|
254
257
|
[lineStart + valueStartInLine, lineStart + valueEndInLine]));
|
|
255
258
|
|
|
256
259
|
// Add stripped blockquote to mdLines
|
|
257
|
-
const cleanBlockquote = `> ${
|
|
260
|
+
const cleanBlockquote = `> ${result.content}`;
|
|
258
261
|
mdLines.push(cleanBlockquote);
|
|
259
262
|
return true;
|
|
260
263
|
}
|
|
@@ -265,12 +268,15 @@ function scanTokens(text) {
|
|
|
265
268
|
// Add stripped paragraph to mdLines
|
|
266
269
|
let cleanPara = line;
|
|
267
270
|
|
|
268
|
-
// Remove inline carrier annotations using
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
271
|
+
// Remove inline carrier annotations using character-based detection
|
|
272
|
+
const carriers = scanInlineCarriers(cleanPara, 0);
|
|
273
|
+
for (const carrier of carriers) {
|
|
274
|
+
if (carrier.attrs && (carrier.type === 'emphasis' || carrier.type === 'code')) {
|
|
275
|
+
// Replace with just the content
|
|
276
|
+
const before = cleanPara.substring(0, carrier.range[0]);
|
|
277
|
+
const after = cleanPara.substring(carrier.range[1]);
|
|
278
|
+
cleanPara = before + (carrier.text || '') + after;
|
|
279
|
+
}
|
|
274
280
|
}
|
|
275
281
|
|
|
276
282
|
// Remove bracket-style annotations [text] {annotation}
|
|
@@ -313,67 +319,7 @@ function scanTokens(text) {
|
|
|
313
319
|
}
|
|
314
320
|
|
|
315
321
|
function extractInlineCarriers(text, baseOffset = 0) {
|
|
316
|
-
|
|
317
|
-
let pos = 0;
|
|
318
|
-
|
|
319
|
-
const CARRIER_EXTRACTORS = {
|
|
320
|
-
'<': (text, pos, baseOffset) => {
|
|
321
|
-
const angleEnd = text.indexOf('>', pos);
|
|
322
|
-
if (angleEnd === -1) return null;
|
|
323
|
-
const url = text.slice(pos + 1, angleEnd);
|
|
324
|
-
if (!URL_REGEX.test(url)) return null;
|
|
325
|
-
const { attrs, attrsRange, finalSpanEnd } = extractAttributesFromText(text, angleEnd + 1, baseOffset);
|
|
326
|
-
return createCarrier('link', url, attrs, attrsRange,
|
|
327
|
-
[baseOffset + pos + 1, baseOffset + angleEnd],
|
|
328
|
-
[baseOffset + pos, baseOffset + finalSpanEnd], finalSpanEnd, { url });
|
|
329
|
-
},
|
|
330
|
-
'[': (text, pos, baseOffset) => {
|
|
331
|
-
const bracketEnd = findMatchingBracket(text, pos);
|
|
332
|
-
if (!bracketEnd) return null;
|
|
333
|
-
const carrierText = text.slice(pos + 1, bracketEnd - 1);
|
|
334
|
-
const { url, spanEnd } = extractUrlFromBrackets(text, bracketEnd);
|
|
335
|
-
const { attrs, attrsRange, finalSpanEnd } = extractAttributesFromText(text, spanEnd, baseOffset);
|
|
336
|
-
const { carrierType, resourceIRI } = determineCarrierType(url);
|
|
337
|
-
if (url?.startsWith('=')) return { skip: true, pos: finalSpanEnd };
|
|
338
|
-
return createCarrier(carrierType, carrierText, attrs, attrsRange,
|
|
339
|
-
[baseOffset + pos + 1, baseOffset + bracketEnd - 1],
|
|
340
|
-
[baseOffset + pos, baseOffset + finalSpanEnd], finalSpanEnd, { url: resourceIRI });
|
|
341
|
-
}
|
|
342
|
-
};
|
|
343
|
-
|
|
344
|
-
const extractCarrier = (text, pos, baseOffset) => {
|
|
345
|
-
const extractor = CARRIER_EXTRACTORS[text[pos]];
|
|
346
|
-
if (extractor) return extractor(text, pos, baseOffset);
|
|
347
|
-
|
|
348
|
-
// Use pre-compiled patterns instead of Object.entries()
|
|
349
|
-
for (const [type, pattern] of CARRIER_PATTERN_ARRAY) {
|
|
350
|
-
pattern.lastIndex = pos;
|
|
351
|
-
const match = pattern.exec(text);
|
|
352
|
-
if (match) {
|
|
353
|
-
const ranges = calcCarrierRanges(match, baseOffset, match.index);
|
|
354
|
-
const carrierType = type === 'EMPHASIS' ? 'emphasis' : 'code';
|
|
355
|
-
return createCarrier(carrierType, match[1], `{${match[2]}}`,
|
|
356
|
-
ranges.attrsRange, ranges.valueRange, ranges.range, ranges.pos);
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
return null;
|
|
360
|
-
};
|
|
361
|
-
|
|
362
|
-
while (pos < text.length) {
|
|
363
|
-
const carrier = extractCarrier(text, pos, baseOffset);
|
|
364
|
-
if (carrier) {
|
|
365
|
-
if (carrier.skip) {
|
|
366
|
-
pos = carrier.pos;
|
|
367
|
-
} else {
|
|
368
|
-
carriers.push(carrier);
|
|
369
|
-
pos = carrier.pos;
|
|
370
|
-
}
|
|
371
|
-
} else {
|
|
372
|
-
pos++;
|
|
373
|
-
}
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
return carriers;
|
|
322
|
+
return scanInlineCarriers(text, baseOffset);
|
|
377
323
|
}
|
|
378
324
|
|
|
379
325
|
|
|
@@ -743,14 +689,14 @@ function processTokenAnnotations(token, state, tokenType) {
|
|
|
743
689
|
}
|
|
744
690
|
|
|
745
691
|
function processStandaloneSubject(token, state) {
|
|
746
|
-
const
|
|
747
|
-
if (!
|
|
692
|
+
const result = detectStandaloneSubject(token.text);
|
|
693
|
+
if (!result) return;
|
|
748
694
|
|
|
749
|
-
const sem = parseSemCached(`{=${
|
|
695
|
+
const sem = parseSemCached(`{=${result.content}}`);
|
|
750
696
|
const attrsStart = token.range[0] + token.text.indexOf('{=');
|
|
751
697
|
processAnnotation({
|
|
752
698
|
type: 'standalone', text: '', range: token.range,
|
|
753
|
-
attrsRange: [attrsStart, attrsStart + (
|
|
699
|
+
attrsRange: [attrsStart, attrsStart + (result.content ? result.content.length : 0)],
|
|
754
700
|
valueRange: null
|
|
755
701
|
}, sem, state);
|
|
756
702
|
}
|
|
@@ -0,0 +1,594 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Character-based Tokenizers for MDLD
|
|
3
|
+
*
|
|
4
|
+
* Replaces regex patterns with direct character scanning for better performance.
|
|
5
|
+
* Each tokenizer follows the EBNF grammar exactly.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
// ============================================================================
|
|
9
|
+
// BLOCK-LEVEL TOKENIZERS (Line-based)
|
|
10
|
+
// ============================================================================
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Detect code fence: 3+ backticks or tildes at line start
|
|
14
|
+
* EBNF: backticks = "`", "`", "`", { "`" } | tildes = "~", "~", "~", { "~" }
|
|
15
|
+
*/
|
|
16
|
+
export function detectFence(line) {
|
|
17
|
+
if (line.length < 3) return null;
|
|
18
|
+
|
|
19
|
+
const char = line[0];
|
|
20
|
+
if (char !== '`' && char !== '~') return null;
|
|
21
|
+
|
|
22
|
+
// Count consecutive fence characters
|
|
23
|
+
let count = 1;
|
|
24
|
+
while (count < line.length && line[count] === char) {
|
|
25
|
+
count++;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
if (count < 3) return null;
|
|
29
|
+
|
|
30
|
+
// Extract info string (everything after fence chars)
|
|
31
|
+
const rest = line.slice(count).trimStart();
|
|
32
|
+
const infoString = rest;
|
|
33
|
+
|
|
34
|
+
// Parse info string for language and attributes
|
|
35
|
+
const langMatch = infoString.match(/^([^\s{]+)/);
|
|
36
|
+
const lang = langMatch ? langMatch[1] : '';
|
|
37
|
+
|
|
38
|
+
// Find attributes in info string
|
|
39
|
+
const attrsMatch = infoString.match(/\{([^}]+)\}/);
|
|
40
|
+
const attrs = attrsMatch ? attrsMatch[1] : null;
|
|
41
|
+
|
|
42
|
+
return {
|
|
43
|
+
fenceChar: char,
|
|
44
|
+
fenceLength: count,
|
|
45
|
+
lang,
|
|
46
|
+
attrs,
|
|
47
|
+
infoString
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Detect prefix declaration: [prefix] <IRI>
|
|
53
|
+
* EBNF: contextDecl = "[", contextKey, "]", whitespace, "<", contextIri, ">"
|
|
54
|
+
*/
|
|
55
|
+
export function detectPrefix(line) {
|
|
56
|
+
if (line[0] !== '[') return null;
|
|
57
|
+
|
|
58
|
+
// Find closing bracket
|
|
59
|
+
const closeBracket = line.indexOf(']', 1);
|
|
60
|
+
if (closeBracket === -1) return null;
|
|
61
|
+
|
|
62
|
+
const prefix = line.slice(1, closeBracket).trim();
|
|
63
|
+
if (!prefix) return null;
|
|
64
|
+
|
|
65
|
+
// Find < after bracket (skip whitespace)
|
|
66
|
+
let pos = closeBracket + 1;
|
|
67
|
+
while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
|
|
68
|
+
pos++;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (pos >= line.length || line[pos] !== '<') return null;
|
|
72
|
+
|
|
73
|
+
// Find closing >
|
|
74
|
+
const closeAngle = line.indexOf('>', pos + 1);
|
|
75
|
+
if (closeAngle === -1) return null;
|
|
76
|
+
|
|
77
|
+
const iri = line.slice(pos + 1, closeAngle).trim();
|
|
78
|
+
if (!iri) return null;
|
|
79
|
+
|
|
80
|
+
return { prefix, iri };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Detect heading: 1-6 # characters followed by space and text
|
|
85
|
+
* EBNF: heading = "#", { "#" }, whitespace, text
|
|
86
|
+
*/
|
|
87
|
+
export function detectHeading(line) {
|
|
88
|
+
if (line[0] !== '#') return null;
|
|
89
|
+
|
|
90
|
+
// Count # characters (max 6)
|
|
91
|
+
let depth = 1;
|
|
92
|
+
while (depth < line.length && depth < 6 && line[depth] === '#') {
|
|
93
|
+
depth++;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Must be followed by whitespace
|
|
97
|
+
if (depth >= line.length || (line[depth] !== ' ' && line[depth] !== '\t')) {
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Extract content (skip # and whitespace)
|
|
102
|
+
let contentStart = depth;
|
|
103
|
+
while (contentStart < line.length && (line[contentStart] === ' ' || line[contentStart] === '\t')) {
|
|
104
|
+
contentStart++;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Check for trailing attributes {=...} or {...}
|
|
108
|
+
const trimmed = line.slice(contentStart);
|
|
109
|
+
const attrsMatch = trimmed.match(/\s*\{([^}]+)\}\s*$/);
|
|
110
|
+
|
|
111
|
+
let content = trimmed;
|
|
112
|
+
let attrs = null;
|
|
113
|
+
|
|
114
|
+
if (attrsMatch) {
|
|
115
|
+
attrs = attrsMatch[1];
|
|
116
|
+
content = trimmed.slice(0, -attrsMatch[0].length).trim();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return { depth, content, attrs };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Detect list item: -, *, +, or digits followed by .
|
|
124
|
+
* EBNF: listMarker = "-" | "*" | "+" | digit, { digit }, "."
|
|
125
|
+
*/
|
|
126
|
+
export function detectList(line) {
|
|
127
|
+
// Skip leading whitespace for indentation detection
|
|
128
|
+
let pos = 0;
|
|
129
|
+
while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
|
|
130
|
+
pos++;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const indent = pos;
|
|
134
|
+
if (pos >= line.length) return null;
|
|
135
|
+
|
|
136
|
+
const char = line[pos];
|
|
137
|
+
let marker;
|
|
138
|
+
let contentStart = pos + 1;
|
|
139
|
+
|
|
140
|
+
if (char === '-' || char === '*' || char === '+') {
|
|
141
|
+
marker = char;
|
|
142
|
+
} else if (char >= '0' && char <= '9') {
|
|
143
|
+
// Ordered list: digits followed by .
|
|
144
|
+
let numEnd = pos + 1;
|
|
145
|
+
while (numEnd < line.length && line[numEnd] >= '0' && line[numEnd] <= '9') {
|
|
146
|
+
numEnd++;
|
|
147
|
+
}
|
|
148
|
+
if (numEnd >= line.length || line[numEnd] !== '.') return null;
|
|
149
|
+
marker = line.slice(pos, numEnd + 1);
|
|
150
|
+
contentStart = numEnd + 1;
|
|
151
|
+
} else {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Must be followed by whitespace
|
|
156
|
+
if (contentStart >= line.length || (line[contentStart] !== ' ' && line[contentStart] !== '\t')) {
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Skip whitespace after marker
|
|
161
|
+
while (contentStart < line.length && (line[contentStart] === ' ' || line[contentStart] === '\t')) {
|
|
162
|
+
contentStart++;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Extract content and check for trailing attributes
|
|
166
|
+
let content = line.slice(contentStart);
|
|
167
|
+
let attrs = null;
|
|
168
|
+
|
|
169
|
+
const attrsMatch = content.match(/\s*\{([^}]+)\}\s*$/);
|
|
170
|
+
if (attrsMatch) {
|
|
171
|
+
attrs = attrsMatch[1];
|
|
172
|
+
content = content.slice(0, -attrsMatch[0].length).trim();
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return { indent, marker, content, attrs };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Detect blockquote: > followed by space and text
|
|
180
|
+
* EBNF: blockquote = ">", whitespace, text
|
|
181
|
+
*/
|
|
182
|
+
export function detectBlockquote(line) {
|
|
183
|
+
if (line[0] !== '>') return null;
|
|
184
|
+
|
|
185
|
+
// Must be followed by whitespace or end of line
|
|
186
|
+
if (line.length > 1 && line[1] !== ' ' && line[1] !== '\t') {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// Extract content
|
|
191
|
+
let contentStart = 1;
|
|
192
|
+
while (contentStart < line.length && (line[contentStart] === ' ' || line[contentStart] === '\t')) {
|
|
193
|
+
contentStart++;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
let content = line.slice(contentStart);
|
|
197
|
+
let attrs = null;
|
|
198
|
+
|
|
199
|
+
// Check for trailing attributes
|
|
200
|
+
const attrsMatch = content.match(/\s*\{([^}]+)\}\s*$/);
|
|
201
|
+
if (attrsMatch) {
|
|
202
|
+
attrs = attrsMatch[1];
|
|
203
|
+
content = content.slice(0, -attrsMatch[0].length).trim();
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
return { content, attrs };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Detect standalone subject: {=...} with optional whitespace
|
|
211
|
+
* EBNF: standaloneSubject = whitespace*, "{", "=", ( iriRef | "#", fragment ), "}", whitespace*
|
|
212
|
+
*/
|
|
213
|
+
export function detectStandaloneSubject(line) {
|
|
214
|
+
let pos = 0;
|
|
215
|
+
|
|
216
|
+
// Skip leading whitespace
|
|
217
|
+
while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
|
|
218
|
+
pos++;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
if (pos >= line.length || line[pos] !== '{') return null;
|
|
222
|
+
pos++;
|
|
223
|
+
|
|
224
|
+
if (pos >= line.length || line[pos] !== '=') return null;
|
|
225
|
+
pos++;
|
|
226
|
+
|
|
227
|
+
// Extract content until }
|
|
228
|
+
const contentStart = pos;
|
|
229
|
+
let braceCount = 1;
|
|
230
|
+
|
|
231
|
+
while (pos < line.length && braceCount > 0) {
|
|
232
|
+
if (line[pos] === '{') braceCount++;
|
|
233
|
+
if (line[pos] === '}') braceCount--;
|
|
234
|
+
if (braceCount > 0) pos++;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (braceCount > 0) return null; // No closing brace
|
|
238
|
+
|
|
239
|
+
const content = line.slice(contentStart, pos).trim();
|
|
240
|
+
|
|
241
|
+
// Skip trailing whitespace
|
|
242
|
+
pos++;
|
|
243
|
+
while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
|
|
244
|
+
pos++;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// Must consume entire line (after optional trailing whitespace)
|
|
248
|
+
if (pos < line.length) return null;
|
|
249
|
+
|
|
250
|
+
return { content };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// ============================================================================
|
|
254
|
+
// INLINE CARRIER TOKENIZERS (Character-based within block content)
|
|
255
|
+
// ============================================================================
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Find matching closing bracket, accounting for nesting
|
|
259
|
+
*/
|
|
260
|
+
export function findMatchingBracket(text, startPos, openChar = '[', closeChar = ']') {
|
|
261
|
+
let depth = 1;
|
|
262
|
+
let pos = startPos + 1;
|
|
263
|
+
|
|
264
|
+
while (pos < text.length && depth > 0) {
|
|
265
|
+
if (text[pos] === openChar) depth++;
|
|
266
|
+
if (text[pos] === closeChar) depth--;
|
|
267
|
+
if (depth > 0) pos++;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
return depth === 0 ? pos : null;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Extract attributes block after a position: {...}
|
|
275
|
+
*/
|
|
276
|
+
export function extractAttributes(text, startPos) {
|
|
277
|
+
let pos = startPos;
|
|
278
|
+
|
|
279
|
+
// Skip whitespace
|
|
280
|
+
while (pos < text.length && (text[pos] === ' ' || text[pos] === '\t')) {
|
|
281
|
+
pos++;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
if (pos >= text.length || text[pos] !== '{') return null;
|
|
285
|
+
|
|
286
|
+
// Find closing brace (simple version - no nesting for attribute blocks)
|
|
287
|
+
const closeBrace = text.indexOf('}', pos + 1);
|
|
288
|
+
if (closeBrace === -1) return null;
|
|
289
|
+
|
|
290
|
+
const attrs = text.slice(pos + 1, closeBrace);
|
|
291
|
+
const endPos = closeBrace + 1;
|
|
292
|
+
|
|
293
|
+
return { attrs, endPos };
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Detect URL in angle brackets: <URL>
|
|
298
|
+
* Returns null if not a valid URL per EBNF scheme rules
|
|
299
|
+
*/
|
|
300
|
+
export function detectAngleBracketUrl(text, startPos) {
|
|
301
|
+
if (text[startPos] !== '<') return null;
|
|
302
|
+
|
|
303
|
+
const closeAngle = text.indexOf('>', startPos + 1);
|
|
304
|
+
if (closeAngle === -1) return null;
|
|
305
|
+
|
|
306
|
+
const url = text.slice(startPos + 1, closeAngle).trim();
|
|
307
|
+
|
|
308
|
+
// Validate URL scheme (letter followed by letters/digits/+/ - /.)
|
|
309
|
+
const schemeMatch = url.match(/^[a-zA-Z][a-zA-Z0-9+\-.]*:/);
|
|
310
|
+
if (!schemeMatch) return null;
|
|
311
|
+
|
|
312
|
+
return { url, endPos: closeAngle + 1 };
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Detect emphasis/strong spans: *text* or _text_
|
|
317
|
+
* EBNF: emphasisSpan = "*", text, "*" | "_", text, "_"
|
|
318
|
+
* strongSpan = "**", text, "**" | "__", text, "__"
|
|
319
|
+
*/
|
|
320
|
+
export function detectEmphasis(text, startPos) {
|
|
321
|
+
const char = text[startPos];
|
|
322
|
+
if (char !== '*' && char !== '_') return null;
|
|
323
|
+
|
|
324
|
+
// Count delimiter characters
|
|
325
|
+
let delimCount = 1;
|
|
326
|
+
while (startPos + delimCount < text.length &&
|
|
327
|
+
text[startPos + delimCount] === char) {
|
|
328
|
+
delimCount++;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
if (delimCount > 2) return null; // Only 1 or 2 allowed
|
|
332
|
+
|
|
333
|
+
const type = delimCount === 1 ? 'emphasis' : 'strong';
|
|
334
|
+
const closer = char.repeat(delimCount);
|
|
335
|
+
|
|
336
|
+
// Find closing delimiter
|
|
337
|
+
const contentStart = startPos + delimCount;
|
|
338
|
+
let pos = contentStart;
|
|
339
|
+
|
|
340
|
+
while (pos < text.length) {
|
|
341
|
+
// Check for closing delimiter
|
|
342
|
+
if (text.slice(pos, pos + delimCount) === closer) {
|
|
343
|
+
// Must not be followed by same char (would be longer delimiter)
|
|
344
|
+
if (pos + delimCount < text.length && text[pos + delimCount] === char) {
|
|
345
|
+
pos++;
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
const content = text.slice(contentStart, pos);
|
|
350
|
+
const endPos = pos + delimCount;
|
|
351
|
+
|
|
352
|
+
// Check for attributes after
|
|
353
|
+
const attrsResult = extractAttributes(text, endPos);
|
|
354
|
+
const finalPos = attrsResult ? attrsResult.endPos : endPos;
|
|
355
|
+
|
|
356
|
+
return {
|
|
357
|
+
type,
|
|
358
|
+
content,
|
|
359
|
+
attrs: attrsResult?.attrs || null,
|
|
360
|
+
endPos: finalPos,
|
|
361
|
+
contentEnd: endPos
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
pos++;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
return null; // No closing delimiter found
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Detect code span: `text` or ``text``
|
|
372
|
+
* EBNF: codeSpan = "`", text, "`" | "``", text, "``"
|
|
373
|
+
*/
|
|
374
|
+
export function detectCodeSpan(text, startPos) {
|
|
375
|
+
if (text[startPos] !== '`') return null;
|
|
376
|
+
|
|
377
|
+
// Count opening backticks
|
|
378
|
+
let backtickCount = 1;
|
|
379
|
+
while (startPos + backtickCount < text.length &&
|
|
380
|
+
text[startPos + backtickCount] === '`') {
|
|
381
|
+
backtickCount++;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
if (backtickCount > 2) return null; // Only 1 or 2 allowed in current grammar
|
|
385
|
+
|
|
386
|
+
const opener = '`'.repeat(backtickCount);
|
|
387
|
+
const contentStart = startPos + backtickCount;
|
|
388
|
+
|
|
389
|
+
// Find closing backticks
|
|
390
|
+
let pos = contentStart;
|
|
391
|
+
|
|
392
|
+
while (pos < text.length) {
|
|
393
|
+
if (text.slice(pos, pos + backtickCount) === opener) {
|
|
394
|
+
const content = text.slice(contentStart, pos);
|
|
395
|
+
const endPos = pos + backtickCount;
|
|
396
|
+
|
|
397
|
+
// Check for attributes after
|
|
398
|
+
const attrsResult = extractAttributes(text, endPos);
|
|
399
|
+
const finalPos = attrsResult ? attrsResult.endPos : endPos;
|
|
400
|
+
|
|
401
|
+
return {
|
|
402
|
+
type: 'code',
|
|
403
|
+
content,
|
|
404
|
+
attrs: attrsResult?.attrs || null,
|
|
405
|
+
endPos: finalPos,
|
|
406
|
+
contentEnd: endPos
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
pos++;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
return null;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Detect bracket link: [text](url) or [text]{...} or [text] alone
|
|
417
|
+
* EBNF: linkSpan = "[", text, "](", text, ")" | "[", text, "]"
|
|
418
|
+
*/
|
|
419
|
+
export function detectBracketLink(text, startPos) {
|
|
420
|
+
if (text[startPos] !== '[') return null;
|
|
421
|
+
|
|
422
|
+
const bracketEnd = findMatchingBracket(text, startPos, '[', ']');
|
|
423
|
+
if (!bracketEnd) return null;
|
|
424
|
+
|
|
425
|
+
const linkText = text.slice(startPos + 1, bracketEnd);
|
|
426
|
+
let pos = bracketEnd + 1;
|
|
427
|
+
let url = null;
|
|
428
|
+
|
|
429
|
+
// Check for (url) after ]
|
|
430
|
+
if (pos < text.length && text[pos] === '(') {
|
|
431
|
+
const parenEnd = text.indexOf(')', pos + 1);
|
|
432
|
+
if (parenEnd !== -1) {
|
|
433
|
+
url = text.slice(pos + 1, parenEnd);
|
|
434
|
+
pos = parenEnd + 1;
|
|
435
|
+
}
|
|
436
|
+
} else if (pos < text.length && text[pos] === '<') {
|
|
437
|
+
// Check for <url> after ]
|
|
438
|
+
const closeAngle = text.indexOf('>', pos + 1);
|
|
439
|
+
if (closeAngle !== -1) {
|
|
440
|
+
const angleUrl = text.slice(pos + 1, closeAngle).trim();
|
|
441
|
+
// Validate URL scheme
|
|
442
|
+
const schemeMatch = angleUrl.match(/^[a-zA-Z][a-zA-Z0-9+\-.]*:/);
|
|
443
|
+
if (schemeMatch) {
|
|
444
|
+
url = angleUrl;
|
|
445
|
+
pos = closeAngle + 1;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
// Check for attributes
|
|
451
|
+
const attrsResult = extractAttributes(text, pos);
|
|
452
|
+
const finalPos = attrsResult ? attrsResult.endPos : pos;
|
|
453
|
+
|
|
454
|
+
// Determine carrier type: 'link' if URL present, 'span' otherwise
|
|
455
|
+
const carrierType = url ? 'link' : 'span';
|
|
456
|
+
|
|
457
|
+
return {
|
|
458
|
+
type: carrierType,
|
|
459
|
+
text: linkText,
|
|
460
|
+
url,
|
|
461
|
+
attrs: attrsResult?.attrs || null,
|
|
462
|
+
endPos: finalPos,
|
|
463
|
+
bracketEnd: bracketEnd + 1
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
// ============================================================================
|
|
468
|
+
// RANGE CALCULATION HELPERS
|
|
469
|
+
// ============================================================================
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* Calculate absolute ranges for a token given base offset
|
|
473
|
+
*/
|
|
474
|
+
export function calcRanges(token, baseOffset) {
|
|
475
|
+
return {
|
|
476
|
+
range: [baseOffset + token.start, baseOffset + token.end],
|
|
477
|
+
valueRange: token.valueStart !== undefined
|
|
478
|
+
? [baseOffset + token.valueStart, baseOffset + token.valueEnd]
|
|
479
|
+
: null,
|
|
480
|
+
attrsRange: token.attrsStart !== undefined
|
|
481
|
+
? [baseOffset + token.attrsStart, baseOffset + token.attrsEnd]
|
|
482
|
+
: null
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
// ============================================================================
|
|
487
|
+
// COMPLETE INLINE CARRIER SCANNER (replaces extractInlineCarriers)
|
|
488
|
+
// ============================================================================
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* Scan text for all inline carriers using character-based detection
|
|
492
|
+
* Memory-optimized version - minimizes object allocations
|
|
493
|
+
*/
|
|
494
|
+
export function scanInlineCarriers(text, baseOffset = 0) {
|
|
495
|
+
const carriers = [];
|
|
496
|
+
const len = text.length;
|
|
497
|
+
let pos = 0;
|
|
498
|
+
|
|
499
|
+
while (pos < len) {
|
|
500
|
+
const char = text[pos];
|
|
501
|
+
let detected = null;
|
|
502
|
+
|
|
503
|
+
// Fast path: only check carrier start characters
|
|
504
|
+
switch (char) {
|
|
505
|
+
case '<':
|
|
506
|
+
detected = detectAngleBracketUrl(text, pos);
|
|
507
|
+
if (detected) {
|
|
508
|
+
const attrsResult = extractAttributes(text, detected.endPos);
|
|
509
|
+
if (attrsResult) {
|
|
510
|
+
detected.attrs = attrsResult.attrs;
|
|
511
|
+
detected.endPos = attrsResult.endPos;
|
|
512
|
+
}
|
|
513
|
+
detected.type = 'link';
|
|
514
|
+
detected.text = detected.url;
|
|
515
|
+
}
|
|
516
|
+
break;
|
|
517
|
+
|
|
518
|
+
case '[':
|
|
519
|
+
detected = detectBracketLink(text, pos);
|
|
520
|
+
break;
|
|
521
|
+
|
|
522
|
+
case '*':
|
|
523
|
+
case '_':
|
|
524
|
+
detected = detectEmphasis(text, pos);
|
|
525
|
+
break;
|
|
526
|
+
|
|
527
|
+
case '`':
|
|
528
|
+
detected = detectCodeSpan(text, pos);
|
|
529
|
+
break;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
if (!detected) {
|
|
533
|
+
pos++;
|
|
534
|
+
continue;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// Skip annotation-only patterns
|
|
538
|
+
const url = detected.url;
|
|
539
|
+
const type = detected.type;
|
|
540
|
+
const attrs = detected.attrs;
|
|
541
|
+
|
|
542
|
+
if (url?.startsWith('=') || (type === 'link' && !attrs && !url)) {
|
|
543
|
+
pos = detected.endPos;
|
|
544
|
+
continue;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// Calculate ranges once
|
|
548
|
+
const absStart = baseOffset + pos;
|
|
549
|
+
const absEnd = baseOffset + detected.endPos;
|
|
550
|
+
const contentEndOffset = detected.contentEnd || detected.bracketEnd || detected.endPos;
|
|
551
|
+
const absContentEnd = baseOffset + contentEndOffset;
|
|
552
|
+
|
|
553
|
+
// Build carrier object with minimal properties
|
|
554
|
+
const carrier = {
|
|
555
|
+
type: type,
|
|
556
|
+
text: detected.content !== undefined ? detected.content : (detected.text !== undefined ? detected.text : url),
|
|
557
|
+
range: [absStart, absEnd],
|
|
558
|
+
valueRange: [absStart + (type === 'url' ? 1 : 0), absContentEnd - (type === 'url' ? 1 : 0)],
|
|
559
|
+
attrs: attrs,
|
|
560
|
+
url: url,
|
|
561
|
+
pos: detected.endPos
|
|
562
|
+
};
|
|
563
|
+
|
|
564
|
+
// Only add attrsRange if attrs exist (saves memory for carriers without attrs)
|
|
565
|
+
if (attrs) {
|
|
566
|
+
carrier.attrsRange = [absContentEnd, absEnd];
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
carriers.push(carrier);
|
|
570
|
+
pos = detected.endPos;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
return carriers;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
// ============================================================================
|
|
577
|
+
// TESTING EXPORTS (for validation)
|
|
578
|
+
// ============================================================================
|
|
579
|
+
|
|
580
|
+
export const TEST = {
|
|
581
|
+
detectFence,
|
|
582
|
+
detectPrefix,
|
|
583
|
+
detectHeading,
|
|
584
|
+
detectList,
|
|
585
|
+
detectBlockquote,
|
|
586
|
+
detectStandaloneSubject,
|
|
587
|
+
detectAngleBracketUrl,
|
|
588
|
+
detectEmphasis,
|
|
589
|
+
detectCodeSpan,
|
|
590
|
+
detectBracketLink,
|
|
591
|
+
extractAttributes,
|
|
592
|
+
findMatchingBracket,
|
|
593
|
+
scanInlineCarriers
|
|
594
|
+
};
|