mdld-parse 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/.nojekyll +0 -0
  2. package/dist/CNAME +1 -0
  3. package/dist/README.md +425 -0
  4. package/dist/docs/API.md +397 -0
  5. package/dist/docs/Architecture.md +190 -0
  6. package/dist/docs/Guide.md +322 -0
  7. package/dist/docs/LLM-MDLD-Inducer.md +550 -0
  8. package/dist/docs/Parser.md +654 -0
  9. package/dist/docs/Performance.md +238 -0
  10. package/dist/docs/Spec-render.md +1439 -0
  11. package/dist/docs/Subject.md +180 -0
  12. package/dist/docs/Syntax.md +463 -0
  13. package/dist/docs/TodoList.md +215 -0
  14. package/dist/docs/Use-Cases.md +354 -0
  15. package/dist/docs/index.html +108 -0
  16. package/dist/docs/index.md +56 -0
  17. package/dist/docs/origin.md +459 -0
  18. package/dist/docs/polarity.md +586 -0
  19. package/dist/docs/primary-subject.md +199 -0
  20. package/dist/docs/statements.md +226 -0
  21. package/dist/docs/tokenization-performance-report.md +164 -0
  22. package/dist/examples/Cookbook.md +32 -0
  23. package/dist/examples/Status-SHACL.md +33 -0
  24. package/dist/examples/demo/index.md +35 -0
  25. package/dist/examples/demo/journal.md +18 -0
  26. package/dist/examples/demo/minimal.md +11 -0
  27. package/dist/examples/demo/project.md +15 -0
  28. package/dist/examples/demo/recipe.md +31 -0
  29. package/dist/examples/demo/research.md +46 -0
  30. package/dist/examples/demo/space-mission.md +48 -0
  31. package/dist/examples/dogfood.md +288 -0
  32. package/dist/examples/few-shot.md +309 -0
  33. package/dist/examples/graph/rdf-graph.html +473 -0
  34. package/dist/examples/graph/rdf_force_graph_web_component.html +506 -0
  35. package/dist/examples/index.md +45 -0
  36. package/dist/examples/llm-time-workflow.md +173 -0
  37. package/dist/examples/prov-o.md +2367 -0
  38. package/dist/examples/rdf.md +827 -0
  39. package/dist/examples/shacl.md +3399 -0
  40. package/dist/examples/statements.md +43 -0
  41. package/dist/examples/workflow.md +150 -0
  42. package/dist/index.html +2827 -0
  43. package/dist/index.js +1548 -0
  44. package/dist/logo.svg +7 -0
  45. package/dist/spec/Spec-compact.md +363 -0
  46. package/dist/spec/Spec-ultra.md +221 -0
  47. package/dist/spec/Spec.md +1068 -0
  48. package/dist/spec/index.md +46 -0
  49. package/package.json +7 -8
  50. package/src/constants.js +0 -31
  51. package/src/generate.js +0 -245
  52. package/src/highlight.js +0 -362
  53. package/src/index.js +0 -13
  54. package/src/locate.js +0 -23
  55. package/src/merge.js +0 -157
  56. package/src/parse.js +0 -711
  57. package/src/render.js +0 -576
  58. package/src/shared.js +0 -577
  59. package/src/tokenizers.js +0 -594
  60. package/src/utils.js +0 -430
package/src/tokenizers.js DELETED
@@ -1,594 +0,0 @@
1
- /**
2
- * Character-based Tokenizers for MDLD
3
- *
4
- * Replaces regex patterns with direct character scanning for better performance.
5
- * Each tokenizer follows the EBNF grammar exactly.
6
- */
7
-
8
- // ============================================================================
9
- // BLOCK-LEVEL TOKENIZERS (Line-based)
10
- // ============================================================================
11
-
12
- /**
13
- * Detect code fence: 3+ backticks or tildes at line start
14
- * EBNF: backticks = "`", "`", "`", { "`" } | tildes = "~", "~", "~", { "~" }
15
- */
16
- export function detectFence(line) {
17
- if (line.length < 3) return null;
18
-
19
- const char = line[0];
20
- if (char !== '`' && char !== '~') return null;
21
-
22
- // Count consecutive fence characters
23
- let count = 1;
24
- while (count < line.length && line[count] === char) {
25
- count++;
26
- }
27
-
28
- if (count < 3) return null;
29
-
30
- // Extract info string (everything after fence chars)
31
- const rest = line.slice(count).trimStart();
32
- const infoString = rest;
33
-
34
- // Parse info string for language and attributes
35
- const langMatch = infoString.match(/^([^\s{]+)/);
36
- const lang = langMatch ? langMatch[1] : '';
37
-
38
- // Find attributes in info string
39
- const attrsMatch = infoString.match(/\{([^}]+)\}/);
40
- const attrs = attrsMatch ? attrsMatch[1] : null;
41
-
42
- return {
43
- fenceChar: char,
44
- fenceLength: count,
45
- lang,
46
- attrs,
47
- infoString
48
- };
49
- }
50
-
51
- /**
52
- * Detect prefix declaration: [prefix] <IRI>
53
- * EBNF: contextDecl = "[", contextKey, "]", whitespace, "<", contextIri, ">"
54
- */
55
- export function detectPrefix(line) {
56
- if (line[0] !== '[') return null;
57
-
58
- // Find closing bracket
59
- const closeBracket = line.indexOf(']', 1);
60
- if (closeBracket === -1) return null;
61
-
62
- const prefix = line.slice(1, closeBracket).trim();
63
- if (!prefix) return null;
64
-
65
- // Find < after bracket (skip whitespace)
66
- let pos = closeBracket + 1;
67
- while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
68
- pos++;
69
- }
70
-
71
- if (pos >= line.length || line[pos] !== '<') return null;
72
-
73
- // Find closing >
74
- const closeAngle = line.indexOf('>', pos + 1);
75
- if (closeAngle === -1) return null;
76
-
77
- const iri = line.slice(pos + 1, closeAngle).trim();
78
- if (!iri) return null;
79
-
80
- return { prefix, iri };
81
- }
82
-
83
- /**
84
- * Detect heading: 1-6 # characters followed by space and text
85
- * EBNF: heading = "#", { "#" }, whitespace, text
86
- */
87
- export function detectHeading(line) {
88
- if (line[0] !== '#') return null;
89
-
90
- // Count # characters (max 6)
91
- let depth = 1;
92
- while (depth < line.length && depth < 6 && line[depth] === '#') {
93
- depth++;
94
- }
95
-
96
- // Must be followed by whitespace
97
- if (depth >= line.length || (line[depth] !== ' ' && line[depth] !== '\t')) {
98
- return null;
99
- }
100
-
101
- // Extract content (skip # and whitespace)
102
- let contentStart = depth;
103
- while (contentStart < line.length && (line[contentStart] === ' ' || line[contentStart] === '\t')) {
104
- contentStart++;
105
- }
106
-
107
- // Check for trailing attributes {=...} or {...}
108
- const trimmed = line.slice(contentStart);
109
- const attrsMatch = trimmed.match(/\s*\{([^}]+)\}\s*$/);
110
-
111
- let content = trimmed;
112
- let attrs = null;
113
-
114
- if (attrsMatch) {
115
- attrs = attrsMatch[1];
116
- content = trimmed.slice(0, -attrsMatch[0].length).trim();
117
- }
118
-
119
- return { depth, content, attrs };
120
- }
121
-
122
- /**
123
- * Detect list item: -, *, +, or digits followed by .
124
- * EBNF: listMarker = "-" | "*" | "+" | digit, { digit }, "."
125
- */
126
- export function detectList(line) {
127
- // Skip leading whitespace for indentation detection
128
- let pos = 0;
129
- while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
130
- pos++;
131
- }
132
-
133
- const indent = pos;
134
- if (pos >= line.length) return null;
135
-
136
- const char = line[pos];
137
- let marker;
138
- let contentStart = pos + 1;
139
-
140
- if (char === '-' || char === '*' || char === '+') {
141
- marker = char;
142
- } else if (char >= '0' && char <= '9') {
143
- // Ordered list: digits followed by .
144
- let numEnd = pos + 1;
145
- while (numEnd < line.length && line[numEnd] >= '0' && line[numEnd] <= '9') {
146
- numEnd++;
147
- }
148
- if (numEnd >= line.length || line[numEnd] !== '.') return null;
149
- marker = line.slice(pos, numEnd + 1);
150
- contentStart = numEnd + 1;
151
- } else {
152
- return null;
153
- }
154
-
155
- // Must be followed by whitespace
156
- if (contentStart >= line.length || (line[contentStart] !== ' ' && line[contentStart] !== '\t')) {
157
- return null;
158
- }
159
-
160
- // Skip whitespace after marker
161
- while (contentStart < line.length && (line[contentStart] === ' ' || line[contentStart] === '\t')) {
162
- contentStart++;
163
- }
164
-
165
- // Extract content and check for trailing attributes
166
- let content = line.slice(contentStart);
167
- let attrs = null;
168
-
169
- const attrsMatch = content.match(/\s*\{([^}]+)\}\s*$/);
170
- if (attrsMatch) {
171
- attrs = attrsMatch[1];
172
- content = content.slice(0, -attrsMatch[0].length).trim();
173
- }
174
-
175
- return { indent, marker, content, attrs };
176
- }
177
-
178
- /**
179
- * Detect blockquote: > followed by space and text
180
- * EBNF: blockquote = ">", whitespace, text
181
- */
182
- export function detectBlockquote(line) {
183
- if (line[0] !== '>') return null;
184
-
185
- // Must be followed by whitespace or end of line
186
- if (line.length > 1 && line[1] !== ' ' && line[1] !== '\t') {
187
- return null;
188
- }
189
-
190
- // Extract content
191
- let contentStart = 1;
192
- while (contentStart < line.length && (line[contentStart] === ' ' || line[contentStart] === '\t')) {
193
- contentStart++;
194
- }
195
-
196
- let content = line.slice(contentStart);
197
- let attrs = null;
198
-
199
- // Check for trailing attributes
200
- const attrsMatch = content.match(/\s*\{([^}]+)\}\s*$/);
201
- if (attrsMatch) {
202
- attrs = attrsMatch[1];
203
- content = content.slice(0, -attrsMatch[0].length).trim();
204
- }
205
-
206
- return { content, attrs };
207
- }
208
-
209
- /**
210
- * Detect standalone subject: {=...} with optional whitespace
211
- * EBNF: standaloneSubject = whitespace*, "{", "=", ( iriRef | "#", fragment ), "}", whitespace*
212
- */
213
- export function detectStandaloneSubject(line) {
214
- let pos = 0;
215
-
216
- // Skip leading whitespace
217
- while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
218
- pos++;
219
- }
220
-
221
- if (pos >= line.length || line[pos] !== '{') return null;
222
- pos++;
223
-
224
- if (pos >= line.length || line[pos] !== '=') return null;
225
- pos++;
226
-
227
- // Extract content until }
228
- const contentStart = pos;
229
- let braceCount = 1;
230
-
231
- while (pos < line.length && braceCount > 0) {
232
- if (line[pos] === '{') braceCount++;
233
- if (line[pos] === '}') braceCount--;
234
- if (braceCount > 0) pos++;
235
- }
236
-
237
- if (braceCount > 0) return null; // No closing brace
238
-
239
- const content = line.slice(contentStart, pos).trim();
240
-
241
- // Skip trailing whitespace
242
- pos++;
243
- while (pos < line.length && (line[pos] === ' ' || line[pos] === '\t')) {
244
- pos++;
245
- }
246
-
247
- // Must consume entire line (after optional trailing whitespace)
248
- if (pos < line.length) return null;
249
-
250
- return { content };
251
- }
252
-
253
- // ============================================================================
254
- // INLINE CARRIER TOKENIZERS (Character-based within block content)
255
- // ============================================================================
256
-
257
- /**
258
- * Find matching closing bracket, accounting for nesting
259
- */
260
- export function findMatchingBracket(text, startPos, openChar = '[', closeChar = ']') {
261
- let depth = 1;
262
- let pos = startPos + 1;
263
-
264
- while (pos < text.length && depth > 0) {
265
- if (text[pos] === openChar) depth++;
266
- if (text[pos] === closeChar) depth--;
267
- if (depth > 0) pos++;
268
- }
269
-
270
- return depth === 0 ? pos : null;
271
- }
272
-
273
- /**
274
- * Extract attributes block after a position: {...}
275
- */
276
- export function extractAttributes(text, startPos) {
277
- let pos = startPos;
278
-
279
- // Skip whitespace
280
- while (pos < text.length && (text[pos] === ' ' || text[pos] === '\t')) {
281
- pos++;
282
- }
283
-
284
- if (pos >= text.length || text[pos] !== '{') return null;
285
-
286
- // Find closing brace (simple version - no nesting for attribute blocks)
287
- const closeBrace = text.indexOf('}', pos + 1);
288
- if (closeBrace === -1) return null;
289
-
290
- const attrs = text.slice(pos + 1, closeBrace);
291
- const endPos = closeBrace + 1;
292
-
293
- return { attrs, endPos };
294
- }
295
-
296
- /**
297
- * Detect URL in angle brackets: <URL>
298
- * Returns null if not a valid URL per EBNF scheme rules
299
- */
300
- export function detectAngleBracketUrl(text, startPos) {
301
- if (text[startPos] !== '<') return null;
302
-
303
- const closeAngle = text.indexOf('>', startPos + 1);
304
- if (closeAngle === -1) return null;
305
-
306
- const url = text.slice(startPos + 1, closeAngle).trim();
307
-
308
- // Validate URL scheme (letter followed by letters/digits/+/ - /.)
309
- const schemeMatch = url.match(/^[a-zA-Z][a-zA-Z0-9+\-.]*:/);
310
- if (!schemeMatch) return null;
311
-
312
- return { url, endPos: closeAngle + 1 };
313
- }
314
-
315
- /**
316
- * Detect emphasis/strong spans: *text* or _text_
317
- * EBNF: emphasisSpan = "*", text, "*" | "_", text, "_"
318
- * strongSpan = "**", text, "**" | "__", text, "__"
319
- */
320
- export function detectEmphasis(text, startPos) {
321
- const char = text[startPos];
322
- if (char !== '*' && char !== '_') return null;
323
-
324
- // Count delimiter characters
325
- let delimCount = 1;
326
- while (startPos + delimCount < text.length &&
327
- text[startPos + delimCount] === char) {
328
- delimCount++;
329
- }
330
-
331
- if (delimCount > 2) return null; // Only 1 or 2 allowed
332
-
333
- const type = delimCount === 1 ? 'emphasis' : 'strong';
334
- const closer = char.repeat(delimCount);
335
-
336
- // Find closing delimiter
337
- const contentStart = startPos + delimCount;
338
- let pos = contentStart;
339
-
340
- while (pos < text.length) {
341
- // Check for closing delimiter
342
- if (text.slice(pos, pos + delimCount) === closer) {
343
- // Must not be followed by same char (would be longer delimiter)
344
- if (pos + delimCount < text.length && text[pos + delimCount] === char) {
345
- pos++;
346
- continue;
347
- }
348
-
349
- const content = text.slice(contentStart, pos);
350
- const endPos = pos + delimCount;
351
-
352
- // Check for attributes after
353
- const attrsResult = extractAttributes(text, endPos);
354
- const finalPos = attrsResult ? attrsResult.endPos : endPos;
355
-
356
- return {
357
- type,
358
- content,
359
- attrs: attrsResult?.attrs || null,
360
- endPos: finalPos,
361
- contentEnd: endPos
362
- };
363
- }
364
- pos++;
365
- }
366
-
367
- return null; // No closing delimiter found
368
- }
369
-
370
- /**
371
- * Detect code span: `text` or ``text``
372
- * EBNF: codeSpan = "`", text, "`" | "``", text, "``"
373
- */
374
- export function detectCodeSpan(text, startPos) {
375
- if (text[startPos] !== '`') return null;
376
-
377
- // Count opening backticks
378
- let backtickCount = 1;
379
- while (startPos + backtickCount < text.length &&
380
- text[startPos + backtickCount] === '`') {
381
- backtickCount++;
382
- }
383
-
384
- if (backtickCount > 2) return null; // Only 1 or 2 allowed in current grammar
385
-
386
- const opener = '`'.repeat(backtickCount);
387
- const contentStart = startPos + backtickCount;
388
-
389
- // Find closing backticks
390
- let pos = contentStart;
391
-
392
- while (pos < text.length) {
393
- if (text.slice(pos, pos + backtickCount) === opener) {
394
- const content = text.slice(contentStart, pos);
395
- const endPos = pos + backtickCount;
396
-
397
- // Check for attributes after
398
- const attrsResult = extractAttributes(text, endPos);
399
- const finalPos = attrsResult ? attrsResult.endPos : endPos;
400
-
401
- return {
402
- type: 'code',
403
- content,
404
- attrs: attrsResult?.attrs || null,
405
- endPos: finalPos,
406
- contentEnd: endPos
407
- };
408
- }
409
- pos++;
410
- }
411
-
412
- return null;
413
- }
414
-
415
- /**
416
- * Detect bracket link: [text](url) or [text]{...} or [text] alone
417
- * EBNF: linkSpan = "[", text, "](", text, ")" | "[", text, "]"
418
- */
419
- export function detectBracketLink(text, startPos) {
420
- if (text[startPos] !== '[') return null;
421
-
422
- const bracketEnd = findMatchingBracket(text, startPos, '[', ']');
423
- if (!bracketEnd) return null;
424
-
425
- const linkText = text.slice(startPos + 1, bracketEnd);
426
- let pos = bracketEnd + 1;
427
- let url = null;
428
-
429
- // Check for (url) after ]
430
- if (pos < text.length && text[pos] === '(') {
431
- const parenEnd = text.indexOf(')', pos + 1);
432
- if (parenEnd !== -1) {
433
- url = text.slice(pos + 1, parenEnd);
434
- pos = parenEnd + 1;
435
- }
436
- } else if (pos < text.length && text[pos] === '<') {
437
- // Check for <url> after ]
438
- const closeAngle = text.indexOf('>', pos + 1);
439
- if (closeAngle !== -1) {
440
- const angleUrl = text.slice(pos + 1, closeAngle).trim();
441
- // Validate URL scheme
442
- const schemeMatch = angleUrl.match(/^[a-zA-Z][a-zA-Z0-9+\-.]*:/);
443
- if (schemeMatch) {
444
- url = angleUrl;
445
- pos = closeAngle + 1;
446
- }
447
- }
448
- }
449
-
450
- // Check for attributes
451
- const attrsResult = extractAttributes(text, pos);
452
- const finalPos = attrsResult ? attrsResult.endPos : pos;
453
-
454
- // Determine carrier type: 'link' if URL present, 'span' otherwise
455
- const carrierType = url ? 'link' : 'span';
456
-
457
- return {
458
- type: carrierType,
459
- text: linkText,
460
- url,
461
- attrs: attrsResult?.attrs || null,
462
- endPos: finalPos,
463
- bracketEnd: bracketEnd + 1
464
- };
465
- }
466
-
467
- // ============================================================================
468
- // RANGE CALCULATION HELPERS
469
- // ============================================================================
470
-
471
- /**
472
- * Calculate absolute ranges for a token given base offset
473
- */
474
- export function calcRanges(token, baseOffset) {
475
- return {
476
- range: [baseOffset + token.start, baseOffset + token.end],
477
- valueRange: token.valueStart !== undefined
478
- ? [baseOffset + token.valueStart, baseOffset + token.valueEnd]
479
- : null,
480
- attrsRange: token.attrsStart !== undefined
481
- ? [baseOffset + token.attrsStart, baseOffset + token.attrsEnd]
482
- : null
483
- };
484
- }
485
-
486
- // ============================================================================
487
- // COMPLETE INLINE CARRIER SCANNER (replaces extractInlineCarriers)
488
- // ============================================================================
489
-
490
- /**
491
- * Scan text for all inline carriers using character-based detection
492
- * Memory-optimized version - minimizes object allocations
493
- */
494
- export function scanInlineCarriers(text, baseOffset = 0) {
495
- const carriers = [];
496
- const len = text.length;
497
- let pos = 0;
498
-
499
- while (pos < len) {
500
- const char = text[pos];
501
- let detected = null;
502
-
503
- // Fast path: only check carrier start characters
504
- switch (char) {
505
- case '<':
506
- detected = detectAngleBracketUrl(text, pos);
507
- if (detected) {
508
- const attrsResult = extractAttributes(text, detected.endPos);
509
- if (attrsResult) {
510
- detected.attrs = attrsResult.attrs;
511
- detected.endPos = attrsResult.endPos;
512
- }
513
- detected.type = 'link';
514
- detected.text = detected.url;
515
- }
516
- break;
517
-
518
- case '[':
519
- detected = detectBracketLink(text, pos);
520
- break;
521
-
522
- case '*':
523
- case '_':
524
- detected = detectEmphasis(text, pos);
525
- break;
526
-
527
- case '`':
528
- detected = detectCodeSpan(text, pos);
529
- break;
530
- }
531
-
532
- if (!detected) {
533
- pos++;
534
- continue;
535
- }
536
-
537
- // Skip annotation-only patterns
538
- const url = detected.url;
539
- const type = detected.type;
540
- const attrs = detected.attrs;
541
-
542
- if (url?.startsWith('=') || (type === 'link' && !attrs && !url)) {
543
- pos = detected.endPos;
544
- continue;
545
- }
546
-
547
- // Calculate ranges once
548
- const absStart = baseOffset + pos;
549
- const absEnd = baseOffset + detected.endPos;
550
- const contentEndOffset = detected.contentEnd || detected.bracketEnd || detected.endPos;
551
- const absContentEnd = baseOffset + contentEndOffset;
552
-
553
- // Build carrier object with minimal properties
554
- const carrier = {
555
- type: type,
556
- text: detected.content !== undefined ? detected.content : (detected.text !== undefined ? detected.text : url),
557
- range: [absStart, absEnd],
558
- valueRange: [absStart + (type === 'url' ? 1 : 0), absContentEnd - (type === 'url' ? 1 : 0)],
559
- attrs: attrs,
560
- url: url,
561
- pos: detected.endPos
562
- };
563
-
564
- // Only add attrsRange if attrs exist (saves memory for carriers without attrs)
565
- if (attrs) {
566
- carrier.attrsRange = [absContentEnd, absEnd];
567
- }
568
-
569
- carriers.push(carrier);
570
- pos = detected.endPos;
571
- }
572
-
573
- return carriers;
574
- }
575
-
576
- // ============================================================================
577
- // TESTING EXPORTS (for validation)
578
- // ============================================================================
579
-
580
- export const TEST = {
581
- detectFence,
582
- detectPrefix,
583
- detectHeading,
584
- detectList,
585
- detectBlockquote,
586
- detectStandaloneSubject,
587
- detectAngleBracketUrl,
588
- detectEmphasis,
589
- detectCodeSpan,
590
- detectBracketLink,
591
- extractAttributes,
592
- findMatchingBracket,
593
- scanInlineCarriers
594
- };