mdld-parse 0.9.2 → 0.9.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/.nojekyll +0 -0
  2. package/dist/CNAME +1 -0
  3. package/dist/README.md +425 -0
  4. package/dist/docs/API.md +397 -0
  5. package/dist/docs/Architecture.md +190 -0
  6. package/dist/docs/Guide.md +816 -0
  7. package/dist/docs/LLM-MDLD-Inducer.md +550 -0
  8. package/dist/docs/Parser.md +654 -0
  9. package/dist/docs/Performance.md +238 -0
  10. package/dist/docs/Spec-render.md +1439 -0
  11. package/dist/docs/Subject.md +180 -0
  12. package/dist/docs/Syntax.md +463 -0
  13. package/dist/docs/TodoList.md +215 -0
  14. package/dist/docs/Use-Cases.md +354 -0
  15. package/dist/docs/guide-m.md +56 -0
  16. package/dist/docs/index.html +108 -0
  17. package/dist/docs/index.md +56 -0
  18. package/dist/docs/one-page.md +322 -0
  19. package/dist/docs/origin.md +459 -0
  20. package/dist/docs/polarity.md +586 -0
  21. package/dist/docs/primary-subject.md +199 -0
  22. package/dist/docs/statements.md +226 -0
  23. package/dist/docs/tokenization-performance-report.md +164 -0
  24. package/dist/examples/Cookbook.md +32 -0
  25. package/dist/examples/Status-SHACL.md +31 -0
  26. package/dist/examples/demo/index.md +35 -0
  27. package/dist/examples/demo/journal.md +119 -0
  28. package/dist/examples/demo/minimal.md +23 -0
  29. package/dist/examples/demo/project.md +98 -0
  30. package/dist/examples/demo/prov.md +107 -0
  31. package/dist/examples/demo/rdf.md +69 -0
  32. package/dist/examples/demo/recipe.md +95 -0
  33. package/dist/examples/demo/research.md +126 -0
  34. package/dist/examples/demo/shacl.md +161 -0
  35. package/dist/examples/demo/space-mission.md +129 -0
  36. package/dist/examples/demo/tasks.md +175 -0
  37. package/dist/examples/demo/xsd.md +91 -0
  38. package/dist/examples/dogfood.md +288 -0
  39. package/dist/examples/few-shot.md +309 -0
  40. package/dist/examples/graph/rdf-graph.html +473 -0
  41. package/dist/examples/graph/rdf_force_graph_web_component.html +506 -0
  42. package/dist/examples/index.md +45 -0
  43. package/dist/examples/llm-time-workflow.md +173 -0
  44. package/dist/examples/statements.md +43 -0
  45. package/dist/examples/workflow.md +150 -0
  46. package/dist/index.html +3253 -0
  47. package/dist/index.js +1770 -0
  48. package/dist/logo.svg +7 -0
  49. package/dist/spec/Spec-compact.md +363 -0
  50. package/dist/spec/Spec-ultra.md +221 -0
  51. package/dist/spec/Spec.md +1068 -0
  52. package/dist/spec/index.md +46 -0
  53. package/package.json +7 -8
  54. package/src/constants.js +0 -31
  55. package/src/generate.js +0 -245
  56. package/src/highlight.js +0 -362
  57. package/src/index.js +0 -13
  58. package/src/locate.js +0 -23
  59. package/src/merge.js +0 -157
  60. package/src/parse.js +0 -711
  61. package/src/render.js +0 -576
  62. package/src/shared.js +0 -577
  63. package/src/tokenizers.js +0 -594
  64. package/src/utils.js +0 -430
@@ -0,0 +1,654 @@
1
+ # MD-LD Parser Architecture
2
+
3
+ ## Overview
4
+
5
+ The MD-LD parser implements a **single-pass, streaming-friendly** architecture that transforms Markdown with semantic annotations into RDF/JS quads while maintaining complete provenance and supporting elevated statements extraction.
6
+
7
+ ## Character-Based Tokenization (v0.10.0)
8
+
9
+ The parser uses a unified character-based tokenization system for optimal performance:
10
+
11
+ ### Tokenizer Architecture
12
+
13
+ ```javascript
14
+ // src/tokenizers.js - Centralized tokenization logic
15
+ export function detectFence(line) // ```code blocks
16
+ export function detectPrefix(line) // [prefix] <uri>
17
+ export function detectHeading(line) // # Headings
18
+ export function detectList(line) // - List items
19
+ export function detectBlockquote(line) // > Blockquotes
20
+ export function detectStandaloneSubject(line) // {=subject}
21
+ export function scanInlineCarriers(text) // [text], **bold**, `code`, <URL>
22
+ ```
23
+
24
+ ### Performance Benefits
25
+
26
+ - **20-28% faster parsing** than regex-based approaches
27
+ - **Memory-efficient** with ~640 bytes per quad retained
28
+ - **O(n) linear time complexity** maintained
29
+ - **Better error handling** with precise edge case detection
30
+ - **Cleaner code structure** - No complex regex patterns
31
+
32
+ ### Token Detection Flow
33
+
34
+ ```javascript
35
+ // Block-level detection in scanTokens()
36
+ const PROCESSORS = [
37
+ { type: 'fence', test: line => detectHeading(line), process: handleFence },
38
+ { type: 'prefix', test: line => detectPrefix(line), process: handlePrefix },
39
+ { type: 'heading', test: line => detectHeading(line), process: handleHeading },
40
+ { type: 'list', test: line => detectList(line), process: handleList },
41
+ { type: 'blockquote', test: line => detectBlockquote(line), process: handleBlockquote },
42
+ { type: 'para', test: line => line.trim(), process: handlePara }
43
+ ];
44
+
45
+ // Inline carrier detection in getCarriers()
46
+ function getCarriers(token) {
47
+ if (token.type === 'code') return [];
48
+ return token._carriers || (token._carriers = scanInlineCarriers(token.text, token.range[0]));
49
+ }
50
+ ```
51
+
52
+ ## Core Parser Structures
53
+
54
+ ### 1. Parser State Object
55
+
56
+ ```javascript
57
+ const state = {
58
+ ctx: {}, // Prefix context for IRI expansion
59
+ df: DataFactory, // RDF/JS DataFactory instance
60
+ quads: [], // Final resolved graph state
61
+ quadBuffer: new Map(), // Current parsing buffer
62
+ removeSet: new Set(), // External retractions
63
+ origin: { quadIndex: new Map() }, // Lean origin tracking
64
+ currentSubject: null, // Current subject context
65
+ tokens: null, // Tokenized input
66
+ currentTokenIndex: -1, // Current processing position
67
+ statements: [], // Elevated statements array
68
+ statementCandidates: new Map() // Incomplete rdf:Statement patterns
69
+ };
70
+ ```
71
+
72
+ #### State Components
73
+
74
+ | Component | Purpose | Lifecycle |
75
+ |-----------|---------|-----------|
76
+ | **ctx** | Prefix mappings for IRI expansion | Persistent, updated during prefix processing |
77
+ | **quads** | Final graph output | Accumulated throughout parsing |
78
+ | **quadBuffer** | Intra-document cancel tracking | Modified during quad emission |
79
+ | **removeSet** | External retractions | Accumulated for removal operations |
80
+ | **origin** | Provenance tracking | Updated with each quad |
81
+ | **currentSubject** | Subject context management | Reset and updated during parsing |
82
+ | **statements** | Elevated statements | Accumulated during pattern detection |
83
+ | **statementCandidates** | Pattern completion tracking | Modified during rdf:Statement detection |
84
+
85
+ ### 2. Token Processing Pipeline
86
+
87
+ #### Token Types
88
+
89
+ ```javascript
90
+ const TOKEN_TYPES = {
91
+ heading: 'heading', // ## Heading {=ex:subject .type}
92
+ para: 'para', // Paragraph with annotations
93
+ list: 'list', // - List item {+ex:predicate}
94
+ blockquote: 'blockquote', // > Quote {ex:property}
95
+ code: 'code', // ```code {=ex:codeblock}
96
+ prefix: 'prefix', // [prefix] <uri>
97
+ fence: 'fence' // Code block delimiters
98
+ };
99
+ ```
100
+
101
+ #### Token Structure
102
+
103
+ ```javascript
104
+ const token = {
105
+ type: 'para', // Token type
106
+ range: [start, end], // Character positions in source
107
+ text: 'content', // Raw text content
108
+ attrs: '{+ex:predicate}', // Annotation block
109
+ attrsRange: [start, end], // Annotation positions
110
+ valueRange: [start, end], // Value positions
111
+ _carriers: [...] // Lazy-loaded inline carriers
112
+ };
113
+ ```
114
+
115
+ #### Token Processors
116
+
117
+ ```javascript
118
+ const TOKEN_PROCESSORS = {
119
+ heading: (token, state) => processTokenAnnotations(token, state, 'heading'),
120
+ para: (token, state) => {
121
+ processStandaloneSubject(token, state);
122
+ processTokenAnnotations(token, state, 'para');
123
+ },
124
+ blockquote: (token, state) => processTokenAnnotations(token, state, 'blockquote'),
125
+ code: (token, state) => processTokenAnnotations(token, state, 'code'),
126
+ list: (token, state) => processTokenAnnotations(token, state, 'list')
127
+ };
128
+ ```
129
+
130
+ ### 3. Semantic Block Processing
131
+
132
+ #### Semantic Block Structure
133
+
134
+ ```javascript
135
+ const semanticBlock = {
136
+ subjects: ['ex:subject'], // Zero or more subjects
137
+ predicates: ['ex:predicate'], // Zero or more predicates
138
+ types: ['ex:Type'], // Zero or more types
139
+ object: 'literal', // Optional object value
140
+ remove: false, // Polarity flag
141
+ meta: { ... } // Additional metadata
142
+ };
143
+ ```
144
+
145
+ #### Parsing Pipeline
146
+
147
+ 1. **Annotation Extraction** - Identify annotation blocks in tokens
148
+ 2. **Semantic Parsing** - Parse annotation blocks into semantic structures
149
+ 3. **Subject Resolution** - Resolve subjects using current context
150
+ 4. **Quad Generation** - Create RDF/JS quads from semantic structures
151
+ 5. **Origin Tracking** - Record provenance for each quad
152
+
153
+ ### 4. Retraction System
154
+
155
+ Retractions are first-class citizens in MD-LD, enabling sophisticated diff authoring and document evolution through explicit polarity handling.
156
+
157
+ #### Retraction Types
158
+
159
+ ```javascript
160
+ const RETRACTION_TYPES = {
161
+ INTRA_DOCUMENT: 'intra-document', // Cancel within same document
162
+ EXTERNAL_RETRACT: 'external', // Remove from prior state
163
+ TYPE_MIGRATION: 'migration' // Change type assertions
164
+ };
165
+ ```
166
+
167
+ #### Retraction State Management
168
+
169
+ ```javascript
170
+ const retractionState = {
171
+ quadBuffer: new Map(), // Current document state
172
+ removeSet: new Set(), // External retractions
173
+ quads: [], // Final resolved state
174
+ };
175
+ ```
176
+
177
+ #### Retraction Processing Pipeline
178
+
179
+ ```javascript
180
+ function processRetraction(quad, quadBuffer, removeSet, quads) {
181
+ const quadKey = quadIndexKey(quad.subject, quad.predicate, quad.object);
182
+
183
+ if (quadBuffer.has(quadKey)) {
184
+ // Intra-document cancel: remove from current state
185
+ quadBuffer.delete(quadKey);
186
+ removeFromQuadsArray(quads, quad);
187
+ } else {
188
+ // External retract: target prior document state
189
+ removeSet.add(quad);
190
+ }
191
+ }
192
+ ```
193
+
194
+ #### Polarity Forms
195
+
196
+ | Form | Positive | Negative | Use Case |
197
+ |------|-----------|-----------|-----------|
198
+ | `p` | `{p}` | `{-p}` | Literal properties |
199
+ | `?p` | `{?p}` | `{-?p}` | Object properties |
200
+ | `!p` | `{!p}` | `{-!p}` | Reverse properties |
201
+ | `.Class` | `{.Class}` | `-.Class}` | Type declarations |
202
+
203
+ #### Retraction Examples
204
+
205
+ ```javascript
206
+ // Intra-document cancel
207
+ "Alice has name 'Alice'" // {+ex:name "Alice"}
208
+ "Alice {-ex:name 'Alice'}" // Cancels above, removed from final state
209
+
210
+ // External retract
211
+ "Bob {-ex:name 'Robert'}" // Targets Bob's name from prior documents
212
+
213
+ // Type migration
214
+ "Alice .foaf:Person" // Initial type
215
+ "Alice {-foaf:Person}" // Remove old type
216
+ "Alice .foaf:Agent" // Add new type
217
+ ```
218
+
219
+ #### Hard Invariant Enforcement
220
+
221
+ ```javascript
222
+ function enforceHardInvariant(quads, removeSet) {
223
+ // Ensure: quads ∩ remove = ∅ (no overlap)
224
+ const quadKeys = new Set();
225
+ for (const quad of quads) {
226
+ quadKeys.add(quadIndexKey(quad.subject, quad.predicate, quad.object));
227
+ }
228
+
229
+ // Filter removeSet to ensure no overlap
230
+ return Array.from(removeSet).filter(quad => {
231
+ const key = quadIndexKey(quad.subject, quad.predicate, quad.object);
232
+ return !quadKeys.has(key);
233
+ });
234
+ }
235
+ ```
236
+
237
+ ### 5. Carrier Extraction System
238
+
239
+ #### Carrier Types
240
+
241
+ ```javascript
242
+ const CARRIER_TYPES = {
243
+ link: 'link', // [text] {+ex:predicate}
244
+ emphasis: 'emphasis', // **text** {ex:property}
245
+ code: 'code', // ``code`` {ex:property}
246
+ bracket: 'bracket' // <url> {ex:reference}
247
+ };
248
+ ```
249
+
250
+ #### Carrier Structure
251
+
252
+ ```javascript
253
+ const carrier = {
254
+ type: 'link', // Carrier type
255
+ text: 'display text', // Display content
256
+ attrs: '{+ex:predicate}', // Annotation block
257
+ attrsRange: [start, end], // Annotation positions
258
+ valueRange: [start, end], // Value positions
259
+ range: [start, end], // Full span in source
260
+ pos: endPosition, // Next processing position
261
+ url: 'resolved-iri' // Resolved IRI (for links)
262
+ };
263
+ ```
264
+
265
+ #### Extraction Patterns
266
+
267
+ ```javascript
268
+ const CARRIER_EXTRACTORS = {
269
+ '<': extractLinkCarrier, // Angle bracket links
270
+ '[': extractBracketCarrier, // Square bracket carriers
271
+ emphasis: extractEmphasis, // **emphasis** patterns
272
+ code: extractCodeSpan // ``code`` patterns
273
+ };
274
+ ```
275
+
276
+ ### 5. Quad Emission System
277
+
278
+ #### Emit Quad Function
279
+
280
+ ```javascript
281
+ function emitQuad(quads, quadBuffer, removeSet, quadIndex, block,
282
+ subject, predicate, object, dataFactory, meta,
283
+ statements, statementCandidates) {
284
+
285
+ // 1. Create RDF/JS quad
286
+ const quad = dataFactory.quad(subject, predicate, object);
287
+ const remove = meta?.remove || false;
288
+
289
+ // 2. Handle polarity (add/remove)
290
+ if (remove) {
291
+ // Retraction processing
292
+ const quadKey = quadIndexKey(quad.subject, quad.predicate, quad.object);
293
+
294
+ if (quadBuffer.has(quadKey)) {
295
+ // Intra-document cancel: remove from current state
296
+ quadBuffer.delete(quadKey);
297
+ removeFromQuadsArray(quads, quad);
298
+ quadIndex.delete(quadKey);
299
+ } else {
300
+ // External retract: target prior document state
301
+ removeSet.add(quad);
302
+ }
303
+ } else {
304
+ // Add to buffer and quads array
305
+ const quadKey = quadIndexKey(quad.subject, quad.predicate, quad.object);
306
+ quadBuffer.set(quadKey, quad);
307
+ quads.push(quad);
308
+
309
+ // Detect elevated statements
310
+ detectStatementPatternSinglePass(quad, dataFactory, meta, statements, statementCandidates);
311
+
312
+ // Create origin entry
313
+ const originEntry = createOriginEntry(block, subject, predicate, meta);
314
+ quadIndex.set(quadKey, originEntry);
315
+ }
316
+ }
317
+ ```
318
+
319
+ #### Retraction-Aware Quad Processing
320
+
321
+ ```javascript
322
+ function removeFromQuadsArray(quads, targetQuad) {
323
+ const index = quads.findIndex(q =>
324
+ q.subject.value === targetQuad.subject.value &&
325
+ q.predicate.value === targetQuad.predicate.value &&
326
+ q.object.value === targetQuad.object.value
327
+ );
328
+
329
+ if (index !== -1) {
330
+ quads.splice(index, 1);
331
+ }
332
+ }
333
+ ```
334
+
335
+ #### Origin Entry Creation with Polarity
336
+
337
+ ```javascript
338
+ function createOriginEntry(block, subject, predicate, meta) {
339
+ return {
340
+ blockId: block.id,
341
+ range: block.range,
342
+ carrierType: block.carrierType,
343
+ subject: subject.value,
344
+ predicate: predicate.value,
345
+ context: block.context,
346
+ polarity: meta?.remove ? '-' : '+', // Polarity tracking
347
+ value: block.text || ''
348
+ };
349
+ }
350
+ ```
351
+
352
+ #### Quad Index Key Generation
353
+
354
+ ```javascript
355
+ function quadIndexKey(subject, predicate, object) {
356
+ return `${subject.value}|${predicate.value}|${object.value}`;
357
+ }
358
+ ```
359
+
360
+ ### 6. Elevated Statements Detection
361
+
362
+ #### Pattern Detection System
363
+
364
+ ```javascript
365
+ function detectStatementPatternSinglePass(quad, dataFactory, meta,
366
+ statements, statementCandidates) {
367
+ // Only process rdf:Statement related predicates (optimized)
368
+ if (!isRDFStatementPredicate(quad.predicate.value)) return;
369
+
370
+ // Start new pattern: rdf:type rdf:Statement
371
+ if (isStatementType(quad)) {
372
+ statementCandidates.set(quad.subject.value, { spo: {} });
373
+ return;
374
+ }
375
+
376
+ // Complete pattern parts: rdf:subject, rdf:predicate, rdf:object
377
+ const candidate = statementCandidates.get(quad.subject.value);
378
+ if (candidate) {
379
+ updatePattern(candidate, quad);
380
+
381
+ // Create elevated SPO when pattern complete
382
+ if (isPatternComplete(candidate)) {
383
+ const elevatedQuad = createElevatedQuad(candidate, dataFactory);
384
+ statements.push(elevatedQuad);
385
+ statementCandidates.delete(quad.subject.value);
386
+ }
387
+ }
388
+ }
389
+ ```
390
+
391
+ #### Pattern Completion Tracking
392
+
393
+ ```javascript
394
+ const patternCandidate = {
395
+ spo: {
396
+ subject: NamedNode, // rdf:subject value
397
+ predicate: NamedNode, // rdf:predicate value
398
+ object: NamedNode|Literal // rdf:object value
399
+ },
400
+ objectQuad: Quad // Original quad for literal extraction
401
+ };
402
+ ```
403
+
404
+ ### 7. Origin Tracking System
405
+
406
+ #### Origin Entry Structure
407
+
408
+ ```javascript
409
+ const originEntry = {
410
+ blockId: 'block-identifier', // Unique block identifier
411
+ range: [start, end], // Source location
412
+ carrierType: 'para', // Token type
413
+ subject: 'ex:subject', // Subject IRI
414
+ predicate: 'ex:predicate', // Predicate IRI
415
+ context: { ...block.context }, // Prefix context
416
+ polarity: '+', // Add/remove indicator
417
+ value: 'source text' // Original content
418
+ };
419
+ ```
420
+
421
+ #### Quad Index Mapping
422
+
423
+ ```javascript
424
+ const quadIndex = new Map(); // quadIndexKey -> originEntry
425
+ ```
426
+
427
+ ## Processing Pipeline
428
+
429
+ ### Phase 1: Tokenization
430
+
431
+ ```javascript
432
+ function scanTokens(text) {
433
+ const tokens = [];
434
+ const lines = text.split('\n');
435
+
436
+ for (const line of lines) {
437
+ // Identify token types using regex patterns
438
+ // Extract annotations and ranges
439
+ // Create token objects with metadata
440
+ tokens.push(token);
441
+ }
442
+
443
+ return tokens;
444
+ }
445
+ ```
446
+
447
+ ### Phase 2: Prefix Processing
448
+
449
+ ```javascript
450
+ // Single loop optimization for better performance
451
+ for (const token of tokens) {
452
+ if (token.type === 'prefix') {
453
+ // Resolve CURIEs and update context
454
+ state.ctx[token.prefix] = resolveIRI(token.iri, state.ctx);
455
+ }
456
+ }
457
+ ```
458
+
459
+ ### Phase 3: Semantic Processing
460
+
461
+ ```javascript
462
+ for (let i = 0; i < tokens.length; i++) {
463
+ const token = tokens[i];
464
+ state.currentTokenIndex = i;
465
+
466
+ // Process token using appropriate processor
467
+ TOKEN_PROCESSORS[token.type]?.(token, state);
468
+ }
469
+ ```
470
+
471
+ ### Phase 4: Final Resolution
472
+
473
+ ```javascript
474
+ // Process retractions and ensure hard invariant
475
+ const filteredRemove = processRetractions(state.quads, state.removeSet);
476
+
477
+ return {
478
+ quads: state.quads, // Final resolved graph state
479
+ remove: filteredRemove, // External retractions only
480
+ statements: state.statements, // Elevated statements
481
+ origin: state.origin, // Complete provenance
482
+ context: state.ctx // Final prefix context
483
+ };
484
+ ```
485
+
486
+ #### Retraction Processing
487
+
488
+ ```javascript
489
+ function processRetractions(quads, removeSet) {
490
+ // Create quad key index for fast lookup
491
+ const quadKeys = new Set();
492
+ for (const quad of quads) {
493
+ quadKeys.add(quadIndexKey(quad.subject, quad.predicate, quad.object));
494
+ }
495
+
496
+ // Filter removeSet to enforce hard invariant: quads ∩ remove = ∅
497
+ const filteredRemove = [];
498
+ for (const quad of removeSet) {
499
+ const key = quadIndexKey(quad.subject, quad.predicate, quad.object);
500
+ if (!quadKeys.has(key)) {
501
+ filteredRemove.push(quad);
502
+ }
503
+ }
504
+
505
+ return filteredRemove;
506
+ }
507
+ ```
508
+
509
+ ## Performance Optimizations
510
+
511
+ ### Applied Optimizations
512
+
513
+ 1. **Token Processing**: Single loop instead of filter+forEach
514
+ 2. **Regex Pre-compilation**: Pre-compiled carrier patterns
515
+ 3. **String Operations**: Optimized fragment resolution
516
+ 4. **Array Operations**: Direct iteration instead of Array.from()+filter()
517
+ 5. **Object Creation**: Reduced spread operator usage
518
+ 6. **Early Filtering**: Only process rdf:Statement related predicates
519
+ 7. **Module Constants**: RDF constants defined once at module level
520
+
521
+ ### Performance Characteristics
522
+
523
+ - **Single-pass parsing**: No backtracking or multiple iterations
524
+ - **O(1) additional memory**: Constant overhead for elevated statements
525
+ - **Linear scalability**: Performance scales linearly with input size
526
+ - **Streaming compatible**: Can process large files incrementally
527
+
528
+ ## Memory Management
529
+
530
+ ### Memory Usage Patterns
531
+
532
+ ```javascript
533
+ // Temporary objects (cleaned up automatically)
534
+ const statementCandidates = new Map(); // Cleared on pattern completion
535
+ const quadBuffer = new Map(); // Cleared on intra-document cancel
536
+
537
+ // Persistent objects (grow with input)
538
+ state.quads.push(quad); // Final output
539
+ state.origin.quadIndex.set(key, entry); // Provenance tracking
540
+ ```
541
+
542
+ ### Cleanup Strategies
543
+
544
+ 1. **Automatic Cleanup**: Statement candidates deleted on completion
545
+ 2. **Buffer Management**: Quad buffer cleared on intra-document cancel
546
+ 3. **Set Operations**: RemoveSet converted to array at end
547
+ 4. **Lazy Loading**: Inline carriers loaded on-demand
548
+
549
+ ## Error Handling
550
+
551
+ ### Error Recovery Strategies
552
+
553
+ ```javascript
554
+ // Graceful degradation for malformed input
555
+ if (!isValidAnnotation(attrs)) {
556
+ // Log warning and continue processing
557
+ return;
558
+ }
559
+
560
+ // Safe navigation for optional properties
561
+ const subject = resolveSubject(sem.subject, state) || skipProcessing();
562
+
563
+ // Validation with meaningful errors
564
+ if (!state.currentSubject && needsSubject(sem)) {
565
+ throw new ParseError('Subject required but not set', token.range);
566
+ }
567
+ ```
568
+
569
+ ## Extensibility Points
570
+
571
+ ### Custom Token Processors
572
+
573
+ ```javascript
574
+ // Add new token type
575
+ TOKEN_PROCESSORS.custom = (token, state) => {
576
+ // Custom processing logic
577
+ processTokenAnnotations(token, state, 'custom');
578
+ };
579
+ ```
580
+
581
+ ### Custom Carrier Extractors
582
+
583
+ ```javascript
584
+ // Add new carrier pattern
585
+ CARRIER_EXTRACTORS['@'] = (text, pos, baseOffset) => {
586
+ // Custom carrier extraction
587
+ return createCarrier('mention', text, attrs, ranges);
588
+ };
589
+ ```
590
+
591
+ ### Custom Semantic Parsers
592
+
593
+ ```javascript
594
+ // Extend semantic block parsing
595
+ function parseCustomSemantic(attrs) {
596
+ // Custom annotation parsing
597
+ return { customProperty: value };
598
+ }
599
+ ```
600
+
601
+ ## Integration Points
602
+
603
+ ### External Libraries
604
+
605
+ - **RDF/JS DataFactory**: Quad creation and term management
606
+ - **Context Expansion**: IRI resolution and prefix handling
607
+ - **Validation**: Optional SHACL or schema validation
608
+
609
+ ### Output Formats
610
+
611
+ ```javascript
612
+ // Standard RDF/JS output with first-class retractions
613
+ return {
614
+ quads: RDFQuad[], // Primary graph (final resolved state)
615
+ remove: RDFQuad[], // External retractions (targets prior state)
616
+ statements: RDFQuad[], // Elevated statements from rdf:Statement patterns
617
+ origin: OriginObject, // Complete provenance tracking
618
+ context: ContextObject // Final prefix context
619
+ };
620
+ ```
621
+
622
+ #### Retraction Output Characteristics
623
+
624
+ ```javascript
625
+ // External retractions only (intra-document cancels resolved internally)
626
+ const remove = [
627
+ // Quads that target prior document state
628
+ // Never overlap with quads array (hard invariant enforced)
629
+ ];
630
+
631
+ // Usage for document evolution
632
+ const priorResult = parse(previousDocument);
633
+ const currentResult = parse(currentDocument);
634
+
635
+ // Apply retractions to prior state
636
+ const evolvedGraph = applyRetractions(priorResult.quads, currentResult.remove);
637
+ ```
638
+
639
+ ### Streaming Interface
640
+
641
+ ```javascript
642
+ // Future streaming capability
643
+ function parseStream(stream, options) {
644
+ return new TransformStream({
645
+ transform(chunk, controller) {
646
+ // Process chunk incrementally
647
+ const result = parse(chunk.toString(), options);
648
+ controller.enqueue(result);
649
+ }
650
+ });
651
+ }
652
+ ```
653
+
654
+ This architecture provides a solid foundation for MD-LD parsing with excellent performance characteristics, maintainability, and extensibility for future enhancements.