mdld-parse 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.nojekyll +0 -0
- package/dist/CNAME +1 -0
- package/dist/README.md +425 -0
- package/dist/docs/API.md +397 -0
- package/dist/docs/Architecture.md +190 -0
- package/dist/docs/Guide.md +322 -0
- package/dist/docs/LLM-MDLD-Inducer.md +550 -0
- package/dist/docs/Parser.md +654 -0
- package/dist/docs/Performance.md +238 -0
- package/dist/docs/Spec-render.md +1439 -0
- package/dist/docs/Subject.md +180 -0
- package/dist/docs/Syntax.md +463 -0
- package/dist/docs/TodoList.md +215 -0
- package/dist/docs/Use-Cases.md +354 -0
- package/dist/docs/index.html +108 -0
- package/dist/docs/index.md +56 -0
- package/dist/docs/origin.md +459 -0
- package/dist/docs/polarity.md +586 -0
- package/dist/docs/primary-subject.md +199 -0
- package/dist/docs/statements.md +226 -0
- package/dist/docs/tokenization-performance-report.md +164 -0
- package/dist/examples/Cookbook.md +32 -0
- package/dist/examples/Status-SHACL.md +33 -0
- package/dist/examples/demo/index.md +35 -0
- package/dist/examples/demo/journal.md +18 -0
- package/dist/examples/demo/minimal.md +11 -0
- package/dist/examples/demo/project.md +15 -0
- package/dist/examples/demo/recipe.md +31 -0
- package/dist/examples/demo/research.md +46 -0
- package/dist/examples/demo/space-mission.md +48 -0
- package/dist/examples/dogfood.md +288 -0
- package/dist/examples/few-shot.md +309 -0
- package/dist/examples/graph/rdf-graph.html +473 -0
- package/dist/examples/graph/rdf_force_graph_web_component.html +506 -0
- package/dist/examples/index.md +45 -0
- package/dist/examples/llm-time-workflow.md +173 -0
- package/dist/examples/prov-o.md +2367 -0
- package/dist/examples/rdf.md +827 -0
- package/dist/examples/shacl.md +3399 -0
- package/dist/examples/statements.md +43 -0
- package/dist/examples/workflow.md +150 -0
- package/dist/index.html +2827 -0
- package/dist/index.js +1548 -0
- package/dist/logo.svg +7 -0
- package/dist/spec/Spec-compact.md +363 -0
- package/dist/spec/Spec-ultra.md +221 -0
- package/dist/spec/Spec.md +1068 -0
- package/dist/spec/index.md +46 -0
- package/package.json +7 -8
- package/src/constants.js +0 -31
- package/src/generate.js +0 -245
- package/src/highlight.js +0 -362
- package/src/index.js +0 -13
- package/src/locate.js +0 -23
- package/src/merge.js +0 -157
- package/src/parse.js +0 -711
- package/src/render.js +0 -576
- package/src/shared.js +0 -577
- package/src/tokenizers.js +0 -594
- package/src/utils.js +0 -430
|
@@ -0,0 +1,654 @@
|
|
|
1
|
+
# MD-LD Parser Architecture
|
|
2
|
+
|
|
3
|
+
## Overview
|
|
4
|
+
|
|
5
|
+
The MD-LD parser implements a **single-pass, streaming-friendly** architecture that transforms Markdown with semantic annotations into RDF/JS quads while maintaining complete provenance and supporting elevated statements extraction.
|
|
6
|
+
|
|
7
|
+
## Character-Based Tokenization (v0.10.0)
|
|
8
|
+
|
|
9
|
+
The parser uses a unified character-based tokenization system for optimal performance:
|
|
10
|
+
|
|
11
|
+
### Tokenizer Architecture
|
|
12
|
+
|
|
13
|
+
```javascript
|
|
14
|
+
// src/tokenizers.js - Centralized tokenization logic
|
|
15
|
+
export function detectFence(line) // ```code blocks
|
|
16
|
+
export function detectPrefix(line) // [prefix] <uri>
|
|
17
|
+
export function detectHeading(line) // # Headings
|
|
18
|
+
export function detectList(line) // - List items
|
|
19
|
+
export function detectBlockquote(line) // > Blockquotes
|
|
20
|
+
export function detectStandaloneSubject(line) // {=subject}
|
|
21
|
+
export function scanInlineCarriers(text) // [text], **bold**, `code`, <URL>
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
### Performance Benefits
|
|
25
|
+
|
|
26
|
+
- **20-28% faster parsing** than regex-based approaches
|
|
27
|
+
- **Memory-efficient** with ~640 bytes per quad retained
|
|
28
|
+
- **O(n) linear time complexity** maintained
|
|
29
|
+
- **Better error handling** with precise edge case detection
|
|
30
|
+
- **Cleaner code structure** - No complex regex patterns
|
|
31
|
+
|
|
32
|
+
### Token Detection Flow
|
|
33
|
+
|
|
34
|
+
```javascript
|
|
35
|
+
// Block-level detection in scanTokens()
|
|
36
|
+
const PROCESSORS = [
|
|
37
|
+
{ type: 'fence', test: line => detectHeading(line), process: handleFence },
|
|
38
|
+
{ type: 'prefix', test: line => detectPrefix(line), process: handlePrefix },
|
|
39
|
+
{ type: 'heading', test: line => detectHeading(line), process: handleHeading },
|
|
40
|
+
{ type: 'list', test: line => detectList(line), process: handleList },
|
|
41
|
+
{ type: 'blockquote', test: line => detectBlockquote(line), process: handleBlockquote },
|
|
42
|
+
{ type: 'para', test: line => line.trim(), process: handlePara }
|
|
43
|
+
];
|
|
44
|
+
|
|
45
|
+
// Inline carrier detection in getCarriers()
|
|
46
|
+
function getCarriers(token) {
|
|
47
|
+
if (token.type === 'code') return [];
|
|
48
|
+
return token._carriers || (token._carriers = scanInlineCarriers(token.text, token.range[0]));
|
|
49
|
+
}
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Core Parser Structures
|
|
53
|
+
|
|
54
|
+
### 1. Parser State Object
|
|
55
|
+
|
|
56
|
+
```javascript
|
|
57
|
+
const state = {
|
|
58
|
+
ctx: {}, // Prefix context for IRI expansion
|
|
59
|
+
df: DataFactory, // RDF/JS DataFactory instance
|
|
60
|
+
quads: [], // Final resolved graph state
|
|
61
|
+
quadBuffer: new Map(), // Current parsing buffer
|
|
62
|
+
removeSet: new Set(), // External retractions
|
|
63
|
+
origin: { quadIndex: new Map() }, // Lean origin tracking
|
|
64
|
+
currentSubject: null, // Current subject context
|
|
65
|
+
tokens: null, // Tokenized input
|
|
66
|
+
currentTokenIndex: -1, // Current processing position
|
|
67
|
+
statements: [], // Elevated statements array
|
|
68
|
+
statementCandidates: new Map() // Incomplete rdf:Statement patterns
|
|
69
|
+
};
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
#### State Components
|
|
73
|
+
|
|
74
|
+
| Component | Purpose | Lifecycle |
|
|
75
|
+
|-----------|---------|-----------|
|
|
76
|
+
| **ctx** | Prefix mappings for IRI expansion | Persistent, updated during prefix processing |
|
|
77
|
+
| **quads** | Final graph output | Accumulated throughout parsing |
|
|
78
|
+
| **quadBuffer** | Intra-document cancel tracking | Modified during quad emission |
|
|
79
|
+
| **removeSet** | External retractions | Accumulated for removal operations |
|
|
80
|
+
| **origin** | Provenance tracking | Updated with each quad |
|
|
81
|
+
| **currentSubject** | Subject context management | Reset and updated during parsing |
|
|
82
|
+
| **statements** | Elevated statements | Accumulated during pattern detection |
|
|
83
|
+
| **statementCandidates** | Pattern completion tracking | Modified during rdf:Statement detection |
|
|
84
|
+
|
|
85
|
+
### 2. Token Processing Pipeline
|
|
86
|
+
|
|
87
|
+
#### Token Types
|
|
88
|
+
|
|
89
|
+
```javascript
|
|
90
|
+
const TOKEN_TYPES = {
|
|
91
|
+
heading: 'heading', // ## Heading {=ex:subject .type}
|
|
92
|
+
para: 'para', // Paragraph with annotations
|
|
93
|
+
list: 'list', // - List item {+ex:predicate}
|
|
94
|
+
blockquote: 'blockquote', // > Quote {ex:property}
|
|
95
|
+
code: 'code', // ```code {=ex:codeblock}
|
|
96
|
+
prefix: 'prefix', // [prefix] <uri>
|
|
97
|
+
fence: 'fence' // Code block delimiters
|
|
98
|
+
};
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
#### Token Structure
|
|
102
|
+
|
|
103
|
+
```javascript
|
|
104
|
+
const token = {
|
|
105
|
+
type: 'para', // Token type
|
|
106
|
+
range: [start, end], // Character positions in source
|
|
107
|
+
text: 'content', // Raw text content
|
|
108
|
+
attrs: '{+ex:predicate}', // Annotation block
|
|
109
|
+
attrsRange: [start, end], // Annotation positions
|
|
110
|
+
valueRange: [start, end], // Value positions
|
|
111
|
+
_carriers: [...] // Lazy-loaded inline carriers
|
|
112
|
+
};
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
#### Token Processors
|
|
116
|
+
|
|
117
|
+
```javascript
|
|
118
|
+
const TOKEN_PROCESSORS = {
|
|
119
|
+
heading: (token, state) => processTokenAnnotations(token, state, 'heading'),
|
|
120
|
+
para: (token, state) => {
|
|
121
|
+
processStandaloneSubject(token, state);
|
|
122
|
+
processTokenAnnotations(token, state, 'para');
|
|
123
|
+
},
|
|
124
|
+
blockquote: (token, state) => processTokenAnnotations(token, state, 'blockquote'),
|
|
125
|
+
code: (token, state) => processTokenAnnotations(token, state, 'code'),
|
|
126
|
+
list: (token, state) => processTokenAnnotations(token, state, 'list')
|
|
127
|
+
};
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### 3. Semantic Block Processing
|
|
131
|
+
|
|
132
|
+
#### Semantic Block Structure
|
|
133
|
+
|
|
134
|
+
```javascript
|
|
135
|
+
const semanticBlock = {
|
|
136
|
+
subjects: ['ex:subject'], // Zero or more subjects
|
|
137
|
+
predicates: ['ex:predicate'], // Zero or more predicates
|
|
138
|
+
types: ['ex:Type'], // Zero or more types
|
|
139
|
+
object: 'literal', // Optional object value
|
|
140
|
+
remove: false, // Polarity flag
|
|
141
|
+
meta: { ... } // Additional metadata
|
|
142
|
+
};
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
#### Parsing Pipeline
|
|
146
|
+
|
|
147
|
+
1. **Annotation Extraction** - Identify annotation blocks in tokens
|
|
148
|
+
2. **Semantic Parsing** - Parse annotation blocks into semantic structures
|
|
149
|
+
3. **Subject Resolution** - Resolve subjects using current context
|
|
150
|
+
4. **Quad Generation** - Create RDF/JS quads from semantic structures
|
|
151
|
+
5. **Origin Tracking** - Record provenance for each quad
|
|
152
|
+
|
|
153
|
+
### 4. Retraction System
|
|
154
|
+
|
|
155
|
+
Retractions are first-class citizens in MD-LD, enabling sophisticated diff authoring and document evolution through explicit polarity handling.
|
|
156
|
+
|
|
157
|
+
#### Retraction Types
|
|
158
|
+
|
|
159
|
+
```javascript
|
|
160
|
+
const RETRACTION_TYPES = {
|
|
161
|
+
INTRA_DOCUMENT: 'intra-document', // Cancel within same document
|
|
162
|
+
EXTERNAL_RETRACT: 'external', // Remove from prior state
|
|
163
|
+
TYPE_MIGRATION: 'migration' // Change type assertions
|
|
164
|
+
};
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
#### Retraction State Management
|
|
168
|
+
|
|
169
|
+
```javascript
|
|
170
|
+
const retractionState = {
|
|
171
|
+
quadBuffer: new Map(), // Current document state
|
|
172
|
+
removeSet: new Set(), // External retractions
|
|
173
|
+
quads: [], // Final resolved state
|
|
174
|
+
};
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
#### Retraction Processing Pipeline
|
|
178
|
+
|
|
179
|
+
```javascript
|
|
180
|
+
function processRetraction(quad, quadBuffer, removeSet, quads) {
|
|
181
|
+
const quadKey = quadIndexKey(quad.subject, quad.predicate, quad.object);
|
|
182
|
+
|
|
183
|
+
if (quadBuffer.has(quadKey)) {
|
|
184
|
+
// Intra-document cancel: remove from current state
|
|
185
|
+
quadBuffer.delete(quadKey);
|
|
186
|
+
removeFromQuadsArray(quads, quad);
|
|
187
|
+
} else {
|
|
188
|
+
// External retract: target prior document state
|
|
189
|
+
removeSet.add(quad);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
#### Polarity Forms
|
|
195
|
+
|
|
196
|
+
| Form | Positive | Negative | Use Case |
|
|
197
|
+
|------|-----------|-----------|-----------|
|
|
198
|
+
| `p` | `{p}` | `{-p}` | Literal properties |
|
|
199
|
+
| `?p` | `{?p}` | `{-?p}` | Object properties |
|
|
200
|
+
| `!p` | `{!p}` | `{-!p}` | Reverse properties |
|
|
201
|
+
| `.Class` | `{.Class}` | `-.Class}` | Type declarations |
|
|
202
|
+
|
|
203
|
+
#### Retraction Examples
|
|
204
|
+
|
|
205
|
+
```javascript
|
|
206
|
+
// Intra-document cancel
|
|
207
|
+
"Alice has name 'Alice'" // {+ex:name "Alice"}
|
|
208
|
+
"Alice {-ex:name 'Alice'}" // Cancels above, removed from final state
|
|
209
|
+
|
|
210
|
+
// External retract
|
|
211
|
+
"Bob {-ex:name 'Robert'}" // Targets Bob's name from prior documents
|
|
212
|
+
|
|
213
|
+
// Type migration
|
|
214
|
+
"Alice .foaf:Person" // Initial type
|
|
215
|
+
"Alice {-foaf:Person}" // Remove old type
|
|
216
|
+
"Alice .foaf:Agent" // Add new type
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
#### Hard Invariant Enforcement
|
|
220
|
+
|
|
221
|
+
```javascript
|
|
222
|
+
function enforceHardInvariant(quads, removeSet) {
|
|
223
|
+
// Ensure: quads ∩ remove = ∅ (no overlap)
|
|
224
|
+
const quadKeys = new Set();
|
|
225
|
+
for (const quad of quads) {
|
|
226
|
+
quadKeys.add(quadIndexKey(quad.subject, quad.predicate, quad.object));
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
// Filter removeSet to ensure no overlap
|
|
230
|
+
return Array.from(removeSet).filter(quad => {
|
|
231
|
+
const key = quadIndexKey(quad.subject, quad.predicate, quad.object);
|
|
232
|
+
return !quadKeys.has(key);
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
### 5. Carrier Extraction System
|
|
238
|
+
|
|
239
|
+
#### Carrier Types
|
|
240
|
+
|
|
241
|
+
```javascript
|
|
242
|
+
const CARRIER_TYPES = {
|
|
243
|
+
link: 'link', // [text] {+ex:predicate}
|
|
244
|
+
emphasis: 'emphasis', // **text** {ex:property}
|
|
245
|
+
code: 'code', // ``code`` {ex:property}
|
|
246
|
+
bracket: 'bracket' // <url> {ex:reference}
|
|
247
|
+
};
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
#### Carrier Structure
|
|
251
|
+
|
|
252
|
+
```javascript
|
|
253
|
+
const carrier = {
|
|
254
|
+
type: 'link', // Carrier type
|
|
255
|
+
text: 'display text', // Display content
|
|
256
|
+
attrs: '{+ex:predicate}', // Annotation block
|
|
257
|
+
attrsRange: [start, end], // Annotation positions
|
|
258
|
+
valueRange: [start, end], // Value positions
|
|
259
|
+
range: [start, end], // Full span in source
|
|
260
|
+
pos: endPosition, // Next processing position
|
|
261
|
+
url: 'resolved-iri' // Resolved IRI (for links)
|
|
262
|
+
};
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
#### Extraction Patterns
|
|
266
|
+
|
|
267
|
+
```javascript
|
|
268
|
+
const CARRIER_EXTRACTORS = {
|
|
269
|
+
'<': extractLinkCarrier, // Angle bracket links
|
|
270
|
+
'[': extractBracketCarrier, // Square bracket carriers
|
|
271
|
+
emphasis: extractEmphasis, // **emphasis** patterns
|
|
272
|
+
code: extractCodeSpan // ``code`` patterns
|
|
273
|
+
};
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
### 5. Quad Emission System
|
|
277
|
+
|
|
278
|
+
#### Emit Quad Function
|
|
279
|
+
|
|
280
|
+
```javascript
|
|
281
|
+
function emitQuad(quads, quadBuffer, removeSet, quadIndex, block,
|
|
282
|
+
subject, predicate, object, dataFactory, meta,
|
|
283
|
+
statements, statementCandidates) {
|
|
284
|
+
|
|
285
|
+
// 1. Create RDF/JS quad
|
|
286
|
+
const quad = dataFactory.quad(subject, predicate, object);
|
|
287
|
+
const remove = meta?.remove || false;
|
|
288
|
+
|
|
289
|
+
// 2. Handle polarity (add/remove)
|
|
290
|
+
if (remove) {
|
|
291
|
+
// Retraction processing
|
|
292
|
+
const quadKey = quadIndexKey(quad.subject, quad.predicate, quad.object);
|
|
293
|
+
|
|
294
|
+
if (quadBuffer.has(quadKey)) {
|
|
295
|
+
// Intra-document cancel: remove from current state
|
|
296
|
+
quadBuffer.delete(quadKey);
|
|
297
|
+
removeFromQuadsArray(quads, quad);
|
|
298
|
+
quadIndex.delete(quadKey);
|
|
299
|
+
} else {
|
|
300
|
+
// External retract: target prior document state
|
|
301
|
+
removeSet.add(quad);
|
|
302
|
+
}
|
|
303
|
+
} else {
|
|
304
|
+
// Add to buffer and quads array
|
|
305
|
+
const quadKey = quadIndexKey(quad.subject, quad.predicate, quad.object);
|
|
306
|
+
quadBuffer.set(quadKey, quad);
|
|
307
|
+
quads.push(quad);
|
|
308
|
+
|
|
309
|
+
// Detect elevated statements
|
|
310
|
+
detectStatementPatternSinglePass(quad, dataFactory, meta, statements, statementCandidates);
|
|
311
|
+
|
|
312
|
+
// Create origin entry
|
|
313
|
+
const originEntry = createOriginEntry(block, subject, predicate, meta);
|
|
314
|
+
quadIndex.set(quadKey, originEntry);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
#### Retraction-Aware Quad Processing
|
|
320
|
+
|
|
321
|
+
```javascript
|
|
322
|
+
function removeFromQuadsArray(quads, targetQuad) {
|
|
323
|
+
const index = quads.findIndex(q =>
|
|
324
|
+
q.subject.value === targetQuad.subject.value &&
|
|
325
|
+
q.predicate.value === targetQuad.predicate.value &&
|
|
326
|
+
q.object.value === targetQuad.object.value
|
|
327
|
+
);
|
|
328
|
+
|
|
329
|
+
if (index !== -1) {
|
|
330
|
+
quads.splice(index, 1);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
#### Origin Entry Creation with Polarity
|
|
336
|
+
|
|
337
|
+
```javascript
|
|
338
|
+
function createOriginEntry(block, subject, predicate, meta) {
|
|
339
|
+
return {
|
|
340
|
+
blockId: block.id,
|
|
341
|
+
range: block.range,
|
|
342
|
+
carrierType: block.carrierType,
|
|
343
|
+
subject: subject.value,
|
|
344
|
+
predicate: predicate.value,
|
|
345
|
+
context: block.context,
|
|
346
|
+
polarity: meta?.remove ? '-' : '+', // Polarity tracking
|
|
347
|
+
value: block.text || ''
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
#### Quad Index Key Generation
|
|
353
|
+
|
|
354
|
+
```javascript
|
|
355
|
+
function quadIndexKey(subject, predicate, object) {
|
|
356
|
+
return `${subject.value}|${predicate.value}|${object.value}`;
|
|
357
|
+
}
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
### 6. Elevated Statements Detection
|
|
361
|
+
|
|
362
|
+
#### Pattern Detection System
|
|
363
|
+
|
|
364
|
+
```javascript
|
|
365
|
+
function detectStatementPatternSinglePass(quad, dataFactory, meta,
|
|
366
|
+
statements, statementCandidates) {
|
|
367
|
+
// Only process rdf:Statement related predicates (optimized)
|
|
368
|
+
if (!isRDFStatementPredicate(quad.predicate.value)) return;
|
|
369
|
+
|
|
370
|
+
// Start new pattern: rdf:type rdf:Statement
|
|
371
|
+
if (isStatementType(quad)) {
|
|
372
|
+
statementCandidates.set(quad.subject.value, { spo: {} });
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// Complete pattern parts: rdf:subject, rdf:predicate, rdf:object
|
|
377
|
+
const candidate = statementCandidates.get(quad.subject.value);
|
|
378
|
+
if (candidate) {
|
|
379
|
+
updatePattern(candidate, quad);
|
|
380
|
+
|
|
381
|
+
// Create elevated SPO when pattern complete
|
|
382
|
+
if (isPatternComplete(candidate)) {
|
|
383
|
+
const elevatedQuad = createElevatedQuad(candidate, dataFactory);
|
|
384
|
+
statements.push(elevatedQuad);
|
|
385
|
+
statementCandidates.delete(quad.subject.value);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
#### Pattern Completion Tracking
|
|
392
|
+
|
|
393
|
+
```javascript
|
|
394
|
+
const patternCandidate = {
|
|
395
|
+
spo: {
|
|
396
|
+
subject: NamedNode, // rdf:subject value
|
|
397
|
+
predicate: NamedNode, // rdf:predicate value
|
|
398
|
+
object: NamedNode|Literal // rdf:object value
|
|
399
|
+
},
|
|
400
|
+
objectQuad: Quad // Original quad for literal extraction
|
|
401
|
+
};
|
|
402
|
+
```
|
|
403
|
+
|
|
404
|
+
### 7. Origin Tracking System
|
|
405
|
+
|
|
406
|
+
#### Origin Entry Structure
|
|
407
|
+
|
|
408
|
+
```javascript
|
|
409
|
+
const originEntry = {
|
|
410
|
+
blockId: 'block-identifier', // Unique block identifier
|
|
411
|
+
range: [start, end], // Source location
|
|
412
|
+
carrierType: 'para', // Token type
|
|
413
|
+
subject: 'ex:subject', // Subject IRI
|
|
414
|
+
predicate: 'ex:predicate', // Predicate IRI
|
|
415
|
+
context: { ...block.context }, // Prefix context
|
|
416
|
+
polarity: '+', // Add/remove indicator
|
|
417
|
+
value: 'source text' // Original content
|
|
418
|
+
};
|
|
419
|
+
```
|
|
420
|
+
|
|
421
|
+
#### Quad Index Mapping
|
|
422
|
+
|
|
423
|
+
```javascript
|
|
424
|
+
const quadIndex = new Map(); // quadIndexKey -> originEntry
|
|
425
|
+
```
|
|
426
|
+
|
|
427
|
+
## Processing Pipeline
|
|
428
|
+
|
|
429
|
+
### Phase 1: Tokenization
|
|
430
|
+
|
|
431
|
+
```javascript
|
|
432
|
+
function scanTokens(text) {
|
|
433
|
+
const tokens = [];
|
|
434
|
+
const lines = text.split('\n');
|
|
435
|
+
|
|
436
|
+
for (const line of lines) {
|
|
437
|
+
// Identify token types using regex patterns
|
|
438
|
+
// Extract annotations and ranges
|
|
439
|
+
// Create token objects with metadata
|
|
440
|
+
tokens.push(token);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
return tokens;
|
|
444
|
+
}
|
|
445
|
+
```
|
|
446
|
+
|
|
447
|
+
### Phase 2: Prefix Processing
|
|
448
|
+
|
|
449
|
+
```javascript
|
|
450
|
+
// Single loop optimization for better performance
|
|
451
|
+
for (const token of tokens) {
|
|
452
|
+
if (token.type === 'prefix') {
|
|
453
|
+
// Resolve CURIEs and update context
|
|
454
|
+
state.ctx[token.prefix] = resolveIRI(token.iri, state.ctx);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
```
|
|
458
|
+
|
|
459
|
+
### Phase 3: Semantic Processing
|
|
460
|
+
|
|
461
|
+
```javascript
|
|
462
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
463
|
+
const token = tokens[i];
|
|
464
|
+
state.currentTokenIndex = i;
|
|
465
|
+
|
|
466
|
+
// Process token using appropriate processor
|
|
467
|
+
TOKEN_PROCESSORS[token.type]?.(token, state);
|
|
468
|
+
}
|
|
469
|
+
```
|
|
470
|
+
|
|
471
|
+
### Phase 4: Final Resolution
|
|
472
|
+
|
|
473
|
+
```javascript
|
|
474
|
+
// Process retractions and ensure hard invariant
|
|
475
|
+
const filteredRemove = processRetractions(state.quads, state.removeSet);
|
|
476
|
+
|
|
477
|
+
return {
|
|
478
|
+
quads: state.quads, // Final resolved graph state
|
|
479
|
+
remove: filteredRemove, // External retractions only
|
|
480
|
+
statements: state.statements, // Elevated statements
|
|
481
|
+
origin: state.origin, // Complete provenance
|
|
482
|
+
context: state.ctx // Final prefix context
|
|
483
|
+
};
|
|
484
|
+
```
|
|
485
|
+
|
|
486
|
+
#### Retraction Processing
|
|
487
|
+
|
|
488
|
+
```javascript
|
|
489
|
+
function processRetractions(quads, removeSet) {
|
|
490
|
+
// Create quad key index for fast lookup
|
|
491
|
+
const quadKeys = new Set();
|
|
492
|
+
for (const quad of quads) {
|
|
493
|
+
quadKeys.add(quadIndexKey(quad.subject, quad.predicate, quad.object));
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// Filter removeSet to enforce hard invariant: quads ∩ remove = ∅
|
|
497
|
+
const filteredRemove = [];
|
|
498
|
+
for (const quad of removeSet) {
|
|
499
|
+
const key = quadIndexKey(quad.subject, quad.predicate, quad.object);
|
|
500
|
+
if (!quadKeys.has(key)) {
|
|
501
|
+
filteredRemove.push(quad);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
return filteredRemove;
|
|
506
|
+
}
|
|
507
|
+
```
|
|
508
|
+
|
|
509
|
+
## Performance Optimizations
|
|
510
|
+
|
|
511
|
+
### Applied Optimizations
|
|
512
|
+
|
|
513
|
+
1. **Token Processing**: Single loop instead of filter+forEach
|
|
514
|
+
2. **Regex Pre-compilation**: Pre-compiled carrier patterns
|
|
515
|
+
3. **String Operations**: Optimized fragment resolution
|
|
516
|
+
4. **Array Operations**: Direct iteration instead of Array.from()+filter()
|
|
517
|
+
5. **Object Creation**: Reduced spread operator usage
|
|
518
|
+
6. **Early Filtering**: Only process rdf:Statement related predicates
|
|
519
|
+
7. **Module Constants**: RDF constants defined once at module level
|
|
520
|
+
|
|
521
|
+
### Performance Characteristics
|
|
522
|
+
|
|
523
|
+
- **Single-pass parsing**: No backtracking or multiple iterations
|
|
524
|
+
- **O(1) additional memory**: Constant overhead for elevated statements
|
|
525
|
+
- **Linear scalability**: Performance scales linearly with input size
|
|
526
|
+
- **Streaming compatible**: Can process large files incrementally
|
|
527
|
+
|
|
528
|
+
## Memory Management
|
|
529
|
+
|
|
530
|
+
### Memory Usage Patterns
|
|
531
|
+
|
|
532
|
+
```javascript
|
|
533
|
+
// Temporary objects (cleaned up automatically)
|
|
534
|
+
const statementCandidates = new Map(); // Cleared on pattern completion
|
|
535
|
+
const quadBuffer = new Map(); // Cleared on intra-document cancel
|
|
536
|
+
|
|
537
|
+
// Persistent objects (grow with input)
|
|
538
|
+
state.quads.push(quad); // Final output
|
|
539
|
+
state.origin.quadIndex.set(key, entry); // Provenance tracking
|
|
540
|
+
```
|
|
541
|
+
|
|
542
|
+
### Cleanup Strategies
|
|
543
|
+
|
|
544
|
+
1. **Automatic Cleanup**: Statement candidates deleted on completion
|
|
545
|
+
2. **Buffer Management**: Quad buffer cleared on intra-document cancel
|
|
546
|
+
3. **Set Operations**: RemoveSet converted to array at end
|
|
547
|
+
4. **Lazy Loading**: Inline carriers loaded on-demand
|
|
548
|
+
|
|
549
|
+
## Error Handling
|
|
550
|
+
|
|
551
|
+
### Error Recovery Strategies
|
|
552
|
+
|
|
553
|
+
```javascript
|
|
554
|
+
// Graceful degradation for malformed input
|
|
555
|
+
if (!isValidAnnotation(attrs)) {
|
|
556
|
+
// Log warning and continue processing
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// Safe navigation for optional properties
|
|
561
|
+
const subject = resolveSubject(sem.subject, state) || skipProcessing();
|
|
562
|
+
|
|
563
|
+
// Validation with meaningful errors
|
|
564
|
+
if (!state.currentSubject && needsSubject(sem)) {
|
|
565
|
+
throw new ParseError('Subject required but not set', token.range);
|
|
566
|
+
}
|
|
567
|
+
```
|
|
568
|
+
|
|
569
|
+
## Extensibility Points
|
|
570
|
+
|
|
571
|
+
### Custom Token Processors
|
|
572
|
+
|
|
573
|
+
```javascript
|
|
574
|
+
// Add new token type
|
|
575
|
+
TOKEN_PROCESSORS.custom = (token, state) => {
|
|
576
|
+
// Custom processing logic
|
|
577
|
+
processTokenAnnotations(token, state, 'custom');
|
|
578
|
+
};
|
|
579
|
+
```
|
|
580
|
+
|
|
581
|
+
### Custom Carrier Extractors
|
|
582
|
+
|
|
583
|
+
```javascript
|
|
584
|
+
// Add new carrier pattern
|
|
585
|
+
CARRIER_EXTRACTORS['@'] = (text, pos, baseOffset) => {
|
|
586
|
+
// Custom carrier extraction
|
|
587
|
+
return createCarrier('mention', text, attrs, ranges);
|
|
588
|
+
};
|
|
589
|
+
```
|
|
590
|
+
|
|
591
|
+
### Custom Semantic Parsers
|
|
592
|
+
|
|
593
|
+
```javascript
|
|
594
|
+
// Extend semantic block parsing
|
|
595
|
+
function parseCustomSemantic(attrs) {
|
|
596
|
+
// Custom annotation parsing
|
|
597
|
+
return { customProperty: value };
|
|
598
|
+
}
|
|
599
|
+
```
|
|
600
|
+
|
|
601
|
+
## Integration Points
|
|
602
|
+
|
|
603
|
+
### External Libraries
|
|
604
|
+
|
|
605
|
+
- **RDF/JS DataFactory**: Quad creation and term management
|
|
606
|
+
- **Context Expansion**: IRI resolution and prefix handling
|
|
607
|
+
- **Validation**: Optional SHACL or schema validation
|
|
608
|
+
|
|
609
|
+
### Output Formats
|
|
610
|
+
|
|
611
|
+
```javascript
|
|
612
|
+
// Standard RDF/JS output with first-class retractions
|
|
613
|
+
return {
|
|
614
|
+
quads: RDFQuad[], // Primary graph (final resolved state)
|
|
615
|
+
remove: RDFQuad[], // External retractions (targets prior state)
|
|
616
|
+
statements: RDFQuad[], // Elevated statements from rdf:Statement patterns
|
|
617
|
+
origin: OriginObject, // Complete provenance tracking
|
|
618
|
+
context: ContextObject // Final prefix context
|
|
619
|
+
};
|
|
620
|
+
```
|
|
621
|
+
|
|
622
|
+
#### Retraction Output Characteristics
|
|
623
|
+
|
|
624
|
+
```javascript
|
|
625
|
+
// External retractions only (intra-document cancels resolved internally)
|
|
626
|
+
const remove = [
|
|
627
|
+
// Quads that target prior document state
|
|
628
|
+
// Never overlap with quads array (hard invariant enforced)
|
|
629
|
+
];
|
|
630
|
+
|
|
631
|
+
// Usage for document evolution
|
|
632
|
+
const priorResult = parse(previousDocument);
|
|
633
|
+
const currentResult = parse(currentDocument);
|
|
634
|
+
|
|
635
|
+
// Apply retractions to prior state
|
|
636
|
+
const evolvedGraph = applyRetractions(priorResult.quads, currentResult.remove);
|
|
637
|
+
```
|
|
638
|
+
|
|
639
|
+
### Streaming Interface
|
|
640
|
+
|
|
641
|
+
```javascript
|
|
642
|
+
// Future streaming capability
|
|
643
|
+
function parseStream(stream, options) {
|
|
644
|
+
return new TransformStream({
|
|
645
|
+
transform(chunk, controller) {
|
|
646
|
+
// Process chunk incrementally
|
|
647
|
+
const result = parse(chunk.toString(), options);
|
|
648
|
+
controller.enqueue(result);
|
|
649
|
+
}
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
```
|
|
653
|
+
|
|
654
|
+
This architecture provides a solid foundation for MD-LD parsing with excellent performance characteristics, maintainability, and extensibility for future enhancements.
|