mdld-parse 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.nojekyll +0 -0
- package/dist/CNAME +1 -0
- package/dist/README.md +425 -0
- package/dist/docs/API.md +397 -0
- package/dist/docs/Architecture.md +190 -0
- package/dist/docs/Guide.md +322 -0
- package/dist/docs/LLM-MDLD-Inducer.md +550 -0
- package/dist/docs/Parser.md +654 -0
- package/dist/docs/Performance.md +238 -0
- package/dist/docs/Spec-render.md +1439 -0
- package/dist/docs/Subject.md +180 -0
- package/dist/docs/Syntax.md +463 -0
- package/dist/docs/TodoList.md +215 -0
- package/dist/docs/Use-Cases.md +354 -0
- package/dist/docs/index.html +108 -0
- package/dist/docs/index.md +56 -0
- package/dist/docs/origin.md +459 -0
- package/dist/docs/polarity.md +586 -0
- package/dist/docs/primary-subject.md +199 -0
- package/dist/docs/statements.md +226 -0
- package/dist/docs/tokenization-performance-report.md +164 -0
- package/dist/examples/Cookbook.md +32 -0
- package/dist/examples/Status-SHACL.md +33 -0
- package/dist/examples/demo/index.md +35 -0
- package/dist/examples/demo/journal.md +18 -0
- package/dist/examples/demo/minimal.md +11 -0
- package/dist/examples/demo/project.md +15 -0
- package/dist/examples/demo/recipe.md +31 -0
- package/dist/examples/demo/research.md +46 -0
- package/dist/examples/demo/space-mission.md +48 -0
- package/dist/examples/dogfood.md +288 -0
- package/dist/examples/few-shot.md +309 -0
- package/dist/examples/graph/rdf-graph.html +473 -0
- package/dist/examples/graph/rdf_force_graph_web_component.html +506 -0
- package/dist/examples/index.md +45 -0
- package/dist/examples/llm-time-workflow.md +173 -0
- package/dist/examples/prov-o.md +2367 -0
- package/dist/examples/rdf.md +827 -0
- package/dist/examples/shacl.md +3399 -0
- package/dist/examples/statements.md +43 -0
- package/dist/examples/workflow.md +150 -0
- package/dist/index.html +2827 -0
- package/dist/index.js +1548 -0
- package/dist/logo.svg +7 -0
- package/dist/spec/Spec-compact.md +363 -0
- package/dist/spec/Spec-ultra.md +221 -0
- package/dist/spec/Spec.md +1068 -0
- package/dist/spec/index.md +46 -0
- package/package.json +7 -8
- package/src/constants.js +0 -31
- package/src/generate.js +0 -245
- package/src/highlight.js +0 -362
- package/src/index.js +0 -13
- package/src/locate.js +0 -23
- package/src/merge.js +0 -157
- package/src/parse.js +0 -711
- package/src/render.js +0 -576
- package/src/shared.js +0 -577
- package/src/tokenizers.js +0 -594
- package/src/utils.js +0 -430
package/src/parse.js
DELETED
|
@@ -1,711 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
DataFactory,
|
|
3
|
-
expandIRI,
|
|
4
|
-
quadIndexKey,
|
|
5
|
-
createLiteral,
|
|
6
|
-
hash
|
|
7
|
-
} from './utils.js';
|
|
8
|
-
import {
|
|
9
|
-
DEFAULT_CONTEXT,
|
|
10
|
-
} from './constants.js';
|
|
11
|
-
import {
|
|
12
|
-
detectStandaloneSubject,
|
|
13
|
-
detectPrefix,
|
|
14
|
-
detectHeading,
|
|
15
|
-
detectList,
|
|
16
|
-
detectBlockquote,
|
|
17
|
-
detectFence,
|
|
18
|
-
scanInlineCarriers
|
|
19
|
-
} from './tokenizers.js';
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
getFenceClosePattern,
|
|
23
|
-
calcRangeInfo,
|
|
24
|
-
calcAttrsRange,
|
|
25
|
-
createToken,
|
|
26
|
-
createCarrier,
|
|
27
|
-
createListToken,
|
|
28
|
-
parseSemCached,
|
|
29
|
-
parseLangAndAttrs,
|
|
30
|
-
findMatchingBracket,
|
|
31
|
-
extractUrlFromBrackets,
|
|
32
|
-
extractAttributesFromText,
|
|
33
|
-
determineCarrierType,
|
|
34
|
-
calcCarrierRanges,
|
|
35
|
-
extractCleanText,
|
|
36
|
-
RDF_TYPE,
|
|
37
|
-
RDF_STATEMENT,
|
|
38
|
-
RDF_SUBJECT,
|
|
39
|
-
RDF_PREDICATE,
|
|
40
|
-
RDF_OBJECT,
|
|
41
|
-
createLeanOriginEntry,
|
|
42
|
-
resolveSubject,
|
|
43
|
-
resolveObject,
|
|
44
|
-
processTokenWithBlockTracking
|
|
45
|
-
} from './shared.js';
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
export function parse(firstArg, secondArg = {}) {
|
|
49
|
-
// Dual-mode API: backward compatible with (text, options) and new ({ text, context, ... })
|
|
50
|
-
const isNamedParams = typeof firstArg === 'object' && firstArg !== null && 'text' in firstArg;
|
|
51
|
-
|
|
52
|
-
const text = isNamedParams ? firstArg.text : firstArg;
|
|
53
|
-
const options = isNamedParams
|
|
54
|
-
? { context: firstArg.context, dataFactory: firstArg.dataFactory, graph: firstArg.graph }
|
|
55
|
-
: secondArg;
|
|
56
|
-
|
|
57
|
-
const state = {
|
|
58
|
-
ctx: { ...DEFAULT_CONTEXT, ...(options.context || {}) },
|
|
59
|
-
df: options.dataFactory || DataFactory,
|
|
60
|
-
graph: options.graph ? DataFactory.namedNode(options.graph) : DataFactory.defaultGraph(),
|
|
61
|
-
quads: [],
|
|
62
|
-
quadBuffer: new Map(),
|
|
63
|
-
removeSet: new Set(),
|
|
64
|
-
origin: {
|
|
65
|
-
quadIndex: new Map(),
|
|
66
|
-
blocks: new Map(),
|
|
67
|
-
documentStructure: []
|
|
68
|
-
},
|
|
69
|
-
currentSubject: null,
|
|
70
|
-
primarySubject: null,
|
|
71
|
-
tokens: null,
|
|
72
|
-
currentTokenIndex: -1,
|
|
73
|
-
statements: [],
|
|
74
|
-
statementCandidates: new Map(),
|
|
75
|
-
currentBlock: null,
|
|
76
|
-
blockStack: []
|
|
77
|
-
};
|
|
78
|
-
|
|
79
|
-
const scanResult = scanTokens(text);
|
|
80
|
-
state.tokens = scanResult.tokens;
|
|
81
|
-
|
|
82
|
-
// Single-pass processing: resolve prefixes AND process tokens together
|
|
83
|
-
for (let i = 0; i < state.tokens.length; i++) {
|
|
84
|
-
const token = state.tokens[i];
|
|
85
|
-
state.currentTokenIndex = i;
|
|
86
|
-
|
|
87
|
-
// Handle prefix tokens immediately during main pass
|
|
88
|
-
if (token.type === 'prefix') {
|
|
89
|
-
let resolvedIri = token.iri;
|
|
90
|
-
if (token.iri.includes(':')) {
|
|
91
|
-
const colonIndex = token.iri.indexOf(':');
|
|
92
|
-
const potentialPrefix = token.iri.substring(0, colonIndex);
|
|
93
|
-
const reference = token.iri.substring(colonIndex + 1);
|
|
94
|
-
if (state.ctx[potentialPrefix] && potentialPrefix !== '@vocab') {
|
|
95
|
-
resolvedIri = state.ctx[potentialPrefix] + reference;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
state.ctx[token.prefix] = resolvedIri;
|
|
99
|
-
continue; // Skip token processor for prefixes
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
// Process all other tokens
|
|
103
|
-
TOKEN_PROCESSORS[token.type]?.(token, state);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
// Optimized quad filtering - use Set.has() instead of array.includes()
|
|
107
|
-
const quadKeys = new Set();
|
|
108
|
-
for (const quad of state.quads) {
|
|
109
|
-
quadKeys.add(quadIndexKey(quad.subject, quad.predicate, quad.object));
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
// Direct Set iteration - more efficient than filter()
|
|
113
|
-
const filteredRemove = [];
|
|
114
|
-
for (const quad of state.removeSet) {
|
|
115
|
-
const key = quadIndexKey(quad.subject, quad.predicate, quad.object);
|
|
116
|
-
if (!quadKeys.has(key)) {
|
|
117
|
-
filteredRemove.push(quad);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
return {
|
|
122
|
-
quads: state.quads,
|
|
123
|
-
remove: filteredRemove,
|
|
124
|
-
statements: state.statements,
|
|
125
|
-
origin: state.origin,
|
|
126
|
-
context: state.ctx,
|
|
127
|
-
primarySubject: state.primarySubject,
|
|
128
|
-
md: scanResult.md
|
|
129
|
-
};
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
// Cache for fence regex patterns - using shared utility
|
|
134
|
-
|
|
135
|
-
function getCarriers(token) {
|
|
136
|
-
if (token.type === 'code') return [];
|
|
137
|
-
return token._carriers || (token._carriers = extractInlineCarriers(token.text, token.range[0]));
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
function scanTokens(text) {
|
|
141
|
-
const tokens = [];
|
|
142
|
-
const mdLines = [];
|
|
143
|
-
const lines = text.split('\n');
|
|
144
|
-
let pos = 0;
|
|
145
|
-
let codeBlock = null;
|
|
146
|
-
const PROCESSORS = [
|
|
147
|
-
{ type: 'fence', test: line => detectFence(line.trim()), process: handleFence },
|
|
148
|
-
{ type: 'content', test: () => codeBlock, process: line => codeBlock.content.push(line) },
|
|
149
|
-
{ type: 'prefix', test: line => detectPrefix(line), process: handlePrefix },
|
|
150
|
-
{ type: 'standalone', test: line => detectStandaloneSubject(line), process: handleStandaloneSubject },
|
|
151
|
-
{ type: 'heading', test: line => detectHeading(line), process: handleHeading },
|
|
152
|
-
{ type: 'list', test: line => detectList(line), process: handleList },
|
|
153
|
-
{ type: 'blockquote', test: line => detectBlockquote(line), process: handleBlockquote },
|
|
154
|
-
{ type: 'para', test: line => line.trim(), process: handlePara }
|
|
155
|
-
];
|
|
156
|
-
|
|
157
|
-
function handleFence(line, lineStart, pos) {
|
|
158
|
-
const trimmedLine = line.trim();
|
|
159
|
-
if (!codeBlock) {
|
|
160
|
-
const fenceResult = detectFence(trimmedLine);
|
|
161
|
-
if (!fenceResult) return false;
|
|
162
|
-
|
|
163
|
-
const attrsText = fenceResult.attrs;
|
|
164
|
-
const attrsStartInLine = attrsText ? line.indexOf(attrsText) : -1;
|
|
165
|
-
const contentStart = lineStart + line.length + 1;
|
|
166
|
-
|
|
167
|
-
codeBlock = {
|
|
168
|
-
fence: fenceResult.fenceChar.repeat(fenceResult.fenceLength),
|
|
169
|
-
start: lineStart,
|
|
170
|
-
content: [],
|
|
171
|
-
lang: fenceResult.lang,
|
|
172
|
-
attrs: attrsText,
|
|
173
|
-
attrsRange: attrsText && attrsStartInLine >= 0 ? [lineStart + attrsStartInLine, lineStart + attrsStartInLine + attrsText.length] : null,
|
|
174
|
-
valueRangeStart: contentStart
|
|
175
|
-
};
|
|
176
|
-
|
|
177
|
-
// Add stripped fence line to mdLines
|
|
178
|
-
const cleanFence = line.replace(/\s*\{[^}]+\}\s*$/, '');
|
|
179
|
-
mdLines.push(cleanFence);
|
|
180
|
-
} else {
|
|
181
|
-
const fenceChar = codeBlock.fence[0];
|
|
182
|
-
const expectedFence = fenceChar.repeat(codeBlock.fence.length);
|
|
183
|
-
const fenceMatch = trimmedLine.match(getFenceClosePattern(fenceChar));
|
|
184
|
-
|
|
185
|
-
if (fenceMatch && fenceMatch[1] === expectedFence) {
|
|
186
|
-
const valueStart = codeBlock.valueRangeStart;
|
|
187
|
-
const valueEnd = Math.max(valueStart, lineStart - 1);
|
|
188
|
-
tokens.push({
|
|
189
|
-
type: 'code',
|
|
190
|
-
range: [codeBlock.start, lineStart],
|
|
191
|
-
text: codeBlock.content.join('\n'),
|
|
192
|
-
lang: codeBlock.lang,
|
|
193
|
-
attrs: codeBlock.attrs,
|
|
194
|
-
attrsRange: codeBlock.attrsRange,
|
|
195
|
-
valueRange: [valueStart, valueEnd]
|
|
196
|
-
});
|
|
197
|
-
|
|
198
|
-
// Add all code content to mdLines before closing fence
|
|
199
|
-
for (const contentLine of codeBlock.content) {
|
|
200
|
-
mdLines.push(contentLine);
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
codeBlock = null;
|
|
204
|
-
|
|
205
|
-
// Add closing fence to mdLines without trailing newline
|
|
206
|
-
const closingFence = line.replace(/\r?\n.*$/, '');
|
|
207
|
-
mdLines.push(closingFence);
|
|
208
|
-
} else {
|
|
209
|
-
// Code content - don't add to mdLines here, will be added when closing block
|
|
210
|
-
// No need to add individual lines as they're processed when closing
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
return true;
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
function handlePrefix(line, lineStart, pos) {
|
|
217
|
-
const result = detectPrefix(line);
|
|
218
|
-
tokens.push({ type: 'prefix', prefix: result.prefix, iri: result.iri });
|
|
219
|
-
// Prefixes are typically stripped from clean MD
|
|
220
|
-
return true;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
function handleHeading(line, lineStart, pos) {
|
|
224
|
-
const result = detectHeading(line);
|
|
225
|
-
const attrs = result.attrs;
|
|
226
|
-
const afterHashes = result.depth;
|
|
227
|
-
const rangeInfo = calcRangeInfo(line, attrs, lineStart, afterHashes, result.content.length);
|
|
228
|
-
tokens.push(createToken('heading', [lineStart, pos - 1], result.content, attrs,
|
|
229
|
-
rangeInfo.attrsRange, rangeInfo.valueRange, { depth: result.depth }));
|
|
230
|
-
|
|
231
|
-
// Add stripped heading to mdLines
|
|
232
|
-
const cleanHeading = `${'#'.repeat(result.depth)} ${result.content}`;
|
|
233
|
-
mdLines.push(cleanHeading);
|
|
234
|
-
return true;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
function handleList(line, lineStart, pos) {
|
|
238
|
-
const result = detectList(line);
|
|
239
|
-
// Create match array format expected by createListToken
|
|
240
|
-
const indentStr = ' '.repeat(result.indent);
|
|
241
|
-
const match = [line, indentStr, result.marker, result.content, result.attrs];
|
|
242
|
-
tokens.push(createListToken('list', line, lineStart, pos, match));
|
|
243
|
-
|
|
244
|
-
// Add stripped list item to mdLines
|
|
245
|
-
const cleanList = `${indentStr}${result.marker} ${result.content}`;
|
|
246
|
-
mdLines.push(cleanList);
|
|
247
|
-
return true;
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
function handleBlockquote(line, lineStart, pos) {
|
|
251
|
-
const result = detectBlockquote(line);
|
|
252
|
-
const attrs = result.attrs;
|
|
253
|
-
const valueStartInLine = line.startsWith('> ') ? 2 : line.indexOf('>') + 1;
|
|
254
|
-
const valueEndInLine = valueStartInLine + result.content.length;
|
|
255
|
-
tokens.push(createToken('blockquote', [lineStart, pos - 1], result.content, attrs,
|
|
256
|
-
calcAttrsRange(line, attrs, lineStart),
|
|
257
|
-
[lineStart + valueStartInLine, lineStart + valueEndInLine]));
|
|
258
|
-
|
|
259
|
-
// Add stripped blockquote to mdLines
|
|
260
|
-
const cleanBlockquote = `> ${result.content}`;
|
|
261
|
-
mdLines.push(cleanBlockquote);
|
|
262
|
-
return true;
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
function handlePara(line, lineStart, pos) {
|
|
266
|
-
tokens.push(createToken('para', [lineStart, pos - 1], line.trim()));
|
|
267
|
-
|
|
268
|
-
// Add stripped paragraph to mdLines
|
|
269
|
-
let cleanPara = line;
|
|
270
|
-
|
|
271
|
-
// Remove inline carrier annotations using character-based detection
|
|
272
|
-
const carriers = scanInlineCarriers(cleanPara, 0);
|
|
273
|
-
for (const carrier of carriers) {
|
|
274
|
-
if (carrier.attrs && (carrier.type === 'emphasis' || carrier.type === 'code')) {
|
|
275
|
-
// Replace with just the content
|
|
276
|
-
const before = cleanPara.substring(0, carrier.range[0]);
|
|
277
|
-
const after = cleanPara.substring(carrier.range[1]);
|
|
278
|
-
cleanPara = before + (carrier.text || '') + after;
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
// Remove bracket-style annotations [text] {annotation}
|
|
283
|
-
cleanPara = cleanPara.replace(/\[([^\]]+)\]\s*\{[^}]+\}/g, '$1');
|
|
284
|
-
|
|
285
|
-
// Remove any remaining standalone annotations
|
|
286
|
-
cleanPara = cleanPara.replace(/\s*\{[^}]+\}\s*/g, ' ');
|
|
287
|
-
|
|
288
|
-
// Clean up extra whitespace
|
|
289
|
-
cleanPara = cleanPara.replace(/\s+/g, ' ').trim();
|
|
290
|
-
|
|
291
|
-
mdLines.push(cleanPara);
|
|
292
|
-
return true;
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
function handleStandaloneSubject(line, lineStart, pos) {
|
|
296
|
-
tokens.push({ type: 'standalone', text: line.trim(), range: [lineStart, pos - 1] });
|
|
297
|
-
|
|
298
|
-
// Standalone subjects are stripped from MD by default
|
|
299
|
-
// Don't add anything to mdLines
|
|
300
|
-
return true;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
for (let i = 0; i < lines.length; i++) {
|
|
304
|
-
const line = lines[i];
|
|
305
|
-
const lineStart = pos;
|
|
306
|
-
pos += line.length + 1;
|
|
307
|
-
|
|
308
|
-
// Direct processor lookup - O(n) instead of O(n*m)
|
|
309
|
-
for (const processor of PROCESSORS) {
|
|
310
|
-
if (processor.test(line) && processor.process(line, lineStart, pos)) {
|
|
311
|
-
break;
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
// Join without trailing newline to match expected output
|
|
317
|
-
const mdContent = mdLines.join('\n');
|
|
318
|
-
return { tokens, md: mdContent };
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
function extractInlineCarriers(text, baseOffset = 0) {
|
|
322
|
-
return scanInlineCarriers(text, baseOffset);
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
function createBlockEntry(token, state) {
|
|
327
|
-
const blockId = token._blockId || hash(`${token.type}:${token.range?.[0]}:${token.range?.[1]}`);
|
|
328
|
-
token._blockId = blockId; // Store for later reference
|
|
329
|
-
|
|
330
|
-
// Extract inline carriers first to enable clean text extraction
|
|
331
|
-
const carriers = getCarriers(token);
|
|
332
|
-
|
|
333
|
-
const cleanText = extractCleanText(token);
|
|
334
|
-
|
|
335
|
-
const blockEntry = {
|
|
336
|
-
id: blockId,
|
|
337
|
-
type: token.type,
|
|
338
|
-
range: token.range,
|
|
339
|
-
text: cleanText,
|
|
340
|
-
subject: null,
|
|
341
|
-
types: [],
|
|
342
|
-
predicates: [],
|
|
343
|
-
carriers: [],
|
|
344
|
-
listLevel: token.indent || 0,
|
|
345
|
-
parentBlockId: state.blockStack.length > 0 ? state.blockStack[state.blockStack.length - 1] : null,
|
|
346
|
-
quadKeys: [] // Will be populated during quad emission
|
|
347
|
-
};
|
|
348
|
-
|
|
349
|
-
// Process carriers and add to block
|
|
350
|
-
for (const carrier of carriers) {
|
|
351
|
-
const carrierInfo = {
|
|
352
|
-
type: carrier.type,
|
|
353
|
-
range: carrier.range,
|
|
354
|
-
text: carrier.text,
|
|
355
|
-
subject: null,
|
|
356
|
-
predicates: [],
|
|
357
|
-
sem: null
|
|
358
|
-
};
|
|
359
|
-
|
|
360
|
-
// Extract carrier-specific semantics
|
|
361
|
-
if (carrier.attrs) {
|
|
362
|
-
const carrierSem = parseSemCached(carrier.attrs);
|
|
363
|
-
carrierInfo.sem = carrierSem; // Store full semantics
|
|
364
|
-
carrierInfo.predicates = carrierSem.predicates || [];
|
|
365
|
-
carrierInfo.subject = carrierSem.subject;
|
|
366
|
-
carrierInfo.types = carrierSem.types || [];
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
blockEntry.carriers.push(carrierInfo);
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
// Store block and add to document structure
|
|
373
|
-
state.origin.blocks.set(blockId, blockEntry);
|
|
374
|
-
state.origin.documentStructure.push(blockEntry);
|
|
375
|
-
|
|
376
|
-
return blockEntry;
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
function enrichBlockFromAnnotation(blockEntry, sem, carrier, state) {
|
|
380
|
-
// Update subject if available
|
|
381
|
-
if (sem.subject && sem.subject !== 'RESET') {
|
|
382
|
-
const resolvedSubject = resolveSubject(sem, state);
|
|
383
|
-
if (resolvedSubject) {
|
|
384
|
-
blockEntry.subject = resolvedSubject.value;
|
|
385
|
-
}
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
// Add types
|
|
389
|
-
if (sem.types && sem.types.length > 0) {
|
|
390
|
-
sem.types.forEach(t => {
|
|
391
|
-
const typeIRI = typeof t === 'string' ? t : t.iri;
|
|
392
|
-
const expanded = expandIRI(typeIRI, state.ctx);
|
|
393
|
-
if (!blockEntry.types.includes(expanded)) {
|
|
394
|
-
blockEntry.types.push(expanded);
|
|
395
|
-
}
|
|
396
|
-
});
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
// Add predicates
|
|
400
|
-
if (sem.predicates && sem.predicates.length > 0) {
|
|
401
|
-
sem.predicates.forEach(pred => {
|
|
402
|
-
const expandedPred = {
|
|
403
|
-
iri: expandIRI(pred.iri, state.ctx),
|
|
404
|
-
form: pred.form || '',
|
|
405
|
-
object: null // Will be filled during quad emission
|
|
406
|
-
};
|
|
407
|
-
blockEntry.predicates.push(expandedPred);
|
|
408
|
-
});
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
// Add carrier information
|
|
412
|
-
if (carrier) {
|
|
413
|
-
const carrierInfo = {
|
|
414
|
-
type: carrier.type,
|
|
415
|
-
range: carrier.range,
|
|
416
|
-
text: carrier.text,
|
|
417
|
-
subject: null,
|
|
418
|
-
predicates: []
|
|
419
|
-
};
|
|
420
|
-
|
|
421
|
-
// Extract carrier-specific semantics
|
|
422
|
-
if (carrier.attrs) {
|
|
423
|
-
const carrierSem = parseSemCached(carrier.attrs);
|
|
424
|
-
carrierInfo.sem = carrierSem; // Store full semantics
|
|
425
|
-
carrierInfo.predicates = carrierSem.predicates || [];
|
|
426
|
-
carrierInfo.subject = carrierSem.subject;
|
|
427
|
-
carrierInfo.types = carrierSem.types || [];
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
blockEntry.carriers.push(carrierInfo);
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
|
|
434
|
-
function processAnnotationWithBlockTracking(carrier, sem, state, options = {}) {
|
|
435
|
-
const { preserveGlobalSubject = false, implicitSubject = null } = options;
|
|
436
|
-
|
|
437
|
-
if (sem.subject === 'RESET') {
|
|
438
|
-
state.currentSubject = null;
|
|
439
|
-
return;
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
const previousSubject = state.currentSubject;
|
|
443
|
-
const newSubject = resolveSubject(sem, state);
|
|
444
|
-
const localObject = resolveObject(sem, state);
|
|
445
|
-
|
|
446
|
-
// Track primary subject: first non-fragment subject declaration (fixed once detected)
|
|
447
|
-
if (newSubject && !state.primarySubject && !sem.subject.startsWith('=#')) {
|
|
448
|
-
state.primarySubject = newSubject.value; // Store as string IRI
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
const effectiveSubject = implicitSubject || (newSubject && !preserveGlobalSubject ? newSubject : previousSubject);
|
|
452
|
-
if (newSubject && !preserveGlobalSubject && !implicitSubject) {
|
|
453
|
-
state.currentSubject = newSubject;
|
|
454
|
-
}
|
|
455
|
-
const S = preserveGlobalSubject ? (newSubject || previousSubject) : (implicitSubject || state.currentSubject);
|
|
456
|
-
if (!S) return;
|
|
457
|
-
|
|
458
|
-
const block = createBlock(
|
|
459
|
-
S.value, sem.types, sem.predicates,
|
|
460
|
-
carrier.range, carrier.attrsRange || null, carrier.valueRange || null,
|
|
461
|
-
carrier.type || null, state.ctx, carrier.text
|
|
462
|
-
);
|
|
463
|
-
|
|
464
|
-
const L = createLiteral(carrier.text, sem.datatype, sem.language, state.ctx, state.df);
|
|
465
|
-
const carrierO = carrier.url ? state.df.namedNode(expandIRI(carrier.url, state.ctx)) : null;
|
|
466
|
-
const newSubjectOrCarrierO = newSubject || carrierO;
|
|
467
|
-
|
|
468
|
-
// Enrich current block with semantic information
|
|
469
|
-
if (state.currentBlock) {
|
|
470
|
-
enrichBlockFromAnnotation(state.currentBlock, sem, carrier, state);
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
processTypeAnnotations(sem, newSubject, localObject, carrierO, S, block, state, carrier);
|
|
474
|
-
processPredicateAnnotations(sem, newSubject, previousSubject, localObject, newSubjectOrCarrierO, S, L, block, state, carrier);
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
function createBlock(subject, types, predicates, range, attrsRange, valueRange, carrierType, ctx, text) {
|
|
478
|
-
const expanded = {
|
|
479
|
-
subject,
|
|
480
|
-
types: types.map(t => expandIRI(typeof t === 'string' ? t : t.iri, ctx)),
|
|
481
|
-
predicates: predicates.map(p => ({ iri: expandIRI(p.iri, ctx), form: p.form }))
|
|
482
|
-
};
|
|
483
|
-
|
|
484
|
-
const signature = [subject, carrierType || 'unknown', expanded.types.join(','), expanded.predicates.map(p => `${p.form}${p.iri}`).join(',')].join('|');
|
|
485
|
-
const blockId = hash(signature);
|
|
486
|
-
|
|
487
|
-
return {
|
|
488
|
-
id: blockId,
|
|
489
|
-
range: { start: range[0], end: range[1] },
|
|
490
|
-
carrierType: carrierType || null,
|
|
491
|
-
subject,
|
|
492
|
-
types: expanded.types,
|
|
493
|
-
predicates: expanded.predicates,
|
|
494
|
-
context: ctx,
|
|
495
|
-
text: text || ''
|
|
496
|
-
};
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
function emitQuad(quads, quadBuffer, removeSet, quadIndex, block, subject, predicate, object, dataFactory, meta = null, statements = null, statementCandidates = null, state = null) {
|
|
500
|
-
if (!subject || !predicate || !object) return;
|
|
501
|
-
|
|
502
|
-
const quad = dataFactory.quad(subject, predicate, object);
|
|
503
|
-
const remove = meta?.remove || false;
|
|
504
|
-
|
|
505
|
-
if (remove) {
|
|
506
|
-
// Check if quad exists in current buffer
|
|
507
|
-
const quadKey = quadIndexKey(quad.subject, quad.predicate, quad.object);
|
|
508
|
-
if (quadBuffer.has(quadKey)) {
|
|
509
|
-
// In current state → cancel, appears nowhere
|
|
510
|
-
quadBuffer.delete(quadKey);
|
|
511
|
-
// Also remove from quads array if present
|
|
512
|
-
const index = quads.findIndex(q =>
|
|
513
|
-
q.subject.value === quad.subject.value &&
|
|
514
|
-
q.predicate.value === quad.predicate.value &&
|
|
515
|
-
q.object.value === quad.object.value
|
|
516
|
-
);
|
|
517
|
-
if (index !== -1) {
|
|
518
|
-
quads.splice(index, 1);
|
|
519
|
-
}
|
|
520
|
-
// Remove from quadIndex
|
|
521
|
-
quadIndex.delete(quadKey);
|
|
522
|
-
} else {
|
|
523
|
-
// Not in current state → external retract
|
|
524
|
-
removeSet.add(quad);
|
|
525
|
-
}
|
|
526
|
-
} else {
|
|
527
|
-
// Add to buffer and quads
|
|
528
|
-
const quadKey = quadIndexKey(quad.subject, quad.predicate, quad.object);
|
|
529
|
-
quadBuffer.set(quadKey, quad);
|
|
530
|
-
quads.push(quad);
|
|
531
|
-
|
|
532
|
-
// Detect rdf:Statement pattern during single-pass parsing
|
|
533
|
-
detectStatementPatternSinglePass(quad, dataFactory, meta, statements, statementCandidates);
|
|
534
|
-
|
|
535
|
-
// Create lean origin entry using shared utility
|
|
536
|
-
const originEntry = createLeanOriginEntry(block, subject, predicate, meta);
|
|
537
|
-
|
|
538
|
-
quadIndex.set(quadKey, originEntry);
|
|
539
|
-
|
|
540
|
-
// Link block to this quad for reverse lookup during rendering
|
|
541
|
-
if (state.currentBlock && block.id === state.currentBlock.id) {
|
|
542
|
-
if (!state.currentBlock.quadKeys) {
|
|
543
|
-
state.currentBlock.quadKeys = [];
|
|
544
|
-
}
|
|
545
|
-
state.currentBlock.quadKeys.push(quadKey);
|
|
546
|
-
}
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
function detectStatementPatternSinglePass(quad, dataFactory, meta, statements = null, statementCandidates = null) {
|
|
551
|
-
// Skip if not called from parse context (for testing compatibility)
|
|
552
|
-
if (!statements || !statementCandidates) return;
|
|
553
|
-
|
|
554
|
-
const predicate = quad.predicate.value;
|
|
555
|
-
|
|
556
|
-
// Early filter: only process rdf:Statement related predicates
|
|
557
|
-
if (predicate !== RDF_TYPE &&
|
|
558
|
-
predicate !== RDF_SUBJECT &&
|
|
559
|
-
predicate !== RDF_PREDICATE &&
|
|
560
|
-
predicate !== RDF_OBJECT) {
|
|
561
|
-
return;
|
|
562
|
-
}
|
|
563
|
-
|
|
564
|
-
// Check if this quad starts a new rdf:Statement pattern
|
|
565
|
-
if (predicate === RDF_TYPE && quad.object.value === RDF_STATEMENT) {
|
|
566
|
-
statementCandidates.set(quad.subject.value, { spo: {} });
|
|
567
|
-
return;
|
|
568
|
-
}
|
|
569
|
-
|
|
570
|
-
// Check if this quad completes part of an existing rdf:Statement pattern
|
|
571
|
-
const candidate = statementCandidates.get(quad.subject.value);
|
|
572
|
-
if (!candidate) return;
|
|
573
|
-
|
|
574
|
-
// Direct property assignment instead of switch for better performance
|
|
575
|
-
if (predicate === RDF_SUBJECT) {
|
|
576
|
-
candidate.spo.subject = quad.object;
|
|
577
|
-
} else if (predicate === RDF_PREDICATE) {
|
|
578
|
-
candidate.spo.predicate = quad.object;
|
|
579
|
-
} else if (predicate === RDF_OBJECT) {
|
|
580
|
-
candidate.spo.object = quad.object;
|
|
581
|
-
// Store the original quad for potential literal extraction
|
|
582
|
-
candidate.objectQuad = quad;
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
// Check if pattern is complete and create elevated SPO quad
|
|
586
|
-
if (candidate.spo.subject && candidate.spo.predicate && candidate.spo.object) {
|
|
587
|
-
// Use the object directly - literal detection happens at parse time
|
|
588
|
-
const spoQuad = dataFactory.quad(
|
|
589
|
-
candidate.spo.subject,
|
|
590
|
-
candidate.spo.predicate,
|
|
591
|
-
candidate.spo.object
|
|
592
|
-
);
|
|
593
|
-
statements.push(spoQuad);
|
|
594
|
-
// Clean up candidate to avoid duplicate detection
|
|
595
|
-
statementCandidates.delete(quad.subject.value);
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
const createTypeQuad = (typeIRI, subject, state, block, entryIndex = null) => {
|
|
601
|
-
const expandedType = expandIRI(typeIRI, state.ctx);
|
|
602
|
-
const typeInfo = typeof entryIndex === 'object' ? entryIndex : { entryIndex, remove: false };
|
|
603
|
-
emitQuad(
|
|
604
|
-
state.quads, state.quadBuffer, state.removeSet, state.origin.quadIndex, block,
|
|
605
|
-
subject,
|
|
606
|
-
state.df.namedNode(expandIRI('rdf:type', state.ctx)),
|
|
607
|
-
state.df.namedNode(expandedType),
|
|
608
|
-
state.df,
|
|
609
|
-
{ kind: 'type', token: `.${typeIRI}`, expandedType, entryIndex: typeInfo.entryIndex, remove: typeInfo.remove },
|
|
610
|
-
state.statements, state.statementCandidates,
|
|
611
|
-
state
|
|
612
|
-
);
|
|
613
|
-
};
|
|
614
|
-
|
|
615
|
-
function processTypeAnnotations(sem, newSubject, localObject, carrierO, S, block, state, carrier) {
|
|
616
|
-
sem.types.forEach(t => {
|
|
617
|
-
const typeIRI = typeof t === 'string' ? t : t.iri;
|
|
618
|
-
const typeInfo = typeof t === 'string' ? { entryIndex: null, remove: false } : t;
|
|
619
|
-
// Type subject priority: explicit subject > soft object > carrier URL > current subject
|
|
620
|
-
let typeSubject = newSubject || localObject || carrierO || S;
|
|
621
|
-
createTypeQuad(typeIRI, typeSubject, state, block, typeInfo);
|
|
622
|
-
});
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
const determinePredicateRole = (pred, carrier, newSubject, previousSubject, localObject, newSubjectOrCarrierO, S, L) => {
|
|
626
|
-
if (pred.form === '' && carrier?.type === 'link' && carrier?.url && carrier.text === carrier.url) {
|
|
627
|
-
return null;
|
|
628
|
-
}
|
|
629
|
-
switch (pred.form) {
|
|
630
|
-
case '':
|
|
631
|
-
// Literal predicates: explicit subject > current subject, URL only when no explicit subject
|
|
632
|
-
return newSubject ? { subject: localObject || S, object: L }
|
|
633
|
-
: (carrier?.type === 'link' && carrier?.url && carrier.text !== carrier.url)
|
|
634
|
-
? { subject: newSubjectOrCarrierO, object: L }
|
|
635
|
-
: { subject: localObject || S, object: L };
|
|
636
|
-
case '?':
|
|
637
|
-
// Object predicates: use current subject → explicit object or URL
|
|
638
|
-
return { subject: newSubject ? previousSubject : S, object: localObject || newSubjectOrCarrierO };
|
|
639
|
-
case '!':
|
|
640
|
-
// Reverse predicates: explicit object or URL → current subject
|
|
641
|
-
return { subject: localObject || newSubjectOrCarrierO, object: newSubject ? previousSubject : S };
|
|
642
|
-
default:
|
|
643
|
-
return null;
|
|
644
|
-
}
|
|
645
|
-
};
|
|
646
|
-
|
|
647
|
-
function processPredicateAnnotations(sem, newSubject, previousSubject, localObject, newSubjectOrCarrierO, S, L, block, state, carrier) {
|
|
648
|
-
sem.predicates.forEach(pred => {
|
|
649
|
-
const role = determinePredicateRole(pred, carrier, newSubject, previousSubject, localObject, newSubjectOrCarrierO, S, L);
|
|
650
|
-
if (role) {
|
|
651
|
-
const P = state.df.namedNode(expandIRI(pred.iri, state.ctx));
|
|
652
|
-
emitQuad(state.quads, state.quadBuffer, state.removeSet, state.origin.quadIndex, block,
|
|
653
|
-
role.subject, P, role.object, state.df,
|
|
654
|
-
{ kind: 'pred', token: `${pred.form}${pred.iri}`, form: pred.form, expandedPredicate: P.value, entryIndex: pred.entryIndex, remove: pred.remove || false },
|
|
655
|
-
state.statements, state.statementCandidates,
|
|
656
|
-
state
|
|
657
|
-
);
|
|
658
|
-
}
|
|
659
|
-
});
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
function processAnnotation(carrier, sem, state, options = {}) {
|
|
663
|
-
// Use the enhanced block tracking version
|
|
664
|
-
processAnnotationWithBlockTracking(carrier, sem, state, options);
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
function processTokenAnnotations(token, state, tokenType) {
|
|
675
|
-
if (token.attrs) {
|
|
676
|
-
const sem = parseSemCached(token.attrs);
|
|
677
|
-
processAnnotation({
|
|
678
|
-
type: tokenType, text: token.text, range: token.range,
|
|
679
|
-
attrsRange: token.attrsRange || null, valueRange: token.valueRange || null
|
|
680
|
-
}, sem, state);
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
getCarriers(token).forEach(carrier => {
|
|
684
|
-
if (carrier.attrs) {
|
|
685
|
-
const sem = parseSemCached(carrier.attrs);
|
|
686
|
-
processAnnotation(carrier, sem, state);
|
|
687
|
-
}
|
|
688
|
-
});
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
function processStandaloneSubject(token, state) {
|
|
692
|
-
const result = detectStandaloneSubject(token.text);
|
|
693
|
-
if (!result) return;
|
|
694
|
-
|
|
695
|
-
const sem = parseSemCached(`{=${result.content}}`);
|
|
696
|
-
const attrsStart = token.range[0] + token.text.indexOf('{=');
|
|
697
|
-
processAnnotation({
|
|
698
|
-
type: 'standalone', text: '', range: token.range,
|
|
699
|
-
attrsRange: [attrsStart, attrsStart + (result.content ? result.content.length : 0)],
|
|
700
|
-
valueRange: null
|
|
701
|
-
}, sem, state);
|
|
702
|
-
}
|
|
703
|
-
|
|
704
|
-
const TOKEN_PROCESSORS = {
|
|
705
|
-
heading: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
|
|
706
|
-
code: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
|
|
707
|
-
blockquote: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
|
|
708
|
-
para: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry, [processStandaloneSubject]),
|
|
709
|
-
list: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
|
|
710
|
-
standalone: (token, state) => processStandaloneSubject(token, state),
|
|
711
|
-
};
|