mdld-parse 0.8.4 β 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +102 -13
- package/package.json +12 -2
- package/src/generate.js +73 -2
- package/src/index.js +1 -1
- package/src/merge.js +2 -1
- package/src/parse.js +117 -8
- package/src/render.js +400 -277
- package/src/shared.js +16 -2
- package/src/utils.js +2 -2
package/README.md
CHANGED
|
@@ -87,7 +87,8 @@ import { parse } from 'mdld-parse';
|
|
|
87
87
|
const markdown = `# Document {=ex:doc .Article}
|
|
88
88
|
[Alice] {author}`;
|
|
89
89
|
|
|
90
|
-
const result = parse(
|
|
90
|
+
const result = parse({
|
|
91
|
+
text: markdown,
|
|
91
92
|
context: { ex: 'http://example.org/' }
|
|
92
93
|
});
|
|
93
94
|
|
|
@@ -188,17 +189,19 @@ Declare resources inline with `{=iri}`:
|
|
|
188
189
|
|
|
189
190
|
## π§ API Reference
|
|
190
191
|
|
|
191
|
-
### `parse(
|
|
192
|
+
### `parse({ text, context, dataFactory, graph })`
|
|
192
193
|
|
|
193
194
|
Parse MD-LD markdown and return RDF quads with lean origin tracking.
|
|
194
195
|
|
|
195
|
-
**Parameters:**
|
|
196
|
-
- `
|
|
197
|
-
- `
|
|
198
|
-
|
|
199
|
-
|
|
196
|
+
**Parameters (named object):**
|
|
197
|
+
- `text` (string, required) β MD-LD formatted text
|
|
198
|
+
- `context` (object, optional) β Prefix mappings (default: `{ '@vocab': 'http://www.w3.org/2000/01/rdf-schema#', rdf, rdfs, xsd, sh, prov }`)
|
|
199
|
+
- `dataFactory` (object, optional) β Custom RDF/JS DataFactory
|
|
200
|
+
- `graph` (string, optional) β Named graph IRI
|
|
201
|
+
|
|
202
|
+
**Returns:** `{ quads, remove, statements, origin, context, primarySubject, md }`
|
|
200
203
|
|
|
201
|
-
**
|
|
204
|
+
> **Legacy:** `parse(text, options)` still works for backward compatibility
|
|
202
205
|
|
|
203
206
|
- `quads` β Array of RDF/JS Quads (final resolved graph state)
|
|
204
207
|
- `remove` β Array of RDF/JS Quads (external retractions targeting prior state)
|
|
@@ -206,6 +209,7 @@ Parse MD-LD markdown and return RDF quads with lean origin tracking.
|
|
|
206
209
|
- `origin` β Lean origin tracking object with quadIndex for UI navigation
|
|
207
210
|
- `context` β Final context used (includes prefixes)
|
|
208
211
|
- `primarySubject` β String IRI or null (first non-fragment subject declaration)
|
|
212
|
+
- `md` β Clean Markdown with all MD-LD annotations stripped (round-trip safe)
|
|
209
213
|
|
|
210
214
|
### `merge(docs, options)`
|
|
211
215
|
|
|
@@ -224,17 +228,102 @@ Merge multiple MDLD documents with diff polarity resolution.
|
|
|
224
228
|
- `context` β Final merged context
|
|
225
229
|
- `primarySubjects` β Array of string IRIs (primary subjects from each document, in merge order)
|
|
226
230
|
|
|
227
|
-
### `generate(quads, context, primarySubject)`
|
|
231
|
+
### `generate({ quads, context, primarySubject })`
|
|
228
232
|
|
|
229
|
-
Generate deterministic MDLD from RDF quads.
|
|
233
|
+
Generate deterministic MDLD from RDF quads with visual styling.
|
|
230
234
|
|
|
231
|
-
**Parameters:**
|
|
232
|
-
- `quads` (array) β Array of RDF/JS Quads to convert
|
|
235
|
+
**Parameters (named object):**
|
|
236
|
+
- `quads` (array, required) β Array of RDF/JS Quads to convert
|
|
233
237
|
- `context` (object, optional) β Prefix mappings (default: `{}`)
|
|
234
|
-
- `primarySubject` (string, optional) β String IRI to place first in output (ensures round-trip safety)
|
|
238
|
+
- `primarySubject` (string, optional) β String IRI to place first in output (ensures round-trip safety). If not provided, falls back to the first subject from quads.
|
|
235
239
|
|
|
236
240
|
**Returns:** `{ text, context }`
|
|
237
241
|
|
|
242
|
+
**Features:**
|
|
243
|
+
- Visual carrier styles based on datatype (code spans for numbers, bold booleans, etc.)
|
|
244
|
+
- Label-in-heading: Uses `rdfs:label` in subject headings when available
|
|
245
|
+
- Multiple labels: First label in heading, additional labels rendered as literals
|
|
246
|
+
- Round-trip safe: All data preserved through parse β generate β parse
|
|
247
|
+
- Composable: `generate(parse(text))` extracts semantics; `parse(generate({quads}))` normalizes quads
|
|
248
|
+
|
|
249
|
+
### `generateNode({ quads, focusIRI, context })`
|
|
250
|
+
|
|
251
|
+
Generate node-centric MDLD showing all quads where a specific IRI appears in any position.
|
|
252
|
+
|
|
253
|
+
**Parameters (named object):**
|
|
254
|
+
- `quads` (array, required) β Array of RDF/JS Quads to search
|
|
255
|
+
- `focusIRI` (string, required) β The IRI to center the view on
|
|
256
|
+
- `context` (object, optional) β Prefix mappings (default: `{}`)
|
|
257
|
+
|
|
258
|
+
**Returns:** `{ text, context }`
|
|
259
|
+
|
|
260
|
+
**Behavior (Safety-First):**
|
|
261
|
+
- If `focusIRI` is null/undefined: Returns empty text
|
|
262
|
+
- If `focusIRI` not in graph: Returns empty text (never falls back to all data)
|
|
263
|
+
- If `quads` is empty: Returns empty text
|
|
264
|
+
|
|
265
|
+
**Safety rationale:** Prevents accidental rendering of entire databases on misspelled IRIsβcritical for production use with LLM cost per token. Explicit emptiness signals "not found" to the caller.
|
|
266
|
+
|
|
267
|
+
**Use case:** Perfect for exploring a specific node and all its relationshipsβwhere it appears as subject, object, predicate, type, or datatype. Creates an exhaustive view of everything related to the focus IRI. Ideal for node-centric knowledge graph explorers.
|
|
268
|
+
|
|
269
|
+
### Composition Patterns
|
|
270
|
+
|
|
271
|
+
With the unified named parameter API, `parse()` and `generate()` compose seamlessly through object spreading:
|
|
272
|
+
|
|
273
|
+
```javascript
|
|
274
|
+
import { parse, generate, generateNode } from 'mdld-parse';
|
|
275
|
+
|
|
276
|
+
// Pattern 1: parse β generate (semantic extraction)
|
|
277
|
+
const canonical = generate({ ...parse({ text, context }) });
|
|
278
|
+
// text β quads β canonical MDLD (deterministic, visual styling applied)
|
|
279
|
+
|
|
280
|
+
// Pattern 2: generate β parse (normalize external RDF)
|
|
281
|
+
const normalized = parse({ ...generate({ quads: externalQuads, context }) });
|
|
282
|
+
// external quads β MDLD β validated quads (DataFactory-safe, no blank nodes)
|
|
283
|
+
|
|
284
|
+
// Pattern 3: parse β generateNode (node-centric exploration)
|
|
285
|
+
const nodeView = generateNode({ ...parse({ text }), focusIRI: 'http://example.org/alice' });
|
|
286
|
+
// full graph β isolated node view (safe: returns empty if IRI not found)
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
**Why this works:**
|
|
290
|
+
- `parse()` returns `{ quads, context, primarySubject, md, ... }`
|
|
291
|
+
- `generate()` accepts `{ quads, context, primarySubject }`
|
|
292
|
+
- `generateNode()` accepts `{ quads, context, focusIRI }` (with focusIRI override)
|
|
293
|
+
- Perfect shape alignment enables elegant `{ ...spread }` composition
|
|
294
|
+
|
|
295
|
+
### The `md` Field β Clean Markdown Extraction
|
|
296
|
+
|
|
297
|
+
Every `parse()` result includes a `md` field containing the original Markdown with all MD-LD annotations stripped:
|
|
298
|
+
|
|
299
|
+
```javascript
|
|
300
|
+
const result = parse({
|
|
301
|
+
text: `# Document {=ex:doc .Article}\n[Content] {ex:content}`,
|
|
302
|
+
context: { ex: 'http://example.org/' }
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
console.log(result.md);
|
|
306
|
+
// # Document\nContent
|
|
307
|
+
|
|
308
|
+
// Round-trip safety: re-parsing clean MD produces zero quads
|
|
309
|
+
const reparsed = parse({ text: result.md });
|
|
310
|
+
console.log(reparsed.quads.length); // 0
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
**Behavior:**
|
|
314
|
+
- Valid MD-LD annotations (`{=...}`, `{+...}`, `{...}`) are completely removed
|
|
315
|
+
- Content from value carriers (`[text]`, `**bold**`, `` `code` ``) is preserved
|
|
316
|
+
- Invalid syntax (annotations not at end-of-line) is preserved as visible markers
|
|
317
|
+
- Headings, lists, blockquotes, code blocks maintain their structure
|
|
318
|
+
- Prefix declarations at start of line are stripped
|
|
319
|
+
- Standalone subject declarations (`{=ex:subject}`) are stripped
|
|
320
|
+
|
|
321
|
+
**Use cases:**
|
|
322
|
+
- **Content extraction** β Get readable Markdown without semantic markup
|
|
323
|
+
- **Syntax validation** β Remaining `{...}` patterns indicate invalid MD-LD syntax
|
|
324
|
+
- **Round-trip testing** β `parse(md).md` should parse to zero quads
|
|
325
|
+
- **Preview generation** β Show clean document before publishing
|
|
326
|
+
|
|
238
327
|
### `locate(quad, origin)`
|
|
239
328
|
|
|
240
329
|
Locate the origin entry for a quad using the lean origin system.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mdld-parse",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.1",
|
|
4
4
|
"description": "A standards-compliant parser for **MD-LD (Markdown-Linked Data)** β a human-friendly RDF authoring format that extends Markdown with semantic annotations.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
],
|
|
14
14
|
"scripts": {
|
|
15
15
|
"test": "node tests/index.js",
|
|
16
|
-
"dev": "
|
|
16
|
+
"dev": "vite dev",
|
|
17
|
+
"build": "vite build && vite build --config vite.config.lib.js && cp -r docs/ dist/docs/ && cp -r examples/ dist/examples/ && cp -r spec/ dist/spec/"
|
|
17
18
|
},
|
|
18
19
|
"keywords": [
|
|
19
20
|
"mdld",
|
|
@@ -38,5 +39,14 @@
|
|
|
38
39
|
"homepage": "https://mdld.js.org",
|
|
39
40
|
"bugs": {
|
|
40
41
|
"url": "https://github.com/davay42/mdld-parse/issues"
|
|
42
|
+
},
|
|
43
|
+
"devDependencies": {
|
|
44
|
+
"@unocss/extractor-pug": "^66.6.8",
|
|
45
|
+
"@vitejs/plugin-vue": "^6.0.6",
|
|
46
|
+
"unocss": "^66.6.8",
|
|
47
|
+
"vite": "^8.0.10",
|
|
48
|
+
"vite-plugin-singlefile": "^2.3.3",
|
|
49
|
+
"markdown-it": "^14.1.1",
|
|
50
|
+
"n3": "^2.0.3"
|
|
41
51
|
}
|
|
42
52
|
}
|
package/src/generate.js
CHANGED
|
@@ -37,14 +37,46 @@ export function extractLocalName(iri, ctx = {}) {
|
|
|
37
37
|
* Input: RDF quads + context + optional primarySubject (string IRI)
|
|
38
38
|
* Output: MDLD text
|
|
39
39
|
*/
|
|
40
|
-
export function generate(quads, context = {}, primarySubject = null) {
|
|
40
|
+
export function generate({ quads, context = {}, primarySubject = null }) {
|
|
41
41
|
const fullContext = { ...DEFAULT_CONTEXT, ...context };
|
|
42
42
|
|
|
43
43
|
const normalizedQuads = normalizeAndSortQuads(quads);
|
|
44
44
|
|
|
45
45
|
const subjectGroups = groupQuadsBySubject(normalizedQuads);
|
|
46
46
|
|
|
47
|
-
|
|
47
|
+
// Fallback: if no primarySubject provided, use first subject from quads
|
|
48
|
+
let effectivePrimary = primarySubject;
|
|
49
|
+
if (!effectivePrimary && normalizedQuads.length > 0) {
|
|
50
|
+
effectivePrimary = normalizedQuads[0].subject.value;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const { text } = buildDeterministicMDLD(subjectGroups, fullContext, effectivePrimary);
|
|
54
|
+
|
|
55
|
+
return { text, context: fullContext };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Generate node-centric MDLD showing all quads where a specific IRI appears
|
|
60
|
+
* in any position: subject, object, predicate, type, or datatype.
|
|
61
|
+
* Perfect for exploring individual nodes and their complete relationship graph.
|
|
62
|
+
*/
|
|
63
|
+
export function generateNode({ quads, focusIRI, context = {} }) {
|
|
64
|
+
// Validate: must have quads and a focus IRI
|
|
65
|
+
if (!quads?.length || !focusIRI) {
|
|
66
|
+
return { text: '', context: { ...DEFAULT_CONTEXT, ...context } };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const fullContext = { ...DEFAULT_CONTEXT, ...context };
|
|
70
|
+
const normalizedQuads = normalizeAndSortQuads(quads);
|
|
71
|
+
const nodeGroups = groupQuadsByNode(normalizedQuads);
|
|
72
|
+
|
|
73
|
+
// SAFETY: If focusIRI not in graph, return empty - NEVER fall back to all data
|
|
74
|
+
// This prevents accidental rendering of entire databases on misspelled IRIs
|
|
75
|
+
if (!nodeGroups.has(focusIRI)) {
|
|
76
|
+
return { text: '', context: fullContext };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const { text } = buildDeterministicMDLD(nodeGroups, fullContext, focusIRI);
|
|
48
80
|
|
|
49
81
|
return { text, context: fullContext };
|
|
50
82
|
}
|
|
@@ -86,6 +118,42 @@ function groupQuadsBySubject(quads) {
|
|
|
86
118
|
return groups;
|
|
87
119
|
}
|
|
88
120
|
|
|
121
|
+
function groupQuadsByNode(quads) {
|
|
122
|
+
const groups = new Map();
|
|
123
|
+
const RDFS_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type';
|
|
124
|
+
|
|
125
|
+
const ensure = (key) => {
|
|
126
|
+
if (!groups.has(key)) groups.set(key, []);
|
|
127
|
+
return groups.get(key);
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
for (const quad of quads) {
|
|
131
|
+
const { subject, predicate, object } = quad;
|
|
132
|
+
|
|
133
|
+
// 1. Subject (direct properties)
|
|
134
|
+
ensure(subject.value).push(quad);
|
|
135
|
+
|
|
136
|
+
// 2. Object (reverse relations - where this IRI is pointed to)
|
|
137
|
+
if (object.termType === 'NamedNode') {
|
|
138
|
+
ensure(object.value).push(quad);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// 3. Predicate (where this IRI is used as a property)
|
|
142
|
+
ensure(predicate.value).push(quad);
|
|
143
|
+
|
|
144
|
+
// 4. Type (instances - where this IRI is the class/type)
|
|
145
|
+
if (predicate.value === RDFS_TYPE && object.termType === 'NamedNode') {
|
|
146
|
+
ensure(object.value).push(quad);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// 5. Datatype (literals using this as their type)
|
|
150
|
+
if (object.termType === 'Literal' && object.datatype) {
|
|
151
|
+
ensure(object.datatype.value || object.datatype).push(quad);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return groups;
|
|
155
|
+
}
|
|
156
|
+
|
|
89
157
|
function buildDeterministicMDLD(subjectGroups, context, primarySubject = null) {
|
|
90
158
|
let text = '';
|
|
91
159
|
const usedPrefixes = collectUsedPrefixes(subjectGroups, context);
|
|
@@ -117,6 +185,9 @@ function buildDeterministicMDLD(subjectGroups, context, primarySubject = null) {
|
|
|
117
185
|
|
|
118
186
|
for (const subjectIRI of orderedSubjects) {
|
|
119
187
|
const subjectQuads = subjectGroups.get(subjectIRI);
|
|
188
|
+
// Skip if subject not found in groups (e.g., primarySubject provided but no quads for it)
|
|
189
|
+
if (!subjectQuads) continue;
|
|
190
|
+
|
|
120
191
|
const shortSubject = shortenIRI(subjectIRI, context);
|
|
121
192
|
|
|
122
193
|
// Separate types, literals, and objects using shared utility
|
package/src/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { parse } from './parse.js';
|
|
2
2
|
export { merge } from './merge.js';
|
|
3
|
-
export { generate } from './generate.js';
|
|
3
|
+
export { generate, generateNode } from './generate.js';
|
|
4
4
|
export { locate } from './locate.js';
|
|
5
5
|
export { render } from './render.js';
|
|
6
6
|
export { DEFAULT_CONTEXT } from './constants.js';
|
package/src/merge.js
CHANGED
package/src/parse.js
CHANGED
|
@@ -45,11 +45,19 @@ import {
|
|
|
45
45
|
} from './shared.js';
|
|
46
46
|
|
|
47
47
|
|
|
48
|
-
export function parse(
|
|
48
|
+
export function parse(firstArg, secondArg = {}) {
|
|
49
|
+
// Dual-mode API: backward compatible with (text, options) and new ({ text, context, ... })
|
|
50
|
+
const isNamedParams = typeof firstArg === 'object' && firstArg !== null && 'text' in firstArg;
|
|
51
|
+
|
|
52
|
+
const text = isNamedParams ? firstArg.text : firstArg;
|
|
53
|
+
const options = isNamedParams
|
|
54
|
+
? { context: firstArg.context, dataFactory: firstArg.dataFactory, graph: firstArg.graph }
|
|
55
|
+
: secondArg;
|
|
56
|
+
|
|
49
57
|
const state = {
|
|
50
58
|
ctx: { ...DEFAULT_CONTEXT, ...(options.context || {}) },
|
|
51
59
|
df: options.dataFactory || DataFactory,
|
|
52
|
-
graph: DataFactory.namedNode(options.graph)
|
|
60
|
+
graph: options.graph ? DataFactory.namedNode(options.graph) : DataFactory.defaultGraph(),
|
|
53
61
|
quads: [],
|
|
54
62
|
quadBuffer: new Map(),
|
|
55
63
|
removeSet: new Set(),
|
|
@@ -68,7 +76,8 @@ export function parse(text, options = {}) {
|
|
|
68
76
|
blockStack: []
|
|
69
77
|
};
|
|
70
78
|
|
|
71
|
-
|
|
79
|
+
const scanResult = scanTokens(text);
|
|
80
|
+
state.tokens = scanResult.tokens;
|
|
72
81
|
|
|
73
82
|
// Single-pass processing: resolve prefixes AND process tokens together
|
|
74
83
|
for (let i = 0; i < state.tokens.length; i++) {
|
|
@@ -109,7 +118,15 @@ export function parse(text, options = {}) {
|
|
|
109
118
|
}
|
|
110
119
|
}
|
|
111
120
|
|
|
112
|
-
return {
|
|
121
|
+
return {
|
|
122
|
+
quads: state.quads,
|
|
123
|
+
remove: filteredRemove,
|
|
124
|
+
statements: state.statements,
|
|
125
|
+
origin: state.origin,
|
|
126
|
+
context: state.ctx,
|
|
127
|
+
primarySubject: state.primarySubject,
|
|
128
|
+
md: scanResult.md
|
|
129
|
+
};
|
|
113
130
|
}
|
|
114
131
|
|
|
115
132
|
|
|
@@ -122,6 +139,7 @@ function getCarriers(token) {
|
|
|
122
139
|
|
|
123
140
|
function scanTokens(text) {
|
|
124
141
|
const tokens = [];
|
|
142
|
+
const mdLines = [];
|
|
125
143
|
const lines = text.split('\n');
|
|
126
144
|
let pos = 0;
|
|
127
145
|
let codeBlock = null;
|
|
@@ -129,6 +147,7 @@ function scanTokens(text) {
|
|
|
129
147
|
{ type: 'fence', test: line => FENCE_REGEX.test(line.trim()), process: handleFence },
|
|
130
148
|
{ type: 'content', test: () => codeBlock, process: line => codeBlock.content.push(line) },
|
|
131
149
|
{ type: 'prefix', test: line => PREFIX_REGEX.test(line), process: handlePrefix },
|
|
150
|
+
{ type: 'standalone', test: line => STANDALONE_SUBJECT_REGEX.test(line), process: handleStandaloneSubject },
|
|
132
151
|
{ type: 'heading', test: line => HEADING_REGEX.test(line), process: handleHeading },
|
|
133
152
|
{ type: 'list', test: line => UNORDERED_LIST_REGEX.test(line), process: handleList },
|
|
134
153
|
{ type: 'blockquote', test: line => BLOCKQUOTE_REGEX.test(line), process: handleBlockquote },
|
|
@@ -154,6 +173,10 @@ function scanTokens(text) {
|
|
|
154
173
|
attrsRange: attrsText && attrsStartInLine >= 0 ? [lineStart + attrsStartInLine, lineStart + attrsStartInLine + attrsText.length] : null,
|
|
155
174
|
valueRangeStart: contentStart
|
|
156
175
|
};
|
|
176
|
+
|
|
177
|
+
// Add stripped fence line to mdLines
|
|
178
|
+
const cleanFence = line.replace(/\s*\{[^}]+\}\s*$/, '');
|
|
179
|
+
mdLines.push(cleanFence);
|
|
157
180
|
} else {
|
|
158
181
|
const fenceChar = codeBlock.fence[0];
|
|
159
182
|
const expectedFence = fenceChar.repeat(codeBlock.fence.length);
|
|
@@ -171,7 +194,20 @@ function scanTokens(text) {
|
|
|
171
194
|
attrsRange: codeBlock.attrsRange,
|
|
172
195
|
valueRange: [valueStart, valueEnd]
|
|
173
196
|
});
|
|
197
|
+
|
|
198
|
+
// Add all code content to mdLines before closing fence
|
|
199
|
+
for (const contentLine of codeBlock.content) {
|
|
200
|
+
mdLines.push(contentLine);
|
|
201
|
+
}
|
|
202
|
+
|
|
174
203
|
codeBlock = null;
|
|
204
|
+
|
|
205
|
+
// Add closing fence to mdLines without trailing newline
|
|
206
|
+
const closingFence = line.replace(/\r?\n.*$/, '');
|
|
207
|
+
mdLines.push(closingFence);
|
|
208
|
+
} else {
|
|
209
|
+
// Code content - don't add to mdLines here, will be added when closing block
|
|
210
|
+
// No need to add individual lines as they're processed when closing
|
|
175
211
|
}
|
|
176
212
|
}
|
|
177
213
|
return true;
|
|
@@ -180,6 +216,7 @@ function scanTokens(text) {
|
|
|
180
216
|
function handlePrefix(line, lineStart, pos) {
|
|
181
217
|
const match = PREFIX_REGEX.exec(line);
|
|
182
218
|
tokens.push({ type: 'prefix', prefix: match[1], iri: match[2].trim() });
|
|
219
|
+
// Prefixes are typically stripped from clean MD
|
|
183
220
|
return true;
|
|
184
221
|
}
|
|
185
222
|
|
|
@@ -190,12 +227,20 @@ function scanTokens(text) {
|
|
|
190
227
|
const rangeInfo = calcRangeInfo(line, attrs, lineStart, afterHashes, match[2].length);
|
|
191
228
|
tokens.push(createToken('heading', [lineStart, pos - 1], match[2].trim(), attrs,
|
|
192
229
|
rangeInfo.attrsRange, rangeInfo.valueRange, { depth: match[1].length }));
|
|
230
|
+
|
|
231
|
+
// Add stripped heading to mdLines
|
|
232
|
+
const cleanHeading = `${match[1]} ${match[2].trim()}`;
|
|
233
|
+
mdLines.push(cleanHeading);
|
|
193
234
|
return true;
|
|
194
235
|
}
|
|
195
236
|
|
|
196
237
|
function handleList(line, lineStart, pos) {
|
|
197
238
|
const match = UNORDERED_LIST_REGEX.exec(line);
|
|
198
239
|
tokens.push(createListToken('list', line, lineStart, pos, match));
|
|
240
|
+
|
|
241
|
+
// Add stripped list item to mdLines
|
|
242
|
+
const cleanList = `${match[1]}${match[2]} ${match[3].trim()}`;
|
|
243
|
+
mdLines.push(cleanList);
|
|
199
244
|
return true;
|
|
200
245
|
}
|
|
201
246
|
|
|
@@ -207,11 +252,45 @@ function scanTokens(text) {
|
|
|
207
252
|
tokens.push(createToken('blockquote', [lineStart, pos - 1], match[1].trim(), attrs,
|
|
208
253
|
calcAttrsRange(line, attrs, lineStart),
|
|
209
254
|
[lineStart + valueStartInLine, lineStart + valueEndInLine]));
|
|
255
|
+
|
|
256
|
+
// Add stripped blockquote to mdLines
|
|
257
|
+
const cleanBlockquote = `> ${match[1].trim()}`;
|
|
258
|
+
mdLines.push(cleanBlockquote);
|
|
210
259
|
return true;
|
|
211
260
|
}
|
|
212
261
|
|
|
213
262
|
function handlePara(line, lineStart, pos) {
|
|
214
263
|
tokens.push(createToken('para', [lineStart, pos - 1], line.trim()));
|
|
264
|
+
|
|
265
|
+
// Add stripped paragraph to mdLines
|
|
266
|
+
let cleanPara = line;
|
|
267
|
+
|
|
268
|
+
// Remove inline carrier annotations using existing patterns
|
|
269
|
+
for (const [patternName, regex] of CARRIER_PATTERN_ARRAY) {
|
|
270
|
+
const globalRegex = new RegExp(regex.source, 'gy');
|
|
271
|
+
cleanPara = cleanPara.replace(globalRegex, (match, content, attrs) => {
|
|
272
|
+
return content || '';
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// Remove bracket-style annotations [text] {annotation}
|
|
277
|
+
cleanPara = cleanPara.replace(/\[([^\]]+)\]\s*\{[^}]+\}/g, '$1');
|
|
278
|
+
|
|
279
|
+
// Remove any remaining standalone annotations
|
|
280
|
+
cleanPara = cleanPara.replace(/\s*\{[^}]+\}\s*/g, ' ');
|
|
281
|
+
|
|
282
|
+
// Clean up extra whitespace
|
|
283
|
+
cleanPara = cleanPara.replace(/\s+/g, ' ').trim();
|
|
284
|
+
|
|
285
|
+
mdLines.push(cleanPara);
|
|
286
|
+
return true;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function handleStandaloneSubject(line, lineStart, pos) {
|
|
290
|
+
tokens.push({ type: 'standalone', text: line.trim(), range: [lineStart, pos - 1] });
|
|
291
|
+
|
|
292
|
+
// Standalone subjects are stripped from MD by default
|
|
293
|
+
// Don't add anything to mdLines
|
|
215
294
|
return true;
|
|
216
295
|
}
|
|
217
296
|
|
|
@@ -228,7 +307,9 @@ function scanTokens(text) {
|
|
|
228
307
|
}
|
|
229
308
|
}
|
|
230
309
|
|
|
231
|
-
|
|
310
|
+
// Join without trailing newline to match expected output
|
|
311
|
+
const mdContent = mdLines.join('\n');
|
|
312
|
+
return { tokens, md: mdContent };
|
|
232
313
|
}
|
|
233
314
|
|
|
234
315
|
function extractInlineCarriers(text, baseOffset = 0) {
|
|
@@ -300,6 +381,9 @@ function createBlockEntry(token, state) {
|
|
|
300
381
|
const blockId = token._blockId || hash(`${token.type}:${token.range?.[0]}:${token.range?.[1]}`);
|
|
301
382
|
token._blockId = blockId; // Store for later reference
|
|
302
383
|
|
|
384
|
+
// Extract inline carriers first to enable clean text extraction
|
|
385
|
+
const carriers = getCarriers(token);
|
|
386
|
+
|
|
303
387
|
const cleanText = extractCleanText(token);
|
|
304
388
|
|
|
305
389
|
const blockEntry = {
|
|
@@ -316,6 +400,29 @@ function createBlockEntry(token, state) {
|
|
|
316
400
|
quadKeys: [] // Will be populated during quad emission
|
|
317
401
|
};
|
|
318
402
|
|
|
403
|
+
// Process carriers and add to block
|
|
404
|
+
for (const carrier of carriers) {
|
|
405
|
+
const carrierInfo = {
|
|
406
|
+
type: carrier.type,
|
|
407
|
+
range: carrier.range,
|
|
408
|
+
text: carrier.text,
|
|
409
|
+
subject: null,
|
|
410
|
+
predicates: [],
|
|
411
|
+
sem: null
|
|
412
|
+
};
|
|
413
|
+
|
|
414
|
+
// Extract carrier-specific semantics
|
|
415
|
+
if (carrier.attrs) {
|
|
416
|
+
const carrierSem = parseSemCached(carrier.attrs);
|
|
417
|
+
carrierInfo.sem = carrierSem; // Store full semantics
|
|
418
|
+
carrierInfo.predicates = carrierSem.predicates || [];
|
|
419
|
+
carrierInfo.subject = carrierSem.subject;
|
|
420
|
+
carrierInfo.types = carrierSem.types || [];
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
blockEntry.carriers.push(carrierInfo);
|
|
424
|
+
}
|
|
425
|
+
|
|
319
426
|
// Store block and add to document structure
|
|
320
427
|
state.origin.blocks.set(blockId, blockEntry);
|
|
321
428
|
state.origin.documentStructure.push(blockEntry);
|
|
@@ -368,9 +475,10 @@ function enrichBlockFromAnnotation(blockEntry, sem, carrier, state) {
|
|
|
368
475
|
// Extract carrier-specific semantics
|
|
369
476
|
if (carrier.attrs) {
|
|
370
477
|
const carrierSem = parseSemCached(carrier.attrs);
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
478
|
+
carrierInfo.sem = carrierSem; // Store full semantics
|
|
479
|
+
carrierInfo.predicates = carrierSem.predicates || [];
|
|
480
|
+
carrierInfo.subject = carrierSem.subject;
|
|
481
|
+
carrierInfo.types = carrierSem.types || [];
|
|
374
482
|
}
|
|
375
483
|
|
|
376
484
|
blockEntry.carriers.push(carrierInfo);
|
|
@@ -653,4 +761,5 @@ const TOKEN_PROCESSORS = {
|
|
|
653
761
|
blockquote: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
|
|
654
762
|
para: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry, [processStandaloneSubject]),
|
|
655
763
|
list: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
|
|
764
|
+
standalone: (token, state) => processStandaloneSubject(token, state),
|
|
656
765
|
};
|