mdld-parse 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -3
- package/package.json +18 -2
- package/src/highlight.js +362 -0
- package/src/parse.js +120 -100
- package/src/render.js +337 -465
- package/src/tokenizers.js +594 -0
- package/src/utils.js +2 -2
package/src/parse.js
CHANGED
|
@@ -7,16 +7,16 @@ import {
|
|
|
7
7
|
} from './utils.js';
|
|
8
8
|
import {
|
|
9
9
|
DEFAULT_CONTEXT,
|
|
10
|
-
URL_REGEX,
|
|
11
|
-
FENCE_REGEX,
|
|
12
|
-
PREFIX_REGEX,
|
|
13
|
-
HEADING_REGEX,
|
|
14
|
-
UNORDERED_LIST_REGEX,
|
|
15
|
-
BLOCKQUOTE_REGEX,
|
|
16
|
-
STANDALONE_SUBJECT_REGEX,
|
|
17
|
-
CARRIER_PATTERN_ARRAY,
|
|
18
|
-
|
|
19
10
|
} from './constants.js';
|
|
11
|
+
import {
|
|
12
|
+
detectStandaloneSubject,
|
|
13
|
+
detectPrefix,
|
|
14
|
+
detectHeading,
|
|
15
|
+
detectList,
|
|
16
|
+
detectBlockquote,
|
|
17
|
+
detectFence,
|
|
18
|
+
scanInlineCarriers
|
|
19
|
+
} from './tokenizers.js';
|
|
20
20
|
import {
|
|
21
21
|
|
|
22
22
|
getFenceClosePattern,
|
|
@@ -76,7 +76,8 @@ export function parse(firstArg, secondArg = {}) {
|
|
|
76
76
|
blockStack: []
|
|
77
77
|
};
|
|
78
78
|
|
|
79
|
-
|
|
79
|
+
const scanResult = scanTokens(text);
|
|
80
|
+
state.tokens = scanResult.tokens;
|
|
80
81
|
|
|
81
82
|
// Single-pass processing: resolve prefixes AND process tokens together
|
|
82
83
|
for (let i = 0; i < state.tokens.length; i++) {
|
|
@@ -117,7 +118,15 @@ export function parse(firstArg, secondArg = {}) {
|
|
|
117
118
|
}
|
|
118
119
|
}
|
|
119
120
|
|
|
120
|
-
return {
|
|
121
|
+
return {
|
|
122
|
+
quads: state.quads,
|
|
123
|
+
remove: filteredRemove,
|
|
124
|
+
statements: state.statements,
|
|
125
|
+
origin: state.origin,
|
|
126
|
+
context: state.ctx,
|
|
127
|
+
primarySubject: state.primarySubject,
|
|
128
|
+
md: scanResult.md
|
|
129
|
+
};
|
|
121
130
|
}
|
|
122
131
|
|
|
123
132
|
|
|
@@ -130,38 +139,44 @@ function getCarriers(token) {
|
|
|
130
139
|
|
|
131
140
|
function scanTokens(text) {
|
|
132
141
|
const tokens = [];
|
|
142
|
+
const mdLines = [];
|
|
133
143
|
const lines = text.split('\n');
|
|
134
144
|
let pos = 0;
|
|
135
145
|
let codeBlock = null;
|
|
136
146
|
const PROCESSORS = [
|
|
137
|
-
{ type: 'fence', test: line =>
|
|
147
|
+
{ type: 'fence', test: line => detectFence(line.trim()), process: handleFence },
|
|
138
148
|
{ type: 'content', test: () => codeBlock, process: line => codeBlock.content.push(line) },
|
|
139
|
-
{ type: 'prefix', test: line =>
|
|
140
|
-
{ type: '
|
|
141
|
-
{ type: '
|
|
142
|
-
{ type: '
|
|
149
|
+
{ type: 'prefix', test: line => detectPrefix(line), process: handlePrefix },
|
|
150
|
+
{ type: 'standalone', test: line => detectStandaloneSubject(line), process: handleStandaloneSubject },
|
|
151
|
+
{ type: 'heading', test: line => detectHeading(line), process: handleHeading },
|
|
152
|
+
{ type: 'list', test: line => detectList(line), process: handleList },
|
|
153
|
+
{ type: 'blockquote', test: line => detectBlockquote(line), process: handleBlockquote },
|
|
143
154
|
{ type: 'para', test: line => line.trim(), process: handlePara }
|
|
144
155
|
];
|
|
145
156
|
|
|
146
157
|
function handleFence(line, lineStart, pos) {
|
|
147
158
|
const trimmedLine = line.trim();
|
|
148
159
|
if (!codeBlock) {
|
|
149
|
-
const
|
|
150
|
-
if (!
|
|
160
|
+
const fenceResult = detectFence(trimmedLine);
|
|
161
|
+
if (!fenceResult) return false;
|
|
151
162
|
|
|
152
|
-
const
|
|
163
|
+
const attrsText = fenceResult.attrs;
|
|
153
164
|
const attrsStartInLine = attrsText ? line.indexOf(attrsText) : -1;
|
|
154
165
|
const contentStart = lineStart + line.length + 1;
|
|
155
166
|
|
|
156
167
|
codeBlock = {
|
|
157
|
-
fence:
|
|
168
|
+
fence: fenceResult.fenceChar.repeat(fenceResult.fenceLength),
|
|
158
169
|
start: lineStart,
|
|
159
170
|
content: [],
|
|
160
|
-
lang,
|
|
171
|
+
lang: fenceResult.lang,
|
|
161
172
|
attrs: attrsText,
|
|
162
173
|
attrsRange: attrsText && attrsStartInLine >= 0 ? [lineStart + attrsStartInLine, lineStart + attrsStartInLine + attrsText.length] : null,
|
|
163
174
|
valueRangeStart: contentStart
|
|
164
175
|
};
|
|
176
|
+
|
|
177
|
+
// Add stripped fence line to mdLines
|
|
178
|
+
const cleanFence = line.replace(/\s*\{[^}]+\}\s*$/, '');
|
|
179
|
+
mdLines.push(cleanFence);
|
|
165
180
|
} else {
|
|
166
181
|
const fenceChar = codeBlock.fence[0];
|
|
167
182
|
const expectedFence = fenceChar.repeat(codeBlock.fence.length);
|
|
@@ -179,47 +194,109 @@ function scanTokens(text) {
|
|
|
179
194
|
attrsRange: codeBlock.attrsRange,
|
|
180
195
|
valueRange: [valueStart, valueEnd]
|
|
181
196
|
});
|
|
197
|
+
|
|
198
|
+
// Add all code content to mdLines before closing fence
|
|
199
|
+
for (const contentLine of codeBlock.content) {
|
|
200
|
+
mdLines.push(contentLine);
|
|
201
|
+
}
|
|
202
|
+
|
|
182
203
|
codeBlock = null;
|
|
204
|
+
|
|
205
|
+
// Add closing fence to mdLines without trailing newline
|
|
206
|
+
const closingFence = line.replace(/\r?\n.*$/, '');
|
|
207
|
+
mdLines.push(closingFence);
|
|
208
|
+
} else {
|
|
209
|
+
// Code content - don't add to mdLines here, will be added when closing block
|
|
210
|
+
// No need to add individual lines as they're processed when closing
|
|
183
211
|
}
|
|
184
212
|
}
|
|
185
213
|
return true;
|
|
186
214
|
}
|
|
187
215
|
|
|
188
216
|
function handlePrefix(line, lineStart, pos) {
|
|
189
|
-
const
|
|
190
|
-
tokens.push({ type: 'prefix', prefix:
|
|
217
|
+
const result = detectPrefix(line);
|
|
218
|
+
tokens.push({ type: 'prefix', prefix: result.prefix, iri: result.iri });
|
|
219
|
+
// Prefixes are typically stripped from clean MD
|
|
191
220
|
return true;
|
|
192
221
|
}
|
|
193
222
|
|
|
194
223
|
function handleHeading(line, lineStart, pos) {
|
|
195
|
-
const
|
|
196
|
-
const attrs =
|
|
197
|
-
const afterHashes =
|
|
198
|
-
const rangeInfo = calcRangeInfo(line, attrs, lineStart, afterHashes,
|
|
199
|
-
tokens.push(createToken('heading', [lineStart, pos - 1],
|
|
200
|
-
rangeInfo.attrsRange, rangeInfo.valueRange, { depth:
|
|
224
|
+
const result = detectHeading(line);
|
|
225
|
+
const attrs = result.attrs;
|
|
226
|
+
const afterHashes = result.depth;
|
|
227
|
+
const rangeInfo = calcRangeInfo(line, attrs, lineStart, afterHashes, result.content.length);
|
|
228
|
+
tokens.push(createToken('heading', [lineStart, pos - 1], result.content, attrs,
|
|
229
|
+
rangeInfo.attrsRange, rangeInfo.valueRange, { depth: result.depth }));
|
|
230
|
+
|
|
231
|
+
// Add stripped heading to mdLines
|
|
232
|
+
const cleanHeading = `${'#'.repeat(result.depth)} ${result.content}`;
|
|
233
|
+
mdLines.push(cleanHeading);
|
|
201
234
|
return true;
|
|
202
235
|
}
|
|
203
236
|
|
|
204
237
|
function handleList(line, lineStart, pos) {
|
|
205
|
-
const
|
|
238
|
+
const result = detectList(line);
|
|
239
|
+
// Create match array format expected by createListToken
|
|
240
|
+
const indentStr = ' '.repeat(result.indent);
|
|
241
|
+
const match = [line, indentStr, result.marker, result.content, result.attrs];
|
|
206
242
|
tokens.push(createListToken('list', line, lineStart, pos, match));
|
|
243
|
+
|
|
244
|
+
// Add stripped list item to mdLines
|
|
245
|
+
const cleanList = `${indentStr}${result.marker} ${result.content}`;
|
|
246
|
+
mdLines.push(cleanList);
|
|
207
247
|
return true;
|
|
208
248
|
}
|
|
209
249
|
|
|
210
250
|
function handleBlockquote(line, lineStart, pos) {
|
|
211
|
-
const
|
|
212
|
-
const attrs =
|
|
251
|
+
const result = detectBlockquote(line);
|
|
252
|
+
const attrs = result.attrs;
|
|
213
253
|
const valueStartInLine = line.startsWith('> ') ? 2 : line.indexOf('>') + 1;
|
|
214
|
-
const valueEndInLine = valueStartInLine +
|
|
215
|
-
tokens.push(createToken('blockquote', [lineStart, pos - 1],
|
|
254
|
+
const valueEndInLine = valueStartInLine + result.content.length;
|
|
255
|
+
tokens.push(createToken('blockquote', [lineStart, pos - 1], result.content, attrs,
|
|
216
256
|
calcAttrsRange(line, attrs, lineStart),
|
|
217
257
|
[lineStart + valueStartInLine, lineStart + valueEndInLine]));
|
|
258
|
+
|
|
259
|
+
// Add stripped blockquote to mdLines
|
|
260
|
+
const cleanBlockquote = `> ${result.content}`;
|
|
261
|
+
mdLines.push(cleanBlockquote);
|
|
218
262
|
return true;
|
|
219
263
|
}
|
|
220
264
|
|
|
221
265
|
function handlePara(line, lineStart, pos) {
|
|
222
266
|
tokens.push(createToken('para', [lineStart, pos - 1], line.trim()));
|
|
267
|
+
|
|
268
|
+
// Add stripped paragraph to mdLines
|
|
269
|
+
let cleanPara = line;
|
|
270
|
+
|
|
271
|
+
// Remove inline carrier annotations using character-based detection
|
|
272
|
+
const carriers = scanInlineCarriers(cleanPara, 0);
|
|
273
|
+
for (const carrier of carriers) {
|
|
274
|
+
if (carrier.attrs && (carrier.type === 'emphasis' || carrier.type === 'code')) {
|
|
275
|
+
// Replace with just the content
|
|
276
|
+
const before = cleanPara.substring(0, carrier.range[0]);
|
|
277
|
+
const after = cleanPara.substring(carrier.range[1]);
|
|
278
|
+
cleanPara = before + (carrier.text || '') + after;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// Remove bracket-style annotations [text] {annotation}
|
|
283
|
+
cleanPara = cleanPara.replace(/\[([^\]]+)\]\s*\{[^}]+\}/g, '$1');
|
|
284
|
+
|
|
285
|
+
// Remove any remaining standalone annotations
|
|
286
|
+
cleanPara = cleanPara.replace(/\s*\{[^}]+\}\s*/g, ' ');
|
|
287
|
+
|
|
288
|
+
// Clean up extra whitespace
|
|
289
|
+
cleanPara = cleanPara.replace(/\s+/g, ' ').trim();
|
|
290
|
+
|
|
291
|
+
mdLines.push(cleanPara);
|
|
292
|
+
return true;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function handleStandaloneSubject(line, lineStart, pos) {
|
|
296
|
+
tokens.push({ type: 'standalone', text: line.trim(), range: [lineStart, pos - 1] });
|
|
297
|
+
|
|
298
|
+
// Standalone subjects are stripped from MD by default
|
|
299
|
+
// Don't add anything to mdLines
|
|
223
300
|
return true;
|
|
224
301
|
}
|
|
225
302
|
|
|
@@ -236,71 +313,13 @@ function scanTokens(text) {
|
|
|
236
313
|
}
|
|
237
314
|
}
|
|
238
315
|
|
|
239
|
-
|
|
316
|
+
// Join without trailing newline to match expected output
|
|
317
|
+
const mdContent = mdLines.join('\n');
|
|
318
|
+
return { tokens, md: mdContent };
|
|
240
319
|
}
|
|
241
320
|
|
|
242
321
|
function extractInlineCarriers(text, baseOffset = 0) {
|
|
243
|
-
|
|
244
|
-
let pos = 0;
|
|
245
|
-
|
|
246
|
-
const CARRIER_EXTRACTORS = {
|
|
247
|
-
'<': (text, pos, baseOffset) => {
|
|
248
|
-
const angleEnd = text.indexOf('>', pos);
|
|
249
|
-
if (angleEnd === -1) return null;
|
|
250
|
-
const url = text.slice(pos + 1, angleEnd);
|
|
251
|
-
if (!URL_REGEX.test(url)) return null;
|
|
252
|
-
const { attrs, attrsRange, finalSpanEnd } = extractAttributesFromText(text, angleEnd + 1, baseOffset);
|
|
253
|
-
return createCarrier('link', url, attrs, attrsRange,
|
|
254
|
-
[baseOffset + pos + 1, baseOffset + angleEnd],
|
|
255
|
-
[baseOffset + pos, baseOffset + finalSpanEnd], finalSpanEnd, { url });
|
|
256
|
-
},
|
|
257
|
-
'[': (text, pos, baseOffset) => {
|
|
258
|
-
const bracketEnd = findMatchingBracket(text, pos);
|
|
259
|
-
if (!bracketEnd) return null;
|
|
260
|
-
const carrierText = text.slice(pos + 1, bracketEnd - 1);
|
|
261
|
-
const { url, spanEnd } = extractUrlFromBrackets(text, bracketEnd);
|
|
262
|
-
const { attrs, attrsRange, finalSpanEnd } = extractAttributesFromText(text, spanEnd, baseOffset);
|
|
263
|
-
const { carrierType, resourceIRI } = determineCarrierType(url);
|
|
264
|
-
if (url?.startsWith('=')) return { skip: true, pos: finalSpanEnd };
|
|
265
|
-
return createCarrier(carrierType, carrierText, attrs, attrsRange,
|
|
266
|
-
[baseOffset + pos + 1, baseOffset + bracketEnd - 1],
|
|
267
|
-
[baseOffset + pos, baseOffset + finalSpanEnd], finalSpanEnd, { url: resourceIRI });
|
|
268
|
-
}
|
|
269
|
-
};
|
|
270
|
-
|
|
271
|
-
const extractCarrier = (text, pos, baseOffset) => {
|
|
272
|
-
const extractor = CARRIER_EXTRACTORS[text[pos]];
|
|
273
|
-
if (extractor) return extractor(text, pos, baseOffset);
|
|
274
|
-
|
|
275
|
-
// Use pre-compiled patterns instead of Object.entries()
|
|
276
|
-
for (const [type, pattern] of CARRIER_PATTERN_ARRAY) {
|
|
277
|
-
pattern.lastIndex = pos;
|
|
278
|
-
const match = pattern.exec(text);
|
|
279
|
-
if (match) {
|
|
280
|
-
const ranges = calcCarrierRanges(match, baseOffset, match.index);
|
|
281
|
-
const carrierType = type === 'EMPHASIS' ? 'emphasis' : 'code';
|
|
282
|
-
return createCarrier(carrierType, match[1], `{${match[2]}}`,
|
|
283
|
-
ranges.attrsRange, ranges.valueRange, ranges.range, ranges.pos);
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
return null;
|
|
287
|
-
};
|
|
288
|
-
|
|
289
|
-
while (pos < text.length) {
|
|
290
|
-
const carrier = extractCarrier(text, pos, baseOffset);
|
|
291
|
-
if (carrier) {
|
|
292
|
-
if (carrier.skip) {
|
|
293
|
-
pos = carrier.pos;
|
|
294
|
-
} else {
|
|
295
|
-
carriers.push(carrier);
|
|
296
|
-
pos = carrier.pos;
|
|
297
|
-
}
|
|
298
|
-
} else {
|
|
299
|
-
pos++;
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
return carriers;
|
|
322
|
+
return scanInlineCarriers(text, baseOffset);
|
|
304
323
|
}
|
|
305
324
|
|
|
306
325
|
|
|
@@ -670,14 +689,14 @@ function processTokenAnnotations(token, state, tokenType) {
|
|
|
670
689
|
}
|
|
671
690
|
|
|
672
691
|
function processStandaloneSubject(token, state) {
|
|
673
|
-
const
|
|
674
|
-
if (!
|
|
692
|
+
const result = detectStandaloneSubject(token.text);
|
|
693
|
+
if (!result) return;
|
|
675
694
|
|
|
676
|
-
const sem = parseSemCached(`{=${
|
|
695
|
+
const sem = parseSemCached(`{=${result.content}}`);
|
|
677
696
|
const attrsStart = token.range[0] + token.text.indexOf('{=');
|
|
678
697
|
processAnnotation({
|
|
679
698
|
type: 'standalone', text: '', range: token.range,
|
|
680
|
-
attrsRange: [attrsStart, attrsStart + (
|
|
699
|
+
attrsRange: [attrsStart, attrsStart + (result.content ? result.content.length : 0)],
|
|
681
700
|
valueRange: null
|
|
682
701
|
}, sem, state);
|
|
683
702
|
}
|
|
@@ -688,4 +707,5 @@ const TOKEN_PROCESSORS = {
|
|
|
688
707
|
blockquote: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
|
|
689
708
|
para: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry, [processStandaloneSubject]),
|
|
690
709
|
list: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
|
|
710
|
+
standalone: (token, state) => processStandaloneSubject(token, state),
|
|
691
711
|
};
|