mdld-parse 0.9.0 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/parse.js CHANGED
@@ -7,16 +7,16 @@ import {
7
7
  } from './utils.js';
8
8
  import {
9
9
  DEFAULT_CONTEXT,
10
- URL_REGEX,
11
- FENCE_REGEX,
12
- PREFIX_REGEX,
13
- HEADING_REGEX,
14
- UNORDERED_LIST_REGEX,
15
- BLOCKQUOTE_REGEX,
16
- STANDALONE_SUBJECT_REGEX,
17
- CARRIER_PATTERN_ARRAY,
18
-
19
10
  } from './constants.js';
11
+ import {
12
+ detectStandaloneSubject,
13
+ detectPrefix,
14
+ detectHeading,
15
+ detectList,
16
+ detectBlockquote,
17
+ detectFence,
18
+ scanInlineCarriers
19
+ } from './tokenizers.js';
20
20
  import {
21
21
 
22
22
  getFenceClosePattern,
@@ -76,7 +76,8 @@ export function parse(firstArg, secondArg = {}) {
76
76
  blockStack: []
77
77
  };
78
78
 
79
- state.tokens = scanTokens(text);
79
+ const scanResult = scanTokens(text);
80
+ state.tokens = scanResult.tokens;
80
81
 
81
82
  // Single-pass processing: resolve prefixes AND process tokens together
82
83
  for (let i = 0; i < state.tokens.length; i++) {
@@ -117,7 +118,15 @@ export function parse(firstArg, secondArg = {}) {
117
118
  }
118
119
  }
119
120
 
120
- return { quads: state.quads, remove: filteredRemove, statements: state.statements, origin: state.origin, context: state.ctx, primarySubject: state.primarySubject };
121
+ return {
122
+ quads: state.quads,
123
+ remove: filteredRemove,
124
+ statements: state.statements,
125
+ origin: state.origin,
126
+ context: state.ctx,
127
+ primarySubject: state.primarySubject,
128
+ md: scanResult.md
129
+ };
121
130
  }
122
131
 
123
132
 
@@ -130,38 +139,44 @@ function getCarriers(token) {
130
139
 
131
140
  function scanTokens(text) {
132
141
  const tokens = [];
142
+ const mdLines = [];
133
143
  const lines = text.split('\n');
134
144
  let pos = 0;
135
145
  let codeBlock = null;
136
146
  const PROCESSORS = [
137
- { type: 'fence', test: line => FENCE_REGEX.test(line.trim()), process: handleFence },
147
+ { type: 'fence', test: line => detectFence(line.trim()), process: handleFence },
138
148
  { type: 'content', test: () => codeBlock, process: line => codeBlock.content.push(line) },
139
- { type: 'prefix', test: line => PREFIX_REGEX.test(line), process: handlePrefix },
140
- { type: 'heading', test: line => HEADING_REGEX.test(line), process: handleHeading },
141
- { type: 'list', test: line => UNORDERED_LIST_REGEX.test(line), process: handleList },
142
- { type: 'blockquote', test: line => BLOCKQUOTE_REGEX.test(line), process: handleBlockquote },
149
+ { type: 'prefix', test: line => detectPrefix(line), process: handlePrefix },
150
+ { type: 'standalone', test: line => detectStandaloneSubject(line), process: handleStandaloneSubject },
151
+ { type: 'heading', test: line => detectHeading(line), process: handleHeading },
152
+ { type: 'list', test: line => detectList(line), process: handleList },
153
+ { type: 'blockquote', test: line => detectBlockquote(line), process: handleBlockquote },
143
154
  { type: 'para', test: line => line.trim(), process: handlePara }
144
155
  ];
145
156
 
146
157
  function handleFence(line, lineStart, pos) {
147
158
  const trimmedLine = line.trim();
148
159
  if (!codeBlock) {
149
- const fenceMatch = trimmedLine.match(FENCE_REGEX);
150
- if (!fenceMatch) return false;
160
+ const fenceResult = detectFence(trimmedLine);
161
+ if (!fenceResult) return false;
151
162
 
152
- const { lang, attrsText } = parseLangAndAttrs(fenceMatch[2]);
163
+ const attrsText = fenceResult.attrs;
153
164
  const attrsStartInLine = attrsText ? line.indexOf(attrsText) : -1;
154
165
  const contentStart = lineStart + line.length + 1;
155
166
 
156
167
  codeBlock = {
157
- fence: fenceMatch[1],
168
+ fence: fenceResult.fenceChar.repeat(fenceResult.fenceLength),
158
169
  start: lineStart,
159
170
  content: [],
160
- lang,
171
+ lang: fenceResult.lang,
161
172
  attrs: attrsText,
162
173
  attrsRange: attrsText && attrsStartInLine >= 0 ? [lineStart + attrsStartInLine, lineStart + attrsStartInLine + attrsText.length] : null,
163
174
  valueRangeStart: contentStart
164
175
  };
176
+
177
+ // Add stripped fence line to mdLines
178
+ const cleanFence = line.replace(/\s*\{[^}]+\}\s*$/, '');
179
+ mdLines.push(cleanFence);
165
180
  } else {
166
181
  const fenceChar = codeBlock.fence[0];
167
182
  const expectedFence = fenceChar.repeat(codeBlock.fence.length);
@@ -179,47 +194,109 @@ function scanTokens(text) {
179
194
  attrsRange: codeBlock.attrsRange,
180
195
  valueRange: [valueStart, valueEnd]
181
196
  });
197
+
198
+ // Add all code content to mdLines before closing fence
199
+ for (const contentLine of codeBlock.content) {
200
+ mdLines.push(contentLine);
201
+ }
202
+
182
203
  codeBlock = null;
204
+
205
+ // Add closing fence to mdLines without trailing newline
206
+ const closingFence = line.replace(/\r?\n.*$/, '');
207
+ mdLines.push(closingFence);
208
+ } else {
209
+ // Code content - don't add to mdLines here, will be added when closing block
210
+ // No need to add individual lines as they're processed when closing
183
211
  }
184
212
  }
185
213
  return true;
186
214
  }
187
215
 
188
216
  function handlePrefix(line, lineStart, pos) {
189
- const match = PREFIX_REGEX.exec(line);
190
- tokens.push({ type: 'prefix', prefix: match[1], iri: match[2].trim() });
217
+ const result = detectPrefix(line);
218
+ tokens.push({ type: 'prefix', prefix: result.prefix, iri: result.iri });
219
+ // Prefixes are typically stripped from clean MD
191
220
  return true;
192
221
  }
193
222
 
194
223
  function handleHeading(line, lineStart, pos) {
195
- const match = HEADING_REGEX.exec(line);
196
- const attrs = match[3] || null;
197
- const afterHashes = match[1].length;
198
- const rangeInfo = calcRangeInfo(line, attrs, lineStart, afterHashes, match[2].length);
199
- tokens.push(createToken('heading', [lineStart, pos - 1], match[2].trim(), attrs,
200
- rangeInfo.attrsRange, rangeInfo.valueRange, { depth: match[1].length }));
224
+ const result = detectHeading(line);
225
+ const attrs = result.attrs;
226
+ const afterHashes = result.depth;
227
+ const rangeInfo = calcRangeInfo(line, attrs, lineStart, afterHashes, result.content.length);
228
+ tokens.push(createToken('heading', [lineStart, pos - 1], result.content, attrs,
229
+ rangeInfo.attrsRange, rangeInfo.valueRange, { depth: result.depth }));
230
+
231
+ // Add stripped heading to mdLines
232
+ const cleanHeading = `${'#'.repeat(result.depth)} ${result.content}`;
233
+ mdLines.push(cleanHeading);
201
234
  return true;
202
235
  }
203
236
 
204
237
  function handleList(line, lineStart, pos) {
205
- const match = UNORDERED_LIST_REGEX.exec(line);
238
+ const result = detectList(line);
239
+ // Create match array format expected by createListToken
240
+ const indentStr = ' '.repeat(result.indent);
241
+ const match = [line, indentStr, result.marker, result.content, result.attrs];
206
242
  tokens.push(createListToken('list', line, lineStart, pos, match));
243
+
244
+ // Add stripped list item to mdLines
245
+ const cleanList = `${indentStr}${result.marker} ${result.content}`;
246
+ mdLines.push(cleanList);
207
247
  return true;
208
248
  }
209
249
 
210
250
  function handleBlockquote(line, lineStart, pos) {
211
- const match = BLOCKQUOTE_REGEX.exec(line);
212
- const attrs = match[2] || null;
251
+ const result = detectBlockquote(line);
252
+ const attrs = result.attrs;
213
253
  const valueStartInLine = line.startsWith('> ') ? 2 : line.indexOf('>') + 1;
214
- const valueEndInLine = valueStartInLine + match[1].length;
215
- tokens.push(createToken('blockquote', [lineStart, pos - 1], match[1].trim(), attrs,
254
+ const valueEndInLine = valueStartInLine + result.content.length;
255
+ tokens.push(createToken('blockquote', [lineStart, pos - 1], result.content, attrs,
216
256
  calcAttrsRange(line, attrs, lineStart),
217
257
  [lineStart + valueStartInLine, lineStart + valueEndInLine]));
258
+
259
+ // Add stripped blockquote to mdLines
260
+ const cleanBlockquote = `> ${result.content}`;
261
+ mdLines.push(cleanBlockquote);
218
262
  return true;
219
263
  }
220
264
 
221
265
  function handlePara(line, lineStart, pos) {
222
266
  tokens.push(createToken('para', [lineStart, pos - 1], line.trim()));
267
+
268
+ // Add stripped paragraph to mdLines
269
+ let cleanPara = line;
270
+
271
+ // Remove inline carrier annotations using character-based detection
272
+ const carriers = scanInlineCarriers(cleanPara, 0);
273
+ for (const carrier of carriers) {
274
+ if (carrier.attrs && (carrier.type === 'emphasis' || carrier.type === 'code')) {
275
+ // Replace with just the content
276
+ const before = cleanPara.substring(0, carrier.range[0]);
277
+ const after = cleanPara.substring(carrier.range[1]);
278
+ cleanPara = before + (carrier.text || '') + after;
279
+ }
280
+ }
281
+
282
+ // Remove bracket-style annotations [text] {annotation}
283
+ cleanPara = cleanPara.replace(/\[([^\]]+)\]\s*\{[^}]+\}/g, '$1');
284
+
285
+ // Remove any remaining standalone annotations
286
+ cleanPara = cleanPara.replace(/\s*\{[^}]+\}\s*/g, ' ');
287
+
288
+ // Clean up extra whitespace
289
+ cleanPara = cleanPara.replace(/\s+/g, ' ').trim();
290
+
291
+ mdLines.push(cleanPara);
292
+ return true;
293
+ }
294
+
295
+ function handleStandaloneSubject(line, lineStart, pos) {
296
+ tokens.push({ type: 'standalone', text: line.trim(), range: [lineStart, pos - 1] });
297
+
298
+ // Standalone subjects are stripped from MD by default
299
+ // Don't add anything to mdLines
223
300
  return true;
224
301
  }
225
302
 
@@ -236,71 +313,13 @@ function scanTokens(text) {
236
313
  }
237
314
  }
238
315
 
239
- return tokens;
316
+ // Join without trailing newline to match expected output
317
+ const mdContent = mdLines.join('\n');
318
+ return { tokens, md: mdContent };
240
319
  }
241
320
 
242
321
  function extractInlineCarriers(text, baseOffset = 0) {
243
- const carriers = [];
244
- let pos = 0;
245
-
246
- const CARRIER_EXTRACTORS = {
247
- '<': (text, pos, baseOffset) => {
248
- const angleEnd = text.indexOf('>', pos);
249
- if (angleEnd === -1) return null;
250
- const url = text.slice(pos + 1, angleEnd);
251
- if (!URL_REGEX.test(url)) return null;
252
- const { attrs, attrsRange, finalSpanEnd } = extractAttributesFromText(text, angleEnd + 1, baseOffset);
253
- return createCarrier('link', url, attrs, attrsRange,
254
- [baseOffset + pos + 1, baseOffset + angleEnd],
255
- [baseOffset + pos, baseOffset + finalSpanEnd], finalSpanEnd, { url });
256
- },
257
- '[': (text, pos, baseOffset) => {
258
- const bracketEnd = findMatchingBracket(text, pos);
259
- if (!bracketEnd) return null;
260
- const carrierText = text.slice(pos + 1, bracketEnd - 1);
261
- const { url, spanEnd } = extractUrlFromBrackets(text, bracketEnd);
262
- const { attrs, attrsRange, finalSpanEnd } = extractAttributesFromText(text, spanEnd, baseOffset);
263
- const { carrierType, resourceIRI } = determineCarrierType(url);
264
- if (url?.startsWith('=')) return { skip: true, pos: finalSpanEnd };
265
- return createCarrier(carrierType, carrierText, attrs, attrsRange,
266
- [baseOffset + pos + 1, baseOffset + bracketEnd - 1],
267
- [baseOffset + pos, baseOffset + finalSpanEnd], finalSpanEnd, { url: resourceIRI });
268
- }
269
- };
270
-
271
- const extractCarrier = (text, pos, baseOffset) => {
272
- const extractor = CARRIER_EXTRACTORS[text[pos]];
273
- if (extractor) return extractor(text, pos, baseOffset);
274
-
275
- // Use pre-compiled patterns instead of Object.entries()
276
- for (const [type, pattern] of CARRIER_PATTERN_ARRAY) {
277
- pattern.lastIndex = pos;
278
- const match = pattern.exec(text);
279
- if (match) {
280
- const ranges = calcCarrierRanges(match, baseOffset, match.index);
281
- const carrierType = type === 'EMPHASIS' ? 'emphasis' : 'code';
282
- return createCarrier(carrierType, match[1], `{${match[2]}}`,
283
- ranges.attrsRange, ranges.valueRange, ranges.range, ranges.pos);
284
- }
285
- }
286
- return null;
287
- };
288
-
289
- while (pos < text.length) {
290
- const carrier = extractCarrier(text, pos, baseOffset);
291
- if (carrier) {
292
- if (carrier.skip) {
293
- pos = carrier.pos;
294
- } else {
295
- carriers.push(carrier);
296
- pos = carrier.pos;
297
- }
298
- } else {
299
- pos++;
300
- }
301
- }
302
-
303
- return carriers;
322
+ return scanInlineCarriers(text, baseOffset);
304
323
  }
305
324
 
306
325
 
@@ -670,14 +689,14 @@ function processTokenAnnotations(token, state, tokenType) {
670
689
  }
671
690
 
672
691
  function processStandaloneSubject(token, state) {
673
- const match = STANDALONE_SUBJECT_REGEX.exec(token.text);
674
- if (!match) return;
692
+ const result = detectStandaloneSubject(token.text);
693
+ if (!result) return;
675
694
 
676
- const sem = parseSemCached(`{=${match[1]}}`);
695
+ const sem = parseSemCached(`{=${result.content}}`);
677
696
  const attrsStart = token.range[0] + token.text.indexOf('{=');
678
697
  processAnnotation({
679
698
  type: 'standalone', text: '', range: token.range,
680
- attrsRange: [attrsStart, attrsStart + (match[1] ? match[1].length : 0)],
699
+ attrsRange: [attrsStart, attrsStart + (result.content ? result.content.length : 0)],
681
700
  valueRange: null
682
701
  }, sem, state);
683
702
  }
@@ -688,4 +707,5 @@ const TOKEN_PROCESSORS = {
688
707
  blockquote: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
689
708
  para: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry, [processStandaloneSubject]),
690
709
  list: (token, state) => processTokenWithBlockTracking(token, state, processTokenAnnotations, createBlockEntry),
710
+ standalone: (token, state) => processStandaloneSubject(token, state),
691
711
  };