@uniweb/content-writer 0.3.4 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,12 +1,11 @@
1
1
  {
2
2
  "name": "@uniweb/content-writer",
3
- "version": "0.3.4",
3
+ "version": "1.1.0",
4
4
  "description": "ProseMirror document structure to Markdown converter",
5
5
  "type": "module",
6
6
  "main": "src/index.js",
7
7
  "exports": {
8
- ".": "./src/index.js",
9
- "./entities": "./src/entities.js"
8
+ ".": "./src/index.js"
10
9
  },
11
10
  "keywords": [
12
11
  "markdown",
@@ -20,7 +19,7 @@
20
19
  },
21
20
  "devDependencies": {
22
21
  "vitest": "^4.1.7",
23
- "@uniweb/content-reader": "1.2.4"
22
+ "@uniweb/content-reader": "1.3.0"
24
23
  },
25
24
  "directories": {
26
25
  "test": "tests"
@@ -44,12 +44,40 @@ export function resetDiagnostics() {
44
44
  warnedMarks.clear()
45
45
  }
46
46
 
47
+ /** Set while `collectOmittedNodes` runs: the node types the write leaves out. */
48
+ let omitted = null
49
+
50
+ /**
51
+ * Run a write and return the node types it left out, instead of reporting them.
52
+ * For a caller that must refuse rather than lose content — a pull, whose next
53
+ * push would delete whatever the file does not hold.
54
+ *
55
+ * @template T
56
+ * @param {() => T} run
57
+ * @returns {{ result: T, omitted: string[] }}
58
+ */
59
+ export function collectOmittedNodes(run) {
60
+ const previous = omitted
61
+ omitted = new Set()
62
+ try {
63
+ const result = run()
64
+ return { result, omitted: [...omitted] }
65
+ } finally {
66
+ omitted = previous
67
+ }
68
+ }
69
+
47
70
  /**
48
71
  * Report a ProseMirror node the serializer has no markdown form for.
49
72
  * @param {Object} node - The unmapped node ({ type, attrs, ... })
50
73
  */
51
74
  export function warnUnmappedNode(node) {
52
75
  const type = node?.type || 'unknown'
76
+ // A caller collecting what a write leaves out is told every time, not once per type.
77
+ if (omitted) {
78
+ omitted.add(type)
79
+ return
80
+ }
53
81
  if (warnedNodes.has(type)) return
54
82
  warnedNodes.add(type)
55
83
  report(
package/src/index.js CHANGED
@@ -5,11 +5,10 @@
5
5
  * documents back to Markdown text.
6
6
  */
7
7
 
8
- export { serializeDoc as proseMirrorToMarkdown } from './serializer.js'
8
+ export { serializeDoc as proseMirrorToMarkdown, unwritableNodes } from './serializer.js'
9
9
  export { serializeInlineContent } from './marks.js'
10
10
  export { serializeFrontmatter } from './frontmatter.js'
11
11
  export { docToPlainText, docSummaryText } from './plain-text.js'
12
- export { encodeMarkupEntities, decodeMarkupEntities, decodeTextEntities } from './entities.js'
13
12
  export { normalizeDialect } from './normalize.js'
14
13
  export { setDiagnosticsReporter, resetDiagnostics } from './diagnostics.js'
15
14
 
package/src/marks.js CHANGED
@@ -5,7 +5,6 @@
5
5
  */
6
6
 
7
7
  import { serializeAttributes } from './attributes.js'
8
- import { decodeMarkupEntities } from './entities.js'
9
8
  import { warnUnmappedMark, warnUnmappedNode } from './diagnostics.js'
10
9
 
11
10
  /**
@@ -232,13 +231,58 @@ export function serializeBlockImage(node) {
232
231
  * @param {Array} content - Array of text/image nodes with optional marks
233
232
  * @returns {string} Markdown string
234
233
  */
235
- export function serializeInlineContent(content) {
234
+ export function serializeInlineContent(content, { lineBreak = '\\\n' } = {}) {
236
235
  if (!content || content.length === 0) return ''
237
236
 
238
237
  // No silent drops: flag any mark we can't represent before serializing.
239
238
  reportUnknownMarks(content)
240
239
 
241
- return serializeRuns(content)
240
+ return serializeRuns(mergeAdjacentText(content), { lineBreak, literal: false })
241
+ }
242
+
243
+ /**
244
+ * Join neighbouring text nodes that carry the same marks, as ProseMirror itself
245
+ * would. The reader splits a run wherever markdown had an escape — `\<name>`
246
+ * arrives as `<` and `name>` — and the escaping below has to see the run whole,
247
+ * or a `<` that ends one node would go out bare before the tag-like text of the
248
+ * next.
249
+ *
250
+ * @param {Array} content
251
+ * @returns {Array}
252
+ */
253
+ function mergeAdjacentText(content) {
254
+ const out = []
255
+ for (const node of content) {
256
+ const last = out[out.length - 1]
257
+ if (
258
+ node?.type === 'text' &&
259
+ last?.type === 'text' &&
260
+ JSON.stringify(last.marks || []) === JSON.stringify(node.marks || [])
261
+ ) {
262
+ out[out.length - 1] = { ...last, text: (last.text || '') + (node.text || '') }
263
+ } else {
264
+ out.push(node)
265
+ }
266
+ }
267
+ return out
268
+ }
269
+
270
+ /**
271
+ * Escape text so the reader decodes it back to the same characters.
272
+ *
273
+ * A text node holds the characters a reader sees, and content-reader decodes
274
+ * what markdown escapes — so the two characters it would otherwise read as
275
+ * something else are escaped here: a `<` that would open a tag or an autolink,
276
+ * and an `&` that would spell a character reference. `a < b` and `R&D` stay as
277
+ * they are. ⛔ Until 2026-10-14 a text node held HTML and was written verbatim.
278
+ *
279
+ * @param {string} text
280
+ * @returns {string}
281
+ */
282
+ function escapeMarkdownText(text) {
283
+ return String(text)
284
+ .replace(/&(?=#[0-9]+;|#[xX][0-9a-fA-F]+;|[A-Za-z][A-Za-z0-9]*;)/g, '\\&')
285
+ .replace(/<(?=[A-Za-z/!?])/g, '\\<')
242
286
  }
243
287
 
244
288
  /**
@@ -262,7 +306,7 @@ export function serializeInlineContent(content) {
262
306
  * @param {Array} nodes - Inline nodes
263
307
  * @returns {string} Markdown string
264
308
  */
265
- function serializeRuns(nodes) {
309
+ function serializeRuns(nodes, context) {
266
310
  const out = []
267
311
  let i = 0
268
312
 
@@ -271,7 +315,7 @@ function serializeRuns(nodes) {
271
315
  const outer = node.type === 'text' ? outermostMark(node) : null
272
316
 
273
317
  if (!outer) {
274
- out.push(serializePlainNode(node))
318
+ out.push(serializePlainNode(node, context))
275
319
  i++
276
320
  continue
277
321
  }
@@ -286,7 +330,9 @@ function serializeRuns(nodes) {
286
330
  j++
287
331
  }
288
332
 
289
- out.push(applyMark(serializeRuns(group), outer))
333
+ // A code span is literal: what it wraps is written as the characters it is.
334
+ const inner = serializeRuns(group, outer.type === 'code' ? { ...context, literal: true } : context)
335
+ out.push(applyMark(inner, outer))
290
336
  i = j
291
337
  }
292
338
 
@@ -327,9 +373,7 @@ function withoutOutermostMark(node) {
327
373
  function applyMark(inner, mark) {
328
374
  switch (mark.type) {
329
375
  case 'code':
330
- // Leaving HTML-land: a code span is the one place the reader leaves
331
- // `&lt;`/`&gt;` escaped (see entities.js).
332
- return serializeCodeSpan(decodeMarkupEntities(inner))
376
+ return serializeCodeSpan(inner)
333
377
  case 'bold':
334
378
  return `**${inner}**`
335
379
  case 'italic':
@@ -352,7 +396,7 @@ function applyMark(inner, mark) {
352
396
  /**
353
397
  * Serialize a plain node (no wrapping mark).
354
398
  */
355
- function serializePlainNode(node) {
399
+ function serializePlainNode(node, { lineBreak = '\\\n', literal = false } = {}) {
356
400
  if (node.type === 'image') {
357
401
  return serializeInlineImage(node)
358
402
  }
@@ -367,7 +411,9 @@ function serializePlainNode(node) {
367
411
  // Backslash rather than the two-trailing-spaces spelling: it survives
368
412
  // editors, formatters and pre-commit hooks that strip trailing whitespace,
369
413
  // and it is visible in a diff. Both spellings read back as a hard break.
370
- return '\\\n'
414
+ // A heading or a table cell is one line, so there it is `<br/>` — the one
415
+ // tag the reader keeps, as the same hard break.
416
+ return lineBreak
371
417
  }
372
418
  if (node.type !== 'text') {
373
419
  // No silent drops: an inline node we have no markdown form for is
@@ -376,7 +422,9 @@ function serializePlainNode(node) {
376
422
  warnUnmappedNode(node)
377
423
  return ''
378
424
  }
379
- return serializeTextWithMarks(node.text, node.marks || [])
425
+ const marks = node.marks || []
426
+ const keep = literal || marks.some((mark) => mark.type === 'code')
427
+ return serializeTextWithMarks(keep ? node.text : escapeMarkdownText(node.text), marks)
380
428
  }
381
429
 
382
430
  /**
package/src/nodes.js CHANGED
@@ -20,7 +20,8 @@ export { serializeInsetRef }
20
20
  */
21
21
  export function serializeHeading(node) {
22
22
  const prefix = '#'.repeat(node.attrs?.level || 1)
23
- const text = serializeInlineContent(node.content)
23
+ // A heading is one line: a hard break inside it is written `<br/>`.
24
+ const text = serializeInlineContent(node.content, { lineBreak: '<br/>' })
24
25
  // A `#>` label line (pretitle). The hash count carries the authored
25
26
  // spelling only — `#>` and `##>` mean the same thing — so emitting the
26
27
  // stored level keeps the file byte-identical through a round-trip.
@@ -443,7 +444,8 @@ function serializeTableCell(cell) {
443
444
  const text = cell.content
444
445
  .map(child => {
445
446
  if (child.type === 'paragraph') {
446
- return serializeInlineContent(child.content)
447
+ // A table row is one line: a hard break in a cell is written `<br/>`.
448
+ return serializeInlineContent(child.content, { lineBreak: '<br/>' })
447
449
  }
448
450
  return ''
449
451
  })
package/src/normalize.js CHANGED
@@ -55,8 +55,12 @@
55
55
  * regenerated fixture in `_contracts/editor-vocabulary-diff.test.js`
56
56
  * stopped listing them. That fixture is the signal, and it is the only one
57
57
  * there is — no message announces it.
58
- * - colour/highlight/strike/underline marks — need the named-inline-
59
- * style convention decision.
58
+ * - a colour that is not a named style (`textStyle` with a raw value), and
59
+ * `underline` — neither has a markdown form. `highlight` and the named
60
+ * styles (`accent`, `highlight`, `callout`, `muted`) ARE mapped, onto the
61
+ * `span` mark (`mapEditorMark`), and `strike` serializes as `~~text~~`.
62
+ * ⛔ This line listed all of them as awaiting a decision until 2026-10-14,
63
+ * long after the mapping shipped.
60
64
  *
61
65
  * Each unmapped construct reaches content-writer's serializer as-is and
62
66
  * is reported by the no-silent-drop guard (diagnostics.js) — never lost
package/src/plain-text.js CHANGED
@@ -5,14 +5,12 @@
5
5
  * with newlines for structure. Useful for word counting, search
6
6
  * indexing, and generating summaries.
7
7
  *
8
- * "Plain" is meant literally: PM text nodes carry HTML (see `entities.js`),
9
- * so every text run is stripped of raw inline tags and its entities resolved
10
- * to the characters a reader would see. Without that, a summary can ship
11
- * `&lt;` or a stray `<span>` into whatever consumes it.
8
+ * A text node holds the characters a reader sees, so a run is already plain:
9
+ * it is joined as it is. ⛔ Until 2026-10-14 text nodes carried HTML, and each
10
+ * run had its inline tags stripped and its entities resolved here — which, on
11
+ * text that is text, would delete a `<div>` an author wrote about.
12
12
  */
13
13
 
14
- import { toPlainCharacters } from './entities.js'
15
-
16
14
  /**
17
15
  * Extract plain text from a ProseMirror document.
18
16
  *
@@ -36,9 +34,8 @@ export function docToPlainText(doc) {
36
34
  * title as the summary. A page with no paragraph at the top level returns
37
35
  * `''` — callers are expected to have an authored fallback.
38
36
  *
39
- * The result is plain characters (tags stripped, entities resolved), so
40
- * truncating on a sentence boundary lands on a real sentence boundary
41
- * rather than somewhere inside a tag or an entity.
37
+ * The result is plain characters — what a text node holds — so truncating on
38
+ * a sentence boundary lands on a real sentence boundary.
42
39
  *
43
40
  * @param {Object} doc - ProseMirror document
44
41
  * @param {Object} [options]
@@ -132,12 +129,10 @@ function nodeToPlainText(node) {
132
129
  */
133
130
  function inlineToPlainText(content) {
134
131
  if (!content) return ''
135
- return toPlainCharacters(
136
- content
137
- .filter(node => node.type === 'text')
138
- .map(node => node.text || '')
139
- .join('')
140
- )
132
+ return content
133
+ .filter(node => node.type === 'text')
134
+ .map(node => node.text || '')
135
+ .join('')
141
136
  }
142
137
 
143
138
  /**
package/src/serializer.js CHANGED
@@ -22,7 +22,7 @@ import {
22
22
  setSerializer,
23
23
  } from './nodes.js'
24
24
  import { normalizeDialect } from './normalize.js'
25
- import { warnUnmappedNode } from './diagnostics.js'
25
+ import { warnUnmappedNode, collectOmittedNodes } from './diagnostics.js'
26
26
 
27
27
  const NODE_SERIALIZERS = {
28
28
  heading: serializeHeading,
@@ -62,6 +62,19 @@ export function serializeNode(node) {
62
62
  // Wire up the circular reference for blockquotes
63
63
  setSerializer(serializeNode)
64
64
 
65
+ /**
66
+ * The node types a document holds that markdown cannot carry — what writing it
67
+ * to a file would leave out. A caller that writes files from stored content (a
68
+ * pull) asks first, so content is never dropped on its way to a file and then
69
+ * deleted by the push after it.
70
+ *
71
+ * @param {Object} doc - ProseMirror document
72
+ * @returns {string[]} the node types left out; empty when the whole doc fits
73
+ */
74
+ export function unwritableNodes(doc) {
75
+ return collectOmittedNodes(() => serializeDoc(doc)).omitted
76
+ }
77
+
65
78
  /**
66
79
  * Serialize a ProseMirror document to markdown.
67
80
  * @param {Object} doc - ProseMirror document ({ type: "doc", content: [...] })
@@ -8,5 +8,5 @@ Each collection is declared in `site.yml`, and every item compiles into
8
8
  `/data/<name>.json`. See the [collections guide](/docs/authoring/collections)
9
9
  for the full story.
10
10
 
11
- Text can be **bold**, *italic*, or ***both***. A literal &lt;name&gt; stays
11
+ Text can be **bold**, *italic*, or ***both***. A literal \<name> stays
12
12
  escaped in prose, and `a &amp; b` keeps its entity inside code.
@@ -0,0 +1,79 @@
1
+ import { markdownToProseMirror } from '@uniweb/content-reader'
2
+ import { proseMirrorToMarkdown, docToPlainText } from '../src/index.js'
3
+ import { serializeInlineContent } from '../src/marks.js'
4
+
5
+ /**
6
+ * A text node holds the characters a reader sees, so the writer escapes what
7
+ * the reader would otherwise decode — and nothing else.
8
+ *
9
+ * Until 2026-10-14 a text node held HTML and was written verbatim, with a code
10
+ * span's `&lt;` decoded on the way out. Now the reader decodes every value, and
11
+ * the property that matters is the one these pin: read(write(text)) is the
12
+ * same text, whatever characters it holds.
13
+ */
14
+
15
+ const paragraph = (content) => ({ type: 'doc', content: [{ type: 'paragraph', content }] })
16
+ const text = (t, marks) => ({ type: 'text', text: t, ...(marks ? { marks } : {}) })
17
+ const readBack = (markdown) =>
18
+ markdownToProseMirror(markdown)
19
+ .content[0].content.map((n) => (n.type === 'text' ? n.text : `[${n.type}]`))
20
+ .join('')
21
+
22
+ describe('text round-trips as the characters it is', () => {
23
+ test.each([
24
+ 'if a<b then swap',
25
+ 'x < y and y > z',
26
+ 'a <img src=x onerror=alert(1)> b',
27
+ 'the &copy; reference, typed',
28
+ 'R&D and AT&T',
29
+ 'a tag-like </div> and <!-- not a comment -->',
30
+ '<https://x.test> is not a link here',
31
+ ])('%s', (value) => {
32
+ const markdown = proseMirrorToMarkdown(paragraph([text(value)]))
33
+ expect(readBack(markdown)).toBe(value)
34
+ })
35
+
36
+ test('only what the reader would decode is escaped', () => {
37
+ expect(serializeInlineContent([text('a<b, x < y, R&D, &copy;')])).toBe('a\\<b, x < y, R&D, \\&copy;')
38
+ })
39
+
40
+ test('a code span is written as its characters', () => {
41
+ expect(serializeInlineContent([text('Use '), text('<div> & &lt;', [{ type: 'code' }])])).toBe(
42
+ 'Use `<div> & &lt;`',
43
+ )
44
+ expect(readBack('Use `<div> & &lt;`')).toBe('Use <div> & &lt;')
45
+ })
46
+
47
+ test('a run the reader split at an escape is written whole', () => {
48
+ // `\<name>` reads back as two nodes, `<` and `name>`.
49
+ const doc = markdownToProseMirror('A literal \\<name> here.')
50
+ expect(proseMirrorToMarkdown(doc).trim()).toBe('A literal \\<name> here.')
51
+ })
52
+ })
53
+
54
+ describe('a hard break', () => {
55
+ const brk = { type: 'hardBreak' }
56
+
57
+ test('in a paragraph is a trailing backslash', () => {
58
+ expect(serializeInlineContent([text('a'), brk, text('b')])).toBe('a\\\nb')
59
+ })
60
+
61
+ test('in a heading, which is one line, is <br/> — and reads back as the same break', () => {
62
+ const doc = { type: 'doc', content: [{ type: 'heading', attrs: { level: 1 }, content: [text('Launch'), brk, text('yours')] }] }
63
+ const markdown = proseMirrorToMarkdown(doc).trim()
64
+ expect(markdown).toBe('# Launch<br/>yours')
65
+ expect(markdownToProseMirror(markdown).content[0].content.map((n) => n.text ?? n.type)).toEqual([
66
+ 'Launch',
67
+ 'hardBreak',
68
+ 'yours',
69
+ ])
70
+ })
71
+ })
72
+
73
+ describe('plain text', () => {
74
+ test('is the characters, with nothing stripped', () => {
75
+ expect(docToPlainText(paragraph([text('Write '), text('<div>', [{ type: 'code' }]), text(' & go')]))).toBe(
76
+ 'Write <div> & go',
77
+ )
78
+ })
79
+ })
@@ -0,0 +1,39 @@
1
+ import { unwritableNodes, proseMirrorToMarkdown, setDiagnosticsReporter, resetDiagnostics } from '../src/index.js'
2
+
3
+ /**
4
+ * What a write would leave out, asked per document.
5
+ *
6
+ * The serializer reports a node it cannot write once per type per process, so
7
+ * a caller writing many documents could not tell which of them lost content —
8
+ * and a pull went on, writing a file without the node, which the next push then
9
+ * deleted from the site (measured 2026-10-14 with an editor's button node). A caller
10
+ * that must refuse rather than lose content asks first.
11
+ */
12
+
13
+ const doc = (...content) => ({ type: 'doc', content })
14
+ const p = (text) => ({ type: 'paragraph', content: [{ type: 'text', text }] })
15
+
16
+ describe('unwritableNodes', () => {
17
+ beforeEach(() => resetDiagnostics())
18
+ afterEach(() => setDiagnosticsReporter())
19
+
20
+ test('a document markdown can carry has nothing left out', () => {
21
+ expect(unwritableNodes(doc(p('a'), { type: 'heading', attrs: { level: 2 }, content: [{ type: 'text', text: 'b' }] }))).toEqual([])
22
+ })
23
+
24
+ test('a node with no markdown form is named — every time it is asked', () => {
25
+ const lossy = doc(p('a'), { type: 'mysteryWidget', content: [] })
26
+ expect(unwritableNodes(lossy)).toEqual(['mysteryWidget'])
27
+ expect(unwritableNodes(lossy)).toEqual(['mysteryWidget'])
28
+ })
29
+
30
+ test('asking prints nothing, and the ordinary write still reports once', () => {
31
+ const printed = []
32
+ setDiagnosticsReporter((m) => printed.push(m))
33
+ unwritableNodes(doc({ type: 'mysteryWidget' }))
34
+ expect(printed).toEqual([])
35
+ proseMirrorToMarkdown(doc({ type: 'mysteryWidget' }))
36
+ proseMirrorToMarkdown(doc({ type: 'mysteryWidget' }))
37
+ expect(printed).toHaveLength(1)
38
+ })
39
+ })
package/src/entities.js DELETED
@@ -1,107 +0,0 @@
1
- /**
2
- * @fileoverview Undo the HTML escaping that ProseMirror text nodes carry.
3
- *
4
- * Text in a PM node is HTML, because the render path injects it as HTML
5
- * (`kit`'s `SafeHtml` / `Text` use `dangerouslySetInnerHTML`). A literal `<`
6
- * the author typed inside a code span therefore has to be *stored* escaped —
7
- * `<code><x></code>` would have the browser swallow `<x>` as an unknown tag.
8
- *
9
- * `content-reader` encodes that asymmetry deliberately: it decodes `&amp;`,
10
- * `&quot;` and `&#39;` back to plain characters and leaves `&lt;` / `&gt;`
11
- * escaped. Verified against the parser (`parser/inline.js`) — raw HTML an
12
- * author writes in prose is preserved verbatim through the `html` token, so
13
- * only *machine-escaped* text carries entities on the way out.
14
- *
15
- * Two exports, because "leaving HTML-land" means different things depending
16
- * on where the text is going:
17
- *
18
- * - `decodeMarkupEntities` — the markdown lane. Undoes only the pair the
19
- * reader left behind, so serializing is the reader's exact inverse.
20
- * - `decodeTextEntities` — the plain-text lane. The target is not HTML at
21
- * all, so every entity resolves to the character a reader would see.
22
- */
23
-
24
- /**
25
- * Undo the escaping `content-reader` leaves in place (`&lt;` / `&gt;`).
26
- *
27
- * Deliberately narrow. `&amp;`, `&quot;` and `&#39;` are already decoded by
28
- * the reader, so a PM node holding one of those means the author wrote the
29
- * entity themselves — decoding it again would corrupt their source.
30
- *
31
- * @param {string} text
32
- * @returns {string}
33
- */
34
- export function decodeMarkupEntities(text) {
35
- if (!text || !text.includes('&')) return text
36
- return text.replace(/&lt;/g, '<').replace(/&gt;/g, '>')
37
- }
38
-
39
- /**
40
- * Escape a code span's text for storage in a ProseMirror text node.
41
- *
42
- * The exact inverse of {@link decodeMarkupEntities}, and the other half of the
43
- * invariant: a code span's content is rendered by injecting it as HTML, so a
44
- * literal `<` has to be stored escaped or `<code><x></code>` has the browser
45
- * swallow `<x>` and the span renders empty.
46
- *
47
- * Exported for **producers other than content-reader**. Any tool that writes a
48
- * ProseMirror code span — an editor, a converter — needs this rule, and the
49
- * rule living in prose rather than in code is what let two producers diverge.
50
- * Import it instead of reimplementing it.
51
- *
52
- * Only `<` and `>`: every other character is safe inside an HTML text node,
53
- * and escaping `&` here would double-encode an entity the author typed.
54
- *
55
- * @param {string} text - Raw code span content, as the author typed it
56
- * @returns {string} Text safe to store in a PM code-marked node
57
- */
58
- export function encodeMarkupEntities(text) {
59
- if (!text) return text
60
- return text.replace(/</g, '&lt;').replace(/>/g, '&gt;')
61
- }
62
-
63
- /**
64
- * Resolve every entity to the character a reader would actually see.
65
- *
66
- * `&amp;` is decoded **last** so `&amp;lt;` yields `&lt;` rather than `<` —
67
- * one decoding pass, not a cascade.
68
- *
69
- * @param {string} text
70
- * @returns {string}
71
- */
72
- export function decodeTextEntities(text) {
73
- if (!text || !text.includes('&')) return text
74
- return text
75
- .replace(/&lt;/g, '<')
76
- .replace(/&gt;/g, '>')
77
- .replace(/&quot;/g, '"')
78
- .replace(/&#0*39;/g, "'")
79
- .replace(/&apos;/g, "'")
80
- .replace(/&nbsp;/g, ' ')
81
- .replace(/&amp;/g, '&')
82
- }
83
-
84
- /**
85
- * Drop raw inline HTML an author wrote in prose (`<br>`, `<span …>`).
86
- *
87
- * Only for the plain-text lane. Runs **before** entity decoding, so text the
88
- * author escaped on purpose (`&lt;b&gt;`, meaning a literal `<b>`) survives
89
- * as characters instead of being mistaken for a tag and stripped.
90
- *
91
- * @param {string} text
92
- * @returns {string}
93
- */
94
- export function stripInlineHtml(text) {
95
- if (!text || !text.includes('<')) return text
96
- return text.replace(/<\/?[a-zA-Z][^>]*>/g, '')
97
- }
98
-
99
- /**
100
- * The plain-text cleanup, in the order the two steps have to run.
101
- *
102
- * @param {string} text
103
- * @returns {string}
104
- */
105
- export function toPlainCharacters(text) {
106
- return decodeTextEntities(stripInlineHtml(text))
107
- }
@@ -1,175 +0,0 @@
1
- /**
2
- * Entity handling on the way out of ProseMirror.
3
- *
4
- * The invariant these tests pin down (verified against `content-reader`'s
5
- * `parser/inline.js`): a PM text node holds HTML. The reader decodes `&amp;`,
6
- * `&quot;` and `&#39;` and leaves `&lt;`/`&gt;` escaped, because the render
7
- * path injects the string as HTML. Serializing has to undo exactly that, and
8
- * exactly there — decoding more, or decoding in prose, corrupts source.
9
- */
10
-
11
- import { markdownToProseMirror } from '@uniweb/content-reader'
12
- import { proseMirrorToMarkdown, docSummaryText, docToPlainText } from '../src/index.js'
13
- import { encodeMarkupEntities, decodeMarkupEntities, decodeTextEntities } from '../src/entities.js'
14
-
15
- const md = s => proseMirrorToMarkdown(markdownToProseMirror(s)).trim()
16
-
17
- describe('decodeMarkupEntities', () => {
18
- test('undoes the pair the reader leaves escaped', () => {
19
- expect(decodeMarkupEntities('/data/&lt;name&gt;.json')).toBe('/data/<name>.json')
20
- })
21
-
22
- test('leaves entities the reader already decoded alone', () => {
23
- // A PM node holding these means the author typed them.
24
- expect(decodeMarkupEntities('a &amp; b')).toBe('a &amp; b')
25
- expect(decodeMarkupEntities('say &quot;hi&quot;')).toBe('say &quot;hi&quot;')
26
- })
27
-
28
- test('is a no-op on text with no entities', () => {
29
- expect(decodeMarkupEntities('plain text')).toBe('plain text')
30
- })
31
- })
32
-
33
- describe('decodeTextEntities', () => {
34
- test('resolves every entity to the character a reader sees', () => {
35
- expect(decodeTextEntities('&lt;b&gt; &amp; &quot;q&quot; &#39;a&#39;')).toBe(`<b> & "q" 'a'`)
36
- })
37
-
38
- test('decodes &amp; last, so &amp;lt; yields &lt; and not <', () => {
39
- expect(decodeTextEntities('&amp;lt;')).toBe('&lt;')
40
- })
41
- })
42
-
43
- describe('D1 — code spans round-trip a literal < >', () => {
44
- test('the reported defect', () => {
45
- expect(md('Read `/data/<name>.json` for the cascade.')).toBe(
46
- 'Read `/data/<name>.json` for the cascade.'
47
- )
48
- })
49
-
50
- test('inside a heading', () => {
51
- expect(md('# The `<slug>` folder')).toBe('# The `<slug>` folder')
52
- })
53
-
54
- test('inside a table cell', () => {
55
- expect(md('| Col |\n| --- |\n| `<x>` |')).toContain('`<x>`')
56
- })
57
-
58
- test('inside a link', () => {
59
- expect(md('[the `<x>` doc](/a)')).toBe('[the `<x>` doc](/a)')
60
- })
61
- })
62
-
63
- describe('D1 — what the fix must NOT touch', () => {
64
- test('prose entities stay escaped (decoding them would render as a tag)', () => {
65
- expect(md('Literal &lt;name&gt; in prose.')).toBe('Literal &lt;name&gt; in prose.')
66
- })
67
-
68
- test('raw inline HTML in prose is preserved verbatim', () => {
69
- expect(md('A <name> tag in prose.')).toBe('A <name> tag in prose.')
70
- })
71
-
72
- test('an authored &amp; inside a code span survives', () => {
73
- expect(md('Code `a &amp; b` here.')).toBe('Code `a &amp; b` here.')
74
- })
75
-
76
- test('code blocks are raw text and are left alone', () => {
77
- const source = '```js\nconst x = a < b && c > d;\n```'
78
- expect(md(source)).toBe(source)
79
- })
80
-
81
- test('quotes and apostrophes in code spans are untouched', () => {
82
- expect(md(`Code \`say "hi" it's\` here.`)).toBe(`Code \`say "hi" it's\` here.`)
83
- })
84
- })
85
-
86
- describe('docSummaryText', () => {
87
- const summary = (source, opts) => docSummaryText(markdownToProseMirror(source), opts)
88
-
89
- test('skips the H1 and reads the first paragraph', () => {
90
- expect(summary('# Quickstart\n\nCreate your first Uniweb site in 5 minutes.')).toBe(
91
- 'Create your first Uniweb site in 5 minutes.'
92
- )
93
- })
94
-
95
- test('returns plain text — no anchors, no mark tags', () => {
96
- expect(summary('# T\n\nSee **the [docs](/docs)** for *more*.')).toBe(
97
- 'See the docs for more.'
98
- )
99
- })
100
-
101
- test('resolves entities and drops raw inline HTML', () => {
102
- expect(summary('# T\n\nUse `<name>` and &lt;slug&gt; and <br> here.')).toBe(
103
- 'Use <name> and <slug> and here.'
104
- )
105
- })
106
-
107
- test('truncates at a sentence boundary', () => {
108
- const text = 'One sentence here. Two sentence here. Three sentence here.'
109
- expect(summary(`# T\n\n${text}`, { maxChars: 40 })).toBe('One sentence here. Two sentence here.')
110
- })
111
-
112
- test('falls back to a word boundary with an ellipsis', () => {
113
- const text = 'Collections let you manage repeating content across many files'
114
- expect(summary(`# T\n\n${text}`, { maxChars: 40 })).toBe(
115
- 'Collections let you manage repeating…'
116
- )
117
- })
118
-
119
- test('a dot inside a filename does not end a sentence', () => {
120
- expect(summary('# T\n\nEdit site.yml then run the build now please.', { maxChars: 20 }))
121
- .toBe('Edit site.yml then…')
122
- })
123
-
124
- test('returns empty string when there is no paragraph', () => {
125
- expect(summary('# Only a heading')).toBe('')
126
- expect(docSummaryText(null)).toBe('')
127
- })
128
-
129
- test('collapses newlines into one line', () => {
130
- expect(summary('# T\n\nA line\nand another.')).toBe('A line and another.')
131
- })
132
- })
133
-
134
- describe('docToPlainText also yields plain characters', () => {
135
- test('strips tags and resolves entities', () => {
136
- // Note the doubled space where <br> was: this primitive does not
137
- // normalize whitespace, and its consumers (word counts, search tokens)
138
- // do not care. `docSummaryText` is the one that promises a single line.
139
- expect(docToPlainText(markdownToProseMirror('Use &lt;x&gt; and <br> here.'))).toBe(
140
- 'Use <x> and here.'
141
- )
142
- })
143
- })
144
-
145
- describe('encodeMarkupEntities — the producer-side half', () => {
146
- test('is the exact inverse of decodeMarkupEntities', () => {
147
- const raw = '/data/<name>.json'
148
- expect(encodeMarkupEntities(raw)).toBe('/data/&lt;name&gt;.json')
149
- expect(decodeMarkupEntities(encodeMarkupEntities(raw))).toBe(raw)
150
- })
151
-
152
- test('does not touch &, so an authored entity is not double-encoded', () => {
153
- expect(encodeMarkupEntities('a & b')).toBe('a & b')
154
- expect(encodeMarkupEntities('&amp;')).toBe('&amp;')
155
- })
156
-
157
- test('round-trips what an author types in a code span', () => {
158
- for (const raw of ['<div>', 'a < b && c > d', 'a & b', 'plain']) {
159
- expect(decodeMarkupEntities(encodeMarkupEntities(raw))).toBe(raw)
160
- }
161
- })
162
-
163
- test('the one input it does NOT round-trip: literal entity text', () => {
164
- // `<div>` and `&lt;div&gt;` both store as `&lt;div&gt;`, so the pair
165
- // cannot tell them apart and resolves to the decoded form.
166
- //
167
- // Accepted deliberately, because the loss happened earlier and elsewhere:
168
- // a code span is rendered by injecting its text as HTML, so an author who
169
- // typed `&lt;div&gt;` already sees `<div>` on the page. Decoding makes the
170
- // file agree with the page rather than destroying something a reader could
171
- // otherwise see. Escaping `&` as well would make the pair lossless, at the
172
- // cost of changing what every existing code span stores.
173
- expect(decodeMarkupEntities(encodeMarkupEntities('&lt;div&gt;'))).toBe('<div>')
174
- })
175
- })