@uniweb/content-writer 0.3.4 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -4
- package/src/diagnostics.js +28 -0
- package/src/index.js +1 -2
- package/src/marks.js +60 -12
- package/src/nodes.js +4 -2
- package/src/normalize.js +6 -2
- package/src/plain-text.js +10 -15
- package/src/serializer.js +14 -1
- package/tests/fixtures/roundtrip/prose.md +1 -1
- package/tests/text-is-text.test.js +79 -0
- package/tests/unwritable.test.js +39 -0
- package/src/entities.js +0 -107
- package/tests/entities.test.js +0 -175
package/package.json
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@uniweb/content-writer",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "ProseMirror document structure to Markdown converter",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
7
7
|
"exports": {
|
|
8
|
-
".": "./src/index.js"
|
|
9
|
-
"./entities": "./src/entities.js"
|
|
8
|
+
".": "./src/index.js"
|
|
10
9
|
},
|
|
11
10
|
"keywords": [
|
|
12
11
|
"markdown",
|
|
@@ -20,7 +19,7 @@
|
|
|
20
19
|
},
|
|
21
20
|
"devDependencies": {
|
|
22
21
|
"vitest": "^4.1.7",
|
|
23
|
-
"@uniweb/content-reader": "1.
|
|
22
|
+
"@uniweb/content-reader": "1.3.0"
|
|
24
23
|
},
|
|
25
24
|
"directories": {
|
|
26
25
|
"test": "tests"
|
package/src/diagnostics.js
CHANGED
|
@@ -44,12 +44,40 @@ export function resetDiagnostics() {
|
|
|
44
44
|
warnedMarks.clear()
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
/** Set while `collectOmittedNodes` runs: the node types the write leaves out. */
|
|
48
|
+
let omitted = null
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Run a write and return the node types it left out, instead of reporting them.
|
|
52
|
+
* For a caller that must refuse rather than lose content — a pull, whose next
|
|
53
|
+
* push would delete whatever the file does not hold.
|
|
54
|
+
*
|
|
55
|
+
* @template T
|
|
56
|
+
* @param {() => T} run
|
|
57
|
+
* @returns {{ result: T, omitted: string[] }}
|
|
58
|
+
*/
|
|
59
|
+
export function collectOmittedNodes(run) {
|
|
60
|
+
const previous = omitted
|
|
61
|
+
omitted = new Set()
|
|
62
|
+
try {
|
|
63
|
+
const result = run()
|
|
64
|
+
return { result, omitted: [...omitted] }
|
|
65
|
+
} finally {
|
|
66
|
+
omitted = previous
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
47
70
|
/**
|
|
48
71
|
* Report a ProseMirror node the serializer has no markdown form for.
|
|
49
72
|
* @param {Object} node - The unmapped node ({ type, attrs, ... })
|
|
50
73
|
*/
|
|
51
74
|
export function warnUnmappedNode(node) {
|
|
52
75
|
const type = node?.type || 'unknown'
|
|
76
|
+
// A caller collecting what a write leaves out is told every time, not once per type.
|
|
77
|
+
if (omitted) {
|
|
78
|
+
omitted.add(type)
|
|
79
|
+
return
|
|
80
|
+
}
|
|
53
81
|
if (warnedNodes.has(type)) return
|
|
54
82
|
warnedNodes.add(type)
|
|
55
83
|
report(
|
package/src/index.js
CHANGED
|
@@ -5,11 +5,10 @@
|
|
|
5
5
|
* documents back to Markdown text.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
export { serializeDoc as proseMirrorToMarkdown } from './serializer.js'
|
|
8
|
+
export { serializeDoc as proseMirrorToMarkdown, unwritableNodes } from './serializer.js'
|
|
9
9
|
export { serializeInlineContent } from './marks.js'
|
|
10
10
|
export { serializeFrontmatter } from './frontmatter.js'
|
|
11
11
|
export { docToPlainText, docSummaryText } from './plain-text.js'
|
|
12
|
-
export { encodeMarkupEntities, decodeMarkupEntities, decodeTextEntities } from './entities.js'
|
|
13
12
|
export { normalizeDialect } from './normalize.js'
|
|
14
13
|
export { setDiagnosticsReporter, resetDiagnostics } from './diagnostics.js'
|
|
15
14
|
|
package/src/marks.js
CHANGED
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { serializeAttributes } from './attributes.js'
|
|
8
|
-
import { decodeMarkupEntities } from './entities.js'
|
|
9
8
|
import { warnUnmappedMark, warnUnmappedNode } from './diagnostics.js'
|
|
10
9
|
|
|
11
10
|
/**
|
|
@@ -232,13 +231,58 @@ export function serializeBlockImage(node) {
|
|
|
232
231
|
* @param {Array} content - Array of text/image nodes with optional marks
|
|
233
232
|
* @returns {string} Markdown string
|
|
234
233
|
*/
|
|
235
|
-
export function serializeInlineContent(content) {
|
|
234
|
+
export function serializeInlineContent(content, { lineBreak = '\\\n' } = {}) {
|
|
236
235
|
if (!content || content.length === 0) return ''
|
|
237
236
|
|
|
238
237
|
// No silent drops: flag any mark we can't represent before serializing.
|
|
239
238
|
reportUnknownMarks(content)
|
|
240
239
|
|
|
241
|
-
return serializeRuns(content)
|
|
240
|
+
return serializeRuns(mergeAdjacentText(content), { lineBreak, literal: false })
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Join neighbouring text nodes that carry the same marks, as ProseMirror itself
|
|
245
|
+
* would. The reader splits a run wherever markdown had an escape — `\<name>`
|
|
246
|
+
* arrives as `<` and `name>` — and the escaping below has to see the run whole,
|
|
247
|
+
* or a `<` that ends one node would go out bare before the tag-like text of the
|
|
248
|
+
* next.
|
|
249
|
+
*
|
|
250
|
+
* @param {Array} content
|
|
251
|
+
* @returns {Array}
|
|
252
|
+
*/
|
|
253
|
+
function mergeAdjacentText(content) {
|
|
254
|
+
const out = []
|
|
255
|
+
for (const node of content) {
|
|
256
|
+
const last = out[out.length - 1]
|
|
257
|
+
if (
|
|
258
|
+
node?.type === 'text' &&
|
|
259
|
+
last?.type === 'text' &&
|
|
260
|
+
JSON.stringify(last.marks || []) === JSON.stringify(node.marks || [])
|
|
261
|
+
) {
|
|
262
|
+
out[out.length - 1] = { ...last, text: (last.text || '') + (node.text || '') }
|
|
263
|
+
} else {
|
|
264
|
+
out.push(node)
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
return out
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Escape text so the reader decodes it back to the same characters.
|
|
272
|
+
*
|
|
273
|
+
* A text node holds the characters a reader sees, and content-reader decodes
|
|
274
|
+
* what markdown escapes — so the two characters it would otherwise read as
|
|
275
|
+
* something else are escaped here: a `<` that would open a tag or an autolink,
|
|
276
|
+
* and an `&` that would spell a character reference. `a < b` and `R&D` stay as
|
|
277
|
+
* they are. ⛔ Until 2026-10-14 a text node held HTML and was written verbatim.
|
|
278
|
+
*
|
|
279
|
+
* @param {string} text
|
|
280
|
+
* @returns {string}
|
|
281
|
+
*/
|
|
282
|
+
function escapeMarkdownText(text) {
|
|
283
|
+
return String(text)
|
|
284
|
+
.replace(/&(?=#[0-9]+;|#[xX][0-9a-fA-F]+;|[A-Za-z][A-Za-z0-9]*;)/g, '\\&')
|
|
285
|
+
.replace(/<(?=[A-Za-z/!?])/g, '\\<')
|
|
242
286
|
}
|
|
243
287
|
|
|
244
288
|
/**
|
|
@@ -262,7 +306,7 @@ export function serializeInlineContent(content) {
|
|
|
262
306
|
* @param {Array} nodes - Inline nodes
|
|
263
307
|
* @returns {string} Markdown string
|
|
264
308
|
*/
|
|
265
|
-
function serializeRuns(nodes) {
|
|
309
|
+
function serializeRuns(nodes, context) {
|
|
266
310
|
const out = []
|
|
267
311
|
let i = 0
|
|
268
312
|
|
|
@@ -271,7 +315,7 @@ function serializeRuns(nodes) {
|
|
|
271
315
|
const outer = node.type === 'text' ? outermostMark(node) : null
|
|
272
316
|
|
|
273
317
|
if (!outer) {
|
|
274
|
-
out.push(serializePlainNode(node))
|
|
318
|
+
out.push(serializePlainNode(node, context))
|
|
275
319
|
i++
|
|
276
320
|
continue
|
|
277
321
|
}
|
|
@@ -286,7 +330,9 @@ function serializeRuns(nodes) {
|
|
|
286
330
|
j++
|
|
287
331
|
}
|
|
288
332
|
|
|
289
|
-
|
|
333
|
+
// A code span is literal: what it wraps is written as the characters it is.
|
|
334
|
+
const inner = serializeRuns(group, outer.type === 'code' ? { ...context, literal: true } : context)
|
|
335
|
+
out.push(applyMark(inner, outer))
|
|
290
336
|
i = j
|
|
291
337
|
}
|
|
292
338
|
|
|
@@ -327,9 +373,7 @@ function withoutOutermostMark(node) {
|
|
|
327
373
|
function applyMark(inner, mark) {
|
|
328
374
|
switch (mark.type) {
|
|
329
375
|
case 'code':
|
|
330
|
-
|
|
331
|
-
// `<`/`>` escaped (see entities.js).
|
|
332
|
-
return serializeCodeSpan(decodeMarkupEntities(inner))
|
|
376
|
+
return serializeCodeSpan(inner)
|
|
333
377
|
case 'bold':
|
|
334
378
|
return `**${inner}**`
|
|
335
379
|
case 'italic':
|
|
@@ -352,7 +396,7 @@ function applyMark(inner, mark) {
|
|
|
352
396
|
/**
|
|
353
397
|
* Serialize a plain node (no wrapping mark).
|
|
354
398
|
*/
|
|
355
|
-
function serializePlainNode(node) {
|
|
399
|
+
function serializePlainNode(node, { lineBreak = '\\\n', literal = false } = {}) {
|
|
356
400
|
if (node.type === 'image') {
|
|
357
401
|
return serializeInlineImage(node)
|
|
358
402
|
}
|
|
@@ -367,7 +411,9 @@ function serializePlainNode(node) {
|
|
|
367
411
|
// Backslash rather than the two-trailing-spaces spelling: it survives
|
|
368
412
|
// editors, formatters and pre-commit hooks that strip trailing whitespace,
|
|
369
413
|
// and it is visible in a diff. Both spellings read back as a hard break.
|
|
370
|
-
|
|
414
|
+
// A heading or a table cell is one line, so there it is `<br/>` — the one
|
|
415
|
+
// tag the reader keeps, as the same hard break.
|
|
416
|
+
return lineBreak
|
|
371
417
|
}
|
|
372
418
|
if (node.type !== 'text') {
|
|
373
419
|
// No silent drops: an inline node we have no markdown form for is
|
|
@@ -376,7 +422,9 @@ function serializePlainNode(node) {
|
|
|
376
422
|
warnUnmappedNode(node)
|
|
377
423
|
return ''
|
|
378
424
|
}
|
|
379
|
-
|
|
425
|
+
const marks = node.marks || []
|
|
426
|
+
const keep = literal || marks.some((mark) => mark.type === 'code')
|
|
427
|
+
return serializeTextWithMarks(keep ? node.text : escapeMarkdownText(node.text), marks)
|
|
380
428
|
}
|
|
381
429
|
|
|
382
430
|
/**
|
package/src/nodes.js
CHANGED
|
@@ -20,7 +20,8 @@ export { serializeInsetRef }
|
|
|
20
20
|
*/
|
|
21
21
|
export function serializeHeading(node) {
|
|
22
22
|
const prefix = '#'.repeat(node.attrs?.level || 1)
|
|
23
|
-
|
|
23
|
+
// A heading is one line: a hard break inside it is written `<br/>`.
|
|
24
|
+
const text = serializeInlineContent(node.content, { lineBreak: '<br/>' })
|
|
24
25
|
// A `#>` label line (pretitle). The hash count carries the authored
|
|
25
26
|
// spelling only — `#>` and `##>` mean the same thing — so emitting the
|
|
26
27
|
// stored level keeps the file byte-identical through a round-trip.
|
|
@@ -443,7 +444,8 @@ function serializeTableCell(cell) {
|
|
|
443
444
|
const text = cell.content
|
|
444
445
|
.map(child => {
|
|
445
446
|
if (child.type === 'paragraph') {
|
|
446
|
-
|
|
447
|
+
// A table row is one line: a hard break in a cell is written `<br/>`.
|
|
448
|
+
return serializeInlineContent(child.content, { lineBreak: '<br/>' })
|
|
447
449
|
}
|
|
448
450
|
return ''
|
|
449
451
|
})
|
package/src/normalize.js
CHANGED
|
@@ -55,8 +55,12 @@
|
|
|
55
55
|
* regenerated fixture in `_contracts/editor-vocabulary-diff.test.js`
|
|
56
56
|
* stopped listing them. That fixture is the signal, and it is the only one
|
|
57
57
|
* there is — no message announces it.
|
|
58
|
-
* - colour
|
|
59
|
-
*
|
|
58
|
+
* - a colour that is not a named style (`textStyle` with a raw value), and
|
|
59
|
+
* `underline` — neither has a markdown form. `highlight` and the named
|
|
60
|
+
* styles (`accent`, `highlight`, `callout`, `muted`) ARE mapped, onto the
|
|
61
|
+
* `span` mark (`mapEditorMark`), and `strike` serializes as `~~text~~`.
|
|
62
|
+
* ⛔ This line listed all of them as awaiting a decision until 2026-10-14,
|
|
63
|
+
* long after the mapping shipped.
|
|
60
64
|
*
|
|
61
65
|
* Each unmapped construct reaches content-writer's serializer as-is and
|
|
62
66
|
* is reported by the no-silent-drop guard (diagnostics.js) — never lost
|
package/src/plain-text.js
CHANGED
|
@@ -5,14 +5,12 @@
|
|
|
5
5
|
* with newlines for structure. Useful for word counting, search
|
|
6
6
|
* indexing, and generating summaries.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* A text node holds the characters a reader sees, so a run is already plain:
|
|
9
|
+
* it is joined as it is. ⛔ Until 2026-10-14 text nodes carried HTML, and each
|
|
10
|
+
* run had its inline tags stripped and its entities resolved here — which, on
|
|
11
|
+
* text that is text, would delete a `<div>` an author wrote about.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
import { toPlainCharacters } from './entities.js'
|
|
15
|
-
|
|
16
14
|
/**
|
|
17
15
|
* Extract plain text from a ProseMirror document.
|
|
18
16
|
*
|
|
@@ -36,9 +34,8 @@ export function docToPlainText(doc) {
|
|
|
36
34
|
* title as the summary. A page with no paragraph at the top level returns
|
|
37
35
|
* `''` — callers are expected to have an authored fallback.
|
|
38
36
|
*
|
|
39
|
-
* The result is plain characters
|
|
40
|
-
*
|
|
41
|
-
* rather than somewhere inside a tag or an entity.
|
|
37
|
+
* The result is plain characters — what a text node holds — so truncating on
|
|
38
|
+
* a sentence boundary lands on a real sentence boundary.
|
|
42
39
|
*
|
|
43
40
|
* @param {Object} doc - ProseMirror document
|
|
44
41
|
* @param {Object} [options]
|
|
@@ -132,12 +129,10 @@ function nodeToPlainText(node) {
|
|
|
132
129
|
*/
|
|
133
130
|
function inlineToPlainText(content) {
|
|
134
131
|
if (!content) return ''
|
|
135
|
-
return
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
.join('')
|
|
140
|
-
)
|
|
132
|
+
return content
|
|
133
|
+
.filter(node => node.type === 'text')
|
|
134
|
+
.map(node => node.text || '')
|
|
135
|
+
.join('')
|
|
141
136
|
}
|
|
142
137
|
|
|
143
138
|
/**
|
package/src/serializer.js
CHANGED
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
setSerializer,
|
|
23
23
|
} from './nodes.js'
|
|
24
24
|
import { normalizeDialect } from './normalize.js'
|
|
25
|
-
import { warnUnmappedNode } from './diagnostics.js'
|
|
25
|
+
import { warnUnmappedNode, collectOmittedNodes } from './diagnostics.js'
|
|
26
26
|
|
|
27
27
|
const NODE_SERIALIZERS = {
|
|
28
28
|
heading: serializeHeading,
|
|
@@ -62,6 +62,19 @@ export function serializeNode(node) {
|
|
|
62
62
|
// Wire up the circular reference for blockquotes
|
|
63
63
|
setSerializer(serializeNode)
|
|
64
64
|
|
|
65
|
+
/**
|
|
66
|
+
* The node types a document holds that markdown cannot carry — what writing it
|
|
67
|
+
* to a file would leave out. A caller that writes files from stored content (a
|
|
68
|
+
* pull) asks first, so content is never dropped on its way to a file and then
|
|
69
|
+
* deleted by the push after it.
|
|
70
|
+
*
|
|
71
|
+
* @param {Object} doc - ProseMirror document
|
|
72
|
+
* @returns {string[]} the node types left out; empty when the whole doc fits
|
|
73
|
+
*/
|
|
74
|
+
export function unwritableNodes(doc) {
|
|
75
|
+
return collectOmittedNodes(() => serializeDoc(doc)).omitted
|
|
76
|
+
}
|
|
77
|
+
|
|
65
78
|
/**
|
|
66
79
|
* Serialize a ProseMirror document to markdown.
|
|
67
80
|
* @param {Object} doc - ProseMirror document ({ type: "doc", content: [...] })
|
|
@@ -8,5 +8,5 @@ Each collection is declared in `site.yml`, and every item compiles into
|
|
|
8
8
|
`/data/<name>.json`. See the [collections guide](/docs/authoring/collections)
|
|
9
9
|
for the full story.
|
|
10
10
|
|
|
11
|
-
Text can be **bold**, *italic*, or ***both***. A literal
|
|
11
|
+
Text can be **bold**, *italic*, or ***both***. A literal \<name> stays
|
|
12
12
|
escaped in prose, and `a & b` keeps its entity inside code.
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { markdownToProseMirror } from '@uniweb/content-reader'
|
|
2
|
+
import { proseMirrorToMarkdown, docToPlainText } from '../src/index.js'
|
|
3
|
+
import { serializeInlineContent } from '../src/marks.js'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A text node holds the characters a reader sees, so the writer escapes what
|
|
7
|
+
* the reader would otherwise decode — and nothing else.
|
|
8
|
+
*
|
|
9
|
+
* Until 2026-10-14 a text node held HTML and was written verbatim, with a code
|
|
10
|
+
* span's `<` decoded on the way out. Now the reader decodes every value, and
|
|
11
|
+
* the property that matters is the one these pin: read(write(text)) is the
|
|
12
|
+
* same text, whatever characters it holds.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const paragraph = (content) => ({ type: 'doc', content: [{ type: 'paragraph', content }] })
|
|
16
|
+
const text = (t, marks) => ({ type: 'text', text: t, ...(marks ? { marks } : {}) })
|
|
17
|
+
const readBack = (markdown) =>
|
|
18
|
+
markdownToProseMirror(markdown)
|
|
19
|
+
.content[0].content.map((n) => (n.type === 'text' ? n.text : `[${n.type}]`))
|
|
20
|
+
.join('')
|
|
21
|
+
|
|
22
|
+
describe('text round-trips as the characters it is', () => {
|
|
23
|
+
test.each([
|
|
24
|
+
'if a<b then swap',
|
|
25
|
+
'x < y and y > z',
|
|
26
|
+
'a <img src=x onerror=alert(1)> b',
|
|
27
|
+
'the © reference, typed',
|
|
28
|
+
'R&D and AT&T',
|
|
29
|
+
'a tag-like </div> and <!-- not a comment -->',
|
|
30
|
+
'<https://x.test> is not a link here',
|
|
31
|
+
])('%s', (value) => {
|
|
32
|
+
const markdown = proseMirrorToMarkdown(paragraph([text(value)]))
|
|
33
|
+
expect(readBack(markdown)).toBe(value)
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
test('only what the reader would decode is escaped', () => {
|
|
37
|
+
expect(serializeInlineContent([text('a<b, x < y, R&D, ©')])).toBe('a\\<b, x < y, R&D, \\©')
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
test('a code span is written as its characters', () => {
|
|
41
|
+
expect(serializeInlineContent([text('Use '), text('<div> & <', [{ type: 'code' }])])).toBe(
|
|
42
|
+
'Use `<div> & <`',
|
|
43
|
+
)
|
|
44
|
+
expect(readBack('Use `<div> & <`')).toBe('Use <div> & <')
|
|
45
|
+
})
|
|
46
|
+
|
|
47
|
+
test('a run the reader split at an escape is written whole', () => {
|
|
48
|
+
// `\<name>` reads back as two nodes, `<` and `name>`.
|
|
49
|
+
const doc = markdownToProseMirror('A literal \\<name> here.')
|
|
50
|
+
expect(proseMirrorToMarkdown(doc).trim()).toBe('A literal \\<name> here.')
|
|
51
|
+
})
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
describe('a hard break', () => {
|
|
55
|
+
const brk = { type: 'hardBreak' }
|
|
56
|
+
|
|
57
|
+
test('in a paragraph is a trailing backslash', () => {
|
|
58
|
+
expect(serializeInlineContent([text('a'), brk, text('b')])).toBe('a\\\nb')
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
test('in a heading, which is one line, is <br/> — and reads back as the same break', () => {
|
|
62
|
+
const doc = { type: 'doc', content: [{ type: 'heading', attrs: { level: 1 }, content: [text('Launch'), brk, text('yours')] }] }
|
|
63
|
+
const markdown = proseMirrorToMarkdown(doc).trim()
|
|
64
|
+
expect(markdown).toBe('# Launch<br/>yours')
|
|
65
|
+
expect(markdownToProseMirror(markdown).content[0].content.map((n) => n.text ?? n.type)).toEqual([
|
|
66
|
+
'Launch',
|
|
67
|
+
'hardBreak',
|
|
68
|
+
'yours',
|
|
69
|
+
])
|
|
70
|
+
})
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
describe('plain text', () => {
|
|
74
|
+
test('is the characters, with nothing stripped', () => {
|
|
75
|
+
expect(docToPlainText(paragraph([text('Write '), text('<div>', [{ type: 'code' }]), text(' & go')]))).toBe(
|
|
76
|
+
'Write <div> & go',
|
|
77
|
+
)
|
|
78
|
+
})
|
|
79
|
+
})
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { unwritableNodes, proseMirrorToMarkdown, setDiagnosticsReporter, resetDiagnostics } from '../src/index.js'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* What a write would leave out, asked per document.
|
|
5
|
+
*
|
|
6
|
+
* The serializer reports a node it cannot write once per type per process, so
|
|
7
|
+
* a caller writing many documents could not tell which of them lost content —
|
|
8
|
+
* and a pull went on, writing a file without the node, which the next push then
|
|
9
|
+
* deleted from the site (measured 2026-10-14 with an editor's button node). A caller
|
|
10
|
+
* that must refuse rather than lose content asks first.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const doc = (...content) => ({ type: 'doc', content })
|
|
14
|
+
const p = (text) => ({ type: 'paragraph', content: [{ type: 'text', text }] })
|
|
15
|
+
|
|
16
|
+
describe('unwritableNodes', () => {
|
|
17
|
+
beforeEach(() => resetDiagnostics())
|
|
18
|
+
afterEach(() => setDiagnosticsReporter())
|
|
19
|
+
|
|
20
|
+
test('a document markdown can carry has nothing left out', () => {
|
|
21
|
+
expect(unwritableNodes(doc(p('a'), { type: 'heading', attrs: { level: 2 }, content: [{ type: 'text', text: 'b' }] }))).toEqual([])
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
test('a node with no markdown form is named — every time it is asked', () => {
|
|
25
|
+
const lossy = doc(p('a'), { type: 'mysteryWidget', content: [] })
|
|
26
|
+
expect(unwritableNodes(lossy)).toEqual(['mysteryWidget'])
|
|
27
|
+
expect(unwritableNodes(lossy)).toEqual(['mysteryWidget'])
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
test('asking prints nothing, and the ordinary write still reports once', () => {
|
|
31
|
+
const printed = []
|
|
32
|
+
setDiagnosticsReporter((m) => printed.push(m))
|
|
33
|
+
unwritableNodes(doc({ type: 'mysteryWidget' }))
|
|
34
|
+
expect(printed).toEqual([])
|
|
35
|
+
proseMirrorToMarkdown(doc({ type: 'mysteryWidget' }))
|
|
36
|
+
proseMirrorToMarkdown(doc({ type: 'mysteryWidget' }))
|
|
37
|
+
expect(printed).toHaveLength(1)
|
|
38
|
+
})
|
|
39
|
+
})
|
package/src/entities.js
DELETED
|
@@ -1,107 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @fileoverview Undo the HTML escaping that ProseMirror text nodes carry.
|
|
3
|
-
*
|
|
4
|
-
* Text in a PM node is HTML, because the render path injects it as HTML
|
|
5
|
-
* (`kit`'s `SafeHtml` / `Text` use `dangerouslySetInnerHTML`). A literal `<`
|
|
6
|
-
* the author typed inside a code span therefore has to be *stored* escaped —
|
|
7
|
-
* `<code><x></code>` would have the browser swallow `<x>` as an unknown tag.
|
|
8
|
-
*
|
|
9
|
-
* `content-reader` encodes that asymmetry deliberately: it decodes `&`,
|
|
10
|
-
* `"` and `'` back to plain characters and leaves `<` / `>`
|
|
11
|
-
* escaped. Verified against the parser (`parser/inline.js`) — raw HTML an
|
|
12
|
-
* author writes in prose is preserved verbatim through the `html` token, so
|
|
13
|
-
* only *machine-escaped* text carries entities on the way out.
|
|
14
|
-
*
|
|
15
|
-
* Two exports, because "leaving HTML-land" means different things depending
|
|
16
|
-
* on where the text is going:
|
|
17
|
-
*
|
|
18
|
-
* - `decodeMarkupEntities` — the markdown lane. Undoes only the pair the
|
|
19
|
-
* reader left behind, so serializing is the reader's exact inverse.
|
|
20
|
-
* - `decodeTextEntities` — the plain-text lane. The target is not HTML at
|
|
21
|
-
* all, so every entity resolves to the character a reader would see.
|
|
22
|
-
*/
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Undo the escaping `content-reader` leaves in place (`<` / `>`).
|
|
26
|
-
*
|
|
27
|
-
* Deliberately narrow. `&`, `"` and `'` are already decoded by
|
|
28
|
-
* the reader, so a PM node holding one of those means the author wrote the
|
|
29
|
-
* entity themselves — decoding it again would corrupt their source.
|
|
30
|
-
*
|
|
31
|
-
* @param {string} text
|
|
32
|
-
* @returns {string}
|
|
33
|
-
*/
|
|
34
|
-
export function decodeMarkupEntities(text) {
|
|
35
|
-
if (!text || !text.includes('&')) return text
|
|
36
|
-
return text.replace(/</g, '<').replace(/>/g, '>')
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Escape a code span's text for storage in a ProseMirror text node.
|
|
41
|
-
*
|
|
42
|
-
* The exact inverse of {@link decodeMarkupEntities}, and the other half of the
|
|
43
|
-
* invariant: a code span's content is rendered by injecting it as HTML, so a
|
|
44
|
-
* literal `<` has to be stored escaped or `<code><x></code>` has the browser
|
|
45
|
-
* swallow `<x>` and the span renders empty.
|
|
46
|
-
*
|
|
47
|
-
* Exported for **producers other than content-reader**. Any tool that writes a
|
|
48
|
-
* ProseMirror code span — an editor, a converter — needs this rule, and the
|
|
49
|
-
* rule living in prose rather than in code is what let two producers diverge.
|
|
50
|
-
* Import it instead of reimplementing it.
|
|
51
|
-
*
|
|
52
|
-
* Only `<` and `>`: every other character is safe inside an HTML text node,
|
|
53
|
-
* and escaping `&` here would double-encode an entity the author typed.
|
|
54
|
-
*
|
|
55
|
-
* @param {string} text - Raw code span content, as the author typed it
|
|
56
|
-
* @returns {string} Text safe to store in a PM code-marked node
|
|
57
|
-
*/
|
|
58
|
-
export function encodeMarkupEntities(text) {
|
|
59
|
-
if (!text) return text
|
|
60
|
-
return text.replace(/</g, '<').replace(/>/g, '>')
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
/**
|
|
64
|
-
* Resolve every entity to the character a reader would actually see.
|
|
65
|
-
*
|
|
66
|
-
* `&` is decoded **last** so `&lt;` yields `<` rather than `<` —
|
|
67
|
-
* one decoding pass, not a cascade.
|
|
68
|
-
*
|
|
69
|
-
* @param {string} text
|
|
70
|
-
* @returns {string}
|
|
71
|
-
*/
|
|
72
|
-
export function decodeTextEntities(text) {
|
|
73
|
-
if (!text || !text.includes('&')) return text
|
|
74
|
-
return text
|
|
75
|
-
.replace(/</g, '<')
|
|
76
|
-
.replace(/>/g, '>')
|
|
77
|
-
.replace(/"/g, '"')
|
|
78
|
-
.replace(/�*39;/g, "'")
|
|
79
|
-
.replace(/'/g, "'")
|
|
80
|
-
.replace(/ /g, ' ')
|
|
81
|
-
.replace(/&/g, '&')
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
/**
|
|
85
|
-
* Drop raw inline HTML an author wrote in prose (`<br>`, `<span …>`).
|
|
86
|
-
*
|
|
87
|
-
* Only for the plain-text lane. Runs **before** entity decoding, so text the
|
|
88
|
-
* author escaped on purpose (`<b>`, meaning a literal `<b>`) survives
|
|
89
|
-
* as characters instead of being mistaken for a tag and stripped.
|
|
90
|
-
*
|
|
91
|
-
* @param {string} text
|
|
92
|
-
* @returns {string}
|
|
93
|
-
*/
|
|
94
|
-
export function stripInlineHtml(text) {
|
|
95
|
-
if (!text || !text.includes('<')) return text
|
|
96
|
-
return text.replace(/<\/?[a-zA-Z][^>]*>/g, '')
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
/**
|
|
100
|
-
* The plain-text cleanup, in the order the two steps have to run.
|
|
101
|
-
*
|
|
102
|
-
* @param {string} text
|
|
103
|
-
* @returns {string}
|
|
104
|
-
*/
|
|
105
|
-
export function toPlainCharacters(text) {
|
|
106
|
-
return decodeTextEntities(stripInlineHtml(text))
|
|
107
|
-
}
|
package/tests/entities.test.js
DELETED
|
@@ -1,175 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Entity handling on the way out of ProseMirror.
|
|
3
|
-
*
|
|
4
|
-
* The invariant these tests pin down (verified against `content-reader`'s
|
|
5
|
-
* `parser/inline.js`): a PM text node holds HTML. The reader decodes `&`,
|
|
6
|
-
* `"` and `'` and leaves `<`/`>` escaped, because the render
|
|
7
|
-
* path injects the string as HTML. Serializing has to undo exactly that, and
|
|
8
|
-
* exactly there — decoding more, or decoding in prose, corrupts source.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
import { markdownToProseMirror } from '@uniweb/content-reader'
|
|
12
|
-
import { proseMirrorToMarkdown, docSummaryText, docToPlainText } from '../src/index.js'
|
|
13
|
-
import { encodeMarkupEntities, decodeMarkupEntities, decodeTextEntities } from '../src/entities.js'
|
|
14
|
-
|
|
15
|
-
const md = s => proseMirrorToMarkdown(markdownToProseMirror(s)).trim()
|
|
16
|
-
|
|
17
|
-
describe('decodeMarkupEntities', () => {
|
|
18
|
-
test('undoes the pair the reader leaves escaped', () => {
|
|
19
|
-
expect(decodeMarkupEntities('/data/<name>.json')).toBe('/data/<name>.json')
|
|
20
|
-
})
|
|
21
|
-
|
|
22
|
-
test('leaves entities the reader already decoded alone', () => {
|
|
23
|
-
// A PM node holding these means the author typed them.
|
|
24
|
-
expect(decodeMarkupEntities('a & b')).toBe('a & b')
|
|
25
|
-
expect(decodeMarkupEntities('say "hi"')).toBe('say "hi"')
|
|
26
|
-
})
|
|
27
|
-
|
|
28
|
-
test('is a no-op on text with no entities', () => {
|
|
29
|
-
expect(decodeMarkupEntities('plain text')).toBe('plain text')
|
|
30
|
-
})
|
|
31
|
-
})
|
|
32
|
-
|
|
33
|
-
describe('decodeTextEntities', () => {
|
|
34
|
-
test('resolves every entity to the character a reader sees', () => {
|
|
35
|
-
expect(decodeTextEntities('<b> & "q" 'a'')).toBe(`<b> & "q" 'a'`)
|
|
36
|
-
})
|
|
37
|
-
|
|
38
|
-
test('decodes & last, so &lt; yields < and not <', () => {
|
|
39
|
-
expect(decodeTextEntities('&lt;')).toBe('<')
|
|
40
|
-
})
|
|
41
|
-
})
|
|
42
|
-
|
|
43
|
-
describe('D1 — code spans round-trip a literal < >', () => {
|
|
44
|
-
test('the reported defect', () => {
|
|
45
|
-
expect(md('Read `/data/<name>.json` for the cascade.')).toBe(
|
|
46
|
-
'Read `/data/<name>.json` for the cascade.'
|
|
47
|
-
)
|
|
48
|
-
})
|
|
49
|
-
|
|
50
|
-
test('inside a heading', () => {
|
|
51
|
-
expect(md('# The `<slug>` folder')).toBe('# The `<slug>` folder')
|
|
52
|
-
})
|
|
53
|
-
|
|
54
|
-
test('inside a table cell', () => {
|
|
55
|
-
expect(md('| Col |\n| --- |\n| `<x>` |')).toContain('`<x>`')
|
|
56
|
-
})
|
|
57
|
-
|
|
58
|
-
test('inside a link', () => {
|
|
59
|
-
expect(md('[the `<x>` doc](/a)')).toBe('[the `<x>` doc](/a)')
|
|
60
|
-
})
|
|
61
|
-
})
|
|
62
|
-
|
|
63
|
-
describe('D1 — what the fix must NOT touch', () => {
|
|
64
|
-
test('prose entities stay escaped (decoding them would render as a tag)', () => {
|
|
65
|
-
expect(md('Literal <name> in prose.')).toBe('Literal <name> in prose.')
|
|
66
|
-
})
|
|
67
|
-
|
|
68
|
-
test('raw inline HTML in prose is preserved verbatim', () => {
|
|
69
|
-
expect(md('A <name> tag in prose.')).toBe('A <name> tag in prose.')
|
|
70
|
-
})
|
|
71
|
-
|
|
72
|
-
test('an authored & inside a code span survives', () => {
|
|
73
|
-
expect(md('Code `a & b` here.')).toBe('Code `a & b` here.')
|
|
74
|
-
})
|
|
75
|
-
|
|
76
|
-
test('code blocks are raw text and are left alone', () => {
|
|
77
|
-
const source = '```js\nconst x = a < b && c > d;\n```'
|
|
78
|
-
expect(md(source)).toBe(source)
|
|
79
|
-
})
|
|
80
|
-
|
|
81
|
-
test('quotes and apostrophes in code spans are untouched', () => {
|
|
82
|
-
expect(md(`Code \`say "hi" it's\` here.`)).toBe(`Code \`say "hi" it's\` here.`)
|
|
83
|
-
})
|
|
84
|
-
})
|
|
85
|
-
|
|
86
|
-
describe('docSummaryText', () => {
|
|
87
|
-
const summary = (source, opts) => docSummaryText(markdownToProseMirror(source), opts)
|
|
88
|
-
|
|
89
|
-
test('skips the H1 and reads the first paragraph', () => {
|
|
90
|
-
expect(summary('# Quickstart\n\nCreate your first Uniweb site in 5 minutes.')).toBe(
|
|
91
|
-
'Create your first Uniweb site in 5 minutes.'
|
|
92
|
-
)
|
|
93
|
-
})
|
|
94
|
-
|
|
95
|
-
test('returns plain text — no anchors, no mark tags', () => {
|
|
96
|
-
expect(summary('# T\n\nSee **the [docs](/docs)** for *more*.')).toBe(
|
|
97
|
-
'See the docs for more.'
|
|
98
|
-
)
|
|
99
|
-
})
|
|
100
|
-
|
|
101
|
-
test('resolves entities and drops raw inline HTML', () => {
|
|
102
|
-
expect(summary('# T\n\nUse `<name>` and <slug> and <br> here.')).toBe(
|
|
103
|
-
'Use <name> and <slug> and here.'
|
|
104
|
-
)
|
|
105
|
-
})
|
|
106
|
-
|
|
107
|
-
test('truncates at a sentence boundary', () => {
|
|
108
|
-
const text = 'One sentence here. Two sentence here. Three sentence here.'
|
|
109
|
-
expect(summary(`# T\n\n${text}`, { maxChars: 40 })).toBe('One sentence here. Two sentence here.')
|
|
110
|
-
})
|
|
111
|
-
|
|
112
|
-
test('falls back to a word boundary with an ellipsis', () => {
|
|
113
|
-
const text = 'Collections let you manage repeating content across many files'
|
|
114
|
-
expect(summary(`# T\n\n${text}`, { maxChars: 40 })).toBe(
|
|
115
|
-
'Collections let you manage repeating…'
|
|
116
|
-
)
|
|
117
|
-
})
|
|
118
|
-
|
|
119
|
-
test('a dot inside a filename does not end a sentence', () => {
|
|
120
|
-
expect(summary('# T\n\nEdit site.yml then run the build now please.', { maxChars: 20 }))
|
|
121
|
-
.toBe('Edit site.yml then…')
|
|
122
|
-
})
|
|
123
|
-
|
|
124
|
-
test('returns empty string when there is no paragraph', () => {
|
|
125
|
-
expect(summary('# Only a heading')).toBe('')
|
|
126
|
-
expect(docSummaryText(null)).toBe('')
|
|
127
|
-
})
|
|
128
|
-
|
|
129
|
-
test('collapses newlines into one line', () => {
|
|
130
|
-
expect(summary('# T\n\nA line\nand another.')).toBe('A line and another.')
|
|
131
|
-
})
|
|
132
|
-
})
|
|
133
|
-
|
|
134
|
-
describe('docToPlainText also yields plain characters', () => {
|
|
135
|
-
test('strips tags and resolves entities', () => {
|
|
136
|
-
// Note the doubled space where <br> was: this primitive does not
|
|
137
|
-
// normalize whitespace, and its consumers (word counts, search tokens)
|
|
138
|
-
// do not care. `docSummaryText` is the one that promises a single line.
|
|
139
|
-
expect(docToPlainText(markdownToProseMirror('Use <x> and <br> here.'))).toBe(
|
|
140
|
-
'Use <x> and here.'
|
|
141
|
-
)
|
|
142
|
-
})
|
|
143
|
-
})
|
|
144
|
-
|
|
145
|
-
describe('encodeMarkupEntities — the producer-side half', () => {
|
|
146
|
-
test('is the exact inverse of decodeMarkupEntities', () => {
|
|
147
|
-
const raw = '/data/<name>.json'
|
|
148
|
-
expect(encodeMarkupEntities(raw)).toBe('/data/<name>.json')
|
|
149
|
-
expect(decodeMarkupEntities(encodeMarkupEntities(raw))).toBe(raw)
|
|
150
|
-
})
|
|
151
|
-
|
|
152
|
-
test('does not touch &, so an authored entity is not double-encoded', () => {
|
|
153
|
-
expect(encodeMarkupEntities('a & b')).toBe('a & b')
|
|
154
|
-
expect(encodeMarkupEntities('&')).toBe('&')
|
|
155
|
-
})
|
|
156
|
-
|
|
157
|
-
test('round-trips what an author types in a code span', () => {
|
|
158
|
-
for (const raw of ['<div>', 'a < b && c > d', 'a & b', 'plain']) {
|
|
159
|
-
expect(decodeMarkupEntities(encodeMarkupEntities(raw))).toBe(raw)
|
|
160
|
-
}
|
|
161
|
-
})
|
|
162
|
-
|
|
163
|
-
test('the one input it does NOT round-trip: literal entity text', () => {
|
|
164
|
-
// `<div>` and `<div>` both store as `<div>`, so the pair
|
|
165
|
-
// cannot tell them apart and resolves to the decoded form.
|
|
166
|
-
//
|
|
167
|
-
// Accepted deliberately, because the loss happened earlier and elsewhere:
|
|
168
|
-
// a code span is rendered by injecting its text as HTML, so an author who
|
|
169
|
-
// typed `<div>` already sees `<div>` on the page. Decoding makes the
|
|
170
|
-
// file agree with the page rather than destroying something a reader could
|
|
171
|
-
// otherwise see. Escaping `&` as well would make the pair lossless, at the
|
|
172
|
-
// cost of changing what every existing code span stores.
|
|
173
|
-
expect(decodeMarkupEntities(encodeMarkupEntities('<div>'))).toBe('<div>')
|
|
174
|
-
})
|
|
175
|
-
})
|