@lincyaw/compiler 1.0.0-rc.110 → 1.0.0-rc.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -2
- package/src/import.ts +210 -0
- package/src/index.ts +4 -0
- package/src/markdown.ts +336 -0
- package/src/word.ts +42 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lincyaw/compiler",
|
|
3
|
-
"version": "1.0.0-rc.
|
|
3
|
+
"version": "1.0.0-rc.112",
|
|
4
4
|
"description": "Presmium's compiler: structured JSON documents to runtime IR, plugin loading, compile-time execution and its cache, checks and the renderers",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -31,13 +31,22 @@
|
|
|
31
31
|
"dependencies": {
|
|
32
32
|
"content-disposition": "1.0.0",
|
|
33
33
|
"cors": "2.8.5",
|
|
34
|
+
"hast-util-from-html": "2.0.3",
|
|
35
|
+
"hast-util-to-mdast": "^10.1.3",
|
|
34
36
|
"jiti": "^2.7.0",
|
|
37
|
+
"mammoth": "^1.13.0",
|
|
38
|
+
"mdast-util-from-markdown": "2.1.0",
|
|
39
|
+
"mdast-util-gfm": "3.1.0",
|
|
40
|
+
"mdast-util-math": "3.0.0",
|
|
41
|
+
"mdast-util-to-markdown": "2.2.0",
|
|
42
|
+
"micromark-extension-gfm": "3.0.0",
|
|
43
|
+
"micromark-extension-math": "3.1.0",
|
|
35
44
|
"myst-common": "1.10.1",
|
|
36
45
|
"myst-transforms": "1.3.51",
|
|
37
46
|
"semver": "^7.8.5",
|
|
38
47
|
"send": "1.2.1",
|
|
39
48
|
"yaml": "^2.9.1",
|
|
40
|
-
"@lincyaw/kernel": "1.0.0-rc.
|
|
49
|
+
"@lincyaw/kernel": "1.0.0-rc.112"
|
|
41
50
|
},
|
|
42
51
|
"devDependencies": {
|
|
43
52
|
"@types/content-disposition": "^0.5.9",
|
package/src/import.ts
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
// `presmium import`: a folder of Markdown notes (or one note) to documents, one `lecture.json` per note.
|
|
2
|
+
//
|
|
3
|
+
// `a/b/note.md` becomes `<out>/a/b/note/lecture.json`. Wikilinks resolve as note-taking apps resolve them: by path,
|
|
4
|
+
// then by file name or alias (case-insensitive), the nearest note winning when several share a name; a folder's
|
|
5
|
+
// `index` or `README` note also answers to the folder's name. A local file a note links to is copied into its
|
|
6
|
+
// document's `attachments/`, since a document publishes only files inside the workspace; an excluded folder (`raw`,
|
|
7
|
+
// say) is neither imported nor copied from.
|
|
8
|
+
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs'
|
|
9
|
+
import { basename, dirname, extname, join, posix, relative, resolve, sep } from 'node:path'
|
|
10
|
+
import type { MdastNode } from '@lincyaw/kernel'
|
|
11
|
+
import { DOCUMENT_ENTRY } from '@lincyaw/kernel'
|
|
12
|
+
import { headingSlug, markdownDocument, noteTitle, splitFrontmatter } from './markdown.ts'
|
|
13
|
+
|
|
14
|
+
export interface ImportOptions {
|
|
15
|
+
out: string
|
|
16
|
+
/** Names (with `*`) of folders and files to leave out, matched against each part of a path. */
|
|
17
|
+
exclude?: string[]
|
|
18
|
+
lang?: string
|
|
19
|
+
/** The note's folder at this depth becomes the document's first tag, by which the site's sidebar groups documents. */
|
|
20
|
+
group?: number
|
|
21
|
+
/** Retains the exact source beside the editable converted document. */
|
|
22
|
+
preserveOriginal?: boolean
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface ImportReport {
|
|
26
|
+
/** The written documents' folders, relative to `out`. */
|
|
27
|
+
documents: string[]
|
|
28
|
+
files: number
|
|
29
|
+
unresolved: { note: string, link: string }[]
|
|
30
|
+
warnings?: number
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
interface Note { file: string, path: string, dir: string, title: string, aliases: string[], text: string, assets?: Map<string, Uint8Array>, warnings?: number, tree?: MdastNode }
|
|
34
|
+
|
|
35
|
+
const MARKDOWN = new Set(['.md', '.markdown'])
|
|
36
|
+
const SKIPPED = new Set(['node_modules'])
|
|
37
|
+
const RESERVED = new Set(['dist', 'templates', 'node_modules'])
|
|
38
|
+
const FOLDER_NOTES = new Set(['index', 'readme'])
|
|
39
|
+
const ATTACHMENTS = 'attachments'
|
|
40
|
+
|
|
41
|
+
const glob = (pattern: string) => new RegExp(`^${pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.')}$`, 'i')
|
|
42
|
+
const toPosix = (path: string) => path.split(sep).join('/')
|
|
43
|
+
const key = (target: string) => target.trim().replace(/^\/+/, '').replace(/\.(?:md|markdown|docx)$/i, '').toLowerCase()
|
|
44
|
+
/** A document folder's part: letters, digits, `.`, `_` and `-`, not starting with `.`. */
|
|
45
|
+
const part = (name: string) => name.replace(/[^\p{L}\p{N}._-]+/gu, '-').replace(/^[.-]+/, '') || 'note'
|
|
46
|
+
|
|
47
|
+
function discover(input: string, options: ImportOptions, extensions: ReadonlySet<string>) {
|
|
48
|
+
const source = resolve(input)
|
|
49
|
+
const out = resolve(options.out)
|
|
50
|
+
const excluded = (options.exclude ?? []).map(glob)
|
|
51
|
+
const isExcluded = (path: string) => path.split('/').some(name => excluded.some(pattern => pattern.test(name)))
|
|
52
|
+
const single = statSync(source).isFile()
|
|
53
|
+
const root = single ? dirname(source) : source
|
|
54
|
+
|
|
55
|
+
const files: string[] = []
|
|
56
|
+
const seen = new Set<string>()
|
|
57
|
+
const walk = (dir: string) => {
|
|
58
|
+
const real = realpathSync(dir)
|
|
59
|
+
if (existsSync(join(dir, DOCUMENT_ENTRY)) || (dir !== root && resolve(dir) === out) || seen.has(real)) return
|
|
60
|
+
seen.add(real)
|
|
61
|
+
for (const name of readdirSync(dir).sort()) {
|
|
62
|
+
const file = join(dir, name)
|
|
63
|
+
if (name.startsWith('.') || SKIPPED.has(name) || isExcluded(toPosix(relative(root, file)))) continue
|
|
64
|
+
const stat = statSync(file, { throwIfNoEntry: false })
|
|
65
|
+
if (stat?.isDirectory()) walk(file)
|
|
66
|
+
else if (stat?.isFile() && extensions.has(extname(name).toLowerCase())) files.push(file)
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (single) { if (extensions.has(extname(source).toLowerCase())) files.push(source) }
|
|
70
|
+
else walk(source)
|
|
71
|
+
|
|
72
|
+
return { root, out, files, isExcluded }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function importMarkdown(input: string, options: ImportOptions): ImportReport {
|
|
76
|
+
const found = discover(input, options, MARKDOWN)
|
|
77
|
+
return writeNotes(found, options, found.files.map(file => ({ file, text: new TextDecoder('utf-8', { fatal: true }).decode(readFileSync(file)) })))
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** One editable document per Markdown or Word source, with shared link and attachment resolution. */
|
|
81
|
+
export async function importDocuments(input: string, options: ImportOptions): Promise<ImportReport> {
|
|
82
|
+
const found = discover(input, options, new Set([...MARKDOWN, '.docx']))
|
|
83
|
+
const sources: { file: string, text: string, assets?: Map<string, Uint8Array>, warnings?: number, tree?: MdastNode }[] = []
|
|
84
|
+
for (const file of found.files) sources.push(extname(file).toLowerCase() === '.docx'
|
|
85
|
+
? { file, ...await (await import('./word.ts')).wordMarkdown(readFileSync(file)) }
|
|
86
|
+
: { file, text: new TextDecoder('utf-8', { fatal: true }).decode(readFileSync(file)) })
|
|
87
|
+
return writeNotes(found, options, sources)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function writeNotes({ root, out, isExcluded }: ReturnType<typeof discover>, options: ImportOptions, sources: { file: string, text: string, assets?: Map<string, Uint8Array>, warnings?: number, tree?: MdastNode }[]): ImportReport {
|
|
91
|
+
|
|
92
|
+
const taken = new Set<string>()
|
|
93
|
+
const notes: Note[] = sources.map(({ file, text, assets, warnings, tree }) => {
|
|
94
|
+
const path = toPosix(relative(root, file))
|
|
95
|
+
const parts = path.replace(/\.(?:md|markdown|docx)$/i, '').split('/').map(part)
|
|
96
|
+
if (RESERVED.has(parts[0]!)) parts[0] = `${parts[0]}-notes`
|
|
97
|
+
let dir = parts.join('/')
|
|
98
|
+
for (let n = 2; taken.has(dir.toLowerCase()); n++) dir = `${parts.join('/')}-${n}`
|
|
99
|
+
taken.add(dir.toLowerCase())
|
|
100
|
+
return { file, path, dir, text, assets, warnings, tree, title: noteTitle(text) ?? basename(file, extname(file)), aliases: splitFrontmatter(text).frontmatter.aliases }
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
const pathKey = (path: string) => posix.normalize(path).replace(/^\/+/, '').toLowerCase()
|
|
104
|
+
const byPath = new Map(notes.map(note => [pathKey(note.path), note]))
|
|
105
|
+
const byStem = new Map<string, Note[]>()
|
|
106
|
+
for (const note of notes) { const stem = key(note.path); byStem.set(stem, [...byStem.get(stem) ?? [], note]) }
|
|
107
|
+
const pathNote = (path: string) => {
|
|
108
|
+
const exact = byPath.get(pathKey(path))
|
|
109
|
+
const candidates = byStem.get(key(posix.normalize(path))) ?? []
|
|
110
|
+
return exact ?? (!/\.(?:md|markdown|docx)$/i.test(path) && candidates.length === 1 ? candidates[0] : undefined)
|
|
111
|
+
}
|
|
112
|
+
const byName = new Map<string, Note[]>()
|
|
113
|
+
const register = (name: string, note: Note) => {
|
|
114
|
+
const k = key(name)
|
|
115
|
+
if (k && !byName.get(k)?.includes(note)) byName.set(k, [...byName.get(k) ?? [], note])
|
|
116
|
+
}
|
|
117
|
+
for (const note of notes) {
|
|
118
|
+
const stem = posix.basename(note.path).replace(/\.(?:md|markdown|docx)$/i, '')
|
|
119
|
+
register(stem, note)
|
|
120
|
+
note.aliases.forEach(alias => register(alias, note))
|
|
121
|
+
if (FOLDER_NOTES.has(stem.toLowerCase()) && posix.dirname(note.path) !== '.') register(posix.basename(posix.dirname(note.path)), note)
|
|
122
|
+
}
|
|
123
|
+
/** The note nearest to `from` among `candidates`: the one sharing the longest folder path, if only one does. */
|
|
124
|
+
const nearest = (from: Note, candidates: Note[]) => {
|
|
125
|
+
const shared = (note: Note) => {
|
|
126
|
+
const a = posix.dirname(from.path).split('/'), b = posix.dirname(note.path).split('/')
|
|
127
|
+
let n = 0
|
|
128
|
+
while (n < a.length && a[n] === b[n]) n++
|
|
129
|
+
return n
|
|
130
|
+
}
|
|
131
|
+
const best = Math.max(...candidates.map(shared))
|
|
132
|
+
const winners = candidates.filter(note => shared(note) === best)
|
|
133
|
+
return winners.length === 1 ? winners[0] : undefined
|
|
134
|
+
}
|
|
135
|
+
const href = (from: Note, to: Note, fragment?: string) => {
|
|
136
|
+
const path = posix.relative(from.dir, to.dir)
|
|
137
|
+
return `${path ? `${path}/` : ''}${DOCUMENT_ENTRY}${fragment ? `#${headingSlug(fragment)}` : ''}`
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const report: ImportReport = { documents: [], files: 0, unresolved: [] }
|
|
141
|
+
const copied = new Map<string, string | Uint8Array>()
|
|
142
|
+
for (const note of notes) {
|
|
143
|
+
const docDir = join(out, note.dir)
|
|
144
|
+
const names = new Map<string, string>()
|
|
145
|
+
/** A file inside the imported folder to its copy in the document's attachments; undefined for anything else. */
|
|
146
|
+
const attach = (url: string) => {
|
|
147
|
+
const embedded = note.assets?.get(url)
|
|
148
|
+
if (embedded) {
|
|
149
|
+
const copy = `${ATTACHMENTS}/${basename(url)}`
|
|
150
|
+
copied.set(join(docDir, copy), embedded)
|
|
151
|
+
return copy
|
|
152
|
+
}
|
|
153
|
+
const [path = '', ...fragment] = url.split('#')
|
|
154
|
+
let name = path
|
|
155
|
+
try {
|
|
156
|
+
name = decodeURIComponent(path)
|
|
157
|
+
} catch {
|
|
158
|
+
}
|
|
159
|
+
const file = resolve(dirname(note.file), name)
|
|
160
|
+
const rel = toPosix(relative(root, file))
|
|
161
|
+
if (!name || rel.startsWith('../') || isExcluded(rel) || !statSync(file, { throwIfNoEntry: false })?.isFile()) return undefined
|
|
162
|
+
const target = byPath.get(pathKey(rel))
|
|
163
|
+
if (target) return href(note, target, fragment.length ? fragment.join('#') : undefined)
|
|
164
|
+
let copy = names.get(file)
|
|
165
|
+
if (!copy) {
|
|
166
|
+
const base = basename(file)
|
|
167
|
+
copy = [...names.values()].includes(`${ATTACHMENTS}/${base}`) ? `${ATTACHMENTS}/${names.size}-${base}` : `${ATTACHMENTS}/${base}`
|
|
168
|
+
names.set(file, copy)
|
|
169
|
+
}
|
|
170
|
+
copied.set(join(docDir, copy), file)
|
|
171
|
+
return `${encodeURI(copy)}${fragment.length ? `#${fragment.join('#')}` : ''}`
|
|
172
|
+
}
|
|
173
|
+
const { document, unresolved } = markdownDocument(note.text, {
|
|
174
|
+
id: note.dir.replace(/\//g, '--'),
|
|
175
|
+
title: note.title,
|
|
176
|
+
tree: note.tree,
|
|
177
|
+
lang: options.lang,
|
|
178
|
+
links: {
|
|
179
|
+
page: (target, fragment) => {
|
|
180
|
+
const direct = pathNote(posix.join(posix.dirname(note.path), target)) ?? pathNote(target)
|
|
181
|
+
const candidates = byName.get(key(target)) ?? []
|
|
182
|
+
const found = direct ?? (candidates.length === 1 ? candidates[0] : candidates.length > 1 ? nearest(note, candidates) : undefined)
|
|
183
|
+
return found && href(note, found, fragment)
|
|
184
|
+
},
|
|
185
|
+
url: attach,
|
|
186
|
+
image: attach,
|
|
187
|
+
},
|
|
188
|
+
})
|
|
189
|
+
const folder = options.group ? posix.dirname(note.path).split('/').slice(0, options.group).at(-1) : undefined
|
|
190
|
+
if (folder && folder !== '.') document.metadata.tags = [folder, ...(document.metadata.tags ?? []).filter(tag => tag !== folder)]
|
|
191
|
+
mkdirSync(docDir, { recursive: true })
|
|
192
|
+
writeFileSync(join(docDir, DOCUMENT_ENTRY), `${JSON.stringify(document, null, 2)}\n`)
|
|
193
|
+
report.documents.push(note.dir)
|
|
194
|
+
report.unresolved.push(...unresolved.map(link => ({ note: note.path, link })))
|
|
195
|
+
if (note.warnings) report.warnings = (report.warnings ?? 0) + note.warnings
|
|
196
|
+
if (options.preserveOriginal) copied.set(join(docDir, 'originals', basename(note.file)), note.file)
|
|
197
|
+
}
|
|
198
|
+
for (const [copy, file] of copied) {
|
|
199
|
+
mkdirSync(dirname(copy), { recursive: true })
|
|
200
|
+
if (typeof file !== 'string') writeFileSync(copy, file)
|
|
201
|
+
else if (!existsSync(copy) || statSync(copy).mtimeMs < statSync(file).mtimeMs || statSync(copy).size !== statSync(file).size) copyFileSync(file, copy)
|
|
202
|
+
}
|
|
203
|
+
report.files = copied.size
|
|
204
|
+
return report
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** The `documents` patterns that cover these folders: one per depth, as workspace patterns do not cross folders. */
|
|
208
|
+
export function documentPatterns(dirs: string[]): string[] {
|
|
209
|
+
return [...new Set(dirs.map(dir => dir.split('/').length))].sort((a, b) => a - b).map(depth => `${'*/'.repeat(depth)}${DOCUMENT_ENTRY}`)
|
|
210
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
export { directoryAsset, fileHash, publishesDirectory } from './assets.ts'
|
|
2
|
+
export { documentPatterns, importDocuments, importMarkdown } from './import.ts'
|
|
3
|
+
export type { ImportOptions, ImportReport } from './import.ts'
|
|
4
|
+
export { headingSlug, markdownDocument, noteTitle, splitFrontmatter } from './markdown.ts'
|
|
5
|
+
export type { Frontmatter, ImportedMarkdown, MarkdownLinks } from './markdown.ts'
|
|
2
6
|
export { availableRenderers, build } from './build.ts'
|
|
3
7
|
export type { BuildOptions, BuildReport } from './build.ts'
|
|
4
8
|
export { buildSummary, buildToDist, checkContext, runBlock } from './compilation-actions.ts'
|
package/src/markdown.ts
ADDED
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
// One Markdown note to one document (`presmium import`). Pure: the caller resolves links and copies files.
|
|
2
|
+
//
|
|
3
|
+
// Each level-1 or level-2 heading starts a page (the first page holds what comes before it, under the note's title);
|
|
4
|
+
// every page has one step that shows all its blocks. Paragraphs, headings, lists, quotes, code, math and tables become
|
|
5
|
+
// rich text; a Mermaid diagram becomes a code block of its own, which is what draws it; a paragraph holding only an
|
|
6
|
+
// image becomes an image block. What has no rich-text node (raw HTML, footnotes, rules, callouts) is kept as an mdast
|
|
7
|
+
// extension that the viewer renders. `[[target]]` and `[[target|label]]` wikilinks, relative links and images go through
|
|
8
|
+
// the caller's `links`. Node ids come from the content, so importing an unchanged paragraph again keeps its id.
|
|
9
|
+
import { fromMarkdown } from 'mdast-util-from-markdown'
|
|
10
|
+
import { gfmFromMarkdown } from 'mdast-util-gfm'
|
|
11
|
+
import { mathFromMarkdown } from 'mdast-util-math'
|
|
12
|
+
import { gfm } from 'micromark-extension-gfm'
|
|
13
|
+
import { math } from 'micromark-extension-math'
|
|
14
|
+
import { parse as parseYaml } from 'yaml'
|
|
15
|
+
import type { Asset, DocumentBlock, InlineNode, MdastNode, RichTextNode, Section, Step, TeachingDocument, TextMark } from '@lincyaw/kernel'
|
|
16
|
+
|
|
17
|
+
export interface Frontmatter {
|
|
18
|
+
title?: string
|
|
19
|
+
description?: string
|
|
20
|
+
lang?: string
|
|
21
|
+
tags: string[]
|
|
22
|
+
aliases: string[]
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface MarkdownLinks {
|
|
26
|
+
/** A wikilink's target (without `#fragment`) to the href of the document it names; undefined when there is none. */
|
|
27
|
+
page(target: string, fragment: string | undefined): string | undefined
|
|
28
|
+
/** A relative URL of a link to its new href; undefined when it leads nowhere that is published. */
|
|
29
|
+
url(url: string): string | undefined
|
|
30
|
+
/** A relative URL of an image to the location of its copy, relative to the document. */
|
|
31
|
+
image(url: string): string | undefined
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface ImportedMarkdown {
|
|
35
|
+
document: TeachingDocument
|
|
36
|
+
/** Wikilinks and relative links that `links` could not resolve, as written. */
|
|
37
|
+
unresolved: string[]
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/
|
|
41
|
+
/** Stands in for the `|` of `[[target|label]]` while parsing, so a GFM table does not split the cell at it. */
|
|
42
|
+
const LABEL = '\uE000'
|
|
43
|
+
const WIKILINK = new RegExp(`\\[\\[([^\\]|${LABEL}\\n]+)(?:[|${LABEL}]([^\\]\\n]+))?\\]\\]`, 'g')
|
|
44
|
+
const HIGHLIGHT = /==(?=\S)([^=\n]*\S)==/g
|
|
45
|
+
const CALLOUT = /^\[!([A-Za-z]+)\][+-]?[ \t]*/
|
|
46
|
+
const SCHEME = /^(?:[a-z][a-z\d+.-]*:|\/\/|#)/i
|
|
47
|
+
const MARKS: Record<string, TextMark['type']> = { strong: 'bold', emphasis: 'italic', delete: 'strike' }
|
|
48
|
+
const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu
|
|
49
|
+
const LETTER = /\p{L}/gu
|
|
50
|
+
|
|
51
|
+
export function splitFrontmatter(source: string): { frontmatter: Frontmatter, body: string } {
|
|
52
|
+
const match = FRONTMATTER.exec(source)
|
|
53
|
+
let data: unknown
|
|
54
|
+
if (match) {
|
|
55
|
+
try {
|
|
56
|
+
data = parseYaml(match[1]!)
|
|
57
|
+
} catch {
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
const record = data !== null && typeof data === 'object' && !Array.isArray(data) ? data as Record<string, unknown> : {}
|
|
61
|
+
const text = (key: string) => (typeof record[key] === 'string' && (record[key] as string).trim() ? (record[key] as string).trim() : undefined)
|
|
62
|
+
const list = (key: string) => {
|
|
63
|
+
const value = record[key]
|
|
64
|
+
return (Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : [])
|
|
65
|
+
.flatMap(item => (typeof item === 'string' || typeof item === 'number' ? [String(item).trim()] : [])).filter(Boolean)
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
frontmatter: { title: text('title'), description: text('description'), lang: text('lang'), tags: list('tags'), aliases: list('aliases') },
|
|
69
|
+
body: match ? source.slice(match[0].length) : source,
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** The title of a note: its frontmatter title, else its first level-1 heading. */
|
|
74
|
+
export function noteTitle(source: string): string | undefined {
|
|
75
|
+
const { frontmatter, body } = splitFrontmatter(source)
|
|
76
|
+
return frontmatter.title ?? /^#[ \t]+(.+?)[ \t#]*$/m.exec(body)?.[1]?.trim()
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The anchor of a heading, as GitHub writes it: lower case, punctuation dropped, spaces to hyphens. */
|
|
80
|
+
export function headingSlug(text: string): string {
|
|
81
|
+
return text.trim().toLowerCase().replace(/[^\p{L}\p{N}\s_-]/gu, '').replace(/\s/g, '-')
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function guessLang(text: string): string {
|
|
85
|
+
const letters = text.match(LETTER)?.length ?? 0
|
|
86
|
+
return letters > 0 && (text.match(CJK)?.length ?? 0) / letters > 0.3 ? 'zh-CN' : 'en'
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function plain(node: MdastNode): string {
|
|
90
|
+
if (typeof node.value === 'string') return node.value
|
|
91
|
+
return (node.children ?? []).map(plain).join('')
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function hash(text: string): string {
|
|
95
|
+
let h = 0x811c9dc5
|
|
96
|
+
for (let i = 0; i < text.length; i++) h = Math.imul(h ^ text.charCodeAt(i), 0x01000193)
|
|
97
|
+
return (h >>> 0).toString(36)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function withoutPositions(node: MdastNode): unknown {
|
|
101
|
+
const { position: _position, children, ...rest } = node
|
|
102
|
+
return children ? { ...rest, children: children.map(withoutPositions) } : rest
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function restoreLabels(node: MdastNode): void {
|
|
106
|
+
if (typeof node.value === 'string') node.value = node.value.replaceAll(LABEL, '|')
|
|
107
|
+
node.children?.forEach(restoreLabels)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export function markdownDocument(source: string, options: { id: string, title: string, lang?: string, tree?: MdastNode, links: MarkdownLinks }): ImportedMarkdown {
|
|
111
|
+
const { frontmatter, body } = splitFrontmatter(source)
|
|
112
|
+
const protectedBody = body.replace(/\[\[([^\]|\n]+)\|([^\]\n]+)\]\]/g, (_, target: string, label: string) => `[[${target}${LABEL}${label}]]`)
|
|
113
|
+
const root = options.tree ? structuredClone(options.tree) : fromMarkdown(protectedBody, { extensions: [gfm(), math()], mdastExtensions: [gfmFromMarkdown(), mathFromMarkdown()] }) as unknown as MdastNode
|
|
114
|
+
const unresolved: string[] = []
|
|
115
|
+
const definitions = new Map<string, string>()
|
|
116
|
+
const collect = (node: MdastNode) => {
|
|
117
|
+
if (node.type === 'definition') definitions.set(String(node.identifier).toLowerCase(), String(node.url))
|
|
118
|
+
node.children?.forEach(collect)
|
|
119
|
+
}
|
|
120
|
+
collect(root)
|
|
121
|
+
|
|
122
|
+
// Links are rewritten in the mdast first, so callouts and other extensions carry the same hrefs as rich text.
|
|
123
|
+
const relink = (node: MdastNode): MdastNode[] => {
|
|
124
|
+
if (node.type === 'text' && typeof node.value === 'string' && node.value.includes('[[')) return wikilinks(node.value)
|
|
125
|
+
if ((node.type === 'link' || node.type === 'definition') && typeof node.url === 'string' && !SCHEME.test(node.url)) {
|
|
126
|
+
const href = options.links.url(node.url)
|
|
127
|
+
if (href === undefined) {
|
|
128
|
+
unresolved.push(node.url)
|
|
129
|
+
if (node.type === 'link') return (node.children ?? []).flatMap(relink)
|
|
130
|
+
}
|
|
131
|
+
node.url = href ?? node.url
|
|
132
|
+
}
|
|
133
|
+
if (node.type === 'imageReference') {
|
|
134
|
+
const url = definitions.get(String(node.identifier).toLowerCase())
|
|
135
|
+
if (url !== undefined) return relink({ type: 'image', url, alt: node.alt, title: node.title })
|
|
136
|
+
}
|
|
137
|
+
if (node.type === 'image' && typeof node.url === 'string') {
|
|
138
|
+
const location = /^(?:https?:|\/\/)/i.test(node.url) ? node.url : options.links.image(node.url)
|
|
139
|
+
if (location === undefined) { unresolved.push(node.url); return [{ type: 'text', value: String(node.alt ?? '') }] }
|
|
140
|
+
node.url = location
|
|
141
|
+
}
|
|
142
|
+
if (node.type === 'linkReference') {
|
|
143
|
+
const url = definitions.get(String(node.identifier).toLowerCase())
|
|
144
|
+
if (url !== undefined) return relink({ type: 'link', url, children: node.children ?? [] })
|
|
145
|
+
}
|
|
146
|
+
if (node.children) node.children = node.children.flatMap(relink)
|
|
147
|
+
return [node]
|
|
148
|
+
}
|
|
149
|
+
const wikilinks = (value: string): MdastNode[] => {
|
|
150
|
+
const out: MdastNode[] = []
|
|
151
|
+
let cursor = 0
|
|
152
|
+
for (const match of value.matchAll(WIKILINK)) {
|
|
153
|
+
if (match.index > cursor) out.push({ type: 'text', value: value.slice(cursor, match.index) })
|
|
154
|
+
const written = match[1]!.trim()
|
|
155
|
+
const [target = '', ...fragment] = written.split('#')
|
|
156
|
+
const label = (match[2] ?? written).trim()
|
|
157
|
+
const href = options.links.page(target.trim(), fragment.length ? fragment.join('#') : undefined)
|
|
158
|
+
if (href === undefined) unresolved.push(`[[${written}]]`)
|
|
159
|
+
out.push(href === undefined ? { type: 'text', value: label } : { type: 'link', url: href, children: [{ type: 'text', value: label }] })
|
|
160
|
+
cursor = match.index + match[0].length
|
|
161
|
+
}
|
|
162
|
+
if (cursor < value.length) out.push({ type: 'text', value: value.slice(cursor) })
|
|
163
|
+
return out
|
|
164
|
+
}
|
|
165
|
+
root.children = (root.children ?? []).flatMap(relink)
|
|
166
|
+
restoreLabels(root)
|
|
167
|
+
|
|
168
|
+
const used = new Set<string>()
|
|
169
|
+
const unique = (base: string) => {
|
|
170
|
+
let id = base.slice(0, 180) || 'node'
|
|
171
|
+
for (let n = 2; used.has(id); n++) id = `${base.slice(0, 180)}-${n}`
|
|
172
|
+
used.add(id)
|
|
173
|
+
return id
|
|
174
|
+
}
|
|
175
|
+
const idOf = (prefix: string, node: MdastNode) => unique(`${prefix}-${hash(JSON.stringify(withoutPositions(node)))}`)
|
|
176
|
+
const extension = (node: MdastNode): RichTextNode => {
|
|
177
|
+
const text = plain(node).trim()
|
|
178
|
+
return { id: idOf(node.type, node), type: 'extension', kind: 'presmium.mdast', schemaVersion: 1, data: { node: withoutPositions(node) as never },
|
|
179
|
+
fallback: text ? [{ id: idOf('fallback', node), type: 'paragraph', children: [{ type: 'text', text }] }] : [] }
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const inline = (nodes: MdastNode[], marks: TextMark[] = []): InlineNode[] => nodes.flatMap((node): InlineNode[] => {
|
|
183
|
+
const styled = (text: string, extra: TextMark[] = []): InlineNode => ({ type: 'text', text, ...marks.length + extra.length ? { marks: [...marks, ...extra] } : {} })
|
|
184
|
+
if (node.type === 'text') {
|
|
185
|
+
const value = String(node.value ?? '')
|
|
186
|
+
if (!value.includes('==')) return value ? [styled(value)] : []
|
|
187
|
+
const out: InlineNode[] = []
|
|
188
|
+
let cursor = 0
|
|
189
|
+
for (const match of value.matchAll(HIGHLIGHT)) {
|
|
190
|
+
if (match.index > cursor) out.push(styled(value.slice(cursor, match.index)))
|
|
191
|
+
out.push(styled(match[1]!, [{ type: 'background', value: 'warn' }]))
|
|
192
|
+
cursor = match.index + match[0].length
|
|
193
|
+
}
|
|
194
|
+
if (cursor < value.length) out.push(styled(value.slice(cursor)))
|
|
195
|
+
return out
|
|
196
|
+
}
|
|
197
|
+
if (node.type === 'break') return [{ type: 'break' }]
|
|
198
|
+
if (node.type === 'inlineMath') return [{ type: 'math', value: String(node.value ?? '') }]
|
|
199
|
+
if (node.type === 'inlineCode') return [styled(String(node.value ?? ''), [{ type: 'code' }])]
|
|
200
|
+
if (MARKS[node.type]) return inline(node.children ?? [], [...marks, { type: MARKS[node.type]! } as TextMark])
|
|
201
|
+
if (node.type === 'link') return [{ type: 'link', href: String(node.url), children: inline(node.children ?? [], marks) }]
|
|
202
|
+
if (node.type === 'image' && typeof node.url === 'string') {
|
|
203
|
+
return [{ type: 'extension', kind: 'presmium.mdast', schemaVersion: 1, data: { node: withoutPositions(node) as never }, fallback: String(node.alt ?? '') }]
|
|
204
|
+
}
|
|
205
|
+
return [{ type: 'extension', kind: 'presmium.mdast', schemaVersion: 1, data: { node: withoutPositions(node) as never }, fallback: plain(node) }]
|
|
206
|
+
})
|
|
207
|
+
|
|
208
|
+
const rich = (node: MdastNode): RichTextNode[] => {
|
|
209
|
+
const children = node.children ?? []
|
|
210
|
+
switch (node.type) {
|
|
211
|
+
case 'paragraph': {
|
|
212
|
+
const content = inline(children)
|
|
213
|
+
return content.length ? [{ id: idOf('p', node), type: 'paragraph', children: content }] : []
|
|
214
|
+
}
|
|
215
|
+
case 'heading':
|
|
216
|
+
return [{ id: unique(headingSlug(plain(node)) || `h-${hash(plain(node))}`), type: 'heading', level: Math.min(6, Math.max(1, Number(node.depth))) as 1, children: inline(children) }]
|
|
217
|
+
case 'blockquote': {
|
|
218
|
+
const callout = calloutOf(node)
|
|
219
|
+
return callout ? [extension(callout)] : [{ id: idOf('quote', node), type: 'blockquote', children: children.flatMap(rich) }]
|
|
220
|
+
}
|
|
221
|
+
case 'list':
|
|
222
|
+
return [{ id: idOf('list', node), type: 'list', ordered: Boolean(node.ordered), ...Number(node.start) >= 1 ? { start: Number(node.start) } : {},
|
|
223
|
+
children: children.map(item => ({ id: idOf('item', item), type: 'listItem', ...typeof item.checked === 'boolean' ? { checked: item.checked } : {}, children: (item.children ?? []).flatMap(rich) })) }]
|
|
224
|
+
case 'code':
|
|
225
|
+
return [{ id: idOf('code', node), type: 'code', language: String(node.lang ?? ''), value: String(node.value ?? '') }]
|
|
226
|
+
case 'math':
|
|
227
|
+
return [{ id: idOf('math', node), type: 'math', value: String(node.value ?? '') }]
|
|
228
|
+
case 'table': {
|
|
229
|
+
const align = Array.isArray(node.align) ? node.align as (string | null)[] : []
|
|
230
|
+
return [{ id: idOf('table', node), type: 'table', headerRows: node.headerRows === 0 ? 0 : 1, children: children.map(row => ({ id: idOf('row', row), type: 'tableRow',
|
|
231
|
+
children: (row.children ?? []).map((cell, c) => ({ id: idOf('cell', cell), type: 'tableCell', children: inline(cell.children ?? []),
|
|
232
|
+
...align[c] ? { style: { alignment: align[c] as 'left' } } : {} })) })) }]
|
|
233
|
+
}
|
|
234
|
+
case 'definition':
|
|
235
|
+
return []
|
|
236
|
+
default:
|
|
237
|
+
return [extension(node)]
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Pages: a level-1 or level-2 heading starts one. The note's own title heading opens the first page and is not repeated.
|
|
242
|
+
const nodes = root.children ?? []
|
|
243
|
+
const title = frontmatter.title ?? (nodes[0]?.type === 'heading' && nodes[0].depth === 1 ? plain(nodes[0]).trim() : undefined) ?? options.title
|
|
244
|
+
const pages: { title: string, level: number, nodes: MdastNode[] }[] = [{ title, level: 1, nodes: [] }]
|
|
245
|
+
nodes.forEach((node, i) => {
|
|
246
|
+
const opensPage = node.type === 'heading' && Number(node.depth) <= 2
|
|
247
|
+
if (opensPage && i === 0 && node.depth === 1 && plain(node).trim() === title) return
|
|
248
|
+
if (opensPage) pages.push({ title: plain(node).trim() || title, level: Number(node.depth), nodes: [] })
|
|
249
|
+
else pages.at(-1)!.nodes.push(node)
|
|
250
|
+
})
|
|
251
|
+
if (pages.length > 1 && pages[0]!.nodes.length === 0 && pages[1]!.level === 1) pages.shift()
|
|
252
|
+
|
|
253
|
+
const sections: Record<string, Section> = {}, blocks: Record<string, DocumentBlock> = {}, steps: Record<string, Step> = {}, assets: Record<string, Asset> = {}
|
|
254
|
+
const sectionOrder: string[] = []
|
|
255
|
+
for (const page of pages) {
|
|
256
|
+
const sectionId = unique(headingSlug(page.title) || `page-${hash(page.title)}`)
|
|
257
|
+
const children: string[] = []
|
|
258
|
+
let text: RichTextNode[] = []
|
|
259
|
+
const flush = () => {
|
|
260
|
+
if (!text.length) return
|
|
261
|
+
const id = unique(`${sectionId}--text-${children.length}`)
|
|
262
|
+
blocks[id] = { id, type: 'text', content: text }
|
|
263
|
+
children.push(id)
|
|
264
|
+
text = []
|
|
265
|
+
}
|
|
266
|
+
for (const node of page.nodes) {
|
|
267
|
+
const image = node.type === 'paragraph' ? soleImage(node) : undefined
|
|
268
|
+
const location = image && typeof image.url === 'string' ? image.url : undefined
|
|
269
|
+
if (node.type === 'code' && (node.lang === 'mermaid' || node.lang === 'mmd')) {
|
|
270
|
+
flush()
|
|
271
|
+
const id = unique(`${sectionId}--diagram-${children.length}`)
|
|
272
|
+
blocks[id] = { id, type: 'code', language: 'mermaid', value: String(node.value ?? '') }
|
|
273
|
+
children.push(id)
|
|
274
|
+
} else if (image && location) {
|
|
275
|
+
flush()
|
|
276
|
+
const id = unique(`${sectionId}--image-${children.length}`), assetId = unique(`asset-${hash(location)}`)
|
|
277
|
+
assets[assetId] = { id: assetId, kind: 'image', location, ...image.title ? { title: String(image.title) } : {} }
|
|
278
|
+
blocks[id] = { id, type: 'image', assetId, alt: String(image.alt ?? '') }
|
|
279
|
+
children.push(id)
|
|
280
|
+
} else {
|
|
281
|
+
if (image && !location && typeof image.url === 'string') unresolved.push(image.url)
|
|
282
|
+
text.push(...rich(node))
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
flush()
|
|
286
|
+
const stepId = unique(`${sectionId}--step`)
|
|
287
|
+
sections[sectionId] = { id: sectionId, title: page.title, level: Math.min(2, page.level) as 1, children, stepOrder: [stepId], layout: { type: 'stack' } }
|
|
288
|
+
steps[stepId] = { id: stepId, sectionId, visuals: children.map(blockId => ({ blockId, visible: true })), narrationIds: [] }
|
|
289
|
+
sectionOrder.push(sectionId)
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
return {
|
|
293
|
+
document: {
|
|
294
|
+
schemaVersion: 1,
|
|
295
|
+
id: unique(options.id),
|
|
296
|
+
metadata: {
|
|
297
|
+
title,
|
|
298
|
+
lang: frontmatter.lang ?? options.lang ?? guessLang(body),
|
|
299
|
+
...frontmatter.description ? { description: frontmatter.description } : {},
|
|
300
|
+
...frontmatter.tags.length ? { tags: frontmatter.tags } : {},
|
|
301
|
+
plugins: [],
|
|
302
|
+
renderers: { web: {} },
|
|
303
|
+
},
|
|
304
|
+
sectionOrder, sections, blocks, steps, narrations: {}, assets, definitions: {},
|
|
305
|
+
production: { defaults: {}, stepOverrides: {}, mediaClips: [] },
|
|
306
|
+
} as TeachingDocument,
|
|
307
|
+
unresolved: [...new Set(unresolved)],
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function soleImage(paragraph: MdastNode): MdastNode | undefined {
|
|
312
|
+
const content = (paragraph.children ?? []).filter(child => !(child.type === 'text' && !String(child.value ?? '').trim()))
|
|
313
|
+
return content.length === 1 && content[0]!.type === 'image' ? content[0] : undefined
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/** `> [!note] Title` (as GitHub and Obsidian write callouts) as a MyST admonition, which the viewer draws as one. */
|
|
317
|
+
function calloutOf(quote: MdastNode): MdastNode | undefined {
|
|
318
|
+
const first = quote.children?.[0]
|
|
319
|
+
const lead = first?.type === 'paragraph' ? first.children?.[0] : undefined
|
|
320
|
+
const match = lead?.type === 'text' ? CALLOUT.exec(String(lead.value ?? '')) : null
|
|
321
|
+
if (!first || !lead || !match) return undefined
|
|
322
|
+
const rest = String(lead.value).slice(match[0].length)
|
|
323
|
+
const breakAt = rest.indexOf('\n')
|
|
324
|
+
const titleText = (breakAt < 0 ? rest : rest.slice(0, breakAt)).trim()
|
|
325
|
+
const bodyLead = breakAt < 0 ? '' : rest.slice(breakAt + 1)
|
|
326
|
+
const inlineRest = [...bodyLead ? [{ type: 'text', value: bodyLead }] : [], ...(first.children ?? []).slice(1)]
|
|
327
|
+
const kind = match[1]!.toLowerCase()
|
|
328
|
+
return {
|
|
329
|
+
type: 'admonition', kind,
|
|
330
|
+
children: [
|
|
331
|
+
{ type: 'admonitionTitle', children: [{ type: 'text', value: titleText || kind.charAt(0).toUpperCase() + kind.slice(1) }] },
|
|
332
|
+
...inlineRest.length ? [{ type: 'paragraph', children: inlineRest }] : [],
|
|
333
|
+
...(quote.children ?? []).slice(1),
|
|
334
|
+
],
|
|
335
|
+
}
|
|
336
|
+
}
|
package/src/word.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import mammoth from 'mammoth'
|
|
2
|
+
import { fromHtml } from 'hast-util-from-html'
|
|
3
|
+
import type { MdastNode } from '@lincyaw/kernel'
|
|
4
|
+
import { defaultHandlers, toMdast } from 'hast-util-to-mdast'
|
|
5
|
+
import { toMarkdown } from 'mdast-util-to-markdown'
|
|
6
|
+
import { gfmToMarkdown } from 'mdast-util-gfm'
|
|
7
|
+
|
|
8
|
+
/** DOCX contributes semantic content; page layout, fonts and external files do not enter the document. */
|
|
9
|
+
export async function wordMarkdown(buffer: Buffer): Promise<{ text: string, assets: Map<string, Uint8Array>, warnings: number, tree: MdastNode }> {
|
|
10
|
+
const assets = new Map<string, Uint8Array>()
|
|
11
|
+
let imageNumber = 0
|
|
12
|
+
const extensions: Record<string, string> = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/gif': 'gif', 'image/webp': 'webp', 'image/svg+xml': 'svg', 'image/tiff': 'tiff' }
|
|
13
|
+
const converted = await mammoth.convertToHtml({ buffer }, {
|
|
14
|
+
includeEmbeddedStyleMap: false,
|
|
15
|
+
styleMap: ["p[style-name='Title'] => h1:fresh"],
|
|
16
|
+
externalFileAccess: false,
|
|
17
|
+
convertImage: mammoth.images.imgElement(async image => {
|
|
18
|
+
const name = `word-image-${++imageNumber}.${extensions[image.contentType] ?? 'bin'}`
|
|
19
|
+
const bytes = await image.readAsBuffer()
|
|
20
|
+
assets.set(name, bytes)
|
|
21
|
+
return { src: name }
|
|
22
|
+
}),
|
|
23
|
+
})
|
|
24
|
+
const tree = toMdast(fromHtml(converted.value, { fragment: true }), {
|
|
25
|
+
handlers: {
|
|
26
|
+
table(state, node) {
|
|
27
|
+
const result = defaultHandlers.table!(state, node)
|
|
28
|
+
if (result && !Array.isArray(result) && result.type === 'table') {
|
|
29
|
+
const hasHeader = (element: typeof node): boolean => element.tagName === 'thead' || element.tagName === 'th'
|
|
30
|
+
|| element.children.some(child => child.type === 'element' && hasHeader(child))
|
|
31
|
+
// The HTML converter inserts an empty Markdown header for Word tables without one.
|
|
32
|
+
// Keep the original rows in the editable tree; the text copy only supplies its title.
|
|
33
|
+
if (!hasHeader(node)) result.children.shift()
|
|
34
|
+
Object.assign(result, { headerRows: hasHeader(node) ? 1 : 0 })
|
|
35
|
+
}
|
|
36
|
+
return result
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
})
|
|
40
|
+
const text = toMarkdown(tree, { extensions: [gfmToMarkdown()], bullet: '-', fences: true })
|
|
41
|
+
return { text, tree: tree as unknown as MdastNode, assets, warnings: converted.messages.length }
|
|
42
|
+
}
|