@lincyaw/cli 1.0.0-rc.110 → 1.0.0-rc.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -11
- package/src/cli.ts +4 -3
- package/src/import.ts +2 -169
- package/src/markdown.ts +2 -328
- package/src/messages.ts +7 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lincyaw/cli",
|
|
3
|
-
"version": "1.0.0-rc.
|
|
3
|
+
"version": "1.0.0-rc.111",
|
|
4
4
|
"description": "The presmium command: build, dev, ir, check and serve for Presmium documents",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -36,16 +36,11 @@
|
|
|
36
36
|
],
|
|
37
37
|
"dependencies": {
|
|
38
38
|
"jiti": "^2.7.0",
|
|
39
|
-
"mdast-util-from-markdown": "2.1.0",
|
|
40
|
-
"mdast-util-gfm": "3.1.0",
|
|
41
|
-
"mdast-util-math": "3.0.0",
|
|
42
|
-
"micromark-extension-gfm": "3.0.0",
|
|
43
|
-
"micromark-extension-math": "3.1.0",
|
|
44
39
|
"yaml": "^2.9.1",
|
|
45
|
-
"@lincyaw/compiler": "1.0.0-rc.
|
|
46
|
-
"@lincyaw/
|
|
47
|
-
"@lincyaw/
|
|
48
|
-
"@lincyaw/
|
|
49
|
-
"@lincyaw/
|
|
40
|
+
"@lincyaw/compiler": "1.0.0-rc.111",
|
|
41
|
+
"@lincyaw/workspace": "1.0.0-rc.111",
|
|
42
|
+
"@lincyaw/server": "1.0.0-rc.111",
|
|
43
|
+
"@lincyaw/kernel": "1.0.0-rc.111",
|
|
44
|
+
"@lincyaw/skills": "1.0.0-rc.111"
|
|
50
45
|
}
|
|
51
46
|
}
|
package/src/cli.ts
CHANGED
|
@@ -15,7 +15,7 @@ import type { ApprovalPolicy } from '@lincyaw/server'
|
|
|
15
15
|
import { buildWorkspace, isWorkspaceRoot, serveWorkspace } from '@lincyaw/workspace'
|
|
16
16
|
import type { WorkspaceBuild } from '@lincyaw/workspace'
|
|
17
17
|
import { runCheck } from './check.ts'
|
|
18
|
-
import { documentPatterns,
|
|
18
|
+
import { documentPatterns, importDocuments } from '@lincyaw/compiler'
|
|
19
19
|
import type { CheckResult } from './check.ts'
|
|
20
20
|
import { devHost } from './dev-host.ts'
|
|
21
21
|
import { DocumentFileError, readDocumentFile, saveDocumentFile } from './document-file.ts'
|
|
@@ -335,7 +335,7 @@ async function runNewCommand(argv: string[], bundled: string): Promise<number> {
|
|
|
335
335
|
}
|
|
336
336
|
|
|
337
337
|
/** Markdown notes to documents; prints what it wrote, the links it could not resolve and the patterns that list them. */
|
|
338
|
-
function runImportCommand(argv: string[]): number {
|
|
338
|
+
async function runImportCommand(argv: string[]): Promise<number> {
|
|
339
339
|
let parsed
|
|
340
340
|
try {
|
|
341
341
|
parsed = parseArgs({ args: argv, allowPositionals: true, options: {
|
|
@@ -369,7 +369,8 @@ function runImportCommand(argv: string[]): number {
|
|
|
369
369
|
return 2
|
|
370
370
|
}
|
|
371
371
|
const out = resolve(values.out ?? '.')
|
|
372
|
-
const report =
|
|
372
|
+
const report = await importDocuments(input, { out, exclude: values.exclude, lang: values.lang, group, preserveOriginal: true }).catch(() => undefined)
|
|
373
|
+
if (!report) return fail(t.importFailed, 1)
|
|
373
374
|
for (const { note, link } of report.unresolved) process.stderr.write(`${t.importUnresolved(note, link)}\n`)
|
|
374
375
|
process.stdout.write(`${t.imported(report.documents.length, report.files, relative(process.cwd(), out) || '.')}\n`)
|
|
375
376
|
if (report.documents.length > 0) process.stdout.write(`${t.importPatterns(documentPatterns(report.documents))}\n`)
|
package/src/import.ts
CHANGED
|
@@ -1,169 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
// `a/b/note.md` becomes `<out>/a/b/note/lecture.json`. Wikilinks resolve as note-taking apps resolve them: by path,
|
|
4
|
-
// then by file name or alias (case-insensitive), the nearest note winning when several share a name; a folder's
|
|
5
|
-
// `index` or `README` note also answers to the folder's name. A local file a note links to is copied into its
|
|
6
|
-
// document's `attachments/`, since a document publishes only files inside the workspace; an excluded folder (`raw`,
|
|
7
|
-
// say) is neither imported nor copied from.
|
|
8
|
-
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs'
|
|
9
|
-
import { basename, dirname, extname, join, posix, relative, resolve, sep } from 'node:path'
|
|
10
|
-
import { DOCUMENT_ENTRY } from '@lincyaw/kernel'
|
|
11
|
-
import { headingSlug, markdownDocument, noteTitle, splitFrontmatter } from './markdown.ts'
|
|
12
|
-
|
|
13
|
-
export interface ImportOptions {
|
|
14
|
-
out: string
|
|
15
|
-
/** Names (with `*`) of folders and files to leave out, matched against each part of a path. */
|
|
16
|
-
exclude?: string[]
|
|
17
|
-
lang?: string
|
|
18
|
-
/** The note's folder at this depth becomes the document's first tag, by which the site's sidebar groups documents. */
|
|
19
|
-
group?: number
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export interface ImportReport {
|
|
23
|
-
/** The written documents' folders, relative to `out`. */
|
|
24
|
-
documents: string[]
|
|
25
|
-
files: number
|
|
26
|
-
unresolved: { note: string, link: string }[]
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
interface Note { file: string, path: string, dir: string, title: string, aliases: string[] }
|
|
30
|
-
|
|
31
|
-
const MARKDOWN = new Set(['.md', '.markdown'])
|
|
32
|
-
const SKIPPED = new Set(['node_modules'])
|
|
33
|
-
const RESERVED = new Set(['dist', 'templates', 'node_modules'])
|
|
34
|
-
const FOLDER_NOTES = new Set(['index', 'readme'])
|
|
35
|
-
const ATTACHMENTS = 'attachments'
|
|
36
|
-
|
|
37
|
-
const glob = (pattern: string) => new RegExp(`^${pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.')}$`, 'i')
|
|
38
|
-
const toPosix = (path: string) => path.split(sep).join('/')
|
|
39
|
-
const key = (target: string) => target.trim().replace(/^\/+/, '').replace(/\.(?:md|markdown)$/i, '').toLowerCase()
|
|
40
|
-
/** A document folder's part: letters, digits, `.`, `_` and `-`, not starting with `.`. */
|
|
41
|
-
const part = (name: string) => name.replace(/[^\p{L}\p{N}._-]+/gu, '-').replace(/^[.-]+/, '') || 'note'
|
|
42
|
-
|
|
43
|
-
export function importMarkdown(input: string, options: ImportOptions): ImportReport {
|
|
44
|
-
const source = resolve(input)
|
|
45
|
-
const out = resolve(options.out)
|
|
46
|
-
const excluded = (options.exclude ?? []).map(glob)
|
|
47
|
-
const isExcluded = (path: string) => path.split('/').some(name => excluded.some(pattern => pattern.test(name)))
|
|
48
|
-
const single = statSync(source).isFile()
|
|
49
|
-
const root = single ? dirname(source) : source
|
|
50
|
-
|
|
51
|
-
const files: string[] = []
|
|
52
|
-
const seen = new Set<string>()
|
|
53
|
-
const walk = (dir: string) => {
|
|
54
|
-
const real = realpathSync(dir)
|
|
55
|
-
if (seen.has(real)) return
|
|
56
|
-
seen.add(real)
|
|
57
|
-
for (const name of readdirSync(dir).sort()) {
|
|
58
|
-
const file = join(dir, name)
|
|
59
|
-
if (name.startsWith('.') || SKIPPED.has(name) || isExcluded(toPosix(relative(root, file)))) continue
|
|
60
|
-
const stat = statSync(file, { throwIfNoEntry: false })
|
|
61
|
-
if (stat?.isDirectory()) walk(file)
|
|
62
|
-
else if (stat?.isFile() && MARKDOWN.has(extname(name).toLowerCase())) files.push(file)
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
if (single) files.push(source)
|
|
66
|
-
else walk(source)
|
|
67
|
-
|
|
68
|
-
const taken = new Set<string>()
|
|
69
|
-
const notes: Note[] = files.map((file) => {
|
|
70
|
-
const path = toPosix(relative(root, file))
|
|
71
|
-
const parts = path.replace(/\.(?:md|markdown)$/i, '').split('/').map(part)
|
|
72
|
-
if (RESERVED.has(parts[0]!)) parts[0] = `${parts[0]}-notes`
|
|
73
|
-
let dir = parts.join('/')
|
|
74
|
-
for (let n = 2; taken.has(dir.toLowerCase()); n++) dir = `${parts.join('/')}-${n}`
|
|
75
|
-
taken.add(dir.toLowerCase())
|
|
76
|
-
const text = readFileSync(file, 'utf8')
|
|
77
|
-
return { file, path, dir, title: noteTitle(text) ?? basename(file, extname(file)), aliases: splitFrontmatter(text).frontmatter.aliases }
|
|
78
|
-
})
|
|
79
|
-
|
|
80
|
-
const byPath = new Map(notes.map(note => [key(note.path), note]))
|
|
81
|
-
const byName = new Map<string, Note[]>()
|
|
82
|
-
const register = (name: string, note: Note) => {
|
|
83
|
-
const k = key(name)
|
|
84
|
-
if (k && !byName.get(k)?.includes(note)) byName.set(k, [...byName.get(k) ?? [], note])
|
|
85
|
-
}
|
|
86
|
-
for (const note of notes) {
|
|
87
|
-
const stem = posix.basename(note.path).replace(/\.(?:md|markdown)$/i, '')
|
|
88
|
-
register(stem, note)
|
|
89
|
-
note.aliases.forEach(alias => register(alias, note))
|
|
90
|
-
if (FOLDER_NOTES.has(stem.toLowerCase()) && posix.dirname(note.path) !== '.') register(posix.basename(posix.dirname(note.path)), note)
|
|
91
|
-
}
|
|
92
|
-
/** The note nearest to `from` among `candidates`: the one sharing the longest folder path, if only one does. */
|
|
93
|
-
const nearest = (from: Note, candidates: Note[]) => {
|
|
94
|
-
const shared = (note: Note) => {
|
|
95
|
-
const a = posix.dirname(from.path).split('/'), b = posix.dirname(note.path).split('/')
|
|
96
|
-
let n = 0
|
|
97
|
-
while (n < a.length && a[n] === b[n]) n++
|
|
98
|
-
return n
|
|
99
|
-
}
|
|
100
|
-
const best = Math.max(...candidates.map(shared))
|
|
101
|
-
const winners = candidates.filter(note => shared(note) === best)
|
|
102
|
-
return winners.length === 1 ? winners[0] : undefined
|
|
103
|
-
}
|
|
104
|
-
const href = (from: Note, to: Note, fragment?: string) => {
|
|
105
|
-
const path = posix.relative(from.dir, to.dir)
|
|
106
|
-
return `${path ? `${path}/` : ''}${DOCUMENT_ENTRY}${fragment ? `#${headingSlug(fragment)}` : ''}`
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const report: ImportReport = { documents: [], files: 0, unresolved: [] }
|
|
110
|
-
const copied = new Map<string, string>()
|
|
111
|
-
for (const note of notes) {
|
|
112
|
-
const docDir = join(out, note.dir)
|
|
113
|
-
const names = new Map<string, string>()
|
|
114
|
-
/** A file inside the imported folder to its copy in the document's attachments; undefined for anything else. */
|
|
115
|
-
const attach = (url: string) => {
|
|
116
|
-
const [path = '', ...fragment] = url.split('#')
|
|
117
|
-
let name = path
|
|
118
|
-
try {
|
|
119
|
-
name = decodeURIComponent(path)
|
|
120
|
-
} catch {
|
|
121
|
-
}
|
|
122
|
-
const file = resolve(dirname(note.file), name)
|
|
123
|
-
const rel = toPosix(relative(root, file))
|
|
124
|
-
if (!name || rel.startsWith('../') || isExcluded(rel) || !statSync(file, { throwIfNoEntry: false })?.isFile()) return undefined
|
|
125
|
-
const target = byPath.get(key(rel))
|
|
126
|
-
if (target) return href(note, target, fragment.length ? fragment.join('#') : undefined)
|
|
127
|
-
let copy = names.get(file)
|
|
128
|
-
if (!copy) {
|
|
129
|
-
const base = basename(file)
|
|
130
|
-
copy = [...names.values()].includes(`${ATTACHMENTS}/${base}`) ? `${ATTACHMENTS}/${names.size}-${base}` : `${ATTACHMENTS}/${base}`
|
|
131
|
-
names.set(file, copy)
|
|
132
|
-
}
|
|
133
|
-
copied.set(join(docDir, copy), file)
|
|
134
|
-
return `${encodeURI(copy)}${fragment.length ? `#${fragment.join('#')}` : ''}`
|
|
135
|
-
}
|
|
136
|
-
const { document, unresolved } = markdownDocument(readFileSync(note.file, 'utf8'), {
|
|
137
|
-
id: note.dir.replace(/\//g, '--'),
|
|
138
|
-
title: note.title,
|
|
139
|
-
lang: options.lang,
|
|
140
|
-
links: {
|
|
141
|
-
page: (target, fragment) => {
|
|
142
|
-
const direct = byPath.get(key(target)) ?? byPath.get(key(posix.join(posix.dirname(note.path), target)))
|
|
143
|
-
const candidates = byName.get(key(target)) ?? []
|
|
144
|
-
const found = direct ?? (candidates.length === 1 ? candidates[0] : candidates.length > 1 ? nearest(note, candidates) : undefined)
|
|
145
|
-
return found && href(note, found, fragment)
|
|
146
|
-
},
|
|
147
|
-
url: attach,
|
|
148
|
-
image: attach,
|
|
149
|
-
},
|
|
150
|
-
})
|
|
151
|
-
const folder = options.group ? posix.dirname(note.path).split('/').slice(0, options.group).at(-1) : undefined
|
|
152
|
-
if (folder && folder !== '.') document.metadata.tags = [folder, ...(document.metadata.tags ?? []).filter(tag => tag !== folder)]
|
|
153
|
-
mkdirSync(docDir, { recursive: true })
|
|
154
|
-
writeFileSync(join(docDir, DOCUMENT_ENTRY), `${JSON.stringify(document, null, 2)}\n`)
|
|
155
|
-
report.documents.push(note.dir)
|
|
156
|
-
report.unresolved.push(...unresolved.map(link => ({ note: note.path, link })))
|
|
157
|
-
}
|
|
158
|
-
for (const [copy, file] of copied) {
|
|
159
|
-
mkdirSync(dirname(copy), { recursive: true })
|
|
160
|
-
if (!existsSync(copy) || statSync(copy).mtimeMs < statSync(file).mtimeMs || statSync(copy).size !== statSync(file).size) copyFileSync(file, copy)
|
|
161
|
-
}
|
|
162
|
-
report.files = copied.size
|
|
163
|
-
return report
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
/** The `documents` patterns that cover these folders: one per depth, as workspace patterns do not cross folders. */
|
|
167
|
-
export function documentPatterns(dirs: string[]): string[] {
|
|
168
|
-
return [...new Set(dirs.map(dir => dir.split('/').length))].sort((a, b) => a - b).map(depth => `${'*/'.repeat(depth)}${DOCUMENT_ENTRY}`)
|
|
169
|
-
}
|
|
1
|
+
export { documentPatterns, importMarkdown } from '@lincyaw/compiler'
|
|
2
|
+
export type { ImportOptions, ImportReport } from '@lincyaw/compiler'
|
package/src/markdown.ts
CHANGED
|
@@ -1,328 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
// Each level-1 or level-2 heading starts a page (the first page holds what comes before it, under the note's title);
|
|
4
|
-
// every page has one step that shows all its blocks. Paragraphs, headings, lists, quotes, code, math and tables become
|
|
5
|
-
// rich text; a Mermaid diagram becomes a code block of its own, which is what draws it; a paragraph holding only an
|
|
6
|
-
// image becomes an image block. What has no rich-text node (raw HTML, footnotes, rules, callouts) is kept as an mdast
|
|
7
|
-
// extension that the viewer renders. `[[target]]` and `[[target|label]]` wikilinks, relative links and images go through
|
|
8
|
-
// the caller's `links`. Node ids come from the content, so importing an unchanged paragraph again keeps its id.
|
|
9
|
-
import { fromMarkdown } from 'mdast-util-from-markdown'
|
|
10
|
-
import { gfmFromMarkdown } from 'mdast-util-gfm'
|
|
11
|
-
import { mathFromMarkdown } from 'mdast-util-math'
|
|
12
|
-
import { gfm } from 'micromark-extension-gfm'
|
|
13
|
-
import { math } from 'micromark-extension-math'
|
|
14
|
-
import { parse as parseYaml } from 'yaml'
|
|
15
|
-
import type { Asset, DocumentBlock, InlineNode, MdastNode, RichTextNode, Section, Step, TeachingDocument, TextMark } from '@lincyaw/kernel'
|
|
16
|
-
|
|
17
|
-
export interface Frontmatter {
|
|
18
|
-
title?: string
|
|
19
|
-
description?: string
|
|
20
|
-
lang?: string
|
|
21
|
-
tags: string[]
|
|
22
|
-
aliases: string[]
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
export interface MarkdownLinks {
|
|
26
|
-
/** A wikilink's target (without `#fragment`) to the href of the document it names; undefined when there is none. */
|
|
27
|
-
page(target: string, fragment: string | undefined): string | undefined
|
|
28
|
-
/** A relative URL of a link to its new href; undefined when it leads nowhere that is published. */
|
|
29
|
-
url(url: string): string | undefined
|
|
30
|
-
/** A relative URL of an image to the location of its copy, relative to the document. */
|
|
31
|
-
image(url: string): string | undefined
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
export interface ImportedMarkdown {
|
|
35
|
-
document: TeachingDocument
|
|
36
|
-
/** Wikilinks and relative links that `links` could not resolve, as written. */
|
|
37
|
-
unresolved: string[]
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/
|
|
41
|
-
/** Stands in for the `|` of `[[target|label]]` while parsing, so a GFM table does not split the cell at it. */
|
|
42
|
-
const LABEL = '\uE000'
|
|
43
|
-
const WIKILINK = new RegExp(`\\[\\[([^\\]|${LABEL}\\n]+)(?:[|${LABEL}]([^\\]\\n]+))?\\]\\]`, 'g')
|
|
44
|
-
const HIGHLIGHT = /==(?=\S)([^=\n]*\S)==/g
|
|
45
|
-
const CALLOUT = /^\[!([A-Za-z]+)\][+-]?[ \t]*/
|
|
46
|
-
const SCHEME = /^(?:[a-z][a-z\d+.-]*:|\/\/|#)/i
|
|
47
|
-
const MARKS: Record<string, TextMark['type']> = { strong: 'bold', emphasis: 'italic', delete: 'strike' }
|
|
48
|
-
const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu
|
|
49
|
-
const LETTER = /\p{L}/gu
|
|
50
|
-
|
|
51
|
-
export function splitFrontmatter(source: string): { frontmatter: Frontmatter, body: string } {
|
|
52
|
-
const match = FRONTMATTER.exec(source)
|
|
53
|
-
let data: unknown
|
|
54
|
-
if (match) {
|
|
55
|
-
try {
|
|
56
|
-
data = parseYaml(match[1]!)
|
|
57
|
-
} catch {
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
const record = data !== null && typeof data === 'object' && !Array.isArray(data) ? data as Record<string, unknown> : {}
|
|
61
|
-
const text = (key: string) => (typeof record[key] === 'string' && (record[key] as string).trim() ? (record[key] as string).trim() : undefined)
|
|
62
|
-
const list = (key: string) => {
|
|
63
|
-
const value = record[key]
|
|
64
|
-
return (Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : [])
|
|
65
|
-
.flatMap(item => (typeof item === 'string' || typeof item === 'number' ? [String(item).trim()] : [])).filter(Boolean)
|
|
66
|
-
}
|
|
67
|
-
return {
|
|
68
|
-
frontmatter: { title: text('title'), description: text('description'), lang: text('lang'), tags: list('tags'), aliases: list('aliases') },
|
|
69
|
-
body: match ? source.slice(match[0].length) : source,
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
/** The title of a note: its frontmatter title, else its first level-1 heading. */
|
|
74
|
-
export function noteTitle(source: string): string | undefined {
|
|
75
|
-
const { frontmatter, body } = splitFrontmatter(source)
|
|
76
|
-
return frontmatter.title ?? /^#[ \t]+(.+?)[ \t#]*$/m.exec(body)?.[1]?.trim()
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
/** The anchor of a heading, as GitHub writes it: lower case, punctuation dropped, spaces to hyphens. */
|
|
80
|
-
export function headingSlug(text: string): string {
|
|
81
|
-
return text.trim().toLowerCase().replace(/[^\p{L}\p{N}\s_-]/gu, '').replace(/\s/g, '-')
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
function guessLang(text: string): string {
|
|
85
|
-
const letters = text.match(LETTER)?.length ?? 0
|
|
86
|
-
return letters > 0 && (text.match(CJK)?.length ?? 0) / letters > 0.3 ? 'zh-CN' : 'en'
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
function plain(node: MdastNode): string {
|
|
90
|
-
if (typeof node.value === 'string') return node.value
|
|
91
|
-
return (node.children ?? []).map(plain).join('')
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
function hash(text: string): string {
|
|
95
|
-
let h = 0x811c9dc5
|
|
96
|
-
for (let i = 0; i < text.length; i++) h = Math.imul(h ^ text.charCodeAt(i), 0x01000193)
|
|
97
|
-
return (h >>> 0).toString(36)
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
function withoutPositions(node: MdastNode): unknown {
|
|
101
|
-
const { position: _position, children, ...rest } = node
|
|
102
|
-
return children ? { ...rest, children: children.map(withoutPositions) } : rest
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
function restoreLabels(node: MdastNode): void {
|
|
106
|
-
if (typeof node.value === 'string') node.value = node.value.replaceAll(LABEL, '|')
|
|
107
|
-
node.children?.forEach(restoreLabels)
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
export function markdownDocument(source: string, options: { id: string, title: string, lang?: string, links: MarkdownLinks }): ImportedMarkdown {
|
|
111
|
-
const { frontmatter, body } = splitFrontmatter(source)
|
|
112
|
-
const protectedBody = body.replace(/\[\[([^\]|\n]+)\|([^\]\n]+)\]\]/g, (_, target: string, label: string) => `[[${target}${LABEL}${label}]]`)
|
|
113
|
-
const root = fromMarkdown(protectedBody, { extensions: [gfm(), math()], mdastExtensions: [gfmFromMarkdown(), mathFromMarkdown()] }) as unknown as MdastNode
|
|
114
|
-
const unresolved: string[] = []
|
|
115
|
-
const definitions = new Map<string, string>()
|
|
116
|
-
const collect = (node: MdastNode) => {
|
|
117
|
-
if (node.type === 'definition') definitions.set(String(node.identifier).toLowerCase(), String(node.url))
|
|
118
|
-
node.children?.forEach(collect)
|
|
119
|
-
}
|
|
120
|
-
collect(root)
|
|
121
|
-
|
|
122
|
-
// Links are rewritten in the mdast first, so callouts and other extensions carry the same hrefs as rich text.
|
|
123
|
-
const relink = (node: MdastNode): MdastNode[] => {
|
|
124
|
-
if (node.type === 'text' && typeof node.value === 'string' && node.value.includes('[[')) return wikilinks(node.value)
|
|
125
|
-
if ((node.type === 'link' || node.type === 'definition') && typeof node.url === 'string' && !SCHEME.test(node.url)) {
|
|
126
|
-
const href = options.links.url(node.url)
|
|
127
|
-
if (href === undefined) {
|
|
128
|
-
unresolved.push(node.url)
|
|
129
|
-
if (node.type === 'link') return (node.children ?? []).flatMap(relink)
|
|
130
|
-
}
|
|
131
|
-
node.url = href ?? node.url
|
|
132
|
-
}
|
|
133
|
-
if (node.type === 'linkReference') {
|
|
134
|
-
const url = definitions.get(String(node.identifier).toLowerCase())
|
|
135
|
-
if (url !== undefined) return relink({ type: 'link', url, children: node.children ?? [] })
|
|
136
|
-
}
|
|
137
|
-
if (node.children) node.children = node.children.flatMap(relink)
|
|
138
|
-
return [node]
|
|
139
|
-
}
|
|
140
|
-
const wikilinks = (value: string): MdastNode[] => {
|
|
141
|
-
const out: MdastNode[] = []
|
|
142
|
-
let cursor = 0
|
|
143
|
-
for (const match of value.matchAll(WIKILINK)) {
|
|
144
|
-
if (match.index > cursor) out.push({ type: 'text', value: value.slice(cursor, match.index) })
|
|
145
|
-
const written = match[1]!.trim()
|
|
146
|
-
const [target = '', ...fragment] = written.split('#')
|
|
147
|
-
const label = (match[2] ?? written).trim()
|
|
148
|
-
const href = options.links.page(target.trim(), fragment.length ? fragment.join('#') : undefined)
|
|
149
|
-
if (href === undefined) unresolved.push(`[[${written}]]`)
|
|
150
|
-
out.push(href === undefined ? { type: 'text', value: label } : { type: 'link', url: href, children: [{ type: 'text', value: label }] })
|
|
151
|
-
cursor = match.index + match[0].length
|
|
152
|
-
}
|
|
153
|
-
if (cursor < value.length) out.push({ type: 'text', value: value.slice(cursor) })
|
|
154
|
-
return out
|
|
155
|
-
}
|
|
156
|
-
root.children = (root.children ?? []).flatMap(relink)
|
|
157
|
-
restoreLabels(root)
|
|
158
|
-
|
|
159
|
-
const used = new Set<string>()
|
|
160
|
-
const unique = (base: string) => {
|
|
161
|
-
let id = base.slice(0, 180) || 'node'
|
|
162
|
-
for (let n = 2; used.has(id); n++) id = `${base.slice(0, 180)}-${n}`
|
|
163
|
-
used.add(id)
|
|
164
|
-
return id
|
|
165
|
-
}
|
|
166
|
-
const idOf = (prefix: string, node: MdastNode) => unique(`${prefix}-${hash(JSON.stringify(withoutPositions(node)))}`)
|
|
167
|
-
const extension = (node: MdastNode): RichTextNode => {
|
|
168
|
-
const text = plain(node).trim()
|
|
169
|
-
return { id: idOf(node.type, node), type: 'extension', kind: 'presmium.mdast', schemaVersion: 1, data: { node: withoutPositions(node) as never },
|
|
170
|
-
fallback: text ? [{ id: idOf('fallback', node), type: 'paragraph', children: [{ type: 'text', text }] }] : [] }
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
const inline = (nodes: MdastNode[], marks: TextMark[] = []): InlineNode[] => nodes.flatMap((node): InlineNode[] => {
|
|
174
|
-
const styled = (text: string, extra: TextMark[] = []): InlineNode => ({ type: 'text', text, ...marks.length + extra.length ? { marks: [...marks, ...extra] } : {} })
|
|
175
|
-
if (node.type === 'text') {
|
|
176
|
-
const value = String(node.value ?? '')
|
|
177
|
-
if (!value.includes('==')) return value ? [styled(value)] : []
|
|
178
|
-
const out: InlineNode[] = []
|
|
179
|
-
let cursor = 0
|
|
180
|
-
for (const match of value.matchAll(HIGHLIGHT)) {
|
|
181
|
-
if (match.index > cursor) out.push(styled(value.slice(cursor, match.index)))
|
|
182
|
-
out.push(styled(match[1]!, [{ type: 'background', value: 'warn' }]))
|
|
183
|
-
cursor = match.index + match[0].length
|
|
184
|
-
}
|
|
185
|
-
if (cursor < value.length) out.push(styled(value.slice(cursor)))
|
|
186
|
-
return out
|
|
187
|
-
}
|
|
188
|
-
if (node.type === 'break') return [{ type: 'break' }]
|
|
189
|
-
if (node.type === 'inlineMath') return [{ type: 'math', value: String(node.value ?? '') }]
|
|
190
|
-
if (node.type === 'inlineCode') return [styled(String(node.value ?? ''), [{ type: 'code' }])]
|
|
191
|
-
if (MARKS[node.type]) return inline(node.children ?? [], [...marks, { type: MARKS[node.type]! } as TextMark])
|
|
192
|
-
if (node.type === 'link') return [{ type: 'link', href: String(node.url), children: inline(node.children ?? [], marks) }]
|
|
193
|
-
if (node.type === 'image' && typeof node.url === 'string') {
|
|
194
|
-
const href = SCHEME.test(node.url) ? node.url : options.links.image(node.url)
|
|
195
|
-
return href ? [{ type: 'link', href, children: [styled(String(node.alt || node.url))] }] : [styled(String(node.alt ?? ''))]
|
|
196
|
-
}
|
|
197
|
-
return [{ type: 'extension', kind: 'presmium.mdast', schemaVersion: 1, data: { node: withoutPositions(node) as never }, fallback: plain(node) }]
|
|
198
|
-
})
|
|
199
|
-
|
|
200
|
-
const rich = (node: MdastNode): RichTextNode[] => {
|
|
201
|
-
const children = node.children ?? []
|
|
202
|
-
switch (node.type) {
|
|
203
|
-
case 'paragraph': {
|
|
204
|
-
const content = inline(children)
|
|
205
|
-
return content.length ? [{ id: idOf('p', node), type: 'paragraph', children: content }] : []
|
|
206
|
-
}
|
|
207
|
-
case 'heading':
|
|
208
|
-
return [{ id: unique(headingSlug(plain(node)) || `h-${hash(plain(node))}`), type: 'heading', level: Math.min(6, Math.max(1, Number(node.depth))) as 1, children: inline(children) }]
|
|
209
|
-
case 'blockquote': {
|
|
210
|
-
const callout = calloutOf(node)
|
|
211
|
-
return callout ? [extension(callout)] : [{ id: idOf('quote', node), type: 'blockquote', children: children.flatMap(rich) }]
|
|
212
|
-
}
|
|
213
|
-
case 'list':
|
|
214
|
-
return [{ id: idOf('list', node), type: 'list', ordered: Boolean(node.ordered), ...Number(node.start) >= 1 ? { start: Number(node.start) } : {},
|
|
215
|
-
children: children.map(item => ({ id: idOf('item', item), type: 'listItem', ...typeof item.checked === 'boolean' ? { checked: item.checked } : {}, children: (item.children ?? []).flatMap(rich) })) }]
|
|
216
|
-
case 'code':
|
|
217
|
-
return [{ id: idOf('code', node), type: 'code', language: String(node.lang ?? ''), value: String(node.value ?? '') }]
|
|
218
|
-
case 'math':
|
|
219
|
-
return [{ id: idOf('math', node), type: 'math', value: String(node.value ?? '') }]
|
|
220
|
-
case 'table': {
|
|
221
|
-
const align = Array.isArray(node.align) ? node.align as (string | null)[] : []
|
|
222
|
-
return [{ id: idOf('table', node), type: 'table', headerRows: 1, children: children.map(row => ({ id: idOf('row', row), type: 'tableRow',
|
|
223
|
-
children: (row.children ?? []).map((cell, c) => ({ id: idOf('cell', cell), type: 'tableCell', children: inline(cell.children ?? []),
|
|
224
|
-
...align[c] ? { style: { alignment: align[c] as 'left' } } : {} })) })) }]
|
|
225
|
-
}
|
|
226
|
-
case 'definition':
|
|
227
|
-
return []
|
|
228
|
-
default:
|
|
229
|
-
return [extension(node)]
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
// Pages: a level-1 or level-2 heading starts one. The note's own title heading opens the first page and is not repeated.
|
|
234
|
-
const nodes = root.children ?? []
|
|
235
|
-
const title = frontmatter.title ?? (nodes[0]?.type === 'heading' && nodes[0].depth === 1 ? plain(nodes[0]).trim() : undefined) ?? options.title
|
|
236
|
-
const pages: { title: string, level: number, nodes: MdastNode[] }[] = [{ title, level: 1, nodes: [] }]
|
|
237
|
-
nodes.forEach((node, i) => {
|
|
238
|
-
const opensPage = node.type === 'heading' && Number(node.depth) <= 2
|
|
239
|
-
if (opensPage && i === 0 && node.depth === 1 && plain(node).trim() === title) return
|
|
240
|
-
if (opensPage) pages.push({ title: plain(node).trim() || title, level: Number(node.depth), nodes: [] })
|
|
241
|
-
else pages.at(-1)!.nodes.push(node)
|
|
242
|
-
})
|
|
243
|
-
if (pages.length > 1 && pages[0]!.nodes.length === 0 && pages[1]!.level === 1) pages.shift()
|
|
244
|
-
|
|
245
|
-
const sections: Record<string, Section> = {}, blocks: Record<string, DocumentBlock> = {}, steps: Record<string, Step> = {}, assets: Record<string, Asset> = {}
|
|
246
|
-
const sectionOrder: string[] = []
|
|
247
|
-
for (const page of pages) {
|
|
248
|
-
const sectionId = unique(headingSlug(page.title) || `page-${hash(page.title)}`)
|
|
249
|
-
const children: string[] = []
|
|
250
|
-
let text: RichTextNode[] = []
|
|
251
|
-
const flush = () => {
|
|
252
|
-
if (!text.length) return
|
|
253
|
-
const id = unique(`${sectionId}--text-${children.length}`)
|
|
254
|
-
blocks[id] = { id, type: 'text', content: text }
|
|
255
|
-
children.push(id)
|
|
256
|
-
text = []
|
|
257
|
-
}
|
|
258
|
-
for (const node of page.nodes) {
|
|
259
|
-
const image = node.type === 'paragraph' ? soleImage(node) : undefined
|
|
260
|
-
const location = image && typeof image.url === 'string' && !SCHEME.test(image.url) ? options.links.image(image.url) : undefined
|
|
261
|
-
if (node.type === 'code' && (node.lang === 'mermaid' || node.lang === 'mmd')) {
|
|
262
|
-
flush()
|
|
263
|
-
const id = unique(`${sectionId}--diagram-${children.length}`)
|
|
264
|
-
blocks[id] = { id, type: 'code', language: 'mermaid', value: String(node.value ?? '') }
|
|
265
|
-
children.push(id)
|
|
266
|
-
} else if (image && location) {
|
|
267
|
-
flush()
|
|
268
|
-
const id = unique(`${sectionId}--image-${children.length}`), assetId = unique(`asset-${hash(location)}`)
|
|
269
|
-
assets[assetId] = { id: assetId, kind: 'image', location, ...image.title ? { title: String(image.title) } : {} }
|
|
270
|
-
blocks[id] = { id, type: 'image', assetId, alt: String(image.alt ?? '') }
|
|
271
|
-
children.push(id)
|
|
272
|
-
} else {
|
|
273
|
-
if (image && !location && typeof image.url === 'string') unresolved.push(image.url)
|
|
274
|
-
text.push(...rich(node))
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
flush()
|
|
278
|
-
const stepId = unique(`${sectionId}--step`)
|
|
279
|
-
sections[sectionId] = { id: sectionId, title: page.title, level: Math.min(2, page.level) as 1, children, stepOrder: [stepId], layout: { type: 'stack' } }
|
|
280
|
-
steps[stepId] = { id: stepId, sectionId, visuals: children.map(blockId => ({ blockId, visible: true })), narrationIds: [] }
|
|
281
|
-
sectionOrder.push(sectionId)
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
return {
|
|
285
|
-
document: {
|
|
286
|
-
schemaVersion: 1,
|
|
287
|
-
id: options.id,
|
|
288
|
-
metadata: {
|
|
289
|
-
title,
|
|
290
|
-
lang: frontmatter.lang ?? options.lang ?? guessLang(body),
|
|
291
|
-
...frontmatter.description ? { description: frontmatter.description } : {},
|
|
292
|
-
...frontmatter.tags.length ? { tags: frontmatter.tags } : {},
|
|
293
|
-
plugins: [],
|
|
294
|
-
renderers: { web: {} },
|
|
295
|
-
},
|
|
296
|
-
sectionOrder, sections, blocks, steps, narrations: {}, assets, definitions: {},
|
|
297
|
-
production: { defaults: {}, stepOverrides: {}, mediaClips: [] },
|
|
298
|
-
} as TeachingDocument,
|
|
299
|
-
unresolved: [...new Set(unresolved)],
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
function soleImage(paragraph: MdastNode): MdastNode | undefined {
|
|
304
|
-
const content = (paragraph.children ?? []).filter(child => !(child.type === 'text' && !String(child.value ?? '').trim()))
|
|
305
|
-
return content.length === 1 && content[0]!.type === 'image' ? content[0] : undefined
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
/** `> [!note] Title` (as GitHub and Obsidian write callouts) as a MyST admonition, which the viewer draws as one. */
|
|
309
|
-
function calloutOf(quote: MdastNode): MdastNode | undefined {
|
|
310
|
-
const first = quote.children?.[0]
|
|
311
|
-
const lead = first?.type === 'paragraph' ? first.children?.[0] : undefined
|
|
312
|
-
const match = lead?.type === 'text' ? CALLOUT.exec(String(lead.value ?? '')) : null
|
|
313
|
-
if (!first || !lead || !match) return undefined
|
|
314
|
-
const rest = String(lead.value).slice(match[0].length)
|
|
315
|
-
const breakAt = rest.indexOf('\n')
|
|
316
|
-
const titleText = (breakAt < 0 ? rest : rest.slice(0, breakAt)).trim()
|
|
317
|
-
const bodyLead = breakAt < 0 ? '' : rest.slice(breakAt + 1)
|
|
318
|
-
const inlineRest = [...bodyLead ? [{ type: 'text', value: bodyLead }] : [], ...(first.children ?? []).slice(1)]
|
|
319
|
-
const kind = match[1]!.toLowerCase()
|
|
320
|
-
return {
|
|
321
|
-
type: 'admonition', kind,
|
|
322
|
-
children: [
|
|
323
|
-
{ type: 'admonitionTitle', children: [{ type: 'text', value: titleText || kind.charAt(0).toUpperCase() + kind.slice(1) }] },
|
|
324
|
-
...inlineRest.length ? [{ type: 'paragraph', children: inlineRest }] : [],
|
|
325
|
-
...(quote.children ?? []).slice(1),
|
|
326
|
-
],
|
|
327
|
-
}
|
|
328
|
-
}
|
|
1
|
+
export { headingSlug, markdownDocument, noteTitle, splitFrontmatter } from '@lincyaw/compiler'
|
|
2
|
+
export type { Frontmatter, ImportedMarkdown, MarkdownLinks } from '@lincyaw/compiler'
|
package/src/messages.ts
CHANGED
|
@@ -39,6 +39,7 @@ export interface CliMessages {
|
|
|
39
39
|
notRendered(blocks: number): string
|
|
40
40
|
codeRuns(executed: number, cached: number, sandbox?: string): string
|
|
41
41
|
importMissing: string
|
|
42
|
+
importFailed: string
|
|
42
43
|
badGroup(value: string): string
|
|
43
44
|
imported(documents: number, files: number, out: string): string
|
|
44
45
|
importUnresolved(note: string, link: string): string
|
|
@@ -135,7 +136,7 @@ export const CLI_MESSAGES: Record<Locale, CliMessages> = {
|
|
|
135
136
|
--allow <种类> 无需询问即允许的操作:operate、spend、write、agent
|
|
136
137
|
ir 以 JSON 打印文稿的运行快照
|
|
137
138
|
--out <文件> 写入文件,而不是打印
|
|
138
|
-
import 把 Markdown
|
|
139
|
+
import 把 Markdown 或 Word 转成文稿(参数是 .md、.markdown、.docx 文件或文件夹):每个文件成为
|
|
139
140
|
<输出>/<路径>/${DOCUMENT_ENTRY},一级与二级标题各开一个页面;[[笔记]] 与 [[笔记|文字]]
|
|
140
141
|
按文件名或 frontmatter 的 aliases 找到对应的文稿,笔记链接的本地文件复制到文稿的
|
|
141
142
|
attachments/;再次导入时覆盖写过的文稿
|
|
@@ -231,7 +232,8 @@ export const CLI_MESSAGES: Record<Locale, CliMessages> = {
|
|
|
231
232
|
notRendered: blocks => ` (${blocks} 个内容块未显示)`,
|
|
232
233
|
codeRuns: (executed, cached, sandbox) => `代码运行:执行 ${executed} 个,取自缓存 ${cached} 个${sandbox ? `,隔离方式为 ${sandbox}` : ''}`,
|
|
233
234
|
badGroup: value => `--group 需要 1 或更大的整数,而不是“${value}”。`,
|
|
234
|
-
importMissing: '
|
|
235
|
+
importMissing: '缺少要导入的文件。请给出 .md、.markdown、.docx 文件或文件夹,例如 presmium import notes --out site。',
|
|
236
|
+
importFailed: '文件未能导入。请检查文件编码、Word 文件是否完整,以及输出目录是否可写。',
|
|
235
237
|
imported: (documents, files, out) => `已导入 ${documents} 份文稿,复制了 ${files} 个文件,写在 ${out}`,
|
|
236
238
|
importUnresolved: (note, link) => `${note}:链接 ${link} 没有对应的笔记或文件,已改为普通文字`,
|
|
237
239
|
importPatterns: patterns => `presmium.workspace.yaml 的 documents 需要包含:\n${patterns.map(pattern => ` - '${pattern}'`).join('\n')}`,
|
|
@@ -357,7 +359,7 @@ Commands:
|
|
|
357
359
|
--allow <kinds> approvals given without asking: operate, spend, write, agent
|
|
358
360
|
ir print the runtime snapshot as JSON
|
|
359
361
|
--out <file> write it to a file instead
|
|
360
|
-
import Markdown
|
|
362
|
+
import Markdown or Word to documents (.md, .markdown, .docx files or folders): each file becomes
|
|
361
363
|
<out>/<path>/${DOCUMENT_ENTRY}, a page for each level-1 and level-2 heading; [[note]] and
|
|
362
364
|
[[note|text]] find their document by file name or the aliases of the frontmatter, and
|
|
363
365
|
the local files a note links to are copied into the document's attachments/; importing
|
|
@@ -464,7 +466,8 @@ Exit Codes: 0 no errors, 1 error findings, 2 usage or internal error.`,
|
|
|
464
466
|
notRendered: blocks => ` (${blocks} blocks not rendered)`,
|
|
465
467
|
codeRuns: (executed, cached, sandbox) => `code runs: ${executed} executed, ${cached} from cache${sandbox ? `, isolated by ${sandbox}` : ''}`,
|
|
466
468
|
badGroup: value => `--group needs a whole number of 1 or more, not “${value}”.`,
|
|
467
|
-
importMissing: 'No
|
|
469
|
+
importMissing: 'No files to import. Give a .md, .markdown, .docx file or folder, for example presmium import notes --out site.',
|
|
470
|
+
importFailed: 'Import failed. Check the file encoding, whether the Word file is complete, and output directory permissions.',
|
|
468
471
|
imported: (documents, files, out) => `imported ${documents} documents and copied ${files} files into ${out}`,
|
|
469
472
|
importUnresolved: (note, link) => `${note}: the link ${link} leads to no note or file; it is plain text now`,
|
|
470
473
|
importPatterns: patterns => `the documents of presmium.workspace.yaml need:\n${patterns.map(pattern => ` - '${pattern}'`).join('\n')}`,
|