@lincyaw/compiler 1.0.0-rc.118 → 1.0.0-rc.119
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/package.json +4 -3
- package/src/compiler.ts +18 -10
- package/src/document-lowering.ts +48 -8
- package/src/document-tree.ts +142 -0
- package/src/import.ts +9 -8
- package/src/index.ts +5 -3
- package/src/markdown.ts +7 -5
- package/src/migrate.ts +224 -0
- package/src/translations.ts +7 -6
- package/src/workspace-settings.ts +40 -10
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @lincyaw/compiler
|
|
2
2
|
|
|
3
|
-
Presmium's compiler validates the canonical `
|
|
3
|
+
Presmium's compiler validates the canonical `index.pmd` teaching document and lowers it to a runtime snapshot. Plugin loading, execution, checks and renderers consume that derived snapshot.
|
|
4
4
|
|
|
5
5
|
Part of [Presmium](https://github.com/Lincyaw/presmium), which publishes structured teaching documents as web pages, slides, Jupyter notebooks and narrated lectures.
|
|
6
6
|
|
|
@@ -13,13 +13,13 @@ It comes with `@lincyaw/cli`. Plugins that use its helpers declare it as a peer
|
|
|
13
13
|
```ts
|
|
14
14
|
import { build, Compiler } from '@lincyaw/compiler'
|
|
15
15
|
|
|
16
|
-
const compilation = await new Compiler('
|
|
16
|
+
const compilation = await new Compiler('index.pmd').compile()
|
|
17
17
|
console.log(compilation.doc.sections.map(section => section.title))
|
|
18
18
|
const findings = await compilation.check()
|
|
19
19
|
await build(compilation, { outDir: 'dist', to: ['web'] })
|
|
20
20
|
```
|
|
21
21
|
|
|
22
|
-
`
|
|
22
|
+
`index.pmd` is the only runtime authoring entry. The compiler uses the same `validateDocument` contract as UI and agent changes. MyST files require a one-time offline migration; they are not an alternate entry. Rich text lowers directly from its typed tree, without a Markdown round trip.
|
|
23
23
|
|
|
24
24
|
Stable authoring step IDs and complete visual states are retained in the derived IR. A JSON pointer in diagnostics identifies the source entity; it is not a mutable source-line binding.
|
|
25
25
|
|
|
@@ -27,7 +27,7 @@ Stable authoring step IDs and complete visual states are retained in the derived
|
|
|
27
27
|
|
|
28
28
|
## Documentation
|
|
29
29
|
|
|
30
|
-
- [Plugin guide](https://github.com/Lincyaw/presmium/blob/main/site/plugins/locales/en/
|
|
30
|
+
- [Plugin guide](https://github.com/Lincyaw/presmium/blob/main/site/plugins/locales/en/index.pmd)
|
|
31
31
|
|
|
32
32
|
## License
|
|
33
33
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lincyaw/compiler",
|
|
3
|
-
"version": "1.0.0-rc.
|
|
3
|
+
"version": "1.0.0-rc.119",
|
|
4
4
|
"description": "Presmium's compiler: structured JSON documents to runtime IR, plugin loading, compile-time execution and its cache, checks and the renderers",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -23,7 +23,8 @@
|
|
|
23
23
|
"access": "public"
|
|
24
24
|
},
|
|
25
25
|
"exports": {
|
|
26
|
-
".": "./src/index.ts"
|
|
26
|
+
".": "./src/index.ts",
|
|
27
|
+
"./migrate": "./src/migrate.ts"
|
|
27
28
|
},
|
|
28
29
|
"files": [
|
|
29
30
|
"src"
|
|
@@ -46,7 +47,7 @@
|
|
|
46
47
|
"semver": "^7.8.5",
|
|
47
48
|
"send": "1.2.1",
|
|
48
49
|
"yaml": "^2.9.1",
|
|
49
|
-
"@lincyaw/kernel": "1.0.0-rc.
|
|
50
|
+
"@lincyaw/kernel": "1.0.0-rc.119"
|
|
50
51
|
},
|
|
51
52
|
"devDependencies": {
|
|
52
53
|
"@types/content-disposition": "^0.5.9",
|
package/src/compiler.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { readFile } from 'node:fs/promises'
|
|
|
4
4
|
import { basename, dirname, extname, join, relative, resolve } from 'node:path'
|
|
5
5
|
import { ANCHOR_LOCK_PATH, AnchorLock, DEFAULT_LANG, IR_VERSION, PROCESS_SANDBOX, createAnchorResolverRegistry, docBlocks, environmentExecutor, errorMessage, failedRun, fnv1a64, isRecord, productionAssetIds, validateDocument } from '@lincyaw/kernel'
|
|
6
6
|
import { operatorSpeechSecrets, productionAssetFile } from '@lincyaw/kernel/node'
|
|
7
|
-
import type { AnchorResolverRegistry, CanonicalSource, CodeNode, EnvironmentContext, EnvironmentProvider, EnvironmentSpec, ExecuteConfig, Executor, Finding, IRDocument, IRPass, IRSection, NodeServiceContext, PassContext, RunContext, RunRequest, RunResult, ServiceToken, SourceCapabilities, SourceReader,
|
|
7
|
+
import type { AnchorResolverRegistry, CanonicalSource, CodeNode, EnvironmentContext, EnvironmentProvider, EnvironmentSpec, ExecuteConfig, Executor, Finding, IRDocument, IRPass, IRSection, NodeServiceContext, PassContext, RunContext, RunRequest, RunResult, ServiceToken, SourceCapabilities, SourceReader, PresmiumDocument } from '@lincyaw/kernel'
|
|
8
8
|
import { audienceDocument, emptyReferences, filterForAudience, scanOutputs, withheld } from './audience.ts'
|
|
9
9
|
import type { AudienceMark, Withheld } from './audience.ts'
|
|
10
10
|
import { runAllChecks } from './compilation-actions.ts'
|
|
@@ -27,7 +27,8 @@ import { resolveReferences } from './references.ts'
|
|
|
27
27
|
import { applyTranslations, translationFields } from './translations.ts'
|
|
28
28
|
import { validateIR } from './validate.ts'
|
|
29
29
|
import { createSourceReader, findWorkspaceRoot } from './workspace.ts'
|
|
30
|
-
import { operatorDir, operatorPlugins, workspaceSettings } from './workspace-settings.ts'
|
|
30
|
+
import { operatorDir, operatorPlugins, WORKSPACE_CONFIG, workspaceSettings } from './workspace-settings.ts'
|
|
31
|
+
import { effectiveDocumentIn } from './document-tree.ts'
|
|
31
32
|
import type { OperatorSettings } from './workspace-settings.ts'
|
|
32
33
|
|
|
33
34
|
export { DECK_ENTRY }
|
|
@@ -124,6 +125,7 @@ interface Workspace {
|
|
|
124
125
|
src: SourceReader
|
|
125
126
|
settings: OperatorSettings
|
|
126
127
|
problem?: string
|
|
128
|
+
needsMigration: boolean
|
|
127
129
|
sources: PluginSources
|
|
128
130
|
operatorDir?: string
|
|
129
131
|
operatorProblem?: string
|
|
@@ -162,7 +164,9 @@ interface CompileRun {
|
|
|
162
164
|
report: Report
|
|
163
165
|
at: HeadmatterLine
|
|
164
166
|
signal?: AbortSignal
|
|
165
|
-
document
|
|
167
|
+
/** The document with its inherited settings; `stored` is it as saved. */
|
|
168
|
+
document: PresmiumDocument
|
|
169
|
+
stored: PresmiumDocument
|
|
166
170
|
rawSource: string
|
|
167
171
|
}
|
|
168
172
|
|
|
@@ -275,12 +279,15 @@ export class Compiler {
|
|
|
275
279
|
try { parsed = JSON.parse(text) } catch (error) { throw new Error(`Invalid ${DECK_ENTRY}: ${errorMessage(error)}`) }
|
|
276
280
|
const validated = validateDocument(parsed)
|
|
277
281
|
if (!validated.document) throw new Error(validated.issues.map(issue => `${issue.path || '/'}: ${issue.message}`).join('\n'))
|
|
278
|
-
|
|
282
|
+
if (workspace.needsMigration) report({ checkId: 'presmium.workspace', severity: 'error', message: `${WORKSPACE_CONFIG} still lists documents:, which documents no longer use; run presmium migrate to place them in the document tree`, source: { file: join(workspaceRoot, WORKSPACE_CONFIG), line: 1 } })
|
|
283
|
+
// Settings the document leaves out come from its ancestors and the workspace defaults.
|
|
284
|
+
const stored = validated.document
|
|
285
|
+
const document = effectiveDocumentIn(existsSync(join(workspaceRoot, WORKSPACE_CONFIG)) ? workspaceRoot : this.deckDir, this.entry, stored)
|
|
279
286
|
const headmatter = withHeadmatter({ ...document.metadata }, this.options.headmatter)
|
|
280
287
|
const defaultVoice = document.production.defaults.voiceSource
|
|
281
288
|
const declared = isRecord((headmatter.renderers as Record<string, unknown> | undefined)?.lecture)
|
|
282
289
|
const lecture = declared ? (headmatter.renderers as Record<string, Record<string, unknown>>).lecture : {}
|
|
283
|
-
if (lecture && Object.hasOwn(lecture, 'script')) throw new Error('/metadata/renderers/lecture/script: External lecture scripts are not supported. Author narration in
|
|
290
|
+
if (lecture && Object.hasOwn(lecture, 'script')) throw new Error('/metadata/renderers/lecture/script: External lecture scripts are not supported. Author narration in the document narrations and bind it through each step narrationIds.')
|
|
284
291
|
// The production defaults configure the lecture form where the document builds it; they never add the form.
|
|
285
292
|
if (declared && (defaultVoice?.type === 'tts' || document.production.defaults.trailingPauseMs !== undefined || Object.keys(lecture ?? {}).length)) headmatter.renderers = { ...(isRecord(headmatter.renderers) ? headmatter.renderers : {}), lecture: { ...lecture, ...defaultVoice?.type === 'tts' ? { voice: defaultVoice.voice } : {}, ...document.production.defaults.trailingPauseMs !== undefined ? { gap: document.production.defaults.trailingPauseMs } : {} } }
|
|
286
293
|
const lang = typeof headmatter.lang === 'string' && headmatter.lang.trim() ? headmatter.lang.trim() : DEFAULT_LANG
|
|
@@ -316,7 +323,7 @@ export class Compiler {
|
|
|
316
323
|
}
|
|
317
324
|
const execute = toExecuteConfig(headmatter.execute, report, at)
|
|
318
325
|
checkSecrets(headmatter, report, at)
|
|
319
|
-
return { workspace, headmatter, lang, plugins, configOf, service, execute, inputs, findings, report, at, signal: options.signal, document, rawSource: read }
|
|
326
|
+
return { workspace, headmatter, lang, plugins, configOf, service, execute, inputs, findings, report, at, signal: options.signal, document, stored, rawSource: read }
|
|
320
327
|
}
|
|
321
328
|
|
|
322
329
|
/** Collects the plugins' anchor resolvers, executors, environment providers and components. */
|
|
@@ -352,7 +359,7 @@ export class Compiler {
|
|
|
352
359
|
|
|
353
360
|
/** Validated JSON is the sole runtime source; compiler output remains a derived snapshot. */
|
|
354
361
|
async #compileSections(run: CompileRun, _contributions: Contributions): Promise<{ sections: IRSection[], marks: AudienceMark[], sourceFiles: string[], sourceFingerprints: Record<string, string> }> {
|
|
355
|
-
const sections = lowerDocument(run.document, this.entry, this.deckDir, run.report, (...files) => run.inputs.watch(...files))
|
|
362
|
+
const sections = lowerDocument(run.document, this.entry, this.deckDir, run.report, (...files) => run.inputs.watch(...files), run.stored)
|
|
356
363
|
for (const id of productionAssetIds(run.document.production, run.document.sectionOrder.map(id => ({ id, stepIds: run.document.sections[id]!.stepOrder })))) {
|
|
357
364
|
const asset = run.document.assets[id], file = asset && productionAssetFile(this.deckDir, asset.location)
|
|
358
365
|
if (file) run.inputs.watch(file)
|
|
@@ -506,9 +513,10 @@ export class Compiler {
|
|
|
506
513
|
|
|
507
514
|
async #openWorkspace(): Promise<Workspace> {
|
|
508
515
|
const root = this.options.workspaceRoot ? resolve(this.options.workspaceRoot) : await findWorkspaceRoot(this.deckDir)
|
|
509
|
-
const read =
|
|
510
|
-
const
|
|
511
|
-
const
|
|
516
|
+
const read = workspaceSettings(root)
|
|
517
|
+
const settings = this.options.settings ?? read.settings
|
|
518
|
+
const sources: PluginSources = { workspaceRoot: root, listed: operatorPlugins(settings), bundled: this.options.bundled }
|
|
519
|
+
const workspace: Workspace = { root, src: createSourceReader(root), settings, problem: this.options.settings ? undefined : read.problem, needsMigration: Boolean(read.needsMigration), sources }
|
|
512
520
|
try {
|
|
513
521
|
workspace.operatorDir = operatorDir(root, this.options.operatorDir ?? process.env.PRESMIUM_OPERATOR_DIR)
|
|
514
522
|
} catch (error) {
|
package/src/document-lowering.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { captionSourceFingerprint, documentColorValue, isRecord, voiceTarget } from '@lincyaw/kernel'
|
|
2
|
-
import type { CodeNode, ComponentNode, DocumentBlock, Finding, InlineNode, IRBlock, IRSection, JsonObject, MdastNode, NarrationNode, RichTextNode, SourcePos,
|
|
1
|
+
import { captionSourceFingerprint, DEFAULT_LANG, documentColorValue, isRecord, voiceTarget } from '@lincyaw/kernel'
|
|
2
|
+
import type { CodeNode, ComponentNode, DocumentBlock, ReferenceNode, Finding, InlineNode, IRBlock, IRSection, JsonObject, MdastNode, NarrationNode, RichTextNode, SourcePos, PresmiumDocument, TextStyle, VisualState } from '@lincyaw/kernel'
|
|
3
3
|
import { resolve } from 'node:path'
|
|
4
4
|
|
|
5
5
|
const pointerKey = (id: string) => id.replaceAll('~', '~0').replaceAll('/', '~1')
|
|
@@ -7,7 +7,11 @@ const source = (entry: string, pointer: string): SourcePos => ({ file: entry, li
|
|
|
7
7
|
const root = (children: MdastNode[]): MdastNode => ({ type: 'root', children })
|
|
8
8
|
|
|
9
9
|
/** Lower the validated authoring graph directly; no Markdown is generated or parsed. */
|
|
10
|
-
|
|
10
|
+
/** `stored` is the document as saved, before inherited settings are filled in: caption timings were fingerprinted from it. */
|
|
11
|
+
export function lowerDocument(doc: PresmiumDocument, entry: string, deckDir: string, report: (finding: Finding) => void, watch: (...files: string[]) => void, stored: PresmiumDocument = doc): IRSection[] {
|
|
12
|
+
const english = (doc.metadata.lang ?? DEFAULT_LANG).startsWith('en')
|
|
13
|
+
const inlineText = (nodes: InlineNode[]) => inlineNodes(nodes, doc)
|
|
14
|
+
const richText = (node: RichTextNode) => richNode(node, doc)
|
|
11
15
|
const resolved = new Map<string, Map<string, VisualState>>()
|
|
12
16
|
const visuals = (id: string): Map<string, VisualState> => {
|
|
13
17
|
const known = resolved.get(id)
|
|
@@ -42,7 +46,7 @@ export function lowerDocument(doc: TeachingDocument, entry: string, deckDir: str
|
|
|
42
46
|
case 'text': return { type: 'markdown', mdast: root(block.content.map(richText)), ...block.reveal ? { reveal: 'items' } : {} }
|
|
43
47
|
case 'image': {
|
|
44
48
|
if (!block.assetId) {
|
|
45
|
-
const message =
|
|
49
|
+
const message = english ? 'No image selected.' : '未选择图片。'
|
|
46
50
|
report({ checkId: 'presmium.document.image', severity: 'warning', message, source: where })
|
|
47
51
|
return { type: 'markdown', mdast: root([{ type: 'paragraph', children: [{ type: 'text', value: message }] }]) }
|
|
48
52
|
}
|
|
@@ -62,6 +66,12 @@ export function lowerDocument(doc: TeachingDocument, entry: string, deckDir: str
|
|
|
62
66
|
if (block.variants) node.variants = block.variants.map(variant => ({ label: variant.title, value: variant.value ?? '' }))
|
|
63
67
|
return node
|
|
64
68
|
}
|
|
69
|
+
case 'embed': {
|
|
70
|
+
// Synced blocks are shown in place by a later version; until then the page says so where the block is.
|
|
71
|
+
const message = english ? `Synced block from ${block.source.document}: synced blocks are shown in a later version of Presmium.` : `来自 ${block.source.document} 的同步块:同步块将在后续版本中显示。`
|
|
72
|
+
report({ checkId: 'embed.unresolved', severity: 'warning', message, source: where })
|
|
73
|
+
return { type: 'markdown', mdast: root([{ type: 'paragraph', children: [{ type: 'text', value: message }], data: { hProperties: { className: ['pm-embed-placeholder'] } } }]) }
|
|
74
|
+
}
|
|
65
75
|
case 'container': return { type: 'component', plugin: 'presmium.layout', name: block.layout === 'columns' ? 'PmColumns' : block.layout === 'tabs' ? 'PmTabs' : 'PmStack', props: { ...block.properties }, children: block.children.map(child) }
|
|
66
76
|
case 'component': {
|
|
67
77
|
const definition = doc.definitions[block.definitionId]!
|
|
@@ -105,7 +115,7 @@ export function lowerDocument(doc: TeachingDocument, entry: string, deckDir: str
|
|
|
105
115
|
const target = advances ? undefined : voiceTarget({ id: sectionId, steps: section.stepOrder.length - 1, stepIds: section.stepOrder, blocks }, block.at)
|
|
106
116
|
return [block.id, { ...target ? { target } : {}, ...advances && section.stepOrder.some(id => doc.production.stepOverrides[id]?.voiceSource) ? { issue: 'cue-step' } : voice?.type === 'tts' ? { voice: voice.voice } : { issue: voice?.type === 'recording' ? 'recording-unavailable' : 'voice-unset' } }]
|
|
107
117
|
}))
|
|
108
|
-
return { id: sectionId, index: index + 1, title: section.title, depth: index === 0 && (section.level ?? 1) === 1 ? 0 : section.level ?? 1, source: source(entry, `/sections/${pointerKey(sectionId)}`), steps: Math.max(0, section.stepOrder.length - 1), stepIds: [...section.stepOrder], stepNarrations, captionSources: Object.fromEntries(section.stepOrder.map(id => [id, captionSourceFingerprint(
|
|
118
|
+
return { id: sectionId, index: index + 1, title: section.title, depth: index === 0 && (section.level ?? 1) === 1 ? 0 : section.level ?? 1, source: source(entry, `/sections/${pointerKey(sectionId)}`), steps: Math.max(0, section.stepOrder.length - 1), stepIds: [...section.stepOrder], stepNarrations, captionSources: Object.fromEntries(section.stepOrder.map(id => [id, captionSourceFingerprint(stored, id)])), hints: section.layout.renderers ?? {}, ...section.audience ? { audience: section.audience } : {}, config: { ...section.config, 'presmium.lecture': { ...section.config?.['presmium.lecture'], resolvedNarrationVoices } }, blocks }
|
|
109
119
|
})
|
|
110
120
|
}
|
|
111
121
|
|
|
@@ -117,7 +127,34 @@ function styled(node: MdastNode, style: TextStyle | undefined): MdastNode {
|
|
|
117
127
|
return { ...node, data: { hProperties: properties } }
|
|
118
128
|
}
|
|
119
129
|
|
|
120
|
-
|
|
130
|
+
/** A reference's text: its label, the current title of a target in this document, or the last known title. */
|
|
131
|
+
function referenceText(node: ReferenceNode, doc: PresmiumDocument): string {
|
|
132
|
+
if (node.label) return node.label
|
|
133
|
+
if (node.target.document !== doc.id) return node.fallback
|
|
134
|
+
const section = node.target.section ?? (node.target.block ? Object.values(doc.sections).find(section => sectionOwns(doc, section.children, node.target.block!))?.id : undefined)
|
|
135
|
+
if (node.target.block === undefined && section !== undefined && Object.hasOwn(doc.sections, section)) return doc.sections[section]!.title
|
|
136
|
+
if (node.target.block === undefined && section === undefined) return doc.metadata.title
|
|
137
|
+
return node.fallback
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function sectionOwns(doc: PresmiumDocument, children: string[], id: string): boolean {
|
|
141
|
+
return children.some((child) => {
|
|
142
|
+
if (child === id) return true
|
|
143
|
+
const block = doc.blocks[child]
|
|
144
|
+
return block !== undefined && (block.type === 'container' || block.type === 'component') && sectionOwns(doc, block.children ?? [], id)
|
|
145
|
+
})
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Cross-document references are plain text until the site links documents; a page of this document is linked as before. */
|
|
149
|
+
function referenceNode(node: ReferenceNode, doc: PresmiumDocument): MdastNode {
|
|
150
|
+
const text: MdastNode = { type: 'text', value: referenceText(node, doc) }
|
|
151
|
+
if (node.target.document !== doc.id) return text
|
|
152
|
+
const section = node.target.section ?? (node.target.block ? Object.values(doc.sections).find(section => sectionOwns(doc, section.children, node.target.block!))?.id : undefined)
|
|
153
|
+
return section !== undefined && Object.hasOwn(doc.sections, section) ? { type: 'link', url: `#${section}`, children: [text] } : text
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export function inlineNodes(nodes: InlineNode[], doc: PresmiumDocument): MdastNode[] {
|
|
157
|
+
const inlineText = (nodes: InlineNode[]) => inlineNodes(nodes, doc)
|
|
121
158
|
return nodes.map(node => {
|
|
122
159
|
if (node.type === 'text') {
|
|
123
160
|
let result: MdastNode = { type: node.marks?.some(mark => mark.type === 'code') ? 'inlineCode' : 'text', value: node.text }
|
|
@@ -131,11 +168,14 @@ export function inlineText(nodes: InlineNode[]): MdastNode[] {
|
|
|
131
168
|
if (node.type === 'link') return { type: 'link', url: node.href, children: inlineText(node.children) }
|
|
132
169
|
if (node.type === 'math') return { type: 'inlineMath', value: node.value }
|
|
133
170
|
if (node.type === 'break') return { type: 'break' }
|
|
171
|
+
if (node.type === 'reference') return referenceNode(node, doc)
|
|
134
172
|
return extensionNode(node.kind, node.schemaVersion, node.data, [{ type: 'text', value: node.fallback }])
|
|
135
173
|
})
|
|
136
174
|
}
|
|
137
175
|
|
|
138
|
-
export function
|
|
176
|
+
export function richNode(node: RichTextNode, doc: PresmiumDocument): MdastNode {
|
|
177
|
+
const inlineText = (nodes: InlineNode[]) => inlineNodes(nodes, doc)
|
|
178
|
+
const richText = (node: RichTextNode) => richNode(node, doc)
|
|
139
179
|
let result: MdastNode
|
|
140
180
|
switch (node.type) {
|
|
141
181
|
case 'paragraph': result = styled({ type: 'paragraph', children: inlineText(node.children) }, node.style); break
|
|
@@ -156,7 +196,7 @@ function extensionNode(kind: string, schemaVersion: number, data: JsonObject, fa
|
|
|
156
196
|
return { type: 'presmiumExtension', kind, schemaVersion, data: structuredClone(data), children: fallback }
|
|
157
197
|
}
|
|
158
198
|
|
|
159
|
-
function narrationText(node: NarrationNode, doc:
|
|
199
|
+
function narrationText(node: NarrationNode, doc: PresmiumDocument): MdastNode {
|
|
160
200
|
if (node.type === 'text') return { type: 'text', value: node.text }
|
|
161
201
|
if (node.type === 'pronunciation') return { type: 'presmiumRole', plugin: 'presmium.lecture', name: 'say', options: { value: node.display, alias: node.spoken }, children: [{ type: 'text', value: node.display }] }
|
|
162
202
|
const arg = node.action === 'pause' ? `${doc.production.cueTimings?.[node.id]?.pauseMs ?? node.parameters?.arg ?? 0}${doc.production.cueTimings?.[node.id] ? 'ms' : ''}` : node.parameters?.arg
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'
|
|
2
|
+
import { dirname, join, relative, resolve, sep } from 'node:path'
|
|
3
|
+
import { DOCUMENT_ENTRY, documentTree, effectiveSettings, isRecord, ranksAfter, withEffectiveSettings } from '@lincyaw/kernel'
|
|
4
|
+
import type { DocumentDefaults, DocumentPlacement, DocumentTree, DocumentTreeNode, EffectiveDocument, PresmiumDocument } from '@lincyaw/kernel'
|
|
5
|
+
import { findWorkspaceRoot } from './workspace.ts'
|
|
6
|
+
import { WORKSPACE_CONFIG, workspaceSettings } from './workspace-settings.ts'
|
|
7
|
+
|
|
8
|
+
/** Folders discovery never enters; `locales/<lang>/` holds translations, which are not tree nodes. */
|
|
9
|
+
const SKIPPED = new Set(['node_modules', '.git', '.presmium', 'dist', '.trash'])
|
|
10
|
+
|
|
11
|
+
/** One discovered entry file and what the tree needs from it, read without validating the whole document. */
|
|
12
|
+
export interface ScannedDocument extends DocumentTreeNode {
|
|
13
|
+
/** The schema version the file declares; an older one needs `presmium migrate`. */
|
|
14
|
+
schemaVersion: unknown
|
|
15
|
+
/** Why the file could not be read as a document. */
|
|
16
|
+
problem?: string
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface WorkspaceDocuments {
|
|
20
|
+
root: string
|
|
21
|
+
defaults: DocumentDefaults
|
|
22
|
+
/** The workspace file still lists `documents:`. */
|
|
23
|
+
needsMigration: boolean
|
|
24
|
+
documents: ScannedDocument[]
|
|
25
|
+
tree: DocumentTree
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
interface Read { mtimeMs: number, size: number, value: unknown }
|
|
29
|
+
const reads = new Map<string, Read>()
|
|
30
|
+
|
|
31
|
+
/** The parsed entry, cached until the file changes. */
|
|
32
|
+
function readEntry(file: string): unknown {
|
|
33
|
+
const stat = statSync(file)
|
|
34
|
+
const known = reads.get(file)
|
|
35
|
+
if (known && known.mtimeMs === stat.mtimeMs && known.size === stat.size) return known.value
|
|
36
|
+
let value: unknown
|
|
37
|
+
try { value = JSON.parse(readFileSync(file, 'utf8').replace(/^/, '')) } catch (error) { value = error instanceof Error ? error : new Error(String(error)) }
|
|
38
|
+
reads.set(file, { mtimeMs: stat.mtimeMs, size: stat.size, value })
|
|
39
|
+
return value
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Every workspace-relative entry file under `root`, in path order. */
|
|
43
|
+
export function discoverDocumentEntries(root: string): string[] {
|
|
44
|
+
const found: string[] = []
|
|
45
|
+
// A folder reached twice through symbolic links holds its documents once, at the first path.
|
|
46
|
+
const seen = new Set<string>()
|
|
47
|
+
const walk = (dir: string, rel: string) => {
|
|
48
|
+
let names: string[]
|
|
49
|
+
try { const real = realpathSync(dir); if (seen.has(real)) return; seen.add(real); names = readdirSync(dir).sort() } catch { return }
|
|
50
|
+
for (const name of names) {
|
|
51
|
+
if (SKIPPED.has(name) || (name.startsWith('.') && name !== DOCUMENT_ENTRY)) continue
|
|
52
|
+
const path = join(dir, name), next = rel ? `${rel}/${name}` : name
|
|
53
|
+
let stat
|
|
54
|
+
try { stat = statSync(path) } catch { continue }
|
|
55
|
+
if (stat.isDirectory()) { if (name !== 'locales') walk(path, next) }
|
|
56
|
+
else if (name === DOCUMENT_ENTRY && stat.isFile()) found.push(next)
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
walk(root, '')
|
|
60
|
+
return found
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const placementOf = (value: Record<string, unknown>): DocumentPlacement => {
|
|
64
|
+
const placement = isRecord(value.placement) ? value.placement : {}
|
|
65
|
+
return { parent: typeof placement.parent === 'string' ? placement.parent : null, rank: typeof placement.rank === 'string' ? placement.rank : '' }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function scan(root: string, path: string): ScannedDocument {
|
|
69
|
+
const value = readEntry(join(root, path))
|
|
70
|
+
const fallbackTitle = path.split('/').slice(0, -1).join('/') || path
|
|
71
|
+
if (!isRecord(value) || typeof value.id !== 'string') return { id: `path:${path}`, path, title: fallbackTitle, placement: { parent: null, rank: '' }, schemaVersion: isRecord(value) ? value.schemaVersion : undefined, problem: value instanceof Error ? value.message : 'not a document' }
|
|
72
|
+
const metadata = isRecord(value.metadata) ? value.metadata : {}
|
|
73
|
+
return { id: value.id, path, title: typeof metadata.title === 'string' && metadata.title.trim() ? metadata.title.trim() : fallbackTitle, placement: placementOf(value), schemaVersion: value.schemaVersion }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Every document of the workspace at `root` and the tree their placements make. Cheap to repeat: files are cached until they change. */
|
|
77
|
+
export function workspaceDocuments(root: string): WorkspaceDocuments {
|
|
78
|
+
const settings = workspaceSettings(root)
|
|
79
|
+
const documents = discoverDocumentEntries(root).map(path => scan(root, path))
|
|
80
|
+
return { root, defaults: settings.defaults, needsMigration: Boolean(settings.needsMigration), documents, tree: documentTree(documents) }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const real = (path: string) => { try { return realpathSync(path) } catch { return resolve(path) } }
|
|
84
|
+
|
|
85
|
+
/** The workspace root for a document: the nearest folder above it with a workspace file, otherwise the document's own folder. */
|
|
86
|
+
export async function documentWorkspaceRoot(entry: string): Promise<string> {
|
|
87
|
+
const found = await findWorkspaceRoot(dirname(entry))
|
|
88
|
+
return existsSync(join(found, WORKSPACE_CONFIG)) ? found : dirname(entry)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** The original a translation follows in the tree, or the document itself. */
|
|
92
|
+
function treeEntry(entry: string, document: PresmiumDocument): string {
|
|
93
|
+
const original = document.metadata.translationOf?.entry
|
|
94
|
+
return original ? resolve(dirname(entry), original) : entry
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** The document's ancestors (root first, parent last) and the workspace `defaults` at the root of its settings. */
|
|
98
|
+
function inheritance(root: string, entry: string, document: PresmiumDocument): { ancestors: PresmiumDocument[], defaults: DocumentDefaults } {
|
|
99
|
+
const { defaults } = workspaceSettings(root)
|
|
100
|
+
const own = treeEntry(entry, document)
|
|
101
|
+
const placement = own === entry ? document.placement : (() => { const value = existsSync(own) ? readEntry(own) : undefined; return isRecord(value) ? placementOf(value) : document.placement })()
|
|
102
|
+
if (placement.parent === null) return { ancestors: [], defaults }
|
|
103
|
+
const { tree } = workspaceDocuments(root)
|
|
104
|
+
const rel = relative(real(root), real(own)).split(sep).join('/')
|
|
105
|
+
const id = tree.byId.get(document.id)?.path === rel ? document.id : [...tree.byId.values()].find(node => node.path === rel)?.id
|
|
106
|
+
const ancestors = (id === undefined ? [] : tree.ancestors(id)).flatMap((node) => {
|
|
107
|
+
const value = readEntry(join(root, node.path))
|
|
108
|
+
return isRecord(value) && isRecord(value.metadata) && isRecord(value.production) ? [value as unknown as PresmiumDocument] : []
|
|
109
|
+
})
|
|
110
|
+
return { ancestors, defaults }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The document with its inherited settings: workspace `defaults` → root ancestor → … → parent → itself. `entry` is
|
|
115
|
+
* the document's file, `root` the workspace it belongs to; a document outside every workspace has no ancestors.
|
|
116
|
+
* Ancestors are found by ID through the workspace scan; ancestors that cannot be read are skipped.
|
|
117
|
+
*/
|
|
118
|
+
export function effectiveDocumentIn(root: string, entry: string, document: PresmiumDocument): EffectiveDocument {
|
|
119
|
+
const { ancestors, defaults } = inheritance(root, entry, document)
|
|
120
|
+
return withEffectiveSettings(document, ancestors, defaults)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** What the document inherits, without its own settings: an editor fills it in with `withInheritedSettings` as the author edits. */
|
|
124
|
+
export function inheritedSettingsIn(root: string, entry: string, document: PresmiumDocument): DocumentDefaults {
|
|
125
|
+
const { ancestors, defaults } = inheritance(root, entry, document)
|
|
126
|
+
const { lang: _lang, ...merged } = effectiveSettings(ancestors, defaults)
|
|
127
|
+
const lang = [defaults.lang, ...ancestors.map(ancestor => ancestor.metadata.lang)].filter((value): value is string => typeof value === 'string').at(-1)
|
|
128
|
+
return { ...merged, ...lang !== undefined ? { lang } : {} }
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** `effectiveDocumentIn` for a document whose workspace is found upward from its file, as the compiler finds it. */
|
|
132
|
+
export async function effectiveDocument(entry: string, document: PresmiumDocument, options: { workspaceRoot?: string } = {}): Promise<EffectiveDocument> {
|
|
133
|
+
const file = resolve(entry)
|
|
134
|
+
return effectiveDocumentIn(options.workspaceRoot ? resolve(options.workspaceRoot) : await documentWorkspaceRoot(file), file, document)
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Ranks for `count` new documents placed at the top level of the workspace at `root`, after the last one there. */
|
|
138
|
+
export function topLevelRanks(root: string, count: number): string[] {
|
|
139
|
+
const top = workspaceDocuments(root).tree.children.get(null) ?? []
|
|
140
|
+
const last = top.length ? workspaceDocuments(root).tree.byId.get(top.at(-1)!)!.placement.rank || null : null
|
|
141
|
+
return ranksAfter(last, count)
|
|
142
|
+
}
|
package/src/import.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
// `presmium import`: a folder of Markdown notes (or one note) to documents, one `
|
|
1
|
+
// `presmium import`: a folder of Markdown notes (or one note) to documents, one `index.pmd` per note, placed at the top
|
|
2
|
+
// level of the workspace after its documents, in note order.
|
|
2
3
|
//
|
|
3
|
-
// `a/b/note.md` becomes `<out>/a/b/note/
|
|
4
|
+
// `a/b/note.md` becomes `<out>/a/b/note/index.pmd`. Wikilinks resolve as note-taking apps resolve them: by path,
|
|
4
5
|
// then by file name or alias (case-insensitive), the nearest note winning when several share a name; a folder's
|
|
5
6
|
// `index` or `README` note also answers to the folder's name. A local file a note links to is copied into its
|
|
6
7
|
// document's `attachments/`, since a document publishes only files inside the workspace; an excluded folder (`raw`,
|
|
@@ -9,6 +10,7 @@ import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, realpat
|
|
|
9
10
|
import { basename, dirname, extname, join, posix, relative, resolve, sep } from 'node:path'
|
|
10
11
|
import type { MdastNode } from '@lincyaw/kernel'
|
|
11
12
|
import { DOCUMENT_ENTRY } from '@lincyaw/kernel'
|
|
13
|
+
import { topLevelRanks } from './document-tree.ts'
|
|
12
14
|
import { headingSlug, markdownDocument, noteTitle, splitFrontmatter } from './markdown.ts'
|
|
13
15
|
|
|
14
16
|
export interface ImportOptions {
|
|
@@ -20,6 +22,8 @@ export interface ImportOptions {
|
|
|
20
22
|
group?: number
|
|
21
23
|
/** Retains the exact source beside the editable converted document. */
|
|
22
24
|
preserveOriginal?: boolean
|
|
25
|
+
/** The workspace the documents join, whose top level they are placed at the end of; `out` when not given. */
|
|
26
|
+
workspaceRoot?: string
|
|
23
27
|
}
|
|
24
28
|
|
|
25
29
|
export interface ImportReport {
|
|
@@ -139,7 +143,8 @@ function writeNotes({ root, out, isExcluded }: ReturnType<typeof discover>, opti
|
|
|
139
143
|
|
|
140
144
|
const report: ImportReport = { documents: [], files: 0, unresolved: [] }
|
|
141
145
|
const copied = new Map<string, string | Uint8Array>()
|
|
142
|
-
|
|
146
|
+
const ranks = topLevelRanks(options.workspaceRoot ?? out, notes.length)
|
|
147
|
+
for (const [index, note] of notes.entries()) {
|
|
143
148
|
const docDir = join(out, note.dir)
|
|
144
149
|
const names = new Map<string, string>()
|
|
145
150
|
/** A file inside the imported folder to its copy in the document's attachments; undefined for anything else. */
|
|
@@ -175,6 +180,7 @@ function writeNotes({ root, out, isExcluded }: ReturnType<typeof discover>, opti
|
|
|
175
180
|
title: note.title,
|
|
176
181
|
tree: note.tree,
|
|
177
182
|
lang: options.lang,
|
|
183
|
+
rank: ranks[index]!,
|
|
178
184
|
links: {
|
|
179
185
|
page: (target, fragment) => {
|
|
180
186
|
const direct = pathNote(posix.join(posix.dirname(note.path), target)) ?? pathNote(target)
|
|
@@ -203,8 +209,3 @@ function writeNotes({ root, out, isExcluded }: ReturnType<typeof discover>, opti
|
|
|
203
209
|
report.files = copied.size
|
|
204
210
|
return report
|
|
205
211
|
}
|
|
206
|
-
|
|
207
|
-
/** The `documents` patterns that cover these folders: one per depth, as workspace patterns do not cross folders. */
|
|
208
|
-
export function documentPatterns(dirs: string[]): string[] {
|
|
209
|
-
return [...new Set(dirs.map(dir => dir.split('/').length))].sort((a, b) => a - b).map(depth => `${'*/'.repeat(depth)}${DOCUMENT_ENTRY}`)
|
|
210
|
-
}
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { directoryAsset, fileHash, publishesDirectory } from './assets.ts'
|
|
2
|
-
export {
|
|
2
|
+
export { importDocuments, importMarkdown } from './import.ts'
|
|
3
3
|
export type { ImportOptions, ImportReport } from './import.ts'
|
|
4
4
|
export { headingSlug, markdownDocument, noteTitle, splitFrontmatter } from './markdown.ts'
|
|
5
5
|
export type { Frontmatter, ImportedMarkdown, MarkdownLinks } from './markdown.ts'
|
|
@@ -29,5 +29,7 @@ export { compileTranslations, documentDigest } from './translations.ts'
|
|
|
29
29
|
export type { CompiledTranslation } from './translations.ts'
|
|
30
30
|
export { viewFor } from './views.ts'
|
|
31
31
|
export { findWorkspaceRoot } from './workspace.ts'
|
|
32
|
-
export { operatorDir, WORKSPACE_CONFIG, workspaceSettings } from './workspace-settings.ts'
|
|
33
|
-
export type { OperatorSettings } from './workspace-settings.ts'
|
|
32
|
+
export { operatorDir, WORKSPACE_CONFIG, workspaceDefaults, workspaceSettings } from './workspace-settings.ts'
|
|
33
|
+
export type { OperatorSettings, WorkspaceSettings } from './workspace-settings.ts'
|
|
34
|
+
export { discoverDocumentEntries, documentWorkspaceRoot, effectiveDocument, effectiveDocumentIn, inheritedSettingsIn, topLevelRanks, workspaceDocuments } from './document-tree.ts'
|
|
35
|
+
export type { ScannedDocument, WorkspaceDocuments } from './document-tree.ts'
|
package/src/markdown.ts
CHANGED
|
@@ -12,7 +12,8 @@ import { mathFromMarkdown } from 'mdast-util-math'
|
|
|
12
12
|
import { gfm } from 'micromark-extension-gfm'
|
|
13
13
|
import { math } from 'micromark-extension-math'
|
|
14
14
|
import { parse as parseYaml } from 'yaml'
|
|
15
|
-
import
|
|
15
|
+
import { DOCUMENT_SCHEMA_VERSION } from '@lincyaw/kernel'
|
|
16
|
+
import type { Asset, DocumentBlock, InlineNode, MdastNode, RichTextNode, Section, Step, PresmiumDocument, TextMark } from '@lincyaw/kernel'
|
|
16
17
|
|
|
17
18
|
export interface Frontmatter {
|
|
18
19
|
title?: string
|
|
@@ -32,7 +33,7 @@ export interface MarkdownLinks {
|
|
|
32
33
|
}
|
|
33
34
|
|
|
34
35
|
export interface ImportedMarkdown {
|
|
35
|
-
document:
|
|
36
|
+
document: PresmiumDocument
|
|
36
37
|
/** Wikilinks and relative links that `links` could not resolve, as written. */
|
|
37
38
|
unresolved: string[]
|
|
38
39
|
}
|
|
@@ -107,7 +108,7 @@ function restoreLabels(node: MdastNode): void {
|
|
|
107
108
|
node.children?.forEach(restoreLabels)
|
|
108
109
|
}
|
|
109
110
|
|
|
110
|
-
export function markdownDocument(source: string, options: { id: string, title: string, lang?: string, tree?: MdastNode, links: MarkdownLinks }): ImportedMarkdown {
|
|
111
|
+
export function markdownDocument(source: string, options: { id: string, title: string, lang?: string, rank: string, tree?: MdastNode, links: MarkdownLinks }): ImportedMarkdown {
|
|
111
112
|
const { frontmatter, body } = splitFrontmatter(source)
|
|
112
113
|
const protectedBody = body.replace(/\[\[([^\]|\n]+)\|([^\]\n]+)\]\]/g, (_, target: string, label: string) => `[[${target}${LABEL}${label}]]`)
|
|
113
114
|
const root = options.tree ? structuredClone(options.tree) : fromMarkdown(protectedBody, { extensions: [gfm(), math()], mdastExtensions: [gfmFromMarkdown(), mathFromMarkdown()] }) as unknown as MdastNode
|
|
@@ -291,8 +292,9 @@ export function markdownDocument(source: string, options: { id: string, title: s
|
|
|
291
292
|
|
|
292
293
|
return {
|
|
293
294
|
document: {
|
|
294
|
-
schemaVersion:
|
|
295
|
+
schemaVersion: DOCUMENT_SCHEMA_VERSION,
|
|
295
296
|
id: unique(options.id),
|
|
297
|
+
placement: { parent: null, rank: options.rank },
|
|
296
298
|
metadata: {
|
|
297
299
|
title,
|
|
298
300
|
lang: frontmatter.lang ?? options.lang ?? guessLang(body),
|
|
@@ -303,7 +305,7 @@ export function markdownDocument(source: string, options: { id: string, title: s
|
|
|
303
305
|
},
|
|
304
306
|
sectionOrder, sections, blocks, steps, narrations: {}, assets, definitions: {},
|
|
305
307
|
production: { defaults: {}, stepOverrides: {}, mediaClips: [] },
|
|
306
|
-
} as
|
|
308
|
+
} as PresmiumDocument,
|
|
307
309
|
unresolved: [...new Set(unresolved)],
|
|
308
310
|
}
|
|
309
311
|
}
|
package/src/migrate.ts
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
// `presmium migrate` and Studio's Upgrade: converts documents of schema version 1 to version 2 in place. Runtime code
|
|
2
|
+
// (the compiler, the viewer, the workspace's reads) never imports this module; only the command and the upgrade route do.
|
|
3
|
+
//
|
|
4
|
+
// Version 2 renames each `lecture.json` to `index.pmd` (translations too), gives every document a `placement` in the
|
|
5
|
+
// workspace tree (top level, ranked in the order the old `documents:` list of `presmium.workspace.yaml` gave, then by
|
|
6
|
+
// path), and removes `documents:` from that file. Settings stay explicit in each document: nothing is moved into
|
|
7
|
+
// inheritance. Paths that named `lecture.json` (translations, links, the workspace file) are rewritten.
|
|
8
|
+
import { existsSync, readdirSync, readFileSync, realpathSync, renameSync, statSync, writeFileSync } from 'node:fs'
|
|
9
|
+
import { join, relative, resolve, sep } from 'node:path'
|
|
10
|
+
import { DOCUMENT_ENTRY, DOCUMENT_SCHEMA_VERSION, isRecord, ranksAfter, validateDocument } from '@lincyaw/kernel'
|
|
11
|
+
import type { PresmiumDocument, WorkspaceMigrationNeed } from '@lincyaw/kernel'
|
|
12
|
+
import { isMap, isScalar, isSeq, parseDocument, visit } from 'yaml'
|
|
13
|
+
import { workspaceDocuments } from './document-tree.ts'
|
|
14
|
+
import { documentDigest } from './translations.ts'
|
|
15
|
+
import { WORKSPACE_CONFIG } from './workspace-settings.ts'
|
|
16
|
+
|
|
17
|
+
/** The entry file of schema version 1. */
|
|
18
|
+
export const LEGACY_DOCUMENT_ENTRY = 'lecture.json'
|
|
19
|
+
|
|
20
|
+
const SKIPPED = new Set(['node_modules', '.git', '.presmium', 'dist', '.trash'])
|
|
21
|
+
const LEGACY_PATH = /(^|\/)lecture\.json(?=$|[#?])/
|
|
22
|
+
const legacyPath = (value: string) => value.replace(LEGACY_PATH, `$1${DOCUMENT_ENTRY}`)
|
|
23
|
+
const toPosix = (path: string) => path.split(sep).join('/')
|
|
24
|
+
|
|
25
|
+
export interface MigrationWrite { path: string, content: string }
|
|
26
|
+
export interface MigrationRename { from: string, to: string }
|
|
27
|
+
|
|
28
|
+
/** Everything one migration changes, relative to `root`; `problems` are documents it leaves alone and why. */
|
|
29
|
+
export interface MigrationPlan {
|
|
30
|
+
root: string
|
|
31
|
+
renames: MigrationRename[]
|
|
32
|
+
writes: MigrationWrite[]
|
|
33
|
+
/** The documents converted to version 2, by their new entry path. */
|
|
34
|
+
documents: string[]
|
|
35
|
+
workspaceFile: boolean
|
|
36
|
+
problems: { path: string, message: string }[]
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Every file named `name` under `root`, translations included, in path order. */
|
|
40
|
+
function findEntries(root: string, name: string): string[] {
|
|
41
|
+
const found: string[] = []
|
|
42
|
+
// A folder reached twice through symbolic links is migrated once, by the first path.
|
|
43
|
+
const seen = new Set<string>()
|
|
44
|
+
const walk = (dir: string, rel: string) => {
|
|
45
|
+
let names: string[]
|
|
46
|
+
try { const real = realpathSync(dir); if (seen.has(real)) return; seen.add(real); names = readdirSync(dir).sort() } catch { return }
|
|
47
|
+
for (const entry of names) {
|
|
48
|
+
if (SKIPPED.has(entry) || entry.startsWith('.')) continue
|
|
49
|
+
const path = join(dir, entry), next = rel ? `${rel}/${entry}` : entry
|
|
50
|
+
let stat
|
|
51
|
+
try { stat = statSync(path) } catch { continue }
|
|
52
|
+
if (stat.isDirectory()) walk(path, next)
|
|
53
|
+
else if (entry === name && stat.isFile()) found.push(next)
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
walk(root, '')
|
|
57
|
+
return found
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const isTranslation = (path: string) => /(^|\/)locales\/[^/]+\/[^/]+$/.test(path)
|
|
61
|
+
const dirOf = (path: string) => path.split('/').slice(0, -1).join('/')
|
|
62
|
+
const renamed = (path: string) => path.endsWith(LEGACY_DOCUMENT_ENTRY) ? `${path.slice(0, -LEGACY_DOCUMENT_ENTRY.length)}${DOCUMENT_ENTRY}` : path
|
|
63
|
+
|
|
64
|
+
/** `documents:` patterns of the old workspace file, matched against entry paths. */
|
|
65
|
+
function patternOrder(patterns: string[], paths: string[]): string[] {
|
|
66
|
+
const ordered: string[] = []
|
|
67
|
+
for (const pattern of patterns) {
|
|
68
|
+
const want = pattern.replace(/^\.\//, '').split('/')
|
|
69
|
+
const matcher = want.map(segment => new RegExp(`^${segment.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[^/]*').replace(/\?/g, '[^/]')}$`))
|
|
70
|
+
for (const path of paths) {
|
|
71
|
+
const have = path.split('/')
|
|
72
|
+
if (have.length === matcher.length && matcher.every((test, index) => test.test(have[index]!)) && !ordered.includes(path)) ordered.push(path)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return ordered
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Rewrites the strings of a document that name an entry file: translations, the original of a translation, links. */
|
|
79
|
+
function rewritePaths(document: Record<string, unknown>): void {
|
|
80
|
+
const metadata = isRecord(document.metadata) ? document.metadata : undefined
|
|
81
|
+
if (metadata && isRecord(metadata.translations)) for (const [lang, entry] of Object.entries(metadata.translations)) if (typeof entry === 'string') metadata.translations[lang] = legacyPath(entry)
|
|
82
|
+
if (metadata && isRecord(metadata.translationOf) && typeof metadata.translationOf.entry === 'string') metadata.translationOf.entry = legacyPath(metadata.translationOf.entry)
|
|
83
|
+
const links = (value: unknown): void => {
|
|
84
|
+
if (Array.isArray(value)) { value.forEach(links); return }
|
|
85
|
+
if (!isRecord(value)) return
|
|
86
|
+
if (value.type === 'link' && typeof value.href === 'string') value.href = legacyPath(value.href)
|
|
87
|
+
for (const child of Object.values(value)) links(child)
|
|
88
|
+
}
|
|
89
|
+
links(document.blocks)
|
|
90
|
+
links(document.comments)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** The document as version 2: `placement` right after `id`, every other field as it was. */
|
|
94
|
+
function upgraded(value: Record<string, unknown>, rank: string): Record<string, unknown> {
|
|
95
|
+
const result: Record<string, unknown> = {}
|
|
96
|
+
for (const [key, field] of Object.entries(value)) {
|
|
97
|
+
result[key] = key === 'schemaVersion' ? DOCUMENT_SCHEMA_VERSION : field
|
|
98
|
+
if (key === 'id') result.placement = { parent: null, rank }
|
|
99
|
+
}
|
|
100
|
+
if (!('placement' in result)) result.placement = { parent: null, rank }
|
|
101
|
+
return result
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** The document in the formatting of the file it came from: its indentation, line endings and byte order mark. */
|
|
105
|
+
function serialize(value: unknown, text: string): string {
|
|
106
|
+
const body = text.replace(/^/, '')
|
|
107
|
+
const indent = /^\{\r?\n([ \t]+)"/.exec(body)?.[1] ?? ' '
|
|
108
|
+
const json = `${JSON.stringify(value, null, indent)}\n`
|
|
109
|
+
return `${text.startsWith('') ? '' : ''}${body.includes('\r\n') ? json.replace(/\n/g, '\r\n') : json}`
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** What migrating the workspace or document folder `target` changes, without writing anything. */
|
|
113
|
+
export function planMigration(target: string): MigrationPlan {
|
|
114
|
+
const root = resolve(target)
|
|
115
|
+
const plan: MigrationPlan = { root, renames: [], writes: [], documents: [], workspaceFile: false, problems: [] }
|
|
116
|
+
/** The content digest of each original before and after, by folder, to keep current translations current. */
|
|
117
|
+
const digests = new Map<string, { before: string, after: string }>()
|
|
118
|
+
const yamlFile = join(root, WORKSPACE_CONFIG)
|
|
119
|
+
const yamlText = existsSync(yamlFile) ? readFileSync(yamlFile, 'utf8') : undefined
|
|
120
|
+
const yaml = yamlText === undefined ? undefined : parseDocument(yamlText)
|
|
121
|
+
const declared = yaml && isSeq(yaml.get('documents')) ? (yaml.get('documents') as { items: unknown[] }).items.flatMap(item => isScalar(item) && typeof item.value === 'string' ? [item.value] : []) : []
|
|
122
|
+
|
|
123
|
+
const legacy = findEntries(root, LEGACY_DOCUMENT_ENTRY)
|
|
124
|
+
const current = findEntries(root, DOCUMENT_ENTRY).filter((path) => {
|
|
125
|
+
try { const value: unknown = JSON.parse(readFileSync(join(root, path), 'utf8').replace(/^/, '')); return isRecord(value) && value.schemaVersion === 1 } catch { return false }
|
|
126
|
+
})
|
|
127
|
+
const reads = new Map<string, { text: string, value: Record<string, unknown> }>()
|
|
128
|
+
for (const path of [...legacy, ...current]) {
|
|
129
|
+
const text = readFileSync(join(root, path), 'utf8')
|
|
130
|
+
let value: unknown
|
|
131
|
+
try { value = JSON.parse(text.replace(/^/, '')) } catch (error) { plan.problems.push({ path, message: error instanceof Error ? error.message : String(error) }); continue }
|
|
132
|
+
if (!isRecord(value) || value.schemaVersion !== 1) { plan.problems.push({ path, message: `not a document of schema version 1` }); continue }
|
|
133
|
+
if (path.endsWith(LEGACY_DOCUMENT_ENTRY) && existsSync(join(root, renamed(path)))) { plan.problems.push({ path, message: `${renamed(path)} exists already` }); continue }
|
|
134
|
+
reads.set(path, { text, value })
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const nodes = [...reads.keys()].filter(path => !isTranslation(path))
|
|
138
|
+
const ordered = [...patternOrder(declared, nodes), ...nodes.filter(path => !patternOrder(declared, nodes).includes(path))]
|
|
139
|
+
const placed = workspaceDocuments(root).tree
|
|
140
|
+
const top = placed.children.get(null) ?? []
|
|
141
|
+
const ranks = ranksAfter(top.length ? placed.byId.get(top.at(-1)!)!.placement.rank || null : null, ordered.length)
|
|
142
|
+
const rankOf = new Map(ordered.map((path, index) => [path, ranks[index]!]))
|
|
143
|
+
|
|
144
|
+
for (const [path, { text, value }] of reads) {
|
|
145
|
+
const translation = isTranslation(path)
|
|
146
|
+
// A translation is no tree node; it follows its original, whose rank it takes.
|
|
147
|
+
const original = translation ? `${dirOf(dirOf(dirOf(path)))}${dirOf(dirOf(dirOf(path))) ? '/' : ''}${path.split('/').at(-1)}` : path
|
|
148
|
+
const rank = rankOf.get(original) ?? rankOf.get(path) ?? ranksAfter(null, 1)[0]!
|
|
149
|
+
const before = translation ? undefined : documentDigest(value as unknown as PresmiumDocument)
|
|
150
|
+
const next = upgraded(value, rank)
|
|
151
|
+
rewritePaths(next)
|
|
152
|
+
const checked = validateDocument(next)
|
|
153
|
+
if (!checked.document) { plan.problems.push({ path, message: checked.issues.slice(0, 3).map(issue => `${issue.path || '/'}: ${issue.message}`).join('; ') }); continue }
|
|
154
|
+
reads.set(path, { text, value: next })
|
|
155
|
+
if (before !== undefined) digests.set(dirOf(path), { before, after: documentDigest(next as unknown as PresmiumDocument) })
|
|
156
|
+
}
|
|
157
|
+
for (const [path, { value }] of reads) {
|
|
158
|
+
if (value.schemaVersion !== DOCUMENT_SCHEMA_VERSION || !isTranslation(path)) continue
|
|
159
|
+
const metadata = value.metadata as Record<string, unknown>
|
|
160
|
+
const of = isRecord(metadata.translationOf) ? metadata.translationOf : undefined
|
|
161
|
+
const original = of && typeof of.entry === 'string' ? toPosix(relative(root, resolve(root, dirOf(path), dirOf(of.entry) || '.'))) : undefined
|
|
162
|
+
const digest = original === undefined ? undefined : digests.get(original)
|
|
163
|
+
// A translation that was current for its original stays current; a stale one stays stale.
|
|
164
|
+
if (of && digest && of.digest === digest.before) of.digest = digest.after
|
|
165
|
+
}
|
|
166
|
+
for (const [path, { text, value }] of reads) {
|
|
167
|
+
if (value.schemaVersion !== DOCUMENT_SCHEMA_VERSION) continue
|
|
168
|
+
const to = renamed(path)
|
|
169
|
+
if (to !== path) plan.renames.push({ from: path, to })
|
|
170
|
+
plan.writes.push({ path: to, content: serialize(value, text) })
|
|
171
|
+
if (!isTranslation(path)) plan.documents.push(to)
|
|
172
|
+
const freeze = join(root, dirOf(path), '.presmium', 'freeze')
|
|
173
|
+
if (to !== path && existsSync(freeze)) for (const name of readdirSync(freeze).sort()) {
|
|
174
|
+
if (!name.endsWith('.json')) continue
|
|
175
|
+
const file = join(freeze, name), record: unknown = (() => { try { return JSON.parse(readFileSync(file, 'utf8')) } catch { return undefined } })()
|
|
176
|
+
if (isRecord(record) && typeof record.entry === 'string' && LEGACY_PATH.test(record.entry)) plan.writes.push({ path: toPosix(relative(root, file)), content: `${JSON.stringify({ ...record, entry: legacyPath(record.entry) })}` })
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
if (yaml && yamlText !== undefined && isMap(yaml.contents)) {
|
|
181
|
+
// Edit the text where the parsed file locates it, so that its formatting and comments stay as they are.
|
|
182
|
+
const edits: { from: number, to: number, text: string }[] = []
|
|
183
|
+
const documents = yaml.contents.items.find(pair => isScalar(pair.key) && pair.key.value === 'documents')
|
|
184
|
+
if (documents && isScalar(documents.key) && documents.key.range) {
|
|
185
|
+
const from = yamlText.lastIndexOf('\n', documents.key.range[0] - 1) + 1
|
|
186
|
+
const valueEnd = (documents.value as { range?: [number, number, number] } | null)?.range?.[2] ?? documents.key.range[2]
|
|
187
|
+
const lineEnd = yamlText.indexOf('\n', Math.max(valueEnd - 1, from))
|
|
188
|
+
edits.push({ from, to: lineEnd < 0 ? yamlText.length : lineEnd + 1, text: '' })
|
|
189
|
+
}
|
|
190
|
+
visit(yaml, {
|
|
191
|
+
Scalar(_key, node) {
|
|
192
|
+
if (typeof node.value !== 'string' || !LEGACY_PATH.test(node.value) || !node.range) return
|
|
193
|
+
const [from, to] = node.range
|
|
194
|
+
if (edits.some(edit => from >= edit.from && to <= edit.to)) return
|
|
195
|
+
edits.push({ from, to, text: yamlText.slice(from, to).replaceAll(LEGACY_DOCUMENT_ENTRY, DOCUMENT_ENTRY) })
|
|
196
|
+
},
|
|
197
|
+
})
|
|
198
|
+
if (edits.length) {
|
|
199
|
+
let text = yamlText
|
|
200
|
+
for (const edit of edits.sort((x, y) => y.from - x.from)) text = `${text.slice(0, edit.from)}${edit.text}${text.slice(edit.to)}`
|
|
201
|
+
plan.workspaceFile = true
|
|
202
|
+
plan.writes.push({ path: WORKSPACE_CONFIG, content: text })
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return plan
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Whether the workspace at `root` needs `presmium migrate`, cheaply enough to ask on every document listing. */
|
|
209
|
+
export function migrationNeed(root: string): WorkspaceMigrationNeed | undefined {
|
|
210
|
+
const yamlFile = join(root, WORKSPACE_CONFIG)
|
|
211
|
+
const yaml = existsSync(yamlFile) ? (() => { try { return parseDocument(readFileSync(yamlFile, 'utf8')) } catch { return undefined } })() : undefined
|
|
212
|
+
const workspaceFile = Boolean(yaml?.has('documents'))
|
|
213
|
+
const documents = [
|
|
214
|
+
...findEntries(root, LEGACY_DOCUMENT_ENTRY).filter(path => !isTranslation(path)),
|
|
215
|
+
...workspaceDocuments(root).documents.filter(document => document.schemaVersion !== DOCUMENT_SCHEMA_VERSION && document.problem === undefined).map(document => document.path),
|
|
216
|
+
]
|
|
217
|
+
return workspaceFile || documents.length ? { documents, workspaceFile } : undefined
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** Carries out a plan: renames first (`rename` lets the caller keep history, as `git mv` does), then writes. */
|
|
221
|
+
export function applyMigration(plan: MigrationPlan, rename: (from: string, to: string) => void = (from, to) => renameSync(join(plan.root, from), join(plan.root, to))): void {
|
|
222
|
+
for (const { from, to } of plan.renames) rename(from, to)
|
|
223
|
+
for (const { path, content } of plan.writes) writeFileSync(join(plan.root, path), content)
|
|
224
|
+
}
|
package/src/translations.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'
|
|
|
2
2
|
import { readFileSync, realpathSync, statSync } from 'node:fs'
|
|
3
3
|
import { basename, dirname, relative, resolve, sep } from 'node:path'
|
|
4
4
|
import { DOCUMENT_ENTRY, errorMessage, isRecord, validateDocument } from '@lincyaw/kernel'
|
|
5
|
-
import type { Finding, IRDocument,
|
|
5
|
+
import type { Finding, IRDocument, PresmiumDocument } from '@lincyaw/kernel'
|
|
6
6
|
import { isInside } from '@lincyaw/kernel/node'
|
|
7
7
|
import type { AudienceCompilation, Compilation, Compiler, CompilerOptions } from './compiler.ts'
|
|
8
8
|
|
|
@@ -34,7 +34,7 @@ export function applyTranslations(doc: IRDocument, fields: TranslationFields, re
|
|
|
34
34
|
if (fields.translations) staleTranslations(doc, fields.translations).forEach(report)
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
function documentValue(text: string):
|
|
37
|
+
function documentValue(text: string): PresmiumDocument {
|
|
38
38
|
const checked = validateDocument(JSON.parse(text.replace(/^\uFEFF/, '')))
|
|
39
39
|
if (!checked.document) throw new Error(checked.issues.map(issue => `${issue.path || '/'}: ${issue.message}`).join('\n'))
|
|
40
40
|
return checked.document
|
|
@@ -46,12 +46,13 @@ function sorted(value: unknown): unknown {
|
|
|
46
46
|
return value
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
-
/** Translation bookkeeping and JSON formatting do not change the original's content digest. */
|
|
50
|
-
export function documentDigest(document:
|
|
49
|
+
/** Translation bookkeeping, the document's place in the tree and JSON formatting do not change the original's content digest. */
|
|
50
|
+
export function documentDigest(document: PresmiumDocument): string {
|
|
51
|
+
const { placement: _placement, ...content } = document
|
|
51
52
|
const metadata = { ...document.metadata }
|
|
52
53
|
delete metadata.translations
|
|
53
54
|
delete metadata.translationOf
|
|
54
|
-
return createHash('sha256').update(JSON.stringify(sorted({ ...
|
|
55
|
+
return createHash('sha256').update(JSON.stringify(sorted({ ...content, metadata }))).digest('hex')
|
|
55
56
|
}
|
|
56
57
|
|
|
57
58
|
export function originalDigest(doc: IRDocument): string {
|
|
@@ -93,7 +94,7 @@ function staleTranslations(doc: IRDocument, translations: Record<string, string>
|
|
|
93
94
|
findings.push({ checkId: CHECK_ID, severity: 'warning', message: `${name} is out of date: the original changed since digest ${translationOf.digest.slice(0, 12)}…; check the translation against it, then set translationOf.digest to ${digest}`, source: at('/metadata/translationOf/digest') })
|
|
94
95
|
}
|
|
95
96
|
} catch (error) {
|
|
96
|
-
findings.push({ checkId: CHECK_ID, severity: 'error', message: `${name} is not a valid
|
|
97
|
+
findings.push({ checkId: CHECK_ID, severity: 'error', message: `${name} is not a valid document: ${errorMessage(error)}`, source: at('') })
|
|
97
98
|
}
|
|
98
99
|
}
|
|
99
100
|
return findings
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto'
|
|
2
|
-
import { existsSync, readFileSync, realpathSync } from 'node:fs'
|
|
2
|
+
import { existsSync, readFileSync, realpathSync, statSync } from 'node:fs'
|
|
3
3
|
import { homedir } from 'node:os'
|
|
4
4
|
import { join, resolve } from 'node:path'
|
|
5
5
|
import { errorMessage, isRecord } from '@lincyaw/kernel'
|
|
6
|
+
import type { DocumentDefaults } from '@lincyaw/kernel'
|
|
6
7
|
import { isInside } from '@lincyaw/kernel/node'
|
|
7
8
|
import { parse as parseYaml } from 'yaml'
|
|
8
9
|
|
|
@@ -10,27 +11,56 @@ export const WORKSPACE_CONFIG = 'presmium.workspace.yaml'
|
|
|
10
11
|
|
|
11
12
|
export type OperatorSettings = Record<string, Record<string, unknown>>
|
|
12
13
|
|
|
13
|
-
|
|
14
|
+
export interface WorkspaceSettings {
|
|
15
|
+
settings: OperatorSettings
|
|
16
|
+
/** The root of every document's inherited settings (`defaults:`). */
|
|
17
|
+
defaults: DocumentDefaults
|
|
18
|
+
/** The file still lists `documents:`, which `presmium migrate` turns into document placements. */
|
|
19
|
+
needsMigration?: boolean
|
|
20
|
+
problem?: string
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const cached = new Map<string, { mtimeMs: number, result: WorkspaceSettings }>()
|
|
24
|
+
|
|
25
|
+
/** `defaults:` as written, keeping only the inherited fields of the expected shapes. */
|
|
26
|
+
export function workspaceDefaults(value: unknown): DocumentDefaults {
|
|
27
|
+
if (!isRecord(value)) return {}
|
|
28
|
+
const records = (item: unknown) => isRecord(item) ? Object.fromEntries(Object.entries(item).filter(([, entry]) => isRecord(entry))) as DocumentDefaults['presmium'] : undefined
|
|
29
|
+
const production = isRecord(value.production) ? value.production : {}
|
|
30
|
+
const result: DocumentDefaults = {}
|
|
31
|
+
if (typeof value.lang === 'string' && value.lang.trim()) result.lang = value.lang.trim()
|
|
32
|
+
if (Array.isArray(value.plugins)) result.plugins = value.plugins.filter((name): name is string => typeof name === 'string' && name.length > 0)
|
|
33
|
+
for (const key of ['presmium', 'renderers'] as const) { const found = records(value[key]); if (found) result[key] = found }
|
|
34
|
+
if (isRecord(value.execute)) result.execute = value.execute as DocumentDefaults['execute']
|
|
35
|
+
if (isRecord(value.checks)) result.checks = value.checks as DocumentDefaults['checks']
|
|
36
|
+
if (isRecord(production.defaults) || isRecord(production.captions)) result.production = { ...isRecord(production.defaults) ? { defaults: production.defaults as NonNullable<DocumentDefaults['production']>['defaults'] } : {}, ...isRecord(production.captions) && typeof production.captions.scale === 'number' ? { captions: { scale: production.captions.scale } } : {} }
|
|
37
|
+
return result
|
|
38
|
+
}
|
|
14
39
|
|
|
15
|
-
export function workspaceSettings(root: string):
|
|
40
|
+
export function workspaceSettings(root: string): WorkspaceSettings {
|
|
16
41
|
const file = join(root, WORKSPACE_CONFIG)
|
|
42
|
+
const mtimeMs = existsSync(file) ? statSync(file).mtimeMs : -1
|
|
17
43
|
const known = cached.get(file)
|
|
18
|
-
if (known) return known
|
|
19
|
-
let result: { settings:
|
|
20
|
-
if (
|
|
44
|
+
if (known?.mtimeMs === mtimeMs) return known.result
|
|
45
|
+
let result: WorkspaceSettings = { settings: {}, defaults: {} }
|
|
46
|
+
if (mtimeMs >= 0) {
|
|
21
47
|
try {
|
|
22
48
|
const data: unknown = parseYaml(readFileSync(file, 'utf8'))
|
|
23
49
|
const settings = isRecord(data) && isRecord(data.settings) ? data.settings : {}
|
|
24
50
|
const renderers = isRecord(data) && isRecord(data.renderers) ? data.renderers : {}
|
|
25
51
|
for (const [path, value] of [['renderers.lecture', renderers.lecture], ['settings.presmium.lecture', settings['presmium.lecture']]] as const) {
|
|
26
|
-
if (isRecord(value) && Object.hasOwn(value, 'script')) throw new Error(`${path}.script: External lecture scripts are not supported. Author narration in each document's
|
|
52
|
+
if (isRecord(value) && Object.hasOwn(value, 'script')) throw new Error(`${path}.script: External lecture scripts are not supported. Author narration in each document's narrations.`)
|
|
53
|
+
}
|
|
54
|
+
result = {
|
|
55
|
+
settings: Object.fromEntries(Object.entries(settings).map(([id, value]) => [id, isRecord(value) ? value : {}])),
|
|
56
|
+
defaults: workspaceDefaults(isRecord(data) ? data.defaults : undefined),
|
|
57
|
+
...isRecord(data) && data.documents !== undefined ? { needsMigration: true } : {},
|
|
27
58
|
}
|
|
28
|
-
result = { settings: Object.fromEntries(Object.entries(settings).map(([id, value]) => [id, isRecord(value) ? value : {}])) }
|
|
29
59
|
} catch (error) {
|
|
30
|
-
result = { settings: {}, problem: `${file}: ${errorMessage(error)}` }
|
|
60
|
+
result = { settings: {}, defaults: {}, problem: `${file}: ${errorMessage(error)}` }
|
|
31
61
|
}
|
|
32
62
|
}
|
|
33
|
-
cached.set(file, result)
|
|
63
|
+
cached.set(file, { mtimeMs, result })
|
|
34
64
|
return result
|
|
35
65
|
}
|
|
36
66
|
|