@lincyaw/compiler 0.0.0-stage → 1.0.0-rc.1

package/src/translations.ts

@@ -0,0 +1,133 @@
+import { createHash } from 'node:crypto'
+import { readFileSync, realpathSync, statSync } from 'node:fs'
+import { basename, dirname, relative, resolve, sep } from 'node:path'
+import { DOCUMENT_ENTRY, errorMessage, isRecord, validateDocument } from '@lincyaw/kernel'
+import type { Finding, IRDocument, TeachingDocument } from '@lincyaw/kernel'
+import { isInside } from '@lincyaw/kernel/node'
+import type { AudienceCompilation, Compilation, Compiler, CompilerOptions } from './compiler.ts'
+
+const CHECK_ID = 'presmium.translations'
+const pointerKey = (key: string) => key.replaceAll('~', '~0').replaceAll('/', '~1')
+const sourceAt = (file: string, pointer: string) => ({ file, line: 1, format: 'json' as const, pointer })
+
+export interface TranslationFields {
+  translations?: Record<string, string>
+  translationOf?: { entry: string, digest: string }
+}
+
+export function translationFields(metadata: Record<string, unknown>): TranslationFields {
+  const fields: TranslationFields = {}
+  const { translations, translationOf } = metadata
+  if (isRecord(translations)) {
+    const entries = Object.entries(translations).filter((entry): entry is [string, string] => typeof entry[1] === 'string' && entry[1].length > 0)
+    if (entries.length > 0) fields.translations = Object.fromEntries(entries)
+  }
+  if (isRecord(translationOf) && typeof translationOf.entry === 'string' && typeof translationOf.digest === 'string') {
+    fields.translationOf = { entry: translationOf.entry, digest: translationOf.digest }
+  }
+  return fields
+}
+
+export function applyTranslations(doc: IRDocument, fields: TranslationFields, report: (finding: Finding) => void): void {
+  if (fields.translations) doc.config.translations = fields.translations
+  if (fields.translationOf) doc.meta.translationOf = fields.translationOf
+  if (fields.translations) staleTranslations(doc, fields.translations).forEach(report)
+}
+
+function documentValue(text: string): TeachingDocument {
+  const checked = validateDocument(JSON.parse(text.replace(/^\uFEFF/, '')))
+  if (!checked.document) throw new Error(checked.issues.map(issue => `${issue.path || '/'}: ${issue.message}`).join('\n'))
+  return checked.document
+}
+
+function sorted(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(sorted)
+  if (isRecord(value)) return Object.fromEntries(Object.keys(value).sort().map(key => [key, sorted(value[key])]))
+  return value
+}
+
+/** Translation bookkeeping and JSON formatting do not change the original's content digest. */
+export function documentDigest(document: TeachingDocument): string {
+  const metadata = { ...document.metadata }
+  delete metadata.translations
+  delete metadata.translationOf
+  return createHash('sha256').update(JSON.stringify(sorted({ ...document, metadata }))).digest('hex')
+}
+
+export function originalDigest(doc: IRDocument): string {
+  return documentDigest(documentValue(readFileSync(doc.meta.entry, 'utf8')))
+}
+
+function translationFile(deckDir: string, entry: string): string | undefined {
+  const file = resolve(deckDir, entry)
+  if (basename(file) !== DOCUMENT_ENTRY) return undefined
+  try {
+    return statSync(file).isFile() && isInside(realpathSync(file), realpathSync(deckDir)) ? file : undefined
+  } catch {
+    return undefined
+  }
+}
+
+function staleTranslations(doc: IRDocument, translations: Record<string, string>): Finding[] {
+  const deckDir = dirname(doc.meta.entry)
+  const original = basename(doc.meta.entry)
+  let digest: string | undefined
+  const findings: Finding[] = []
+  for (const [lang, entry] of Object.entries(translations)) {
+    const file = translationFile(deckDir, entry)
+    const name = `translation "${lang}" (${entry})`
+    if (!file || file === doc.meta.entry) {
+      findings.push({ checkId: CHECK_ID, severity: 'warning', message: `${name} must name a distinct ${DOCUMENT_ENTRY} inside the document's directory`, source: sourceAt(doc.meta.entry, `/metadata/translations/${pointerKey(lang)}`) })
+      continue
+    }
+    const at = (pointer: string) => sourceAt(file, pointer)
+    try {
+      digest ??= originalDigest(doc)
+      const { translationOf } = documentValue(readFileSync(file, 'utf8')).metadata
+      const expected = relative(dirname(file), doc.meta.entry).split(sep).join('/')
+      if (!translationOf) {
+        findings.push({ checkId: CHECK_ID, severity: 'warning', message: `${name} records no digest of the original; once it matches the original, set metadata.translationOf to { entry: ${expected}, digest: ${digest} }`, source: at('/metadata') })
+      } else if (resolve(dirname(file), translationOf.entry) !== doc.meta.entry) {
+        findings.push({ checkId: CHECK_ID, severity: 'warning', message: `${name} names ${translationOf.entry} as its original, not ${original}`, source: at('/metadata/translationOf/entry') })
+      } else if (translationOf.digest !== digest) {
+        findings.push({ checkId: CHECK_ID, severity: 'warning', message: `${name} is out of date: the original changed since digest ${translationOf.digest.slice(0, 12)}…; check the translation against it, then set translationOf.digest to ${digest}`, source: at('/metadata/translationOf/digest') })
+      }
+    } catch (error) {
+      findings.push({ checkId: CHECK_ID, severity: 'error', message: `${name} is not a valid teaching document: ${errorMessage(error)}`, source: at('') })
+    }
+  }
+  return findings
+}
+
+export interface CompiledTranslation {
+  lang: string
+  doc: IRDocument
+  files: string[]
+  audience?: AudienceCompilation
+}
+
+export async function compileTranslations(compilation: Compilation, compilers = new Map<string, Compiler>()): Promise<{ translations: CompiledTranslation[], findings: Finding[] }> {
+  const { doc, deckDir } = compilation
+  const translations: CompiledTranslation[] = []
+  const findings: Finding[] = []
+  const { Compiler } = await import('./compiler.ts')
+  const { audience: _audience, headmatter: _headmatter, deckDir: _deckDir, ...base } = compilation.options
+  const audience = compilation.audience?.name
+  for (const [lang, entry] of Object.entries(doc.config.translations ?? {})) {
+    const file = translationFile(deckDir, entry)
+    if (!file || file === doc.meta.entry) continue
+    try {
+      const options: CompilerOptions = { ...base, ...audience ? { audience } : {} }
+      const known = compilers.get(file)
+      const same = known && known.options.audience === options.audience
+      const compiler = same ? known : new Compiler(file, options)
+      compilers.set(file, compiler)
+      const translated = await compiler.compile()
+      findings.push(...translated.doc.diagnostics)
+      translations.push({ lang, doc: translated.doc, files: [...new Set([...translated.sourceFiles, ...translated.inputs.files])], ...translated.audience ? { audience: translated.audience } : {} })
+    } catch (error) {
+      findings.push({ checkId: CHECK_ID, severity: 'error', message: `translation "${lang}" (${entry}) could not be compiled: ${errorMessage(error)}`, source: sourceAt(file, '') })
+    }
+  }
+  return { translations, findings }
+}

package/src/validate.ts

@@ -0,0 +1,107 @@
+import { IR_VERSION, isRecord } from '@lincyaw/kernel'
+import type { Finding, IRBlock, IRDocument, IRSection, MdastNode, SourcePos } from '@lincyaw/kernel'
+
+export function validateIR(doc: IRDocument, entry?: string): Finding[] {
+  const findings: Finding[] = []
+  const file = entry ?? (isRecord(doc.meta) && typeof doc.meta.entry === 'string' ? doc.meta.entry : 'ir.json')
+  const fail = (message: string, source: SourcePos = { file, line: 1 }) =>
+    findings.push({ checkId: 'presmium.ir', severity: 'error', message, source: { file: source.file, line: source.line } })
+
+  if (doc.ir !== IR_VERSION) fail(`ir must be ${IR_VERSION}`)
+  if (!isRecord(doc.meta) || typeof doc.meta.title !== 'string' || typeof doc.meta.entry !== 'string') fail('meta needs a title and an entry')
+  const config = doc.config as unknown
+  if (!isRecord(config) || !Array.isArray(config.plugins) || !isRecord(config.presmium) || !isRecord(config.renderers)) {
+    fail('config needs plugins, presmium and renderers')
+  }
+  if (!Array.isArray(doc.sections)) {
+    fail('sections must be a list')
+    return findings
+  }
+
+  const ids = new Set<string>()
+  doc.sections.forEach((section, i) => {
+    if (!isRecord(section)) return fail(`section ${i + 1} is not an object`)
+    const where = isSource(section.source) ? section.source : undefined
+    if (!where) fail(`section ${i + 1} has no source position`)
+    if (typeof section.id !== 'string' || section.id === '') fail(`section ${i + 1} has no id`, where)
+    else if (ids.has(section.id)) fail(`section id ${section.id} is not unique`, where)
+    ids.add(section.id)
+    if (section.index !== i + 1) fail(`section ${section.id} has index ${section.index}, expected ${i + 1}`, where)
+    if (typeof section.title !== 'string') fail(`section ${section.id} has no title`, where)
+    if (!isCount(section.steps)) fail(`section ${section.id}: steps must be a non-negative integer`, where)
+    if (!isRecord(section.hints) || !Object.values(section.hints).every(isRecord)) fail(`section ${section.id}: hints must map renderer ids to objects`, where)
+    if (section.config !== undefined && !(isRecord(section.config) && Object.values(section.config).every(isRecord))) fail(`section ${section.id}: config must map plugin ids to objects`, where)
+    if (!Array.isArray(section.blocks)) fail(`section ${section.id}: blocks must be a list`, where)
+    else validateBlocks(section.blocks, section, fail)
+  })
+  return findings
+}
+
+function validateBlocks(blocks: IRBlock[], section: IRSection, fail: (message: string, source?: SourcePos) => void): void {
+  const ids = new Set<string>()
+  for (const block of blocks) {
+    if (!isRecord(block)) {
+      fail(`a block of section ${section.id} is not an object`, section.source)
+      continue
+    }
+    const where = isSource(block.source) ? block.source : section.source
+    const name = `block ${block.id} of section ${section.id}`
+    if (!isSource(block.source)) fail(`${name} has no source position`, where)
+    if (typeof block.id !== 'string' || block.id === '') fail(`a block of section ${section.id} has no id`, where)
+    else if (ids.has(block.id)) fail(`${name}: id is not unique in the section`, where)
+    ids.add(block.id)
+    if (block.role !== 'visual' && block.role !== 'narration') fail(`${name}: role must be visual or narration`, where)
+    if (!isCount(block.at)) fail(`${name}: at must be a non-negative integer`, where)
+    if (block.span !== undefined && !isCount(block.span)) fail(`${name}: span must be a non-negative integer`, where)
+    if (isCount(block.at) && block.at + (block.span ?? 0) > section.steps) fail(`${name} ends at step ${block.at + (block.span ?? 0)}, after the last step ${section.steps}`, where)
+    if (block.until !== undefined && !(isCount(block.until) && block.until > block.at)) fail(`${name}: until must be an integer after at`, where)
+    if (block.only !== undefined && !(Array.isArray(block.only) && block.only.every(id => typeof id === 'string'))) fail(`${name}: only must list renderer ids`, where)
+    const problem = nodeProblem(block)
+    if (problem) fail(`${name}: ${problem}`, where)
+    if (block.node?.type === 'component' && block.node.children) validateBlocks(block.node.children, section, fail)
+  }
+}
+
+function nodeProblem({ node }: IRBlock): string | undefined {
+  if (!isRecord(node)) return 'node is missing'
+  switch (node.type) {
+    case 'markdown':
+      if (node.narration !== undefined && (!isRecord(node.narration) || !['during', 'after'].includes(String(node.narration.mode)))) return 'narration mode must be during or after'
+      return isRecord(node.mdast) ? mdastProblem(node.mdast, 'mdast') : 'markdown node needs an mdast tree'
+    case 'code':
+      if (typeof node.lang !== 'string' || typeof node.value !== 'string') return 'code node needs lang and value'
+      if (node.highlight && !node.highlight.every(lines => Array.isArray(lines) && lines.every(n => Number.isInteger(n) && n >= 1))) return 'highlight must list 1-based line numbers per step'
+      if (node.run && !(typeof node.run.lang === 'string' && ['compile', 'live', 'both'].includes(node.run.when))) return 'run needs lang and when (compile, live or both)'
+      if (node.outputs && !node.outputs.every(o => isRecord(o) && typeof o.stdout === 'string' && typeof o.cacheKey === 'string')) return 'outputs must be recorded runs'
+      if (node.variants && !node.variants.every(v => isRecord(v) && typeof v.label === 'string' && typeof v.value === 'string')) return 'variants need label and value'
+      return undefined
+    case 'component':
+      if (typeof node.plugin !== 'string' || typeof node.name !== 'string' || node.name === '') return 'component node needs plugin and name'
+      if (!isRecord(node.props)) return 'component props must be an object'
+      if (node.assets !== undefined && !(isRecord(node.assets) && Object.values(node.assets).every(url => typeof url === 'string'))) return 'component assets must map props to URLs'
+      if (node.durations !== undefined && !(Array.isArray(node.durations) && node.durations.every(ms => typeof ms === 'number' && Number.isFinite(ms) && ms >= 0))) return 'component durations must list non-negative milliseconds'
+      if (node.fallback !== undefined) return isRecord(node.fallback) ? mdastProblem(node.fallback, 'fallback') : 'component fallback must be mdast'
+      return undefined
+    default:
+      return `unknown node type ${String((node as { type?: unknown }).type)}`
+  }
+}
+
+function mdastProblem(node: MdastNode, path: string): string | undefined {
+  if (typeof node.type !== 'string') return `${path} has a node without a type`
+  if (node.children === undefined) return undefined
+  if (!Array.isArray(node.children)) return `${path}: the children of a ${node.type} node must be a list`
+  for (const [i, child] of node.children.entries()) {
+    const problem = isRecord(child) ? mdastProblem(child, `${path} > ${node.type}[${i}]`) : `${path} > ${node.type}[${i}] is not a node`
+    if (problem) return problem
+  }
+  return undefined
+}
+
+function isCount(value: unknown): value is number {
+  return Number.isInteger(value) && (value as number) >= 0
+}
+
+function isSource(value: unknown): value is SourcePos {
+  return isRecord(value) && typeof value.file === 'string' && Number.isInteger(value.line) && (value.line as number) >= 1
+}

package/src/views.ts

@@ -0,0 +1,2 @@
+// The document as one renderer sees it; the kernel holds it, as the web form's server needs it too.
+export { viewFor } from '@lincyaw/kernel'

package/src/workspace-settings.ts

@@ -0,0 +1,59 @@
+import { createHash } from 'node:crypto'
+import { existsSync, readFileSync, realpathSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { errorMessage, isRecord } from '@lincyaw/kernel'
+import { isInside } from '@lincyaw/kernel/node'
+import { parse as parseYaml } from 'yaml'
+
+export const WORKSPACE_CONFIG = 'presmium.workspace.yaml'
+
+export type OperatorSettings = Record<string, Record<string, unknown>>
+
+const cached = new Map<string, { settings: OperatorSettings, problem?: string }>()
+
+export function workspaceSettings(root: string): { settings: OperatorSettings, problem?: string } {
+  const file = join(root, WORKSPACE_CONFIG)
+  const known = cached.get(file)
+  if (known) return known
+  let result: { settings: OperatorSettings, problem?: string } = { settings: {} }
+  if (existsSync(file)) {
+    try {
+      const data: unknown = parseYaml(readFileSync(file, 'utf8'))
+      const settings = isRecord(data) && isRecord(data.settings) ? data.settings : {}
+      const renderers = isRecord(data) && isRecord(data.renderers) ? data.renderers : {}
+      for (const [path, value] of [['renderers.lecture', renderers.lecture], ['settings.presmium.lecture', settings['presmium.lecture']]] as const) {
+        if (isRecord(value) && Object.hasOwn(value, 'script')) throw new Error(`${path}.script: External lecture scripts are not supported. Author narration in each document's lecture.json narrations.`)
+      }
+      result = { settings: Object.fromEntries(Object.entries(settings).map(([id, value]) => [id, isRecord(value) ? value : {}])) }
+    } catch (error) {
+      result = { settings: {}, problem: `${file}: ${errorMessage(error)}` }
+    }
+  }
+  cached.set(file, result)
+  return result
+}
+
+export function operatorPlugins(settings: OperatorSettings): string[] {
+  const list = settings['presmium.workspace']?.plugins
+  return Array.isArray(list) ? list.filter((name): name is string => typeof name === 'string' && name.length > 0) : []
+}
+
+function real(path: string): string {
+  try {
+    return realpathSync(path)
+  } catch {
+    return resolve(path)
+  }
+}
+
+export function operatorDir(workspaceRoot: string, given = process.env.PRESMIUM_OPERATOR_DIR): string {
+  const root = real(workspaceRoot)
+  if (given) {
+    const dir = resolve(given)
+    if (isInside(real(dir), root)) throw new Error(`the operator directory ${dir} lies inside the workspace ${root}; it must be outside every workspace`)
+    return dir
+  }
+  const state = process.env.XDG_STATE_HOME || join(homedir(), '.local', 'state')
+  return join(state, 'presmium', 'operator', createHash('sha256').update(root).digest('hex').slice(0, 16))
+}

package/src/workspace.ts

@@ -0,0 +1,37 @@
+import { execFile } from 'node:child_process'
+import { existsSync } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import { dirname, join, relative, resolve, sep } from 'node:path'
+import { promisify } from 'node:util'
+import type { SourceReader } from '@lincyaw/kernel'
+import { isInside } from '@lincyaw/kernel/node'
+import { WORKSPACE_CONFIG } from './workspace-settings.ts'
+
+const run = promisify(execFile)
+
+export const COMMIT_HASH = /^(?:[\da-f]{40}|[\da-f]{64})$/i
+
+export async function findWorkspaceRoot(dir: string): Promise<string> {
+  const start = resolve(dir)
+  const repository = await run('git', ['rev-parse', '--show-toplevel'], { cwd: start })
+    .then(({ stdout }) => resolve(stdout.trim()), () => undefined)
+  for (let current = start; ; current = dirname(current)) {
+    if (existsSync(join(current, WORKSPACE_CONFIG))) return current
+    if (current === repository || dirname(current) === current) break
+  }
+  return repository ?? start
+}
+
+export function createSourceReader(root: string): SourceReader {
+  return {
+    async read(file, ref) {
+      const path = resolve(root, file)
+      const rel = relative(root, path)
+      if (rel === '' || !isInside(path, root)) return undefined
+      if (ref === undefined) return readFile(join(root, rel), 'utf8').catch(() => undefined)
+      if (ref.startsWith('-')) return undefined
+      return run('git', ['show', `${ref}:./${rel.split(sep).join('/')}`], { cwd: root, maxBuffer: 64 << 20 })
+        .then(({ stdout }) => stdout, () => undefined)
+    },
+  }
+}