@lincyaw/compiler 0.0.0-stage → 1.0.0-rc.0

package/src/translations.ts

@@ -0,0 +1,139 @@
+import { createHash } from 'node:crypto'
+import { readFileSync, realpathSync, statSync } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'
+import { errorMessage, irBlocks, isRecord } from '@lincyaw/kernel'
+import type { Finding, IRDocument } from '@lincyaw/kernel'
+import type { AudienceCompilation, Compilation, Compiler, CompilerOptions } from './compiler.ts'
+import { readFrontmatter } from './source.ts'
+
+const CHECK_ID = 'presmium.translations'
+
+const ORIGINAL_KEYS = ['plugins', 'presmium', 'renderers', 'execute', 'sectionDepth', 'checks']
+
+export interface TranslationFields {
+  translations?: Record<string, string>
+  translationOf?: { entry: string, digest: string }
+}
+
+export function translationFields(headmatter: Record<string, unknown>): TranslationFields {
+  const fields: TranslationFields = {}
+  const { translations, translationOf } = headmatter
+  if (isRecord(translations)) {
+    const entries = Object.entries(translations).filter((entry): entry is [string, string] => typeof entry[1] === 'string' && entry[1].length > 0)
+    if (entries.length > 0) fields.translations = Object.fromEntries(entries)
+  }
+  if (isRecord(translationOf) && typeof translationOf.entry === 'string' && typeof translationOf.digest === 'string') {
+    fields.translationOf = { entry: translationOf.entry, digest: translationOf.digest }
+  }
+  return fields
+}
+
+export function applyTranslations(doc: IRDocument, fields: TranslationFields, report: (finding: Finding) => void): void {
+  if (fields.translations) doc.config.translations = fields.translations
+  if (fields.translationOf) doc.meta.translationOf = fields.translationOf
+  if (fields.translations) staleTranslations(doc, fields.translations).forEach(report)
+}
+
+function body(text: string): string {
+  return text.replace(/^/, '').replace(/^---[ \t]*\r?\n(?:[\s\S]*?\r?\n)?---[ \t]*(?:\r?\n|$)/, '').replace(/\r\n?/g, '\n').trimEnd()
+}
+
+export function originalDigest(doc: IRDocument): string {
+  const files = [doc.meta.entry]
+  for (const section of doc.sections) {
+    for (const { file } of [section.source, ...[...irBlocks(section.blocks)].map(block => block.source)]) if (!files.includes(file)) files.push(file)
+  }
+  const hash = createHash('sha256').update(doc.meta.title)
+  for (const file of files) {
+    let text: string
+    try {
+      text = readFileSync(file, 'utf8')
+    } catch {
+      continue
+    }
+    hash.update('\0').update(body(text))
+  }
+  return hash.digest('hex')
+}
+
+function translationFile(deckDir: string, entry: string): string | undefined {
+  const file = resolve(deckDir, entry)
+  try {
+    const rel = relative(realpathSync(deckDir), realpathSync(file))
+    return statSync(file).isFile() && !rel.startsWith(`..${sep}`) && rel !== '..' && !isAbsolute(rel) ? file : undefined
+  } catch {
+    return undefined
+  }
+}
+
+function lineWith(text: string, needle: string): number {
+  const index = text.split(/\r?\n/).findIndex(line => line.includes(needle))
+  return index < 0 ? 1 : index + 1
+}
+
+function staleTranslations(doc: IRDocument, translations: Record<string, string>): Finding[] {
+  const deckDir = dirname(doc.meta.entry)
+  const original = basename(doc.meta.entry)
+  const entryText = readFileSync(doc.meta.entry, 'utf8')
+  let digest: string | undefined
+  const findings: Finding[] = []
+  for (const [lang, entry] of Object.entries(translations)) {
+    const file = translationFile(deckDir, entry)
+    const name = `translation "${lang}" (${entry})`
+    if (!file) {
+      findings.push({ checkId: CHECK_ID, severity: 'warning', message: `${name} not found in the deck's directory`, source: { file: doc.meta.entry, line: lineWith(entryText, entry) } })
+      continue
+    }
+    const text = readFileSync(file, 'utf8')
+    digest ??= originalDigest(doc)
+    const { translationOf } = translationFields(readFrontmatter(text))
+    const expected = relative(dirname(file), doc.meta.entry).split(sep).join('/')
+    if (!translationOf) {
+      findings.push({ checkId: CHECK_ID, severity: 'warning', message: `${name} records no digest of the original; once it matches the original, add translationOf: { entry: ${expected}, digest: ${digest} } to its headmatter`, source: { file, line: 1 } })
+    } else if (resolve(dirname(file), translationOf.entry) !== doc.meta.entry) {
+      findings.push({ checkId: CHECK_ID, severity: 'warning', message: `${name} names ${translationOf.entry} as its original, not ${original}`, source: { file, line: lineWith(text, 'entry') } })
+    } else if (translationOf.digest !== digest) {
+      findings.push({ checkId: CHECK_ID, severity: 'warning', message: `${name} is out of date: the original changed since digest ${translationOf.digest.slice(0, 12)}…; check the translation against it, then set translationOf.digest to ${digest}`, source: { file, line: lineWith(text, translationOf.digest) } })
+    }
+  }
+  return findings
+}
+
+export interface CompiledTranslation {
+  lang: string
+  doc: IRDocument
+  files: string[]
+  audience?: AudienceCompilation
+}
+
+export async function compileTranslations(compilation: Compilation, compilers = new Map<string, Compiler>()): Promise<{ translations: CompiledTranslation[], findings: Finding[] }> {
+  const { doc, deckDir, frontmatter } = compilation
+  const translations: CompiledTranslation[] = []
+  const findings: Finding[] = []
+  const { Compiler } = await import('./compiler.ts')
+  const { audience: _audience, headmatter: _headmatter, deckDir: _deckDir, ...base } = compilation.options
+  const audience = compilation.audience?.name
+  for (const [lang, entry] of Object.entries(doc.config.translations ?? {})) {
+    const file = translationFile(deckDir, entry)
+    if (!file) continue
+    try {
+      const own = readFrontmatter(await readFile(file, 'utf8'))
+      const headmatter: Record<string, unknown> = { translations: undefined }
+      for (const key of ORIGINAL_KEYS) headmatter[key] = frontmatter[key]
+      if (!(typeof own.lang === 'string' && own.lang.trim())) headmatter.lang = lang
+      if (own.title === undefined) headmatter.title = doc.meta.title
+      const options: CompilerOptions = { ...base, headmatter, deckDir, ...audience ? { audience } : {} }
+      const known = compilers.get(file)
+      const same = known && JSON.stringify(known.options.headmatter) === JSON.stringify(headmatter) && known.options.audience === options.audience
+      const compiler = same ? known : new Compiler(file, options)
+      compilers.set(file, compiler)
+      const translated = await compiler.compile()
+      findings.push(...translated.doc.diagnostics)
+      translations.push({ lang, doc: translated.doc, files: [...new Set([...translated.files, ...translated.dependencies.files])], ...translated.audience ? { audience: translated.audience } : {} })
+    } catch (error) {
+      findings.push({ checkId: CHECK_ID, severity: 'error', message: `translation "${lang}" (${entry}) could not be compiled: ${errorMessage(error)}`, source: { file, line: 1 } })
+    }
+  }
+  return { translations, findings }
+}

package/src/validate.ts

@@ -0,0 +1,105 @@
+import { IR_VERSION, isRecord } from '@lincyaw/kernel'
+import type { Finding, IRBlock, IRDocument, IRSection, MdastNode, SourcePos } from '@lincyaw/kernel'
+
+export function validateIR(doc: IRDocument, entry?: string): Finding[] {
+  const findings: Finding[] = []
+  const file = entry ?? (isRecord(doc.meta) && typeof doc.meta.entry === 'string' ? doc.meta.entry : 'ir.json')
+  const fail = (message: string, source: SourcePos = { file, line: 1 }) =>
+    findings.push({ checkId: 'presmium.ir', severity: 'error', message, source: { file: source.file, line: source.line } })
+
+  if (doc.ir !== IR_VERSION) fail(`ir must be ${IR_VERSION}`)
+  if (!isRecord(doc.meta) || typeof doc.meta.title !== 'string' || typeof doc.meta.entry !== 'string') fail('meta needs a title and an entry')
+  const config = doc.config as unknown
+  if (!isRecord(config) || !Array.isArray(config.plugins) || !isRecord(config.presmium) || !isRecord(config.renderers)) {
+    fail('config needs plugins, presmium and renderers')
+  }
+  if (!Array.isArray(doc.sections)) {
+    fail('sections must be a list')
+    return findings
+  }
+
+  const ids = new Set<string>()
+  doc.sections.forEach((section, i) => {
+    if (!isRecord(section)) return fail(`section ${i + 1} is not an object`)
+    const where = isSource(section.source) ? section.source : undefined
+    if (!where) fail(`section ${i + 1} has no source position`)
+    if (typeof section.id !== 'string' || section.id === '') fail(`section ${i + 1} has no id`, where)
+    else if (ids.has(section.id)) fail(`section id ${section.id} is not unique`, where)
+    ids.add(section.id)
+    if (section.index !== i + 1) fail(`section ${section.id} has index ${section.index}, expected ${i + 1}`, where)
+    if (typeof section.title !== 'string') fail(`section ${section.id} has no title`, where)
+    if (!isCount(section.steps)) fail(`section ${section.id}: steps must be a non-negative integer`, where)
+    if (!isRecord(section.hints) || !Object.values(section.hints).every(isRecord)) fail(`section ${section.id}: hints must map renderer ids to objects`, where)
+    if (section.config !== undefined && !(isRecord(section.config) && Object.values(section.config).every(isRecord))) fail(`section ${section.id}: config must map plugin ids to objects`, where)
+    if (!Array.isArray(section.blocks)) fail(`section ${section.id}: blocks must be a list`, where)
+    else validateBlocks(section.blocks, section, fail)
+  })
+  return findings
+}
+
+function validateBlocks(blocks: IRBlock[], section: IRSection, fail: (message: string, source?: SourcePos) => void): void {
+  const ids = new Set<string>()
+  for (const block of blocks) {
+    if (!isRecord(block)) {
+      fail(`a block of section ${section.id} is not an object`, section.source)
+      continue
+    }
+    const where = isSource(block.source) ? block.source : section.source
+    const name = `block ${block.id} of section ${section.id}`
+    if (!isSource(block.source)) fail(`${name} has no source position`, where)
+    if (typeof block.id !== 'string' || block.id === '') fail(`a block of section ${section.id} has no id`, where)
+    else if (ids.has(block.id)) fail(`${name}: id is not unique in the section`, where)
+    ids.add(block.id)
+    if (block.role !== 'visual' && block.role !== 'narration') fail(`${name}: role must be visual or narration`, where)
+    if (!isCount(block.at)) fail(`${name}: at must be a non-negative integer`, where)
+    if (block.span !== undefined && !isCount(block.span)) fail(`${name}: span must be a non-negative integer`, where)
+    if (isCount(block.at) && block.at + (block.span ?? 0) > section.steps) fail(`${name} ends at step ${block.at + (block.span ?? 0)}, after the last step ${section.steps}`, where)
+    if (block.until !== undefined && !(isCount(block.until) && block.until > block.at)) fail(`${name}: until must be an integer after at`, where)
+    if (block.only !== undefined && !(Array.isArray(block.only) && block.only.every(id => typeof id === 'string'))) fail(`${name}: only must list renderer ids`, where)
+    const problem = nodeProblem(block)
+    if (problem) fail(`${name}: ${problem}`, where)
+    if (block.node?.type === 'component' && block.node.children) validateBlocks(block.node.children, section, fail)
+  }
+}
+
+function nodeProblem({ node }: IRBlock): string | undefined {
+  if (!isRecord(node)) return 'node is missing'
+  switch (node.type) {
+    case 'markdown':
+      return isRecord(node.mdast) ? mdastProblem(node.mdast, 'mdast') : 'markdown node needs an mdast tree'
+    case 'code':
+      if (typeof node.lang !== 'string' || typeof node.value !== 'string') return 'code node needs lang and value'
+      if (node.highlight && !node.highlight.every(lines => Array.isArray(lines) && lines.every(n => Number.isInteger(n) && n >= 1))) return 'highlight must list 1-based line numbers per step'
+      if (node.run && !(typeof node.run.lang === 'string' && ['compile', 'live', 'both'].includes(node.run.when))) return 'run needs lang and when (compile, live or both)'
+      if (node.outputs && !node.outputs.every(o => isRecord(o) && typeof o.stdout === 'string' && typeof o.cacheKey === 'string')) return 'outputs must be recorded runs'
+      if (node.variants && !node.variants.every(v => isRecord(v) && typeof v.label === 'string' && typeof v.value === 'string')) return 'variants need label and value'
+      return undefined
+    case 'component':
+      if (typeof node.plugin !== 'string' || typeof node.name !== 'string' || node.name === '') return 'component node needs plugin and name'
+      if (!isRecord(node.props)) return 'component props must be an object'
+      if (node.fallback !== undefined) return isRecord(node.fallback) ? mdastProblem(node.fallback, 'fallback') : 'component fallback must be mdast'
+      if (node.assets !== undefined && !(isRecord(node.assets) && Object.values(node.assets).every(url => typeof url === 'string'))) return 'component assets must map props to URLs'
+      return undefined
+    default:
+      return `unknown node type ${String((node as { type?: unknown }).type)}`
+  }
+}
+
+function mdastProblem(node: MdastNode, path: string): string | undefined {
+  if (typeof node.type !== 'string') return `${path} has a node without a type`
+  if (node.children === undefined) return undefined
+  if (!Array.isArray(node.children)) return `${path}: the children of a ${node.type} node must be a list`
+  for (const [i, child] of node.children.entries()) {
+    const problem = isRecord(child) ? mdastProblem(child, `${path} > ${node.type}[${i}]`) : `${path} > ${node.type}[${i}] is not a node`
+    if (problem) return problem
+  }
+  return undefined
+}
+
+function isCount(value: unknown): value is number {
+  return Number.isInteger(value) && (value as number) >= 0
+}
+
+function isSource(value: unknown): value is SourcePos {
+  return isRecord(value) && typeof value.file === 'string' && Number.isInteger(value.line) && (value.line as number) >= 1
+}

package/src/workspace.ts

@@ -0,0 +1,27 @@
+import { execFile } from 'node:child_process'
+import { readFile } from 'node:fs/promises'
+import { isAbsolute, join, relative, resolve, sep } from 'node:path'
+import { promisify } from 'node:util'
+import type { SourceReader } from '@lincyaw/kernel'
+
+const run = promisify(execFile)
+
+export const COMMIT = /^(?:[\da-f]{40}|[\da-f]{64})$/i
+
+export async function findWorkspaceRoot(dir: string): Promise<string> {
+  return run('git', ['rev-parse', '--show-toplevel'], { cwd: dir })
+    .then(({ stdout }) => resolve(stdout.trim()), () => resolve(dir))
+}
+
+export function createSourceReader(root: string): SourceReader {
+  return {
+    async read(file, ref) {
+      const rel = relative(root, resolve(root, file))
+      if (rel === '' || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return undefined
+      if (ref === undefined) return readFile(join(root, rel), 'utf8').catch(() => undefined)
+      if (ref.startsWith('-')) return undefined
+      return run('git', ['show', `${ref}:${rel.split(sep).join('/')}`], { cwd: root, maxBuffer: 64 << 20 })
+        .then(({ stdout }) => stdout, () => undefined)
+    },
+  }
+}