@lincyaw/compiler 1.0.0-rc.0 → 1.0.0-rc.2

package/src/compiler.ts

@@ -1,50 +1,49 @@
-import { randomUUID } from 'node:crypto'
+import { createHash, randomUUID } from 'node:crypto'
 import { existsSync, statSync } from 'node:fs'
 import { readFile } from 'node:fs/promises'
 import { basename, dirname, extname, join, relative, resolve } from 'node:path'
-import { ANCHOR_LOCK_PATH, AnchorLock, DEFAULT_LANG, IR_VERSION, PROCESS_SANDBOX, createAnchorResolverRegistry, docBlocks, environmentExecutor, errorMessage, failedRun, isRecord } from '@lincyaw/kernel'
-import type { AnchorResolverRegistry, CodeNode, DirectiveOptionType, EnvironmentContext, EnvironmentProvider, EnvironmentSpec, ExecuteConfig, Executor, Finding, IRDocument, IRPass, IRSection, NodeServiceContext, PassContext, RunContext, RunRequest, RunResult, ServiceToken, SourceReader } from '@lincyaw/kernel'
-import { defaultOptions } from 'myst-parser'
-import { audienceDocument, audienceMarks, AudienceParser, audienceView, emptyReferences, liftScopes, markBlocks, scanOutputs, withheld } from './audience.ts'
-import type { Withheld } from './audience.ts'
-import { BUILTIN_DIRECTIVES, sectionBlocks } from './blocks.ts'
-import type { PluginDirective, PluginRole } from './blocks.ts'
-import { Dependencies } from './dependencies.ts'
-import type { CommittedFiles, Revision } from './dependencies.ts'
-import { checkLanguages, checkRuns, checkSecrets, environmentExecutors, environmentProviders, readExecute } from './environments.ts'
+import { ANCHOR_LOCK_PATH, AnchorLock, DEFAULT_LANG, IR_VERSION, PROCESS_SANDBOX, createAnchorResolverRegistry, docBlocks, environmentExecutor, errorMessage, failedRun, fnv1a64, isRecord, productionAssetIds, validateDocument } from '@lincyaw/kernel'
+import { operatorSpeechSecrets, productionAssetFile } from '@lincyaw/kernel/node'
+import type { AnchorResolverRegistry, CanonicalSource, CodeNode, EnvironmentContext, EnvironmentProvider, EnvironmentSpec, ExecuteConfig, Executor, Finding, IRDocument, IRPass, IRSection, NodeServiceContext, PassContext, RunContext, RunRequest, RunResult, ServiceToken, SourceCapabilities, SourceReader, TeachingDocument } from '@lincyaw/kernel'
+import { audienceDocument, emptyReferences, filterForAudience, scanOutputs, withheld } from './audience.ts'
+import type { AudienceMark, Withheld } from './audience.ts'
+import { runAllChecks } from './compilation-actions.ts'
+import { enqueueCompile } from './compile-queue.ts'
+import { checkLanguages, checkRuns, checkSecrets, environmentExecutors, environmentProviders, toExecuteConfig } from './environments.ts'
 import { executePass, FREEZE_DIR, FreezeCache, runEnvironment } from './execute.ts'
-import type { ExecutionOptions, RunStats } from './execute.ts'
+import type { RunEnvironmentOptions, RunStats } from './execute.ts'
 import { numberFootnotes } from './footnotes.ts'
 import { checkForbidden } from './forbid.ts'
-import { pluginChecks } from './host.ts'
+import { CompileInputs } from './inputs.ts'
+import type { CommittedFiles, GitRead } from './inputs.ts'
 import { checkLinks } from './links.ts'
-import { DECK_PLUGIN, directoryComponents, loadOperatorPlugins, loadPlugins, pluginId } from './plugins.ts'
+import { pluginLogger } from './logging.ts'
+import { NodeServices } from './node-services.ts'
+import { lowerDocument } from './document-lowering.ts'
+import { DECK_ENTRY } from './paths.ts'
+import { currentPluginName, DECK_PLUGIN, directoryComponents, isRetiredPlugin, listedPlugins, loadOperatorPlugins, loadPlugins, pluginId } from './plugins.ts'
 import type { LoadedPlugin, PluginProblem, PluginSources } from './plugins.ts'
 import { resolveReferences } from './references.ts'
-import { splitSections } from './sections.ts'
-import { NodeServices, operatorDir, operatorPlugins, pluginLogger, workspaceSettings } from './settings.ts'
-import type { OperatorSettings } from './settings.ts'
-import { readFrontmatter, sourceOf, withoutBom } from './source.ts'
-import type { Parser } from './source.ts'
-import { scheduleSteps } from './steps.ts'
 import { applyTranslations, translationFields } from './translations.ts'
 import { validateIR } from './validate.ts'
 import { createSourceReader, findWorkspaceRoot } from './workspace.ts'
+import { operatorDir, operatorPlugins, workspaceSettings } from './workspace-settings.ts'
+import type { OperatorSettings } from './workspace-settings.ts'
 
-export const DECK_ENTRY = 'lecture.md'
+export { DECK_ENTRY }
 
 const BUILTIN_RENDERERS = ['web', 'slides', 'notebook']
 
-const MYST_ROLES = new Set((defaultOptions.roles as unknown as { name: string, alias?: string[] }[]).flatMap(role => [role.name, ...role.alias ?? []]))
 
 export interface CompilerOptions {
+  priority?: 'interactive' | 'background'
   settings?: OperatorSettings
   services?: [ServiceToken<unknown>, unknown][]
   workspaceRoot?: string
   bundled?: string
   operatorDir?: string
   secret?: (name: string) => string | undefined
-  execution?: ExecutionOptions
+  execution?: RunEnvironmentOptions
   audience?: string
   headmatter?: Record<string, unknown>
   deckDir?: string
@@ -65,13 +64,19 @@ export interface DeclaredEnvironment {
 
 export interface Compilation {
   doc: IRDocument
+  sourceCapabilities?: SourceCapabilities
   options: CompilerOptions
-  files: string[]
-  dependencies: { files: string[], revisions: Revision[] }
-  runs: RunStats
+  /** The canonical JSON authoring entry read by this compilation. */
+  sourceFiles: string[]
+  /** Fingerprints of exact authoring text, including BOM. Keys are absolute source paths. */
+  sourceFingerprints?: Record<string, string>
+  /** The other files the compilation read, and the files it read at a git ref. */
+  inputs: { files: string[], gitReads: GitRead[] }
+  runStats: RunStats
   deckDir: string
   entry: string
-  frontmatter: Record<string, unknown>
+  /** The entry's headmatter, merged with `CompilerOptions.headmatter`. */
+  headmatter: Record<string, unknown>
   workspaceRoot: string
   src: SourceReader
   resolvers: AnchorResolverRegistry
@@ -88,19 +93,30 @@ export interface Compilation {
   runContext: RunContext
   run(node: CodeNode, signal?: AbortSignal): Promise<RunResult>
   audience?: AudienceCompilation
-  forAudience(name: string): IRDocument
-  audienceOf(name: string): AudienceCompilation
+  /** The document for one audience, with the blocks that leak withheld content removed. */
+  audienceDocument(name: string): IRDocument
+  audienceCompilation(name: string): AudienceCompilation
   publishChecks(options?: { externalLinks?: boolean }): Promise<Finding[]>
   check(options?: { externalLinks?: boolean }): Promise<Finding[]>
 }
 
 export interface CompileOptions {
   pruneCache?: boolean
+  signal?: AbortSignal
+}
+
+/** Plugin services assembled from the exact authoring source, without running compilation passes. */
+export interface DocumentNodeServices {
+  source: CanonicalSource
+  findings: Finding[]
+  service<T>(token: ServiceToken<T>): T | undefined
 }
 
 export function resolveEntry(deck: string): string {
   const path = resolve(deck)
-  return existsSync(path) && statSync(path).isDirectory() ? join(path, DECK_ENTRY) : path
+  const entry = existsSync(path) && statSync(path).isDirectory() ? join(path, DECK_ENTRY) : path
+  if (basename(entry) !== DECK_ENTRY) throw new Error(`The document entry is ${DECK_ENTRY}; migrate the old source before compiling.`)
+  return entry
 }
 
 interface Workspace {
@@ -117,7 +133,46 @@ interface Plugins {
   key: string
   plugins: LoadedPlugin[]
   problems: PluginProblem[]
-  services: NodeServices
+}
+
+type Report = (finding: Finding) => void
+
+type HeadmatterLine = (needle: string) => { file: string, line: number }
+
+/** What the plugins of one compilation contribute to it. */
+interface Contributions {
+  resolvers: AnchorResolverRegistry
+  executors: Executor[]
+  components: Map<string, string>
+  environments: Map<string, EnvironmentProvider>
+  environmentContext: EnvironmentContext
+}
+
+/** The state of one compilation that its steps share. */
+interface CompileRun {
+  workspace: Workspace
+  headmatter: Record<string, unknown>
+  lang: string
+  plugins: LoadedPlugin[]
+  configOf(plugin: LoadedPlugin): Record<string, unknown>
+  service<T>(token: ServiceToken<T>): T | undefined
+  execute: ExecuteConfig
+  inputs: CompileInputs
+  findings: Finding[]
+  report: Report
+  at: HeadmatterLine
+  signal?: AbortSignal
+  document: TeachingDocument
+  rawSource: string
+}
+
+interface PassResults {
+  lock: AnchorLock
+  runStats: RunStats
+  runContext: RunContext
+  cache: FreezeCache
+  filled: WeakSet<object>
+  invalid: Finding[]
 }
 
 export class Compiler {
@@ -127,8 +182,7 @@ export class Compiler {
   #workspace?: Workspace
   readonly #committed: CommittedFiles = new Map()
   #plugins?: Plugins
-  #parser?: { key: string, parser: Parser }
-  readonly #environments = new Map<string, Executor>()
+  readonly #environments = new WeakMap<EnvironmentProvider, Map<string, Executor>>()
 
   constructor(deck: string, options: CompilerOptions = {}) {
     this.entry = resolveEntry(deck)
@@ -136,67 +190,148 @@ export class Compiler {
     this.options = options
   }
 
-  async compile(options: CompileOptions = {}): Promise<Compilation> {
+  compile(options: CompileOptions = {}): Promise<Compilation> {
+    return enqueueCompile(() => this.#compile(options), this.options.priority === 'interactive')
+  }
+
+  /** Authoring services must also work when a scene or a compilation pass cannot be built. */
+  async nodeServices(options: Pick<CompileOptions, 'signal'> = {}): Promise<DocumentNodeServices> {
+    const run = await this.#start(options)
+    options.signal?.throwIfAborted()
+    return {
+      source: { documentId: run.document.id, version: createHash('sha256').update(run.rawSource).digest('hex').slice(0, 20), fingerprint: fnv1a64(run.rawSource) },
+      findings: [...run.findings],
+      service: run.service,
+    }
+  }
+
+  async #compile(options: CompileOptions): Promise<Compilation> {
+    const run = await this.#start(options)
+    const { headmatter, execute, findings, report } = run
+    const contributions = await this.#collectContributions(run)
+    const { sections, marks, sourceFiles, sourceFingerprints } = await this.#compileSections(run, contributions)
+
+    const { title: _title, lang: _lang, plugins: _plugins, presmium, renderers, sectionDepth: _sectionDepth, execute: _execute, translations: _translations, translationOf: _translationOf, ...rest } = headmatter
+    const doc: IRDocument = {
+      ir: IR_VERSION,
+      source: { documentId: run.document.id, version: createHash('sha256').update(run.rawSource).digest('hex').slice(0, 20), fingerprint: fnv1a64(run.rawSource) },
+      meta: { ...rest, title: titleOf(headmatter, this.entry), lang: run.lang, entry: this.entry, documentId: run.document.id },
+      config: {
+        plugins: listedPlugins(stringList(headmatter.plugins)),
+        presmium: isRecord(presmium) ? presmium : {},
+        renderers: isRecord(renderers) ? recordOfRecords(renderers) : {},
+        ...Object.keys(execute).length > 0 ? { execute } : {},
+      },
+      sections,
+      diagnostics: [],
+      production: structuredClone(run.document.production),
+      productionAssets: Object.fromEntries(productionAssetIds(run.document.production, run.document.sectionOrder.map(id => ({ id, stepIds: run.document.sections[id]!.stepOrder }))).map(id => [id, structuredClone(run.document.assets[id]!)])),
+    }
+    applyTranslations(doc, translationFields(headmatter), report)
+    let audience: AudienceCompilation | undefined
+    if (this.options.audience) {
+      const name = this.options.audience
+      const filtered = filterForAudience(doc, name, marks)
+      doc.sections = filtered.doc.sections
+      doc.meta.audience = name
+      doc.production = filtered.doc.production
+      doc.productionAssets = filtered.doc.productionAssets
+      audience = audienceCompilation(name, withheld(filtered.removed, doc))
+      const left = findings.filter(finding => !filtered.keeps(finding))
+      findings.splice(0, findings.length, ...findings.filter(filtered.keeps))
+      if (left.length > 0) {
+        const severity = left.some(f => f.severity === 'error') ? 'error' : left.some(f => f.severity === 'warning') ? 'warning' : 'info'
+        report({ checkId: 'presmium.audience', severity, message: `${left.length} finding${left.length === 1 ? ' is' : 's are'} in the content left out for ${name}, not shown here: presmium check without --audience shows ${left.length === 1 ? 'it' : 'them'}`, source: { file: this.entry, line: 1 } })
+      }
+    }
+    checkRuns(doc, execute, report)
+    const unsupported = checkLanguages(doc, execute, contributions.executors, report)
+
+    numberFootnotes(doc, sourceFiles)
+
+    const passes = await this.#runPasses(doc, run, contributions, unsupported)
+    findings.push(...passes.invalid)
+    options.signal?.throwIfAborted()
+    doc.diagnostics = sortFindings(findings)
+    // Reassert provenance after extension passes; authored metadata cannot replace the source identity.
+    doc.source = { documentId: run.document.id, version: createHash('sha256').update(run.rawSource).digest('hex').slice(0, 20), fingerprint: fnv1a64(run.rawSource) }
+    if (options.pruneCache && !audience && !findings.some(f => f.severity === 'error')) passes.cache.prune()
+    return this.#assemble(doc, run, contributions, { sourceFiles, sourceFingerprints, marks, audience, passes })
+  }
+
+  /** Reads the entry's headmatter, loads the plugins with their Node services and reads the execution settings. */
+  async #start(options: CompileOptions): Promise<CompileRun> {
+    options.signal?.throwIfAborted()
     const workspace = this.#workspace ??= await this.#openWorkspace()
-    const { root: workspaceRoot, src, settings } = workspace
+    const { root: workspaceRoot, settings } = workspace
     const findings: Finding[] = []
     const report = (finding: Finding) => findings.push(finding)
     const read = await readFile(this.entry, 'utf8').catch(() => undefined)
     if (read === undefined) throw new Error(`${this.entry} not found`)
-    const text = withoutBom(read)
+    const text = read.replace(/^\uFEFF/, '')
     if (workspace.problem) report({ checkId: 'presmium.workspace', severity: 'error', message: workspace.problem, source: { file: this.entry, line: 1 } })
     if (workspace.operatorProblem) report({ checkId: 'presmium.workspace', severity: 'error', message: workspace.operatorProblem, source: { file: this.entry, line: 1 } })
-    const headmatter = withHeadmatter(readFrontmatter(text), this.options.headmatter)
+    let parsed: unknown
+    try { parsed = JSON.parse(text) } catch (error) { throw new Error(`Invalid ${DECK_ENTRY}: ${errorMessage(error)}`) }
+    const validated = validateDocument(parsed)
+    if (!validated.document) throw new Error(validated.issues.map(issue => `${issue.path || '/'}: ${issue.message}`).join('\n'))
+    const document = validated.document
+    const headmatter = withHeadmatter({ ...document.metadata }, this.options.headmatter)
+    const defaultVoice = document.production.defaults.voiceSource
+    const lecture = isRecord((headmatter.renderers as Record<string, unknown> | undefined)?.lecture) ? (headmatter.renderers as Record<string, Record<string, unknown>>).lecture : {}
+    if (lecture && Object.hasOwn(lecture, 'script')) throw new Error('/metadata/renderers/lecture/script: External lecture scripts are not supported. Author narration in lecture.json narrations and bind it through each step narrationIds.')
+    if (defaultVoice?.type === 'tts' || document.production.defaults.trailingPauseMs !== undefined || Object.keys(lecture ?? {}).length) headmatter.renderers = { ...(isRecord(headmatter.renderers) ? headmatter.renderers : {}), lecture: { ...lecture, ...defaultVoice?.type === 'tts' ? { voice: defaultVoice.voice } : {}, ...document.production.defaults.trailingPauseMs !== undefined ? { gap: document.production.defaults.trailingPauseMs } : {} } }
     const lang = typeof headmatter.lang === 'string' && headmatter.lang.trim() ? headmatter.lang.trim() : DEFAULT_LANG
-    const dependencies = new Dependencies(workspaceRoot, this.#committed)
-    const watch = (...files: string[]) => dependencies.watch(...files)
+    const inputs = new CompileInputs(workspaceRoot, this.#committed)
     const presmium = isRecord(headmatter.presmium) ? headmatter.presmium : {}
     const configOf = (plugin: LoadedPlugin) => {
       const config = presmium[pluginId(plugin)]
       return isRecord(config) ? config : {}
     }
 
-    const names = stringList(headmatter.plugins)
-    const { plugins, problems, services } = await this.#loadPlugins(names, workspace, (loaded): NodeServiceContext => ({
+    const listed = stringList(headmatter.plugins)
+    const loaded = await this.#loadPlugins(listedPlugins(listed), workspace)
+    const { plugins } = loaded
+    const problems = [...loaded.problems]
+    const services = new NodeServices(this.options.services)
+    for (const plugin of plugins) addServices(services, plugin, {
       deckDir: this.deckDir,
       workspaceRoot,
-      config: configOf(loaded),
-      settings: settings[pluginId(loaded)] ?? {},
+      config: configOf(plugin),
+      settings: settings[pluginId(plugin)] ?? {},
       lang,
-      secret: this.options.secret ?? (name => process.env[name]),
+      secret: this.options.secret ?? operatorSpeechSecrets(workspace.operatorDir, name => process.env[name]),
       ...workspace.operatorDir ? { operatorDir: workspace.operatorDir } : {},
-      log: pluginLogger(pluginId(loaded)),
-    }))
-    const service = <T>(token: ServiceToken<T>) => services.get(token)
-    const at = (name: string) => ({ file: this.entry, line: frontmatterLine(text, name) })
+      log: pluginLogger(pluginId(plugin)),
+      service: token => services.tryGet(token),
+    }, problems)
+    const service = <T>(token: ServiceToken<T>) => services.tryGet(token)
+    const at = (_name: string) => ({ file: this.entry, line: 1, format: 'json' as const, pointer: '/metadata' })
     for (const p of problems) report({ checkId: 'presmium.plugins', severity: p.severity, message: p.message, source: at(p.name) })
-    const execute = readExecute(headmatter.execute, report, at)
+    for (const name of listed) {
+      if (isRetiredPlugin(name)) report({ checkId: 'presmium.plugins', severity: 'info', message: `plugin "${name}" is no longer needed: the viewer shows the slides itself; remove it from plugins`, source: at(name) })
+      else if (currentPluginName(name) !== name) report({ checkId: 'presmium.plugins', severity: 'info', message: `plugin "${name}" is now named "${currentPluginName(name)}"; the old name still works`, source: at(name) })
+    }
+    const execute = toExecuteConfig(headmatter.execute, report, at)
     checkSecrets(headmatter, report, at)
+    return { workspace, headmatter, lang, plugins, configOf, service, execute, inputs, findings, report, at, signal: options.signal, document, rawSource: read }
+  }
 
-    const directives = new Map<string, PluginDirective>()
-    const roles = new Map<string, PluginRole>()
+  /** Registers each plugin's directives, roles, anchor resolvers, executors, environment providers and components. */
+  async #collectContributions(run: CompileRun): Promise<Contributions> {
+    const { plugins, configOf, execute, report, at } = run
     const resolvers = createAnchorResolverRegistry()
     const executors: Executor[] = []
     const providers = new Map<string, EnvironmentProvider>()
     const components = new Map<string, string>()
     for (const loaded of plugins) {
-      for (const name of await directoryComponents(loaded.dir)) if (!components.has(name)) components.set(name, pluginId(loaded))
+      for (const name of await directoryComponents(loaded.dir)) {
+        const owner = components.get(name)
+        if (owner === undefined) components.set(name, pluginId(loaded))
+        else report({ checkId: 'presmium.plugins', severity: 'warning', message: `plugin ${pluginId(loaded)}: component ${name} is already registered by ${owner}; its component is ignored`, source: at(loaded.name) })
+      }
       const { plugin } = loaded
       if (!plugin) continue
-      for (const spec of plugin.directives ?? []) {
-        if (BUILTIN_DIRECTIVES.has(spec.name) || directives.has(spec.name)) {
-          report({ checkId: 'presmium.plugins', severity: 'error', message: `plugin ${plugin.id}: directive {${spec.name}} is already defined`, source: at(loaded.name) })
-        } else {
-          directives.set(spec.name, { plugin: plugin.id, config: configOf(loaded), spec })
-        }
-      }
-      for (const spec of plugin.roles ?? []) {
-        if (MYST_ROLES.has(spec.name) || roles.has(spec.name)) {
-          report({ checkId: 'presmium.plugins', severity: 'error', message: `plugin ${plugin.id}: role {${spec.name}} is already defined`, source: at(loaded.name) })
-        } else {
-          roles.set(spec.name, { plugin: plugin.id, config: configOf(loaded), spec })
-        }
-      }
       try {
         for (const resolver of plugin.resolvers ?? []) resolvers.register(resolver)
       } catch (error) {
@@ -207,84 +342,27 @@ export class Compiler {
     }
     for (const name of await directoryComponents(this.deckDir)) components.set(name, DECK_PLUGIN)
     const secret = this.options.secret ?? (name => process.env[name])
-    const environmentContext: EnvironmentContext = { deckDir: this.deckDir, workspaceRoot, secret, log: pluginLogger('presmium.execute') }
-    const declared = environmentProviders(execute, providers, report, at)
-    executors.push(...await environmentExecutors(execute, declared, environmentContext, this.#environments, report, at))
-
-    const parser = this.#parserFor(directives, roles)
-    const source = await parser.expand(this.entry)
-    findings.push(...source.findings)
-    dependencies.watch(...source.missing)
-    const { title: _title, lang: _lang, plugins: _plugins, presmium: _presmium, renderers, sectionDepth: depth, execute: _execute, translations: _translations, translationOf: _translationOf, ...rest } = headmatter
-    const title = typeof headmatter.title === 'string' ? headmatter.title : basename(this.entry, extname(this.entry))
-    const sectionDepth = Number.isInteger(depth) && (depth as number) >= 1 ? depth as number : 2
-
-    const lifted = liftScopes(source.children, sectionDepth, (node, audience) => report({
-      checkId: 'presmium.audience',
-      severity: 'warning',
-      message: `this content follows an {include} with :audience: whose file opens a section, so it is part of that section, which is only for ${audience.join(', ')}`,
-      hint: 'start a section of its own with a heading after the {include}, or move it before the {include}',
-      source: sourceOf(node, this.entry),
-    }))
-    const marks = audienceMarks(lifted, this.entry)
-    const raw = splitSections(lifted, { entry: this.entry, title, sectionDepth }, report)
-    const rendererIds = new Set([...BUILTIN_RENDERERS, ...isRecord(renderers) ? Object.keys(renderers) : []])
-    const ctx = { entry: this.entry, deckDir: this.deckDir, workspaceRoot, lang, renderers: rendererIds, parser, directives, roles, components, execute, report, watch }
-    const sections: IRSection[] = []
-    for (const [i, section] of raw.entries()) {
-      const { blocks: drafts, hints, config, audience } = await sectionBlocks(section.nodes, ctx)
-      const { blocks, steps } = scheduleSteps(drafts, report)
-      drafts.forEach((draft, j) => {
-        if (draft.audience) blocks[j]!.audience = draft.audience
-      })
-      sections.push({
-        id: section.id,
-        index: i + 1,
-        title: section.title,
-        depth: section.depth,
-        source: section.source,
-        steps,
-        hints,
-        ...Object.keys(config).length > 0 ? { config } : {},
-        blocks,
-        ...audience ? { audience } : {},
-      })
-    }
-
-    markBlocks(sections, marks)
+    const environmentContext: EnvironmentContext = { deckDir: this.deckDir, workspaceRoot: run.workspace.root, secret, log: pluginLogger('presmium.execute') }
+    const environments = environmentProviders(execute, providers, report, at)
+    executors.push(...await environmentExecutors(execute, environments, environmentContext, this.#environments, report, at))
+    return { resolvers, executors, components, environments, environmentContext }
+  }
 
-    const doc: IRDocument = {
-      ir: IR_VERSION,
-      meta: { ...rest, title, lang, entry: this.entry },
-      config: {
-        plugins: names,
-        presmium,
-        renderers: isRecord(renderers) ? recordOfRecords(renderers) : {},
-        ...Object.keys(execute).length > 0 ? { execute } : {},
-      },
-      sections,
-      diagnostics: [],
-    }
-    applyTranslations(doc, translationFields(headmatter), report)
-    let audience: AudienceCompilation | undefined
-    if (this.options.audience) {
-      const name = this.options.audience
-      const view = audienceView(doc, name, marks)
-      doc.sections = view.doc.sections
-      doc.meta.audience = name
-      audience = audienceCompilation(name, withheld(view.removed, doc))
-      const left = findings.filter(finding => !view.keeps(finding))
-      findings.splice(0, findings.length, ...findings.filter(view.keeps))
-      if (left.length > 0) {
-        const severity = left.some(f => f.severity === 'error') ? 'error' : left.some(f => f.severity === 'warning') ? 'warning' : 'info'
-        report({ checkId: 'presmium.audience', severity, message: `${left.length} finding${left.length === 1 ? ' is' : 's are'} in the content left out for ${name}, not shown here: presmium check without --audience shows ${left.length === 1 ? 'it' : 'them'}`, source: { file: this.entry, line: 1 } })
-      }
+  /** Validated JSON is the sole runtime source; compiler output remains a derived snapshot. */
+  async #compileSections(run: CompileRun, _contributions: Contributions): Promise<{ sections: IRSection[], marks: AudienceMark[], sourceFiles: string[], sourceFingerprints: Record<string, string> }> {
+    const sections = lowerDocument(run.document, this.entry, this.deckDir, run.report, (...files) => run.inputs.watch(...files))
+    for (const id of productionAssetIds(run.document.production, run.document.sectionOrder.map(id => ({ id, stepIds: run.document.sections[id]!.stepOrder })))) {
+      const asset = run.document.assets[id], file = asset && productionAssetFile(this.deckDir, asset.location)
+      if (file) run.inputs.watch(file)
     }
-    checkRuns(doc, execute, report)
-    const unsupported = checkLanguages(doc, execute, executors, report)
-
-    numberFootnotes(doc, source.files)
+    run.inputs.watch(this.entry)
+    return { sections, marks: [], sourceFiles: [this.entry], sourceFingerprints: { [this.entry]: fnv1a64(run.rawSource) } }
+  }
 
+  /** Runs the plugins' passes, resolves references and runs the code at compile time, validating the IR in between. */
+  async #runPasses(doc: IRDocument, run: CompileRun, contributions: Contributions, unsupported: ReadonlySet<CodeNode>): Promise<PassResults> {
+    const { plugins, configOf, workspace, inputs, report, at, signal } = run
+    const { executors, resolvers } = contributions
     const lockFile = join(this.deckDir, ANCHOR_LOCK_PATH)
     let lock = AnchorLock.empty()
     try {
@@ -293,31 +371,33 @@ export class Compiler {
       report({ checkId: 'presmium.lock', severity: 'error', message: errorMessage(error), source: { file: lockFile, line: 1 } })
     }
     const cache = new FreezeCache(join(this.deckDir, FREEZE_DIR), relative(this.deckDir, this.entry))
-    const runContext = this.#runContext(workspaceRoot, settings, service, lang)
+    const runContext = this.#runContext(workspace.root, workspace.settings, run.service, run.lang)
     const passes = plugins.flatMap(loaded => (loaded.plugin?.passes ?? []).map(pass => ({ pass, loaded })))
       .sort((a, b) => (a.pass.order ?? 0) - (b.pass.order ?? 0))
     const runPass = async ({ pass, loaded }: { pass: IRPass, loaded: LoadedPlugin }) => {
+      signal?.throwIfAborted()
       const config = configOf(loaded)
-      const pctx: PassContext = {
+      const passContext: PassContext = {
         deckDir: this.deckDir,
-        workspaceRoot,
-        lang,
-        src: dependencies.reader(src),
-        resolvers: dependencies.resolvers(resolvers),
+        workspaceRoot: workspace.root,
+        lang: run.lang,
+        src: inputs.reader(workspace.src),
+        resolvers: inputs.resolvers(resolvers),
         lock: lock.toJSON(),
         executors,
         cache,
         config,
         configFor: section => ({ ...config, ...section.config?.[pluginId(loaded)] }),
         report: finding => report({ ...finding, source: finding.source ?? at(loaded.name) }),
-        watch,
-        service,
+        watch: (...files) => inputs.watch(...files),
+        service: run.service,
       }
       try {
-        await pass.run(doc, pctx)
+        await pass.run(doc, passContext)
       } catch (error) {
         report({ checkId: 'presmium.plugins', severity: 'error', message: `pass ${pass.name} of plugin ${pluginId(loaded)} failed: ${errorMessage(error)}`, source: at(loaded.name) })
       }
+      signal?.throwIfAborted()
     }
     const blame = (findings: Finding[], culprit: string): Finding[] => findings.map(finding => ({ ...finding, hint: culprit }))
     let invalid = blame(validateIR(doc, this.entry), 'the IR is invalid before the plugin passes: a directive wrote it')
@@ -330,52 +410,58 @@ export class Compiler {
       }
     }
 
-    let runs: RunStats = { executed: 0, cached: 0 }
+    let runStats: RunStats = { executed: 0, cached: 0 }
     let filled = new WeakSet<object>()
     if (invalid.length === 0) {
       filled = emptyReferences(doc)
       resolveReferences(doc, report)
-      runs = await executePass(doc, executors, cache, report, { ...this.options.execution, rerun: execute.rerun === 'environment', context: runContext, unsupported })
+      runStats = await executePass(doc, executors, cache, report, { ...this.options.execution, signal, rerun: run.execute.rerun === 'environment', context: runContext, unsupported })
       for (const step of passes.filter(({ pass }) => pass.after === 'execute')) await runPass(step)
       invalid = validateIR(doc, this.entry)
     }
-    findings.push(...invalid)
-    doc.diagnostics = sortFindings(findings)
-    if (options.pruneCache && !audience && !findings.some(f => f.severity === 'error')) cache.prune()
-    const publish = { doc, entry: this.entry, deckDir: this.deckDir, workspaceRoot, renderers: publishedRenderers(doc, plugins) }
+    return { lock, runStats, runContext, cache, filled, invalid }
+  }
 
+  #assemble(doc: IRDocument, run: CompileRun, contributions: Contributions, results: { sourceFiles: string[], sourceFingerprints: Record<string, string>, marks: AudienceMark[], audience?: AudienceCompilation, passes: PassResults }): Compilation {
+    const { workspace, headmatter, plugins, execute, inputs } = run
+    const { executors } = contributions
+    const { marks, audience, passes } = results
+    const { runContext } = passes
+    const publish = { doc, entry: this.entry, deckDir: this.deckDir, workspaceRoot: workspace.root, renderers: Object.keys(run.document.metadata.renderers ?? {}).length ? Object.keys(run.document.metadata.renderers!) : [...new Set([...BUILTIN_RENDERERS, ...plugins.flatMap(loaded => loaded.plugin?.renderer ? [loaded.plugin.renderer.id] : [])])] }
     const compilation: Compilation = {
       doc,
+      sourceCapabilities: {},
       options: this.options,
-      files: source.files,
-      dependencies: { files: dependencies.files, revisions: dependencies.revisions },
-      runs,
+      sourceFiles: results.sourceFiles,
+      sourceFingerprints: results.sourceFingerprints,
+      inputs: { files: inputs.files, gitReads: inputs.gitReads },
+      runStats: passes.runStats,
       deckDir: this.deckDir,
       entry: this.entry,
-      frontmatter: headmatter,
-      workspaceRoot,
-      src,
-      resolvers,
-      lock,
+      headmatter,
+      workspaceRoot: workspace.root,
+      src: workspace.src,
+      resolvers: contributions.resolvers,
+      lock: passes.lock,
       plugins,
-      components,
-      service,
-      settings,
+      components: contributions.components,
+      service: run.service,
+      settings: workspace.settings,
       operatorDir: workspace.operatorDir,
       pluginSources: workspace.sources,
       execute,
-      environments: [...declared].map(([name, provider]) => ({ name, spec: execute.environments![name]!, provider, context: environmentContext })),
+      environments: [...contributions.environments].map(([name, provider]) => ({ name, spec: execute.environments![name]!, provider, context: contributions.environmentContext })),
       executors,
       runContext,
       run: (node, signal) => this.#run(doc, node, executors, runContext, signal),
       ...audience ? { audience } : {},
-      forAudience: name => audienceDocument(doc, name, filled, marks),
-      audienceOf: (name) => {
-        const view = audienceView(doc, name, marks)
-        return audienceCompilation(name, withheld(view.removed, view.doc))
+      audienceDocument: name => audienceDocument(doc, name, passes.filled, marks),
+      audienceCompilation: (name) => {
+        const filtered = filterForAudience(doc, name, marks)
+        return audienceCompilation(name, withheld(filtered.removed, filtered.doc))
       },
       publishChecks: async (checks = {}) => [...await checkLinks(publish, checks), ...checkForbidden(publish, headmatter)],
-      check: checks => pluginChecks(compilation, checks),
+      check: checks => runAllChecks(compilation, checks),
     }
     return compilation
   }
@@ -409,10 +495,10 @@ export class Compiler {
       }
       nodes = [...before, node]
     }
-    const session = run.session ? `run:${randomUUID()}` : undefined
-    const requests = nodes.map(code => ({ ...request(code), ...session ? { session } : {} }))
+    const runSession = run.session ? `run:${randomUUID()}` : undefined
+    const requests = nodes.map(code => ({ ...request(code), ...runSession ? { session: runSession } : {} }))
     const results = await Promise.all(requests.map(r => executor.run(r, signal, context)))
-    if (session) await executor.endSession?.(session)
+    if (runSession) await executor.endSession?.(runSession)
     return results.at(-1)!
   }
 
@@ -429,7 +515,7 @@ export class Compiler {
     return workspace
   }
 
-  async #loadPlugins(names: string[], workspace: Workspace, context: (loaded: LoadedPlugin) => NodeServiceContext): Promise<Plugins> {
+  async #loadPlugins(names: string[], workspace: Workspace): Promise<Plugins> {
     const key = JSON.stringify(names)
     if (this.#plugins?.key === key) return this.#plugins
     const operators = await loadOperatorPlugins(workspace.sources)
@@ -437,19 +523,10 @@ export class Compiler {
     const dirs = new Set(operators.plugins.map(loaded => loaded.dir))
     const plugins = [...operators.plugins, ...deck.plugins.filter(loaded => !dirs.has(loaded.dir))]
     const problems = [...operators.problems, ...deck.problems]
-    const services = new NodeServices(this.options.services)
-    for (const loaded of plugins) addServices(services, loaded, { ...context(loaded), service: token => services.get(token) }, problems)
-    this.#plugins = { key, plugins, problems, services }
+    this.#plugins = { key, plugins, problems }
     return this.#plugins
   }
 
-  #parserFor(directives: Map<string, PluginDirective>, roles: Map<string, PluginRole>): Parser {
-    const syntax = new Map([...BUILTIN_DIRECTIVES, ...[...directives.keys()].map(name => [name, false] as const)])
-    const options = new Map<string, Record<string, DirectiveOptionType> | undefined>([...roles].map(([name, { spec }]) => [name, spec.options]))
-    const key = JSON.stringify([[...syntax].sort(), [...options].sort()])
-    if (this.#parser?.key !== key) this.#parser = { key, parser: new AudienceParser(syntax, options) }
-    return this.#parser.parser
-  }
 }
 
 function addServices(services: NodeServices, loaded: LoadedPlugin, context: NodeServiceContext, problems: PluginProblem[]): void {
@@ -478,13 +555,13 @@ export async function workspacePlugins(root: string, options: CompilerOptions =
       config: {},
       settings: settings[pluginId(loaded)] ?? {},
       lang: DEFAULT_LANG,
-      secret: options.secret ?? (name => process.env[name]),
+      secret: options.secret ?? operatorSpeechSecrets(operator, name => process.env[name]),
       ...operator ? { operatorDir: operator } : {},
-      service: token => services.get(token),
+      service: token => services.tryGet(token),
       log: pluginLogger(pluginId(loaded)),
     }, problems)
   }
-  return { plugins, problems, service: token => services.get(token) }
+  return { plugins, problems, service: token => services.tryGet(token) }
 }
 
 export function audienceCompilation(name: string, left: Withheld[]): AudienceCompilation {
@@ -498,11 +575,8 @@ function withHeadmatter(own: Record<string, unknown>, replaced?: Record<string,
   return merged
 }
 
-export function frontmatterLine(text: string, needle: string): number {
-  const lines = withoutBom(text).split(/\r?\n/)
-  const end = lines.indexOf('---', 1)
-  const index = lines.slice(0, end < 0 ? undefined : end).findIndex(line => line.includes(needle))
-  return index < 0 ? 1 : index + 1
+function titleOf(headmatter: Record<string, unknown>, entry: string): string {
+  return typeof headmatter.title === 'string' ? headmatter.title : basename(entry, extname(entry))
 }
 
 export function sortFindings(findings: Finding[]): Finding[] {