@lincyaw/compiler 0.0.0-stage → 1.0.0-rc.1

package/src/forbid.ts

@@ -0,0 +1,76 @@
+import { irBlocks, isRecord, mdastText, propText } from '@lincyaw/kernel'
+import type { Finding, IRBlock, MdastNode, SourcePos } from '@lincyaw/kernel'
+import type { PublishInput } from './links.ts'
+import { lineFinder } from './source-lines.ts'
+import { viewFor } from './views.ts'
+
+export function checkForbidden(input: PublishInput, headmatter: Record<string, unknown>): Finding[] {
+  const given = isRecord(headmatter.checks) ? headmatter.checks.forbid : undefined
+  if (given === undefined) return []
+  const line = lineFinder()
+  const at = (needle: string): SourcePos => ({ file: input.entry, line: line({ file: input.entry, line: 1, endLine: Number.MAX_SAFE_INTEGER }, [needle]) })
+  if (!Array.isArray(given) || !given.every(item => typeof item === 'string')) {
+    return [{ checkId: 'presmium.forbid', severity: 'error', message: 'checks.forbid must be a list of regular expressions', source: at('forbid') }]
+  }
+  const findings: Finding[] = []
+  const patterns: { text: string, pattern: RegExp }[] = []
+  for (const text of given) {
+    try {
+      patterns.push({ text, pattern: new RegExp(text, 'g') })
+    } catch (error) {
+      findings.push({ checkId: 'presmium.forbid', severity: 'error', message: `checks.forbid: ${(error as Error).message}`, source: at(text) })
+    }
+  }
+
+  const hits = new Map<string, { source: SourcePos, match: string, pattern: string, renderers: string[] }>()
+  for (const renderer of input.renderers) {
+    const view = viewFor(input.doc, renderer)
+    const texts = view.sections.flatMap(section => [
+      { source: section.source, text: section.title },
+      ...[...irBlocks(section.blocks)].map(block => ({ source: block.source, text: searchableText(block) })),
+    ])
+    for (const { source, text } of texts) {
+      for (const { text: name, pattern } of patterns) {
+        for (const [match] of text.matchAll(pattern)) {
+          if (!match) continue
+          const key = `${source.file}\0${source.line}\0${name}\0${match}`
+          const hit = hits.get(key)
+          if (hit && !hit.renderers.includes(renderer)) hit.renderers.push(renderer)
+          else if (!hit) hits.set(key, { source, match, pattern: name, renderers: [renderer] })
+        }
+      }
+    }
+  }
+  for (const { source, match, pattern, renderers } of hits.values()) {
+    findings.push({
+      checkId: 'presmium.forbid',
+      severity: 'error',
+      message: `"${match}" matches ${pattern} of checks.forbid${renderers.length < input.renderers.length ? ` (in ${renderers.join(', ')})` : ''}`,
+      source: { file: source.file, line: line(source, [match]) },
+    })
+  }
+  return findings
+}
+
+/** What checks.forbid searches in a block: its text with link targets, every code variant and the recorded output. */
+function searchableText(block: IRBlock): string {
+  const { node } = block
+  switch (node.type) {
+    case 'markdown': return `${mdastText(node.mdast)}\n${targets(node.mdast).join('\n')}`
+    case 'code': return [node.title ?? '', node.value, ...(node.variants ?? []).map(variant => variant.value), ...(node.outputs ?? []).flatMap(outputText)].join('\n')
+    case 'component': return [...propText(node.props), ...node.fallback ? [mdastText(node.fallback)] : []].join('\n')
+  }
+}
+
+function targets(node: MdastNode): string[] {
+  const own = node.type === 'html' ? node.value : node.url
+  return [...typeof own === 'string' ? [own] : [], ...(node.children ?? []).flatMap(targets)]
+}
+
+function outputText(run: NonNullable<Extract<IRBlock['node'], { type: 'code' }>['outputs']>[number]): string[] {
+  return [run.stdout, run.stderr, ...(run.outputs ?? []).flatMap((output) => {
+    if (output.output_type === 'stream') return [output.text]
+    if (output.output_type === 'error') return [output.evalue, ...output.traceback]
+    return Object.entries(output.data).flatMap(([type, value]) => (type.startsWith('text/') || type === 'application/json' ? propText(value) : []))
+  })]
+}

package/src/frontmatter.ts

@@ -0,0 +1,16 @@
+import { isRecord } from '@lincyaw/kernel'
+import { parse as parseYaml } from 'yaml'
+
+export function withoutBom(text: string): string {
+  return text.charCodeAt(0) === 0xFEFF ? text.slice(1) : text
+}
+
+export function parseFrontmatter(text: string): Record<string, unknown> {
+  const match = /^---[ \t]*\r?\n(?:([\s\S]*?)\r?\n)?---[ \t]*(?:\r?\n|$)/.exec(withoutBom(text))
+  try {
+    const data: unknown = match ? parseYaml(match[1] ?? '') : {}
+    return isRecord(data) ? data : {}
+  } catch {
+    return {}
+  }
+}

package/src/index.ts

@@ -0,0 +1,30 @@
+export { directoryAsset, fileHash, publishesDirectory } from './assets.ts'
+export { availableRenderers, build } from './build.ts'
+export type { BuildOptions, BuildReport } from './build.ts'
+export { buildSummary, buildToDist, checkContext, runBlock } from './compilation-actions.ts'
+export { Compiler, DECK_ENTRY, resolveEntry, sortFindings, workspacePlugins } from './compiler.ts'
+export type { Compilation, CompileOptions, CompilerOptions, DocumentNodeServices } from './compiler.ts'
+export { dev } from './dev.ts'
+export type { DevOptions, DevServers } from './dev.ts'
+export { BUILTIN_DIRECTIVE_INFO, COMMON_DIRECTIVE_OPTIONS, RUN_OPTION } from './directives.ts'
+export type { DirectiveInfo, DirectiveOptionInfo } from './directives.ts'
+export type { RunStats } from './execute.ts'
+export type { GitRead } from './inputs.ts'
+export { pluginLogger } from './logging.ts'
+export { parseFrontmatter } from './frontmatter.ts'
+export { portable, relativeIR } from './paths.ts'
+export { findPackageDir, loadPlugins, pluginId } from './plugins.ts'
+export type { LoadedPlugin, PluginProblem, PluginSources } from './plugins.ts'
+export { exportDocument } from './export.ts'
+export type { ExportOptions } from './export.ts'
+export { documentExporters, nodeVitePlugins } from './renderers.ts'
+export type { DocumentExporter } from './renderers.ts'
+export { playwrightChromium, serveDirectory } from './serve.ts'
+export type { ServeDirectoryOptions } from './serve.ts'
+export type { ServedExtra, StaticServer } from './serve.ts'
+export { frontmatterLine, lineContaining } from './source-lines.ts'
+export { compileTranslations, documentDigest } from './translations.ts'
+export type { CompiledTranslation } from './translations.ts'
+export { viewFor } from './views.ts'
+export { operatorDir, WORKSPACE_CONFIG, workspaceSettings } from './workspace-settings.ts'
+export type { OperatorSettings } from './workspace-settings.ts'

package/src/inputs.ts

@@ -0,0 +1,85 @@
+import { execFile } from 'node:child_process'
+import { existsSync } from 'node:fs'
+import { dirname, resolve } from 'node:path'
+import { promisify } from 'node:util'
+import type { AnchorResolverRegistry, SourceReader } from '@lincyaw/kernel'
+import { COMMIT_HASH } from './workspace.ts'
+
+export type CommittedFiles = Map<string, string>
+
+const run = promisify(execFile)
+
+/** A file the compilation read at a git ref rather than from the working tree. */
+export interface GitRead {
+  file: string
+  ref: string
+}
+
+/** The files and git refs a compilation read besides its sources, to watch for changes. */
+export class CompileInputs {
+  readonly #root: string
+  readonly #committed: CommittedFiles
+  readonly #files = new Set<string>()
+  readonly #gitReads = new Map<string, GitRead>()
+
+  constructor(root: string, committed: CommittedFiles = new Map()) {
+    this.#root = root
+    this.#committed = committed
+  }
+
+  get files(): string[] {
+    return [...this.#files]
+  }
+
+  get gitReads(): GitRead[] {
+    return [...this.#gitReads.values()]
+  }
+
+  watch(...files: string[]): void {
+    for (const file of files) this.#files.add(resolve(this.#root, file))
+  }
+
+  reader(src: SourceReader): SourceReader {
+    return {
+      read: async (file, ref) => {
+        const path = resolve(this.#root, file)
+        if (ref === undefined) {
+          this.#files.add(path)
+          return src.read(file, ref)
+        }
+        const key = `${path}\0${ref}`
+        this.#gitReads.set(key, { file: path, ref })
+        const known = this.#committed.get(key)
+        if (known !== undefined) return known
+        const text = await src.read(file, ref)
+        if (text !== undefined && COMMIT_HASH.test(ref)) this.#committed.set(key, text)
+        return text
+      },
+    }
+  }
+
+  resolvers(registry: AnchorResolverRegistry): AnchorResolverRegistry {
+    return {
+      register: resolver => registry.register(resolver),
+      schemes: () => registry.schemes(),
+      resolve: (anchor, src, recorded, ref) => registry.resolve(anchor, this.reader(src), recorded, ref),
+    }
+  }
+}
+
+/** The files of git's own state that change when one of the refs that were read moves. */
+export async function gitReadFiles(reads: readonly GitRead[]): Promise<string[]> {
+  const refs = new Map<string, Set<string>>()
+  for (const { file, ref } of reads) refs.set(dirname(file), (refs.get(dirname(file)) ?? new Set<string>()).add(ref))
+  return (await Promise.all([...refs].map(([dir, names]) => gitRefFiles(dir, names)))).flat()
+}
+
+async function gitRefFiles(dir: string, refs: Iterable<string>): Promise<string[]> {
+  let cwd = dir
+  while (!existsSync(cwd) && dirname(cwd) !== cwd) cwd = dirname(cwd)
+  const git = (args: string[]) => run('git', args, { cwd }).then(({ stdout }) => stdout.split('\n').filter(Boolean), () => [])
+  const names = new Set(['HEAD', ...[...refs].filter(ref => !ref.startsWith('-') && !COMMIT_HASH.test(ref))])
+  const full = await Promise.all([...names].map(name => git(['rev-parse', '--symbolic-full-name', name])))
+  const files = new Set(['HEAD', 'packed-refs', ...full.flat()])
+  return (await git(['rev-parse', ...[...files].flatMap(name => ['--git-path', name])])).map(path => resolve(cwd, path))
+}

package/src/links.ts

@@ -0,0 +1,179 @@
+import { execFile, spawnSync } from 'node:child_process'
+import { existsSync, readFileSync, statSync } from 'node:fs'
+import { basename, dirname, join, relative, resolve, sep } from 'node:path'
+import { ABSOLUTE_URL, DOCUMENT_ENTRY, htmlAttributes, irBlocks, isRecord, validateDocument } from '@lincyaw/kernel'
+import type { Finding, IRBlock, IRDocument, MdastNode, SourcePos } from '@lincyaw/kernel'
+import { isInside } from '@lincyaw/kernel/node'
+import { publishesDirectory, unpublishedDirectory } from './assets.ts'
+import { lowerDocument } from './document-lowering.ts'
+import { plainText, sectionSlug } from './sections.ts'
+import { lineFinder } from './source-lines.ts'
+import type { LineFinder } from './source-lines.ts'
+import { viewFor } from './views.ts'
+
+export interface PublishInput {
+  doc: IRDocument
+  entry: string
+  deckDir: string
+  workspaceRoot: string
+  renderers: string[]
+}
+
+const EXTERNAL = /^https?:\/\//i
+const URL_ATTRIBUTES = new Set(['href', 'src', 'poster'])
+const ID_ATTRIBUTES = new Set(['id', 'name'])
+const LINKED = new Set(['link', 'image', 'definition', 'card'])
+
+interface Problem {
+  source: SourcePos
+  url: string
+  message: string
+  renderers: string[]
+}
+
+export async function checkLinks(input: PublishInput, options: { externalLinks?: boolean } = {}): Promise<Finding[]> {
+  const problems = new Map<string, Problem>()
+  const external = new Map<string, SourcePos>()
+  const documents = new Map<string, Promise<Set<string>>>()
+  for (const renderer of input.renderers) {
+    const view = viewFor(input.doc, renderer)
+    const anchors = documentAnchors(view)
+    for (const section of view.sections) {
+      for (const block of irBlocks(section.blocks)) {
+        for (const url of blockUrls(block)) {
+          if (EXTERNAL.test(url)) {
+            if (!external.has(url)) external.set(url, block.source)
+            continue
+          }
+          const message = await linkProblem(url, block.source, input, anchors, documents)
+          if (!message) continue
+          const key = `${block.source.file}\0${block.source.line}\0${url}`
+          const known = problems.get(key)
+          if (known) known.renderers.push(renderer)
+          else problems.set(key, { source: block.source, url, message, renderers: [renderer] })
+        }
+      }
+    }
+  }
+  const line = lineFinder()
+  const findings = [...problems.values()].map(({ source, url, message, renderers }): Finding => ({
+    checkId: 'presmium.links',
+    severity: 'error',
+    message: `${message}${renderers.length < input.renderers.length ? ` (in ${renderers.join(', ')})` : ''}`,
+    source: { file: source.file, line: line(source, needles(url)) },
+  }))
+  if (options.externalLinks) findings.push(...await externalLinks(external, input.entry, line))
+  return findings
+}
+
+function blockUrls(block: IRBlock): string[] {
+  const { node } = block
+  const urls: string[] = []
+  const walk = (mdast: MdastNode) => {
+    if (LINKED.has(mdast.type) && typeof mdast.url === 'string') urls.push(mdast.url)
+    if (mdast.type === 'html' && typeof mdast.value === 'string') {
+      for (const { value } of htmlAttributes(mdast.value, URL_ATTRIBUTES)) urls.push(value)
+    }
+    mdast.children?.forEach(walk)
+  }
+  if (node.type === 'markdown') walk(node.mdast)
+  if (node.type === 'component') {
+    if (node.fallback) walk(node.fallback)
+    for (const url of Object.values(node.assets ?? {})) urls.push(url)
+    for (const state of block.stepStates ?? []) for (const url of Object.values(state.assets ?? {})) urls.push(url)
+  }
+  return urls.map(url => url.trim()).filter(Boolean)
+}
+
+async function linkProblem(url: string, source: SourcePos, input: PublishInput, anchors: Set<string>, documents: Map<string, Promise<Set<string>>>): Promise<string | undefined> {
+  if (ABSOLUTE_URL.test(url)) return undefined
+  const hash = url.indexOf('#')
+  const fragment = hash < 0 ? '' : decode(url.slice(hash + 1))
+  const path = decode((hash < 0 ? url : url.slice(0, hash)).replace(/\?.*$/s, ''))
+  if (!path) return fragment && !anchors.has(fragment) ? `#${fragment} is no section, block, label or heading of the deck` : undefined
+  const file = path.startsWith('/') ? join(input.deckDir, 'public', path) : resolve(dirname(source.file), path)
+  if (!isInside(file, input.workspaceRoot)) return `${path} lies outside the repository: it cannot be published`
+  if (!existsSync(file)) return `${path}: no such file`
+  const hidden = publishesDirectory(file, input.deckDir) ? unpublishedDirectory(file, input.deckDir) : undefined
+  if (hidden) return `${path}: ${hidden}`
+  if (!fragment) return undefined
+  const entry = statSync(file).isDirectory() ? join(file, DOCUMENT_ENTRY) : file
+  if (basename(entry) !== DOCUMENT_ENTRY || !existsSync(entry)) return undefined
+  if (entry === input.entry) return anchors.has(fragment) ? undefined : `${path}#${fragment}: #${fragment} is no section, block, label or heading of the deck`
+  if (!documents.has(entry)) documents.set(entry, sourceAnchors(entry))
+  return (await documents.get(entry)!).has(fragment) ? undefined : `${path}#${fragment}: no section, block, label or heading of ${relative(input.workspaceRoot, entry).split(sep).join('/')} has the id ${fragment}`
+}
+
+function documentAnchors(doc: Pick<IRDocument, 'sections'>): Set<string> {
+  const anchors = new Set<string>()
+  for (const section of doc.sections) {
+    anchors.add(section.id)
+    for (const block of irBlocks(section.blocks)) {
+      anchors.add(block.id)
+      const { node } = block
+      if (node.type === 'markdown') mdastAnchors(node.mdast, anchors)
+      if (node.type === 'component' && node.fallback) mdastAnchors(node.fallback, anchors)
+    }
+  }
+  return anchors
+}
+
+function mdastAnchors(node: MdastNode, anchors: Set<string>): void {
+  for (const key of ['identifier', 'label', 'html_id'] as const) if (typeof node[key] === 'string' && node[key]) anchors.add(node[key] as string)
+  if (node.type === 'footnoteDefinition' && typeof node.identifier === 'string') anchors.add(`fn-${node.identifier}`)
+  if (node.type === 'heading') anchors.add(sectionSlug(plainText(node)))
+  if (node.type === 'mystDirective' && isRecord(node.options) && typeof node.options.label === 'string') anchors.add(node.options.label)
+  if (node.type === 'html' && typeof node.value === 'string') {
+    for (const { value } of htmlAttributes(node.value, ID_ATTRIBUTES)) if (value) anchors.add(value)
+  }
+  node.children?.forEach(child => mdastAnchors(child, anchors))
+}
+
+async function sourceAnchors(entry: string): Promise<Set<string>> {
+  try {
+    const checked = validateDocument(JSON.parse(readFileSync(entry, 'utf8').replace(/^\uFEFF/, '')))
+    if (!checked.document) return new Set()
+    const sections = lowerDocument(checked.document, entry, dirname(entry), () => {}, () => {})
+    return documentAnchors({ sections })
+  } catch {
+    return new Set()
+  }
+}
+
+async function externalLinks(urls: Map<string, SourcePos>, entry: string, line: LineFinder): Promise<Finding[]> {
+  if (urls.size === 0) return []
+  const at = { file: entry, line: 1 }
+  if (spawnSync('lychee', ['--version'], { stdio: 'ignore' }).error) {
+    return [{ checkId: 'presmium.links', severity: 'info', message: `lychee is not installed: ${urls.size === 1 ? 'the external link was' : `the ${urls.size} external links were`} not checked (install lychee, https://lychee.cli.rs, to check them)`, source: at }]
+  }
+  const output = await new Promise<{ stdout: string, error?: Error }>((done) => {
+    const child = execFile('lychee', ['--format', 'json', '--no-progress', '-'], { maxBuffer: 64 << 20 }, (error, stdout) => done({ stdout, ...error ? { error } : {} }))
+    child.stdin?.on('error', () => {})
+    child.stdin?.end(`${[...urls.keys()].join('\n')}\n`)
+  })
+  let report: unknown
+  try {
+    report = JSON.parse(output.stdout)
+  } catch {
+    return [{ checkId: 'presmium.links', severity: 'warning', message: `lychee did not check the external links: ${output.error?.message ?? 'it printed no report'}`, source: at }]
+  }
+  const failed = isRecord(report) ? [report.fail_map, report.error_map].flatMap(map => (isRecord(map) ? Object.values(map).flat() : [])) : []
+  return failed.flatMap((item): Finding[] => {
+    if (!isRecord(item) || typeof item.url !== 'string') return []
+    const source = urls.get(item.url) ?? urls.get(item.url.replace(/\/$/, '')) ?? at
+    const status = isRecord(item.status) && typeof item.status.text === 'string' ? item.status.text : 'failed'
+    return [{ checkId: 'presmium.links', severity: 'warning', message: `${item.url}: ${status}`, source: { file: source.file, line: line(source, [item.url]) } }]
+  })
+}
+
+function needles(url: string): string[] {
+  return [...new Set([url, decode(url)])]
+}
+
+function decode(text: string): string {
+  try {
+    return decodeURIComponent(text)
+  } catch {
+    return text
+  }
+}

package/src/logging.ts

@@ -0,0 +1,16 @@
+import { prefixedLogger } from '@lincyaw/kernel'
+import type { Logger } from '@lincyaw/kernel'
+
+const STDERR: Logger = {
+  debug: (...args) => {
+    if (process.env.PRESMIUM_DEBUG) console.error(...args)
+  },
+  info: (...args) => console.error(...args),
+  warn: (...args) => console.error(...args),
+  error: (...args) => console.error(...args),
+}
+
+/** A plugin's logger in Node: everything on stderr, debug only with PRESMIUM_DEBUG, tagged with the plugin's ID. */
+export function pluginLogger(id: string): Logger {
+  return prefixedLogger(STDERR, id)
+}

package/src/node-services.ts

@@ -0,0 +1,33 @@
+import { ServiceRegistry } from '@lincyaw/kernel'
+import type { ServiceToken } from '@lincyaw/kernel'
+
+/** The Node services of one compilation: the host's, then each plugin's; a later plugin cannot replace a service. */
+export class NodeServices {
+  readonly #registry = new ServiceRegistry()
+
+  constructor(host: [ServiceToken<unknown>, unknown][] = []) {
+    const latest = new Map(host.map(entry => [entry[0].id, entry]))
+    for (const [token, implementation] of latest.values()) this.#registry.provide(token, implementation)
+  }
+
+  add(owner: string, entries: [ServiceToken<unknown>, unknown][]): string[] {
+    const problems: string[] = []
+    for (const [token, implementation] of entries) {
+      if (!token || typeof token.id !== 'string') {
+        problems.push(`plugin ${owner}: services() gave an entry without a service token`)
+        continue
+      }
+      const known = this.#registry.owner(token)
+      if (known !== undefined) {
+        problems.push(`plugin ${owner}: ${token.id} is already provided by ${known}; this implementation is ignored`)
+        continue
+      }
+      this.#registry.provide(token, implementation, owner)
+    }
+    return problems
+  }
+
+  tryGet<T>(token: ServiceToken<T>): T | undefined {
+    return this.#registry.tryGet(token)
+  }
+}

package/src/paths.ts

@@ -0,0 +1,42 @@
+import { isAbsolute, relative, sep } from 'node:path'
+import { DOCUMENT_ENTRY, irBlocks } from '@lincyaw/kernel'
+import type { IRDocument, SourcePos } from '@lincyaw/kernel'
+import { isInside } from '@lincyaw/kernel/node'
+
+export const DECK_ENTRY = DOCUMENT_ENTRY
+
+/** The deck's directory of generated files: renderer work directories, staged files, caches. */
+export const WORK_DIR = '.presmium'
+
+function relativePath(from: string, path: string): string {
+  return relative(from, path).split(sep).join('/') || '.'
+}
+
+export function portable(value: unknown, from: string, root: string): unknown {
+  if (typeof value === 'string') return isAbsolute(value) && isInside(value, root) ? relativePath(from, value) : value
+  if (Array.isArray(value)) return value.map(item => portable(item, from, root))
+  if (value !== null && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, portable(item, from, root)]))
+  return value
+}
+
+export function relativeIR(doc: IRDocument, deckDir: string, workspaceRoot: string): IRDocument {
+  const copy = structuredClone(doc)
+  const source = (pos: SourcePos) => {
+    if (isAbsolute(pos.file)) pos.file = relativePath(deckDir, pos.file)
+  }
+  if (isAbsolute(copy.meta.entry)) copy.meta.entry = relativePath(deckDir, copy.meta.entry)
+  copy.config.presmium = portable(copy.config.presmium, deckDir, workspaceRoot) as IRDocument['config']['presmium']
+  for (const section of copy.sections) {
+    source(section.source)
+    if (section.config) section.config = portable(section.config, deckDir, workspaceRoot) as typeof section.config
+    for (const block of irBlocks(section.blocks)) {
+      source(block.source)
+      if (block.node.type === 'component' && block.node.definition && isAbsolute(block.node.definition.file))
+        block.node.definition.file = relativePath(deckDir, block.node.definition.file)
+    }
+  }
+  for (const finding of copy.diagnostics) {
+    if (finding.source) source(finding.source as SourcePos)
+  }
+  return copy
+}