@lincyaw/addon-exec 0.0.0-stage → 1.0.0-rc.10

package/src/node/notebook-cells.ts

@@ -0,0 +1,46 @@
+import { componentCode, jupyterKernelLanguage, languageId } from '@lincyaw/kernel'
+import type { ComponentNode, IRBlock, MdastNode, RecordedRun, RendererContribution, RenderContext } from '@lincyaw/kernel'
+import { paramsPrelude, shownOutput } from '../domain/index.ts'
+
+function lines(text: string): string[] {
+  return text ? text.split(/(?<=\n)/) : []
+}
+
+function notebookOutputs(runs: RecordedRun[]): unknown[] {
+  return runs.flatMap<unknown>(run => run.outputs ? run.outputs.map((output) => {
+    if (output.output_type === 'stream') return { ...output, text: lines(output.text) }
+    if (output.output_type === 'error') return output
+    return { ...output, metadata: output.metadata ?? {}, ...output.output_type === 'execute_result' ? { execution_count: null } : {} }
+  }) : (['stdout', 'stderr'] as const).flatMap((name) => {
+    const text = run[name]
+    return text ? [{ output_type: 'stream', name, text: lines(text) }] : []
+  }))
+}
+
+function notebookCell(node: ComponentNode, _block: IRBlock, ctx: RenderContext) {
+  const source = componentCode(node)
+  if (!source) return undefined
+  const params = node.props.params as Record<string, unknown> | undefined
+  const runs = source.outputs ?? []
+  const recorded = typeof node.props.recorded === 'string' ? node.props.recorded : undefined
+  const output = shownOutput(runs, recorded)
+  if (languageId(source.lang) !== jupyterKernelLanguage(ctx.config.kernel)) {
+    const children: MdastNode[] = [{ type: 'code', lang: source.lang, value: paramsPrelude(source.lang, params) + source.value }]
+    const text = output.text.trimEnd()
+    if (text) children.push({ type: 'code', lang: 'text', value: text })
+    return { cell_type: 'markdown' as const, mdast: { type: 'root', children } }
+  }
+  const outputs = output.recorded ? [{ output_type: 'stream', name: 'stdout', text: lines(output.text) }] : notebookOutputs(runs)
+  const finished = runs.length > 0 && runs.every(run => !run.timedOut)
+  const metadata = node.props.timeout !== undefined && !finished ? { tags: ['skip-execution'] } : undefined
+  return { cell_type: 'code' as const, source: lines(paramsPrelude(source.lang, params) + source.value), outputs, execution_count: null, ...metadata ? { metadata } : {} }
+}
+
+export const rendererExtensions: Record<string, RendererContribution> = {
+  notebook: {
+    components: {
+      PmRunPanel: notebookCell,
+      PmStepRun: notebookCell,
+    },
+  },
+}

package/src/node/presmium.ts

@@ -0,0 +1,27 @@
+import { createRequire } from 'node:module'
+import type { PresmiumNodePlugin } from '@lincyaw/kernel'
+import { EXEC_ID } from '../domain/index.ts'
+import { browserImportsCheck } from './browser-imports.ts'
+import { rendererExtensions } from './notebook-cells.ts'
+import { pythonExecutor } from './python.ts'
+import { runPanelDirective, runPanelFallbackPass, runPanelPass } from './run-panel.ts'
+import { scriptExecutor } from './script.ts'
+import { stepCountPass, stepRunDirective } from './step-run.ts'
+import { timeoutPass } from './timeouts.ts'
+import { pyodideAssets } from './pyodide.ts'
+
+const { version } = createRequire(import.meta.url)('../../package.json') as { version: string }
+
+const plugin: PresmiumNodePlugin = {
+  id: EXEC_ID,
+  version,
+  kernel: '^1.0.0',
+  directives: [runPanelDirective, stepRunDirective],
+  passes: [timeoutPass, runPanelPass, stepCountPass, runPanelFallbackPass],
+  checks: [browserImportsCheck],
+  executors: [pythonExecutor, scriptExecutor],
+  rendererExtensions,
+  assets: () => [pyodideAssets()],
+}
+
+export default plugin

package/src/node/pyodide.ts

@@ -0,0 +1,22 @@
+import { createRequire } from 'node:module'
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import type { RuntimeAssets } from '@lincyaw/kernel'
+
+const PYODIDE_DIR = 'pyodide/'
+const LICENSES = 'LICENSES'
+const LICENSES_FILE = fileURLToPath(new URL('../../LICENSES', import.meta.url))
+const TYPES: Record<string, string> = {
+  'pyodide.mjs': 'text/javascript',
+  'pyodide.asm.mjs': 'text/javascript',
+  'pyodide.asm.wasm': 'application/wasm',
+  'python_stdlib.zip': 'application/zip',
+  'pyodide-lock.json': 'application/json',
+  [LICENSES]: 'text/plain; charset=utf-8',
+}
+
+/** Pyodide's runtime, at `<base>pyodide/`. */
+export function pyodideAssets(): RuntimeAssets {
+  const dir = dirname(createRequire(import.meta.url).resolve('pyodide/package.json'))
+  return { name: 'presmium:pyodide-runtime', prefix: PYODIDE_DIR, types: TYPES, source: file => (file === LICENSES ? LICENSES_FILE : join(dir, file)) }
+}

package/src/node/python.ts

@@ -0,0 +1,40 @@
+import { spawnSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import type { Executor } from '@lincyaw/kernel'
+import { isPython } from '../domain/index.ts'
+import { createChildProcessRuns } from './child-process.ts'
+
+const PYTHON_EXECUTOR_ID = 'presmium.python'
+
+const PYTHON = 'python3'
+const RUNNER = readFileSync(createRequire(import.meta.url).resolve('@lincyaw/kernel/runner.py'), 'utf8')
+const RUNNER_REVISION = `runner ${createHash('sha256').update(RUNNER).digest('hex').slice(0, 16)}`
+
+let probe: { version: string, prefixes: string[] } | undefined
+
+function python(): { version: string, prefixes: string[] } {
+  if (!probe) {
+    const found = spawnSync(PYTHON, ['-c', 'import sys; print(sys.version.split()[0]); print(sys.prefix); print(sys.base_prefix)'], { encoding: 'utf8' })
+    const [version, ...prefixes] = found.status === 0 ? found.stdout.trim().split('\n') : []
+    probe = version ? { version: `Python ${version}`, prefixes: [...new Set(prefixes)] } : { version: '', prefixes: [] }
+  }
+  return probe
+}
+
+const runs = createChildProcessRuns({
+  executorId: PYTHON_EXECUTOR_ID,
+  argv: [PYTHON, '-c', `${RUNNER}\n_pm_main()\n`],
+  read: () => python().prefixes,
+  sessions: true,
+})
+
+export const pythonExecutor: Executor = {
+  id: PYTHON_EXECUTOR_ID,
+  supports: lang => isPython(lang) && python().version !== '',
+  fingerprint: () => python().version,
+  revision: () => RUNNER_REVISION,
+  run: runs.run,
+  endSession: runs.endSession,
+}

package/src/node/run-panel.ts

@@ -0,0 +1,71 @@
+import { componentCode, docBlocks } from '@lincyaw/kernel'
+import type { DirectiveSpec, IRPass, MdastNode } from '@lincyaw/kernel'
+import { EXEC_ID, paramsPrelude, shownOutput } from '../domain/index.ts'
+import { codeComponent } from './code-component.ts'
+
+export const runPanelDirective: DirectiveSpec = {
+  name: 'run-panel',
+  arg: true,
+  body: true,
+  options: { params: 'string', timeout: 'number', recorded: 'string', run: 'string' },
+  toIR(input, ctx) {
+    const when = input.options.run === undefined ? 'both' : String(input.options.run).trim()
+    if (when !== 'both' && when !== 'live') ctx.report({ checkId: 'presmium.exec.syntax', severity: 'warning', message: `{run-panel} :run: ${when} is not both or live; using both` })
+    const node = codeComponent('PmRunPanel', input, ctx, { when: when === 'live' ? 'live' : 'both' })
+    const recorded = input.options.recorded
+    if (typeof recorded === 'string') {
+      node.props.recorded = recorded
+      node.fallback?.children?.push({ type: 'code', lang: 'text', value: recorded })
+    }
+    return { node }
+  },
+}
+
+/** Gives each run panel its place in the document and its code's executor and session, which its runs send. */
+export const runPanelPass: IRPass = {
+  name: 'presmium.exec.panels',
+  order: 10,
+  run(doc, ctx) {
+    for (const { section, block: { id, node, source, stepStates } } of docBlocks(doc)) {
+      if (node.type !== 'component' || node.plugin !== EXEC_ID || !['PmRunPanel', 'PmStepRun'].includes(node.name)) continue
+      const code = componentCode(node)
+      if (!code || node.children?.filter(child => child.node.type === 'code').length !== 1) {
+        ctx.report({ checkId: 'presmium.exec', severity: 'error', message: `${node.name} requires exactly one canonical code child.`, source })
+        continue
+      }
+      node.props.code = code.value
+      node.props.lang = code.lang
+      delete node.props.compiledOutput
+      for (const state of stepStates ?? []) if (state.parameters) {
+        for (const key of ['code', 'lang', 'place', 'executor', 'session', 'compiledOutput']) delete state.parameters[key]
+      }
+      node.props.place = { section: section.id, block: id }
+      const run = code.run
+      if (run?.executor) node.props.executor = run.executor
+      else delete node.props.executor
+      if (run?.session) node.props.session = run.session
+      else delete node.props.session
+      node.fallback = { type: 'root', children: [{ type: 'code', lang: code.lang, value: code.value }, ...typeof node.props.recorded === 'string' ? [{ type: 'code', lang: 'text', value: node.props.recorded }] : []] }
+    }
+  },
+}
+
+/** After the compile-time runs: the panel's fallback shows its code with the output, and a run that succeeded gives `compiledOutput`. */
+export const runPanelFallbackPass: IRPass = {
+  name: 'presmium.exec.fallbacks',
+  after: 'execute',
+  run(doc) {
+    for (const { block: { node } } of docBlocks(doc)) {
+      if (node.type !== 'component' || node.plugin !== EXEC_ID || node.name !== 'PmRunPanel') continue
+      const code = componentCode(node)
+      if (!code) continue
+      const recorded = typeof node.props.recorded === 'string' ? node.props.recorded : undefined
+      const output = shownOutput(code.outputs ?? [], recorded).text.trimEnd()
+      const children: MdastNode[] = [{ type: 'code', lang: code.lang, value: paramsPrelude(code.lang, node.props.params as Record<string, unknown>) + code.value }]
+      if (output) children.push({ type: 'code', lang: 'text', value: output })
+      node.fallback = { type: 'root', children }
+      const runs = code.outputs ?? []
+      if (runs.length > 0 && runs.every(run => run.ok)) node.props.compiledOutput = shownOutput(runs, undefined).text
+    }
+  },
+}

package/src/node/script-main.mjs

@@ -0,0 +1,11 @@
+import { readFileSync, renameSync, writeFileSync } from 'node:fs'
+import { runScript } from '../shared/script-runner.mjs'
+
+const [requestFile, resultPrefix] = process.argv.slice(2)
+const { runs } = JSON.parse(readFileSync(requestFile, 'utf8'))
+for (const [i, run] of runs.entries()) {
+  const result = await runScript({ code: run.code, args: run.args, typescript: /^(ts|typescript)$/i.test(run.lang ?? '') })
+  writeFileSync(`${resultPrefix}${i}.part`, JSON.stringify(result))
+  renameSync(`${resultPrefix}${i}.part`, `${resultPrefix}${i}.json`)
+}
+process.exit(0)

package/src/node/script.ts

@@ -0,0 +1,26 @@
+import { createRequire } from 'node:module'
+import { dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import type { Executor } from '@lincyaw/kernel'
+import { RUN_REQUEST_FILE, RUN_RESULT_PREFIX } from '@lincyaw/kernel/node'
+import { isScript } from '../domain/index.ts'
+import { createChildProcessRuns } from './child-process.ts'
+
+export const NODE_SCRIPT_EXECUTOR_ID = 'presmium.node'
+
+const MAIN = fileURLToPath(new URL('./script-main.mjs', import.meta.url))
+const READ = [fileURLToPath(new URL('../..', import.meta.url)), dirname(createRequire(import.meta.url).resolve('sucrase/package.json')), dirname(dirname(process.execPath))]
+
+const runs = createChildProcessRuns({
+  executorId: NODE_SCRIPT_EXECUTOR_ID,
+  argv: [process.execPath, MAIN, RUN_REQUEST_FILE, RUN_RESULT_PREFIX],
+  read: () => READ,
+  sessions: false,
+})
+
+export const scriptExecutor: Executor = {
+  id: NODE_SCRIPT_EXECUTOR_ID,
+  supports: isScript,
+  fingerprint: () => `node ${process.version}`,
+  run: runs.run,
+}

package/src/node/step-run.ts

@@ -0,0 +1,71 @@
+import { componentCode, docBlocks, PROCESS_SANDBOX } from '@lincyaw/kernel'
+import type { DirectiveSpec, IRPass, ProcessSandbox } from '@lincyaw/kernel'
+import { minimalRunEnv } from '@lincyaw/kernel/node'
+import { configuredTimeout, EXEC_ID, execConfig, isPython, stringArgs } from '../domain/index.ts'
+import { codeComponent } from './code-component.ts'
+import { pythonExecutor } from './python.ts'
+
+export const stepRunDirective: DirectiveSpec = {
+  name: 'step-run',
+  arg: true,
+  body: true,
+  options: { steps: 'number', params: 'string', timeout: 'number' },
+  toIR(input, ctx) {
+    const node = codeComponent('PmStepRun', input, ctx)
+    const steps = Number(input.options.steps)
+    if (!Number.isInteger(steps) || steps < 1) {
+      ctx.report({ checkId: 'presmium.exec.syntax', severity: 'warning', message: '{step-run} needs :steps:, the number of lines the code executes; without it the slide has no steps' })
+      return { node }
+    }
+    node.props.steps = steps
+    return { node, span: steps }
+  },
+}
+
+const traced = new Map<string, Promise<number | undefined>>()
+
+function traceLength(code: string, args: Record<string, string>, maxEvents: number, timeoutMs: number, sandbox: ProcessSandbox | undefined): Promise<number | undefined> {
+  const key = JSON.stringify([code, args, maxEvents])
+  let found = traced.get(key)
+  if (!found) {
+    const program = `import __main__\nprint(__main__._pm_trace(${JSON.stringify(code)}, ${JSON.stringify(JSON.stringify(args))}, '', '{}', ${maxEvents}))\n`
+    found = pythonExecutor.run({ lang: 'python', code: program, timeoutMs }, undefined, { env: minimalRunEnv(), ...sandbox ? { sandbox } : {} }).then((result) => {
+      if (!result.ok) return undefined
+      try {
+        const trace = JSON.parse(result.stdout) as { events: unknown[], truncated: boolean }
+        return trace.truncated ? undefined : trace.events.length
+      } catch {
+        return undefined
+      }
+    })
+    traced.set(key, found)
+  }
+  return found
+}
+
+/** Warns when a step run declares another number of steps than the lines its code executes. */
+export const stepCountPass: IRPass = {
+  name: 'presmium.exec.steps',
+  order: 20,
+  async run(doc, ctx) {
+    if (!pythonExecutor.supports('python')) return
+    const { maxTraceEvents } = execConfig(ctx.config)
+    const sandbox = ctx.service?.(PROCESS_SANDBOX)
+    for (const { section, block } of docBlocks(doc)) {
+      const { node } = block
+      if (node.type !== 'component' || node.plugin !== EXEC_ID || node.name !== 'PmStepRun' || typeof node.props.steps !== 'number') continue
+      const code = componentCode(node)
+      if (!code || !isPython(code.lang)) continue
+      const timeoutMs = (typeof node.props.timeout === 'number' ? node.props.timeout : undefined) ?? configuredTimeout(ctx.configFor(section)) ?? execConfig(ctx.config).timeoutMs
+      const lines = await traceLength(code.value, stringArgs(node.props.params as Record<string, unknown> | undefined), maxTraceEvents, timeoutMs, sandbox)
+      if (lines === undefined || lines === node.props.steps) continue
+      ctx.report({
+        checkId: 'presmium.exec',
+        severity: 'warning',
+        message: `{step-run} declares :steps: ${node.props.steps}, but its code executes ${lines} lines: the slide's steps do not match the lines`,
+        hint: `write :steps: ${lines}`,
+        source: { file: block.source.file, line: block.source.line },
+      })
+    }
+  },
+}

package/src/node/timeouts.ts

@@ -0,0 +1,15 @@
+import { docBlocks } from '@lincyaw/kernel'
+import type { IRPass } from '@lincyaw/kernel'
+import { configuredTimeout } from '../domain/index.ts'
+
+/** Code blocks without their own timeout take the one their section configures. */
+export const timeoutPass: IRPass = {
+  name: 'presmium.exec.timeouts',
+  run(doc, ctx) {
+    for (const { section, block: { node } } of docBlocks(doc)) {
+      if (node.type !== 'code' || !node.run || node.run.timeoutMs !== undefined) continue
+      const timeout = configuredTimeout(ctx.configFor(section))
+      if (timeout) node.run.timeoutMs = timeout
+    }
+  },
+}

package/src/shared/script-runner.d.mts

@@ -0,0 +1,16 @@
+export interface ScriptRequest {
+  code: string
+  args?: Record<string, string>
+  typescript?: boolean
+  imports?: Record<string, string>
+}
+
+export interface ScriptOutput {
+  stdout: string
+  stderr: string
+  exitCode: number
+  outputs: { output_type: 'stream', name: 'stdout' | 'stderr', text: string }[]
+}
+
+export function prepare(request: ScriptRequest): string
+export function runScript(request: ScriptRequest): Promise<ScriptOutput>

package/src/shared/script-runner.mjs

@@ -0,0 +1,168 @@
+import { transform } from 'sucrase'
+
+const IDENTIFIER = /^[A-Za-z_$][\w$]*$/
+
+function show(value) {
+  if (typeof value === 'string') return value
+  if (value instanceof Error) return `${value.name}: ${value.message}`
+  if (typeof value === 'function') return `[Function ${value.name || '(anonymous)'}]`
+  if (typeof value === 'bigint') return `${value}n`
+  try {
+    return JSON.stringify(value) ?? String(value)
+  } catch {
+    return String(value)
+  }
+}
+
+function failure(error) {
+  const line = error instanceof Error ? /^.*data:text\/javascript.*?:(\d+):\d+\)?$/m.exec(error.stack ?? '')?.[1] : undefined
+  return `${show(error)}${line ? ` (line ${line})` : ''}`
+}
+
+function literal(raw) {
+  try {
+    JSON.parse(raw)
+    return raw
+  } catch {
+    return JSON.stringify(raw)
+  }
+}
+
+const SPECIFIER = /(\bfrom\s*|\bimport\s*\(\s*|\bimport\s+)(['"])([^'"\n]+)\2/g
+
+export function prepare({ code, args = {}, typescript = false, imports = {} }) {
+  const js = typescript ? transform(code, { transforms: ['typescript'], disableESTransforms: true }).code : code
+  const constants = Object.entries(args).filter(([name]) => IDENTIFIER.test(name)).map(([name, raw]) => `const ${name} = ${literal(String(raw))};`)
+  const mapped = js.replace(SPECIFIER, (match, keyword, quote, name) => Object.hasOwn(imports, name) ? `${keyword}${quote}${imports[name]}${quote}` : match)
+  return constants.join(' ') + mapped
+}
+
+function trackTimers() {
+  const native = { setTimeout, clearTimeout, setInterval, clearInterval }
+  const { setTimeout: later, clearTimeout: cancel, setInterval: every } = native
+  const pending = new Set()
+  let armed = false
+  let failed
+  let settle
+  const done = new Promise((resolve, reject) => settle = { resolve, reject })
+  const decide = () => {
+    if (failed) settle.reject(failed.reason)
+    else if (pending.size === 0) settle.resolve()
+  }
+  const check = () => later(() => {
+    if (!armed) return
+    if (failed || pending.size > 0) return decide()
+    later(decide)
+  })
+  const fail = (reason) => {
+    failed ??= { reason }
+    check()
+  }
+  const call = (callback, args) => {
+    try {
+      const result = typeof callback === 'function' ? callback(...args) : undefined
+      if (typeof result?.then === 'function') result.then(undefined, fail)
+    } catch (error) {
+      fail(error)
+    }
+  }
+  const clear = (id) => {
+    pending.delete(id)
+    cancel(id)
+    check()
+  }
+  Object.assign(globalThis, {
+    setTimeout(callback, delay, ...args) {
+      const id = later(() => {
+        pending.delete(id)
+        call(callback, args)
+        check()
+      }, delay)
+      pending.add(id)
+      return id
+    },
+    setInterval(callback, delay, ...args) {
+      const id = every(() => call(callback, args), delay)
+      pending.add(id)
+      return id
+    },
+    clearTimeout: clear,
+    clearInterval: clear,
+  })
+  const node = typeof globalThis.process?.on === 'function' ? globalThis.process : undefined
+  const onNode = reason => fail(reason)
+  const onWorker = (event) => {
+    event.preventDefault()
+    fail(event.reason)
+  }
+  if (node) node.on('unhandledRejection', onNode)
+  else globalThis.addEventListener?.('unhandledrejection', onWorker)
+  return {
+    idle() {
+      armed = true
+      check()
+      return done
+    },
+    restore() {
+      for (const id of pending) cancel(id)
+      Object.assign(globalThis, native)
+      if (node) node.off('unhandledRejection', onNode)
+      else globalThis.removeEventListener?.('unhandledrejection', onWorker)
+    },
+  }
+}
+
+function collector() {
+  const text = { stdout: [], stderr: [] }
+  const outputs = []
+  return {
+    text,
+    outputs,
+    stream: name => ({
+      push(chunk) {
+        text[name].push(chunk)
+        const last = outputs.at(-1)
+        if (last?.name === name) last.text += chunk
+        else outputs.push({ output_type: 'stream', name, text: chunk })
+      },
+    }),
+  }
+}
+
+function captureStreams(out, err) {
+  const streams = [[globalThis.process?.stdout, out], [globalThis.process?.stderr, err]].filter(([stream]) => typeof stream?.write === 'function')
+  const saved = streams.map(([stream]) => [stream, stream.write])
+  for (const [stream, target] of streams) {
+    stream.write = (chunk, encoding, callback) => {
+      target.push(typeof chunk === 'string' ? chunk : new TextDecoder().decode(chunk))
+      const done = typeof encoding === 'function' ? encoding : callback
+      if (typeof done === 'function') queueMicrotask(done)
+      return true
+    }
+  }
+  return () => saved.forEach(([stream, write]) => stream.write = write)
+}
+
+export async function runScript(request) {
+  const output = collector()
+  const out = output.stream('stdout')
+  const err = output.stream('stderr')
+  const write = target => (...values) => void target.push(`${values.map(show).join(' ')}\n`)
+  const saved = { log: console.log, info: console.info, debug: console.debug, dir: console.dir, warn: console.warn, error: console.error }
+  Object.assign(console, { log: write(out), info: write(out), debug: write(out), dir: value => write(out)(value), warn: write(err), error: write(err) })
+  const restoreStreams = captureStreams(out, err)
+  const timers = trackTimers()
+  let exitCode = 0
+  try {
+    await import(/* @vite-ignore */ `data:text/javascript;charset=utf-8,${encodeURIComponent(prepare(request))}`)
+    await timers.idle()
+  } catch (error) {
+    err.push(`${failure(error)}\n`)
+    exitCode = 1
+  } finally {
+    timers.restore()
+    restoreStreams()
+    Object.assign(console, saved)
+  }
+  return { stdout: output.text.stdout.join(''), stderr: output.text.stderr.join(''), exitCode, outputs: output.outputs }
+}