dsh-ptc-plus 0.3.2 → 0.4.2

package/internal/user-binding-console-worker.js

@@ -0,0 +1,244 @@
+import { randomUUID } from 'node:crypto'
+import { createPrivateAsyncLocalStorage } from './async-local-storage-intrinsics.js'
+import { create as createDomain } from 'node:domain'
+import { createRequire, registerHooks } from 'node:module'
+import { resolve } from 'node:path'
+import repl from 'node:repl'
+import { PassThrough } from 'node:stream'
+import { inspect } from 'node:util'
+import { pathToFileURL } from 'node:url'
+import { parentPort, workerData } from 'node:worker_threads'
+import { prepareConsoleProgram } from './compiler-service.js'
+import { createStatefulRootRuntime } from './stateful-root-runtime.js'
+import { managedModuleImport, managedRequire, readModuleImport } from './stateful-module-runtime.js'
+import { compileStatefulModule, createUserModuleCompilationHooks } from './stateful-module-compiler.js'
+import { USER_BINDING_TRANSFORM } from './module-transform-contract.js'
+import { moduleRuntimeIntrinsics } from './compiler-intrinsics.js'
+import { stringifyCompilerData } from './compiler-data.js'
+import { createCellCompletionObserver } from './cell-completion.js'
+import { WORKER_SHUTDOWN_ACKNOWLEDGEMENT, WORKER_SHUTDOWN_REQUEST } from './worker-shutdown.js'
+import { WORKER_REPL_OPTIONS, createWorkerControlPromise, createWorkerReplErrorHandler,
+  disableWorkerReplDomain,
+  captureWorkerReplGlobals, protectWorkerReplAsyncContext, restoreWorkerReplGlobals, runInWorkerReplRealm,
+  workerReplContext } from './worker-repl-realm.js'
+
+const { Object, Set, setHas, setAdd, setDelete, appendArray } = moduleRuntimeIntrinsics
+const Promise = globalThis.Promise
+const process = globalThis.process
+const hasProperty = globalThis.Reflect.has
+const ControlPromise = createWorkerControlPromise()
+
+const evaluations = createPrivateAsyncLocalStorage()
+const context = globalThis
+const completionObserver = createCellCompletionObserver(runInWorkerReplRealm('Function'))
+context.console = console
+let retainedEvaluator
+const completionKey = `__ptc_console_${randomUUID().replaceAll('-', '')}`
+Object.defineProperty(context, completionKey, { value: value => {
+  evaluations.getStore().completed = true
+  return value
+} })
+const parentUrl = pathToFileURL(resolve(workerData.cwd, 'ptc-plus-console')).href
+let moduleUrl
+let replParent
+let replModuleRealm
+const adapterParents = new Set()
+const knownBindings = new Set()
+const roots = createStatefulRootRuntime({
+  importModule: (source, options) => managedModuleImport(replParent, source, options, replModuleRealm),
+  errors: runInWorkerReplRealm('({ ReferenceError, TypeError })'),
+  dynamicIntrinsics: runInWorkerReplRealm('({intrinsicEval:eval,realmFunction:Function,globalObject:globalThis,errors:{ReferenceError,TypeError,SyntaxError},reflect:Reflect,object:Object})'),
+  readAmbient: name => runInWorkerReplRealm(name),
+  typeofAmbient: name => runInWorkerReplRealm(`typeof ${name}`),
+  writeAmbient(name, value, strict) {
+    const argument = name === '__ptc_value' ? '__ptc_other_value' : '__ptc_value'
+    runInWorkerReplRealm(`${strict ? '"use strict";' : ''}(${argument}) => (${name} = ${argument})`)(value)
+  },
+  deleteAmbient: name => runInWorkerReplRealm(`delete ${name}`),
+  hasAmbient: name => hasProperty(context, name),
+  publish: name => setAdd(knownBindings, name),
+})
+class ConsoleReturn {
+  constructor(value) { this.value = value }
+}
+const moduleCompilation = createUserModuleCompilationHooks({
+  transformForParent: parent => parent === replParent || parent === parentUrl ? USER_BINDING_TRANSFORM : undefined,
+})
+const providedRequire = managedRequire(parentUrl, createRequire(parentUrl))
+Object.defineProperty(context, 'require', { configurable: true, value: providedRequire })
+const importCanary = 'data:text/javascript,export default 1'
+registerHooks({ resolve(specifier, context, nextResolve) {
+  if (specifier === importCanary) replParent = context.parentURL
+  return moduleCompilation.resolve(specifier, context, (selected, original) =>
+    nextResolve(selected, original.parentURL === moduleUrl || original.parentURL === replParent || setHas(adapterParents, original.parentURL)
+      ? { ...original, parentURL: parentUrl } : original))
+}, load: moduleCompilation.load })
+const describe = value => inspect(value, {
+  colors: false, customInspect: false, getters: false, depth: 4,
+  maxArrayLength: 40, maxStringLength: Math.min(workerData.maxOutputBytes, 8192), breakLength: 100,
+})
+process.on('uncaughtException', error => {
+  const state = evaluations.getStore()
+  if (state?.finish(true, error) === true || state?.settled === true) return
+  parentPort.postMessage({ fatal: true, error: describe(error) })
+})
+
+function evaluate(source) {
+  const prepared = prepareConsoleProgram(source, { languageSemantics: 'stateful-v1', knownBindings })
+  const helpers = [prepared.rootRuntimeName, prepared.returnSignal]
+  Object.defineProperty(context, prepared.rootRuntimeName, {
+    configurable: true, value: roots.begin({ ...prepared.rootBindings, committed: () => {} }),
+  })
+  Object.defineProperty(context, prepared.returnSignal, { configurable: true, value: ConsoleReturn })
+  if (prepared.moduleLoads.length === 0) {
+    const operation = evaluatePrepared(prepared.code, prepared.asyncCompletion)
+    return new ControlPromise((resolve, reject) => operation.then(
+      value => { removeHelpers(helpers); resolve(value) },
+      error => { removeHelpers(helpers); reject(error) },
+    ))
+  }
+  return evaluateWithModules(prepared, helpers)
+}
+
+function removeHelpers(helpers) {
+  for (let index = 0; index < helpers.length; index++) delete context[helpers[index]]
+}
+
+async function evaluateWithModules(prepared, helpers) {
+  try {
+    for (let index = 0; index < prepared.moduleLoads.length; index++) {
+      const load = prepared.moduleLoads[index]
+      let attributes = ''
+      const entries = Object.entries(load.options?.with ?? {})
+      for (let entry = 0; entry < entries.length; entry++) {
+        attributes += `${entry === 0 ? ' with {' : ','}${stringifyCompilerData(entries[entry][0])}:${stringifyCompilerData(entries[entry][1])}`
+      }
+      if (entries.length > 0) attributes += '}'
+      const source = stringifyCompilerData(load.source)
+      let required
+      if (load.requiredExports !== undefined) {
+        required = ''
+        for (let entry = 0; entry < load.requiredExports.length; entry++) {
+          required += `${entry === 0 ? '' : ','}${stringifyCompilerData(load.requiredExports[entry])} as __required_${entry}`
+        }
+      }
+      const adapter = `data:text/javascript,${encodeURIComponent(`import * as namespace from ${source}${attributes};\n${required === undefined ? '' : `export { ${required} } from ${source}${attributes};`}\nexport { namespace };`)}#console-${randomUUID()}`
+      setAdd(adapterParents, adapter)
+      moduleCompilation.mark(adapter, { compiled: true })
+      let namespace
+      try {
+        const native = await import(adapter)
+        namespace = readModuleImport(adapter, load.source, null, () => native.namespace, load.options?.with)
+      } finally { setDelete(adapterParents, adapter) }
+      if (load.global !== undefined) {
+        appendArray(helpers, load.global)
+        Object.defineProperty(context, load.global, { configurable: true, value: namespace })
+      }
+    }
+    return await evaluatePrepared(prepared.code, prepared.asyncCompletion)
+  } finally {
+    removeHelpers(helpers)
+  }
+}
+
+function evaluatePrepared(javascript, asyncCompletion = false) {
+  releaseRetainedEvaluator()
+  const input = new PassThrough()
+  const output = new PassThrough()
+  output.resume()
+  const workerGlobalBaseline = captureWorkerReplGlobals(context)
+  const server = repl.start({ input, output, terminal: false, prompt: '', ...WORKER_REPL_OPTIONS,
+    handleError: createWorkerReplErrorHandler(evaluations, (state, error) => state.finish(true, error)) })
+  protectWorkerReplAsyncContext(server)
+  workerReplContext(server)
+  restoreWorkerReplGlobals(workerGlobalBaseline, context)
+  Object.defineProperty(context, 'require', { configurable: true, value: providedRequire })
+  const errorDomain = disableWorkerReplDomain(server.eval.domain ?? createDomain())
+  const evaluator = { server, input, output, errorDomain }
+  retainedEvaluator = evaluator
+  return new ControlPromise((resolve, reject) => {
+    const complete = `this[${stringifyCompilerData(completionKey)}]`
+    // Preserve expression results while distinguishing rejection with null/undefined.
+    const code = `${javascript}\n;{let completed = ${complete}();}\n;`
+    let settled = false
+    const state = { completed: false, finish(failed, value) {
+      if (settled) return false
+      settled = true
+      state.settled = true
+      if (failed && value instanceof ConsoleReturn) completionObserver.settle(value.value, resolve, reject)
+      else if (failed) reject(value)
+      else completionObserver.settle(value, resolve, reject)
+      return true
+    } }
+    evaluator.state = state
+    errorDomain.on('error', error => state.finish(true, error))
+    evaluations.run(state, () => errorDomain.run(() => {
+      server.eval(`${code}\n`, context, 'ptc-plus-console', (error, value) => {
+        if (asyncCompletion && (error === null || error === undefined)) {
+          completionObserver.observe(value, value => state.finish(false, value), error => state.finish(true, error))
+          return
+        }
+        const failed = !state.completed || error !== null && error !== undefined
+        state.finish(failed, failed ? error : value)
+      })
+    }))
+  })
+}
+
+function releaseRetainedEvaluator() {
+  const evaluator = retainedEvaluator
+  if (evaluator === undefined) return undefined
+  retainedEvaluator = undefined
+  evaluator.errorDomain.removeAllListeners('error')
+  evaluator.server.close()
+  evaluator.input.destroy()
+  evaluator.output.destroy()
+}
+try {
+  replModuleRealm = await evaluatePrepared('({Promise,stringify:value=>`${value}`,importModule:(source,options)=>import(source,options)})')
+  const prepared = compileStatefulModule(workerData.source)
+  moduleUrl = `data:text/javascript,${encodeURIComponent(prepared.code)}#binding`
+  moduleCompilation.mark(moduleUrl, { compiled: true, moduleInterface: prepared.moduleInterface, sourceRegions: prepared.sourceRegions })
+  const namespace = await managedModuleImport(moduleUrl, moduleUrl)
+  await evaluatePrepared(`import(${stringifyCompilerData(importCanary)})`)
+  const names = Object.keys(namespace)
+  for (let index = 0; index < names.length; index++) {
+    const name = names[index]
+    setAdd(knownBindings, name)
+    Object.defineProperty(context, name, { configurable: true, enumerable: true, get: () => namespace[name] })
+  }
+  parentPort.on('message', (message) => {
+    if (message?.type === WORKER_SHUTDOWN_REQUEST) {
+      // Release the realm, REPL and domain this console owns, then report the
+      // release before the host gives up on the thread; see worker-shutdown.js.
+      releaseRetainedEvaluator()
+      parentPort.postMessage({ type: WORKER_SHUTDOWN_ACKNOWLEDGEMENT })
+      parentPort.close()
+      return
+    }
+    const { id, code } = message
+    const send = result => {
+      const flushed = new ControlPromise(resolve => process.stdout.write('', () => process.stderr.write('', resolve)))
+      moduleRuntimeIntrinsics.observeOwnedPromise(flushed, () => {
+        parentPort.postMessage(result)
+        releaseRetainedEvaluator()
+      /* c8 ignore next 3 -- flushed has no reject capability */
+      }, error => {
+        releaseRetainedEvaluator()
+        parentPort.postMessage({ id, error: describe(error) })
+      })
+    }
+    let operation
+    try {
+      operation = evaluate(code)
+    } catch (error) {
+      send({ id, error: describe(error) })
+      return
+    }
+    moduleRuntimeIntrinsics.observeOwnedPromise(operation,
+      value => send({ id, output: describe(value) }),
+      error => send({ id, error: describe(error) }))
+  })
+} catch (error) {
+  parentPort.postMessage({ fatal: true, error: describe(error) })
+}

package/internal/user-binding-console.js

@@ -0,0 +1,148 @@
+import { randomUUID } from 'node:crypto'
+import { processExitDescription } from './failure-reporting.js'
+import { IsolatedOwner } from './isolated-worker.js'
+import {
+  createBoundedWorkerRound,
+  ROUND_CANCELLED,
+  ROUND_OUTPUT_LIMIT,
+  ROUND_TIMEOUT,
+} from './bounded-worker-round.js'
+import { normalizeWorkerEnvironment } from './worker-client.js'
+const CONSOLE_HELPER = new URL('./kernel-child.js', import.meta.url)
+
+export const CONSOLE_IDLE_MS = 10 * 60 * 1000
+const WORKER_URL = new URL('./user-binding-console-worker.js', import.meta.url)
+const MAX_CONSOLES = 4
+const MAX_SOURCE_BYTES = 1024 * 1024
+const ROUND_REASONS = Object.freeze({
+  [ROUND_TIMEOUT]: 'execution timed out',
+  [ROUND_CANCELLED]: 'stopped',
+  [ROUND_OUTPUT_LIMIT]: 'console output limit exceeded',
+})
+
+/** Own only user workbench workers; no Agent state, replay or persistence. */
+export class UserBindingConsole {
+  constructor(options) {
+    this.options = options
+    this.environments = new Map()
+    // Every console transport is owned here until its real exit is observed.
+    this.owner = new IsolatedOwner()
+  }
+
+  release(capability, reason = 'reset') {
+    const environment = this.environments.get(capability)
+    if (environment === undefined) return
+    this.environments.delete(capability)
+    clearTimeout(environment.idleTimer)
+    environment.active?.finish({ error: reason, released: true })
+    // One close per transport owns the release request, the kill deadline and the
+    // real exit; a refusal stays visible instead of becoming an unhandled rejection.
+    // The owner keeps this instance (and any refusal) until a real exit arrives.
+    void this.owner.stop(environment.hostId).catch(() => {})
+  }
+
+  /** Release every environment while this console's owner stays usable. */
+  async releaseAll(reason = 'released') {
+    for (const capability of [...this.environments.keys()]) this.release(capability, reason)
+    return this.owner.releaseAll()
+  }
+
+  async dispose() {
+    for (const capability of this.environments.keys()) this.release(capability, 'disposed')
+    return this.owner.dispose()
+  }
+
+  async reconfigure(options) {
+    // Reconfiguration releases the current environments; the owner stays usable so
+    // a later run can start again.
+    if (['maxWallMs', 'maxOutputBytes', 'maxOldGenerationSizeMb'].some(key => this.options[key] !== options[key])) {
+      for (const capability of [...this.environments.keys()]) this.release(capability, 'reconfigured')
+      const failures = await this.owner.releaseAll()
+      if (failures.length > 0) {
+        throw new AggregateError(failures, 'user binding console reconfiguration failed')
+      }
+    }
+    this.options = options
+  }
+
+  async run({ environment: capability, source, code }, signal) {
+    for (const [label, value] of [['source', source], ['code', code]]) {
+      if (typeof value !== 'string' || value.trim() === '' || Buffer.byteLength(value) > MAX_SOURCE_BYTES) {
+        throw new TypeError(`console ${label} must be non-empty TypeScript within 1 MiB`)
+      }
+    }
+    signal?.throwIfAborted()
+    let environment = this.environments.get(capability)
+    if (environment?.active !== undefined) throw new Error('console is already running')
+    const reset = capability !== undefined && (environment === undefined || environment.source !== source)
+    if (environment !== undefined && environment.source !== source) {
+      this.release(capability, 'source changed')
+      environment = undefined
+    }
+    if (environment === undefined) {
+      if (this.environments.size >= MAX_CONSOLES) throw new Error('too many active code consoles')
+      capability = randomUUID()
+      const started = this.owner.start({
+        helper: CONSOLE_HELPER,
+        entry: WORKER_URL.href,
+        workerData: { source, cwd: this.options.cwd, maxOutputBytes: this.options.maxOutputBytes },
+        env: normalizeWorkerEnvironment(process.env),
+        resourceLimits: { maxOldGenerationSizeMb: this.options.maxOldGenerationSizeMb },
+        protocol: 'parent-port',
+      })
+      const worker = started.transport
+      environment = { worker, hostId: started.id, source, round: undefined, active: undefined, sequence: 0 }
+      this.environments.set(capability, environment)
+      const fail = reason => this.release(capability, reason)
+      for (const [stream, channel] of [[worker.stdout, 'stdout'], [worker.stderr, 'stderr']]) {
+        stream.on('data', chunk => environment.round?.capture(channel, String(chunk)))
+      }
+      worker.on('error', error => fail(error.message))
+      worker.on('exit', (code, signal) => {
+        fail(`console worker exited (${processExitDescription(code, signal)})`)
+      })
+      worker.on('message', message => {
+        if (message === null || typeof message !== 'object') { fail('invalid console result'); return }
+        if (message.fatal === true) { fail(message.error); return }
+        if (message.id !== environment.active?.id) return
+        if (typeof message.output !== 'string' && typeof message.error !== 'string') {
+          fail('invalid console result'); return
+        }
+        const result = typeof message.error === 'string' ? { error: message.error } : { output: message.output }
+        if (environment.round.exceeds(result)) fail('console output limit exceeded')
+        else environment.active.finish(result)
+      })
+    }
+    clearTimeout(environment.idleTimer)
+    const id = ++environment.sequence
+    const started = Date.now()
+    return new Promise(resolve => {
+      const finish = result => {
+        environment.round = undefined
+        environment.active = undefined
+        const live = this.environments.get(capability) === environment
+        const expiresAt = live ? Date.now() + CONSOLE_IDLE_MS : null
+        if (live) {
+          environment.idleTimer = setTimeout(() => this.release(capability, 'idle'), CONSOLE_IDLE_MS)
+          environment.idleTimer.unref()
+        }
+        resolve({ ...result, id: undefined, environment: live ? capability : null,
+          reset, expiresAt, logs: round.logs, durationMs: Date.now() - started })
+      }
+      const round = createBoundedWorkerRound({
+        maxOutputBytes: this.options.maxOutputBytes,
+        maxWallMs: this.options.maxWallMs,
+        signal,
+        settle: ({ ok, result, reason }) => {
+          if (ok) { finish(result); return }
+          this.release(capability, ROUND_REASONS[reason])
+          finish({ error: ROUND_REASONS[reason], released: true })
+        },
+      })
+      environment.round = round
+      environment.active = { id, finish: result => round.succeed(result) }
+      try { environment.worker.postMessage({ id, code }) }
+      catch (error) { this.release(capability, error.message) }
+    })
+  }
+}