@bevel-software/ael 0.0.0-stage → 0.2.0

package/src/executor/docker-cli.ts

@@ -0,0 +1,148 @@
+import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process';
+import type { ContainerDriver, ContainerRunResult, ContainerRunSpec, ContainerVolume } from './driver.js';
+
+/**
+ * The container engine through the `docker` command on the executor's host.
+ *
+ * An image is built from the Dockerfile's text on stdin and nothing else as
+ * context (no COPY from the knowledge base, by design). A run container gets
+ * its input on stdin, its environment through `--env NAME` with the values in
+ * this process's environment (never on the command line), and is stopped
+ * with `docker kill` when it outlives its budget. The executor never mounts
+ * the engine's socket into a run container.
+ *
+ * `runOptions` are extra `docker run` options this server's operator set for
+ * every step's container (`EXECUTOR_RUN_OPTIONS`; a spec says it is a step's
+ * with `operatorOptions`, and the executor's own root command never does): what a step needs that the
+ * defaults withhold, such as the relaxed confinement a rootless Docker daemon
+ * inside the step needs. They come before the executor's own, so where both
+ * name the same thing (the container's name, its user, a limit) the
+ * executor's is the one the engine takes.
+ */
+export class DockerCliDriver implements ContainerDriver {
+  constructor(private readonly runOptions: readonly string[] = []) {}
+
+  hasImage(ref: string): Promise<boolean> {
+    return succeeds(['image', 'inspect', ref]);
+  }
+
+  async pullImage(ref: string): Promise<void> {
+    await docker(['pull', '--quiet', ref]);
+  }
+
+  async buildImage(tag: string, dockerfile: string): Promise<void> {
+    await docker(['build', '--quiet', '-t', tag, '-'], { stdin: dockerfile });
+  }
+
+  async ensureVolume(name: string, labels: Record<string, string>): Promise<boolean> {
+    if (await succeeds(['volume', 'inspect', name])) return false;
+    await docker(['volume', 'create', ...labelArgs(labels), name]);
+    return true;
+  }
+
+  async listVolumes(labelKey: string): Promise<ContainerVolume[]> {
+    const { stdout } = await docker(['volume', 'ls', '--quiet', '--filter', `label=${labelKey}`]);
+    const names = stdout.split('\n').map((line) => line.trim()).filter(Boolean);
+    if (names.length === 0) return [];
+    const inspected = JSON.parse((await docker(['volume', 'inspect', ...names])).stdout) as Array<{ Name: string; CreatedAt?: string; Labels?: Record<string, string> | null }>;
+    return inspected.map((v) => ({ name: v.Name, createdAt: (v.CreatedAt && Date.parse(v.CreatedAt)) || 0, labels: v.Labels ?? {} }));
+  }
+
+  async removeVolume(name: string): Promise<void> {
+    const args = ['volume', 'rm', name];
+    const result = await start(args).done;
+    if (result.code !== 0 && !/no such volume/i.test(result.stderr)) throw failure(args, result);
+  }
+
+  async run(spec: ContainerRunSpec): Promise<ContainerRunResult> {
+    // The operator's options first: where one names something the executor sets itself (the name it stops the
+    // container by, the user, the limits), the engine takes the later one, which is the executor's.
+    const args = ['run', ...(spec.operatorOptions ? this.runOptions : []), '--rm', '--interactive', '--name', spec.name, '--user', spec.user, '--workdir', spec.workdir];
+    if (spec.limits.memory) args.push('--memory', spec.limits.memory);
+    if (spec.limits.cpus) args.push('--cpus', spec.limits.cpus);
+    if (spec.limits.pids !== undefined) args.push('--pids-limit', String(spec.limits.pids));
+    args.push(...labelArgs(spec.labels));
+    for (const m of spec.mounts) args.push('--mount', `type=${m.kind},source=${m.source},target=${m.target}${m.readonly ? ',readonly' : ''}`);
+    // `--env NAME` with no value takes it from the client's environment.
+    for (const name of Object.keys(spec.env)) args.push('--env', name);
+    args.push(spec.image, ...spec.command);
+
+    const { child, done } = start(args, { stdin: spec.stdin, env: { ...process.env, ...spec.env }, onOutput: spec.onAlive });
+    let timedOut = false;
+    const stop = () =>
+      void start(['kill', spec.name]).done.then((killed) => {
+        // No container by that name (it never started): end the client instead.
+        if (killed.code !== 0) child.kill('SIGKILL');
+      });
+    const timer = setTimeout(() => {
+      timedOut = true;
+      stop();
+    }, spec.timeoutMs);
+    // A cancelled run: the container goes the same way it does on timeout.
+    if (spec.signal?.aborted) stop();
+    else spec.signal?.addEventListener('abort', stop, { once: true });
+    try {
+      const result = await done;
+      return { exitCode: result.code, stdout: result.stdout, stderr: result.stderr, timedOut };
+    } finally {
+      clearTimeout(timer);
+      spec.signal?.removeEventListener('abort', stop);
+    }
+  }
+}
+
+interface Output {
+  code: number | null;
+  stdout: string;
+  stderr: string;
+}
+
+interface StartOptions {
+  /** Written to the process's stdin, which is then closed. */
+  stdin?: string;
+  env?: NodeJS.ProcessEnv;
+  /** Called on every chunk of stdout or stderr. */
+  onOutput?: () => void;
+}
+
+/** Start `docker <args>`; `done` settles when the process has ended, whatever its exit code. */
+function start(args: string[], options: StartOptions = {}): { child: ChildProcessWithoutNullStreams; done: Promise<Output> } {
+  const child = spawn('docker', args, { env: options.env ?? process.env });
+  const out: Buffer[] = [];
+  const err: Buffer[] = [];
+  child.stdout.on('data', (chunk: Buffer) => {
+    out.push(chunk);
+    options.onOutput?.();
+  });
+  child.stderr.on('data', (chunk: Buffer) => {
+    err.push(chunk);
+    options.onOutput?.();
+  });
+  // The process may end before it has read its input; its exit says what happened.
+  child.stdin.on('error', () => undefined);
+  child.stdin.end(options.stdin ?? '');
+  const done = new Promise<Output>((resolve, reject) => {
+    child.once('error', reject);
+    child.once('close', (code) => resolve({ code, stdout: Buffer.concat(out).toString('utf8'), stderr: Buffer.concat(err).toString('utf8') }));
+  });
+  return { child, done };
+}
+
+/** Run `docker <args>` to its end; a non-zero exit is an error carrying the engine's message. */
+async function docker(args: string[], options: StartOptions = {}): Promise<Output> {
+  const result = await start(args, options).done;
+  if (result.code !== 0) throw failure(args, result);
+  return result;
+}
+
+async function succeeds(args: string[]): Promise<boolean> {
+  return (await start(args).done).code === 0;
+}
+
+function failure(args: string[], result: Output): Error {
+  return new Error(`docker ${args.slice(0, 2).join(' ')} failed (${result.code ?? 'killed'}): ${result.stderr.trim() || result.stdout.trim()}`);
+}
+
+function labelArgs(labels: Record<string, string>): string[] {
+  return Object.entries(labels).flatMap(([key, value]) => ['--label', `${key}=${value}`]);
+}

package/src/executor/driver.ts

@@ -0,0 +1,80 @@
+/**
+ * What an executor needs from the container engine on its host, and nothing
+ * more: an image comes by pull or by building one Dockerfile's text with no
+ * context; a run gets a labelled volume for its workspace; a step is one
+ * container that reads its input on stdin and answers on stdout. The host's
+ * engine implements it (`docker-cli.ts`); the tests use a fake.
+ */
+
+export interface ContainerMount {
+  /** A host path (`bind`) or a volume name (`volume`). */
+  source: string;
+  target: string;
+  readonly: boolean;
+  kind: 'bind' | 'volume';
+}
+
+export interface ContainerLimits {
+  /** As the engine spells it, e.g. `4g`. */
+  memory?: string;
+  /** e.g. `2` */
+  cpus?: string;
+  pids?: number;
+}
+
+export interface ContainerRunSpec {
+  /** The container's name, unique per step attempt. */
+  name: string;
+  image: string;
+  command: string[];
+  /** Written to the process's stdin, then closed. */
+  stdin: string;
+  /** Passed as the process's environment; never on the command line. */
+  env: Record<string, string>;
+  mounts: ContainerMount[];
+  workdir: string;
+  /** `uid:gid` the process runs as; never root. */
+  user: string;
+  limits: ContainerLimits;
+  labels: Record<string, string>;
+  /**
+   * True for a step's own container: it also gets the options the server's
+   * operator set for steps. Absent for the executor's own short commands
+   * (handing a new workspace to the run user, as root), which get none.
+   */
+  operatorOptions?: boolean;
+  /** Stop the container at this age; the result then says `timedOut`. */
+  timeoutMs: number;
+  /** Stop the container when this fires: the run was cancelled. */
+  signal?: AbortSignal;
+  /** Called whenever the container produces output, for the activity's heartbeat. */
+  onAlive?: () => void;
+}
+
+export interface ContainerRunResult {
+  exitCode: number | null;
+  stdout: string;
+  stderr: string;
+  timedOut: boolean;
+}
+
+export interface ContainerVolume {
+  name: string;
+  /** Epoch milliseconds; 0 when the engine does not say. */
+  createdAt: number;
+  labels: Record<string, string>;
+}
+
+export interface ContainerDriver {
+  hasImage(ref: string): Promise<boolean>;
+  pullImage(ref: string): Promise<void>;
+  /** Build `dockerfile` (its text, nothing else as context) and tag it. */
+  buildImage(tag: string, dockerfile: string): Promise<void>;
+  /** Create the volume when it does not exist; an existing one is kept as it is. True when this call made it. */
+  ensureVolume(name: string, labels: Record<string, string>): Promise<boolean>;
+  /** The volumes carrying this label key. */
+  listVolumes(labelKey: string): Promise<ContainerVolume[]>;
+  /** Remove a volume; one that is already gone is not an error. */
+  removeVolume(name: string): Promise<void>;
+  run(spec: ContainerRunSpec): Promise<ContainerRunResult>;
+}

package/src/executor/harness-mount.ts

@@ -0,0 +1,66 @@
+import { execFile } from 'node:child_process';
+import { hostname } from 'node:os';
+import { promisify } from 'node:util';
+
+/** Where the harness bundle is for the engine: a volume by name, or a path on the host. */
+export interface HarnessMount {
+  kind: 'bind' | 'volume';
+  source: string;
+}
+
+/**
+ * Where the harness bundle really is, asked of the engine: the mount at
+ * `dir` in THIS container, which is what run containers must be handed.
+ * The compose file names the volume outright, but a deployment platform
+ * may still create it under a name of its own — Coolify did: the executor
+ * filled `<app>_bevel-harness` while every run container was handed an
+ * empty `bevel-harness` the engine created on first use, and no step could
+ * find the harness. The configured name is the fallback for an engine that
+ * cannot be asked, and only then.
+ */
+export async function discoverHarnessMount(dir: string, inspect: (containerId: string) => Promise<string> = inspectMounts, self: string = hostname()): Promise<HarnessMount | null> {
+  let mounts: Array<{ Type?: string; Name?: string; Source?: string; Destination?: string }>;
+  try {
+    mounts = JSON.parse(await inspect(self)) as typeof mounts;
+  } catch {
+    return null;
+  }
+  if (!Array.isArray(mounts)) return null;
+  const mount = mounts.find((m) => m.Destination === dir);
+  if (!mount) return null;
+  if (mount.Type === 'volume' && mount.Name) return { kind: 'volume', source: mount.Name };
+  if (mount.Type === 'bind' && mount.Source) return { kind: 'bind', source: mount.Source };
+  return null;
+}
+
+/** The engine's view of a container's mounts, as JSON. The container id is this container's hostname, the engine's default. */
+async function inspectMounts(containerId: string): Promise<string> {
+  const { stdout } = await promisify(execFile)('docker', ['inspect', '--format', '{{json .Mounts}}', containerId]);
+  return stdout;
+}
+
+/** The label Compose puts on every container of a project: one per deployment, also under Coolify (the application's id). */
+const COMPOSE_PROJECT_LABEL = 'com.docker.compose.project';
+
+/**
+ * The deployment THIS executor belongs to, asked of the engine: its compose
+ * project. Several deployments may share one engine (a host running more
+ * than one tenant), and each one's executor must tell its own workspace
+ * volumes from the others'. Null when the engine cannot be asked or the
+ * container was not started by Compose.
+ */
+export async function discoverDeployment(inspect: (containerId: string) => Promise<string> = inspectLabels, self: string = hostname()): Promise<string | null> {
+  try {
+    const labels = JSON.parse(await inspect(self)) as unknown;
+    if (!labels || typeof labels !== 'object' || Array.isArray(labels)) return null;
+    const project = (labels as Record<string, unknown>)[COMPOSE_PROJECT_LABEL];
+    return typeof project === 'string' && project.trim() ? project.trim() : null;
+  } catch {
+    return null;
+  }
+}
+
+async function inspectLabels(containerId: string): Promise<string> {
+  const { stdout } = await promisify(execFile)('docker', ['inspect', '--format', '{{json .Config.Labels}}', containerId]);
+  return stdout;
+}

package/src/executor/in-place.ts

@@ -0,0 +1,180 @@
+import { spawn } from 'node:child_process';
+import { mkdir, rm } from 'node:fs/promises';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ApplicationFailure } from '@temporalio/activity';
+import { parseOutcomeLine } from '../cli/run-step.js';
+import type { AgentStepOutcome, ContainerStepInput } from '../interpreter/contract.js';
+import { Redactor } from '../shared/redact.js';
+import type { Executor } from './activities.js';
+
+/**
+ * An executor that runs each step where it runs itself: a process of its
+ * own, in a directory per run, with no container engine. For a place that IS
+ * the environment already and ends with the work — a CI job, whose image,
+ * services and network its own configuration chose. The workflow's
+ * `environment:` (an image, a Dockerfile) is not used here; the job's is.
+ *
+ * What a container gave and this does not: the step is not confined from
+ * the executor or from the rest of the job. It runs as the same user and
+ * sees the same files, so everything in the job is the step's to read.
+ * The names in `withhold` are taken out of the step's environment, which
+ * keeps the executor's own key out of a session that prints its variables.
+ *
+ * A run's directory lasts as long as this process's host keeps it: a later
+ * job starts with none, and the step is told its workspace is new.
+ */
+
+export interface InPlaceExecutorOptions {
+  /** This executor's own task queue, the one runs are pinned to. */
+  queueName: string;
+  /** Where each run's workspace directory is made. */
+  workspacesDir: string;
+  /** Variables of this process the step does not get: the executor's own secrets. */
+  withhold?: string[];
+  /** The command a step is; `run-step` of this package on this node when absent (a test names another). */
+  command?: string[];
+  /** How long to pause when a provider refuses the account and names no reset time. */
+  defaultPauseMs?: number;
+  log?: (line: string) => void;
+  now?: () => number;
+  /** Says the step is alive: the platform gives a silent one up. */
+  heartbeat?: () => void;
+  /** The step's cancellation, as a signal its process is stopped on. */
+  cancellation?: () => AbortSignal | undefined;
+}
+
+const ALIVE_HEARTBEAT_MS = 30_000;
+const STDERR_TAIL = 2_000;
+const STDOUT_KEPT = 4_000_000;
+/** Variables of the job whose values are kept out of a stored reason: the step inherits them, and a crash may print them. */
+const SECRET_NAME = /TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|PRIVATE|_KEY$|^KEY_|APIKEY/i;
+/** A run id as the platform makes them; anything else is not used to name a directory. */
+const RUN_ID = /^[A-Za-z0-9_-]{1,80}$/;
+
+function runStepCommand(): string[] {
+  const require = createRequire(import.meta.url);
+  return [process.execPath, require.resolve('tsx/cli'), fileURLToPath(new URL('../cli/main.ts', import.meta.url)), 'run-step'];
+}
+
+export function createInPlaceExecutor(options: InPlaceExecutorOptions): Executor {
+  const { queueName } = options;
+  const log = options.log ?? (() => undefined);
+  const now = options.now ?? (() => Date.now());
+  const heartbeat = options.heartbeat ?? (() => undefined);
+  const cancellation = options.cancellation ?? (() => undefined);
+  const defaultPauseMs = options.defaultPauseMs ?? 30 * 60_000;
+  const withhold = new Set(options.withhold ?? []);
+  let pausedUntil: number | null = null;
+
+  const workspaceOf = (runId: string): string | null => (RUN_ID.test(runId) ? path.join(options.workspacesDir, `run-${runId}`) : null);
+
+  /** One step as a process: its input on stdin, its outcome the marked last line of stdout. */
+  function runProcess(input: ContainerStepInput, workspace: string): Promise<{ exitCode: number | null; stdout: string; stderr: string; timedOut: boolean; failed: string | null }> {
+    const env: Record<string, string> = {};
+    for (const [name, value] of Object.entries(process.env)) if (value !== undefined && !withhold.has(name)) env[name] = value;
+    // The session's own files (its settings, its saved conversation) sit with the run, as in a run container.
+    Object.assign(env, input.env, { HOME: workspace });
+    const command = options.command ?? runStepCommand();
+    return new Promise((resolve) => {
+      // Its own process group, so the session's children go with it when it is stopped.
+      const group = process.platform !== 'win32';
+      const child = spawn(command[0]!, command.slice(1), { cwd: workspace, env, stdio: ['pipe', 'pipe', 'pipe'], detached: group });
+      let stdout = '';
+      let stderr = '';
+      let timedOut = false;
+      const stopChild = () => {
+        try {
+          if (group && child.pid) process.kill(-child.pid, 'SIGKILL');
+          else child.kill('SIGKILL');
+        } catch {
+          // already gone
+        }
+      };
+      const timer = setTimeout(() => {
+        timedOut = true;
+        stopChild();
+      }, input.timeoutMs);
+      const alive = setInterval(heartbeat, ALIVE_HEARTBEAT_MS);
+      const signal = cancellation();
+      if (signal?.aborted) stopChild();
+      else signal?.addEventListener('abort', stopChild, { once: true });
+      const finish = (exitCode: number | null, failed: string | null) => {
+        clearTimeout(timer);
+        clearInterval(alive);
+        signal?.removeEventListener('abort', stopChild);
+        resolve({ exitCode, stdout, stderr, timedOut, failed });
+      };
+      // Decoded by the stream, which keeps a character whose bytes arrive in two chunks whole.
+      child.stdout.setEncoding('utf8');
+      child.stderr.setEncoding('utf8');
+      child.stdout.on('data', (chunk: string) => {
+        // The outcome is the last line: the end is kept, however much a step prints before it.
+        stdout = (stdout + chunk).slice(-STDOUT_KEPT);
+        heartbeat();
+      });
+      child.stderr.on('data', (chunk: string) => {
+        // The tail is all that is read, and a session can print a great deal.
+        stderr = (stderr + chunk).slice(-64_000);
+        heartbeat();
+      });
+      // A process that died before reading its input: its exit says so.
+      child.stdin.on('error', () => undefined);
+      child.on('error', (err) => finish(null, err.message));
+      child.on('close', (code) => finish(code, null));
+      child.stdin.end(JSON.stringify(input));
+    });
+  }
+
+  return {
+    pausedUntil: () => (pausedUntil !== null && pausedUntil > now() ? pausedUntil : null),
+
+    async runContainerStep(input: ContainerStepInput): Promise<AgentStepOutcome> {
+      const until = pausedUntil !== null && pausedUntil > now() ? pausedUntil : null;
+      if (until !== null) throw ApplicationFailure.retryable(`executor ${queueName} is paused until ${new Date(until).toISOString()}`, 'ExecutorPaused', until);
+      const pin = (outcome: AgentStepOutcome): AgentStepOutcome =>
+        outcome.kind === 'verdict' || outcome.kind === 'no-verdict' || outcome.kind === 'timeout' || outcome.kind === 'account-limit' || outcome.kind === 'harness-error' ? { ...outcome, executorQueue: queueName } : outcome;
+      const workspace = workspaceOf(input.step.runId);
+      if (!workspace) return { kind: 'harness-error', threadId: input.threadId, reason: `run id "${input.step.runId}" cannot name a workspace directory` };
+      let made: string | undefined;
+      try {
+        made = await mkdir(workspace, { recursive: true });
+      } catch (err) {
+        return { kind: 'harness-error', threadId: input.threadId, reason: `the workspace could not be made: ${err instanceof Error ? err.message : String(err)}` };
+      }
+      // A directory made now for a step that is not the run's first: whatever an earlier step left was in another job.
+      const stepInput: ContainerStepInput = made !== undefined && input.step.stepIndex > 1 ? { ...input, freshWorkspace: true } : input;
+      if (cancellation()?.aborted) return { kind: 'harness-error', threadId: input.threadId, reason: 'the run was cancelled before the step started' };
+
+      log(`step "${input.step.stepName}" of run ${input.step.runId} started`);
+      const result = await runProcess(stepInput, workspace);
+      log(`step "${input.step.stepName}" of run ${input.step.runId} ended`);
+      if (result.timedOut) return pin({ kind: 'timeout', threadId: input.threadId, reason: `the step exceeded its budget of ${Math.round(input.timeoutMs / 60_000)} minutes` });
+      const outcome = result.failed === null ? parseOutcomeLine(result.stdout) : null;
+      if (!outcome) {
+        // What the process printed becomes a stored reason: the secrets it was handed are scrubbed from it first,
+        // the job's own among them (a CI job token, a registry password), which a container step never saw.
+        const inherited = Object.entries(process.env).flatMap(([name, value]) => (value && SECRET_NAME.test(name) ? [value] : []));
+        const tail = new Redactor([...Object.values(input.env), input.actorKey.token, ...inherited]).text(result.stderr).trim().slice(-STDERR_TAIL);
+        const how = result.failed !== null ? `could not be started: ${result.failed}` : `ended (${result.exitCode ?? 'killed'}) without an outcome${tail ? `: ${tail}` : ''}`;
+        return pin({ kind: 'harness-error', threadId: input.threadId, reason: `the step's process ${how}` });
+      }
+      if (outcome.kind === 'account-limit') {
+        pausedUntil = outcome.resetAt ?? now() + defaultPauseMs;
+        log(`paused until ${new Date(pausedUntil).toISOString()}: ${outcome.text}`);
+      }
+      return pin(outcome);
+    },
+
+    async releaseRun(input: { runId: string }): Promise<void> {
+      const workspace = workspaceOf(input.runId);
+      if (!workspace) return;
+      await rm(workspace, { recursive: true, force: true });
+      log(`released the workspace of run ${input.runId}`);
+    },
+
+    // Nothing outlives the place this runs in: no volumes to look for.
+    sweepVolumes: async () => [],
+  };
+}