@bevel-software/ael 0.0.0-stage → 0.1.0

package/src/executor/main.ts

@@ -0,0 +1,197 @@
+import { spawn } from 'node:child_process';
+import { readFile, unlink } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { createExecutor } from './activities.js';
+import { DockerCliDriver } from './docker-cli.js';
+import { createInPlaceExecutor } from './in-place.js';
+import { discoverDeployment, discoverHarnessMount } from './harness-mount.js';
+import { personalExecutorHooks, runPersonalExecutor, runWorkerStarter } from './personal.js';
+import { PERSONAL_EXECUTOR_NAME } from '../workflow/definition.js';
+
+/**
+ * `bevel-agent executor`: one executor on this host, configured from the
+ * environment. Every executor asks the platform for its steps over HTTPS and
+ * never talks to Temporal.
+ *
+ *   PLATFORM_URL            the deployment's address, `https://…` (required)
+ *   EXECUTOR_TOKEN          a connection key its owner made by hand (required): for a deployment
+ *                           executor, an account with the Infrastructure Engineer role
+ *   EXECUTOR_NAME           this executor's name, unique on the deployment (required): what runs and
+ *                           triggers name it by; lowercase letters, digits and dashes
+ *   EXECUTOR_KIND           `deployment` (anyone's steps) or `personal` (its owner's runs that name
+ *                           it; the default)
+ *   EXECUTOR_DEPLOYMENT     the deployment it serves, labelled on its workspace volumes so a sweep never
+ *                           touches another deployment's on a shared engine (default: its compose project)
+ *   HARNESS_BUNDLE_VOLUME   the named volume holding the harness bundle (default `bevel-harness`;
+ *                           the executor image fills it at start), or
+ *   HARNESS_BUNDLE_PATH     a path ON THE HOST holding the bundle instead
+ *   EXECUTOR_MAX_STEPS      container steps at once (default 2)
+ *   EXECUTOR_MEMORY         memory per run container (default `4g`)
+ *   EXECUTOR_CPUS           CPUs per run container (default `2`)
+ *   EXECUTOR_PIDS           pids per run container (default 512)
+ *   EXECUTOR_RUN_OPTIONS    extra `docker run` options for every run container on this server, separated
+ *                           by spaces (default none): the operator's choice of what steps here may do
+ *                           beyond the defaults, such as running a rootless Docker daemon inside a step.
+ *                           Split on spaces and nothing else: no quoting, so no value with a space in it
+ *
+ * An executor inside a CI job, which is the environment already and ends with the work:
+ *
+ *   EXECUTOR_IN_PLACE       `true`: each step runs as a process of this executor, in a directory per run,
+ *                           with no container engine; the engine settings above (bundle, limits, run
+ *                           options, deployment) do not apply. One step at once unless EXECUTOR_MAX_STEPS says more
+ *   EXECUTOR_EXIT_WHEN_IDLE seconds without a step after which the executor leaves (default: never)
+ *   EXECUTOR_RUN            a run id: take that run's steps alone, one after another — a worker started
+ *                           for one run, which leaves (EXECUTOR_EXIT_WHEN_IDLE) when the run waits or is over
+ *   EXECUTOR_START_WORKER   a shell command: this executor runs no step. It looks for runs with a step
+ *                           waiting and runs the command once per run, to start a worker elsewhere (another
+ *                           pipeline); the command gets BEVEL_RUN_ID, BEVEL_WORKFLOW, BEVEL_STEP and
+ *                           BEVEL_ENVIRONMENT (the workflow's `environment:` as written: the worker job to start)
+ *   EXECUTOR_TOKEN_FILE     a file holding the key, in place of EXECUTOR_TOKEN: read once and removed, so
+ *                           a step running beside the executor finds it neither in a variable nor on disk
+ */
+export async function executorMain(): Promise<number> {
+  const env = process.env;
+  const platformUrl = env.PLATFORM_URL?.trim().replace(/\/+$/, '');
+  if (!platformUrl) {
+    process.stderr.write("PLATFORM_URL is required: the platform's address, which every executor asks for its steps\n");
+    return 2;
+  }
+  let token = env.EXECUTOR_TOKEN?.trim();
+  const tokenFile = env.EXECUTOR_TOKEN_FILE?.trim();
+  if (!token && tokenFile) {
+    try {
+      token = (await readFile(tokenFile, 'utf8')).trim();
+    } catch (err) {
+      process.stderr.write(`EXECUTOR_TOKEN_FILE could not be read: ${err instanceof Error ? err.message : String(err)}\n`);
+      return 2;
+    }
+    // Read once: a step running beside this executor must not find the key on disk. A file that
+    // cannot be removed (a read-only mount) would stay there for every step to read, so nothing starts.
+    try {
+      await unlink(tokenFile);
+    } catch (err) {
+      process.stderr.write(`EXECUTOR_TOKEN_FILE was read but could not be removed (${err instanceof Error ? err.message : String(err)}): a step would find the key there, so the executor does not start\n`);
+      return 2;
+    }
+  }
+  if (!token) {
+    process.stderr.write('EXECUTOR_TOKEN (or EXECUTOR_TOKEN_FILE) is required: a connection key made by hand on the platform (for a deployment executor, by an account with the Infrastructure Engineer role)\n');
+    return 2;
+  }
+  const name = env.EXECUTOR_NAME?.trim() ?? '';
+  if (!PERSONAL_EXECUTOR_NAME.test(name)) {
+    process.stderr.write('EXECUTOR_NAME is required: the name runs and triggers give this executor (lowercase letters, digits and dashes, at most 40)\n');
+    return 2;
+  }
+  const rawKind = (env.EXECUTOR_KIND?.trim() || 'personal').toLowerCase();
+  if (rawKind !== 'personal' && rawKind !== 'deployment') {
+    process.stderr.write(`EXECUTOR_KIND is "${rawKind}": it is \`deployment\` or \`personal\`\n`);
+    return 2;
+  }
+  const kind = rawKind;
+  const log = (line: string) => process.stderr.write(`[executor ${name}] ${line}\n`);
+  const idleSeconds = Number(env.EXECUTOR_EXIT_WHEN_IDLE);
+  const exitWhenIdleMs = Number.isFinite(idleSeconds) && idleSeconds > 0 ? idleSeconds * 1000 : undefined;
+  const queueName = kind === 'deployment' ? `executor-${name}` : `personal-${name}`;
+  const stop = new AbortController();
+  process.once('SIGINT', () => stop.abort());
+  process.once('SIGTERM', () => stop.abort());
+  const run = env.EXECUTOR_RUN?.trim() || undefined;
+  const startWorker = env.EXECUTOR_START_WORKER?.trim();
+  if (startWorker) {
+    await runWorkerStarter(
+      {
+        apiUrl: `${platformUrl}/api`,
+        token,
+        name,
+        kind,
+        log,
+        exitWhenIdleMs,
+        // The operator's own command, as they wrote it; the run reaches it in variables, never spliced into the line.
+        startWorker: (waiting) => shell(startWorker, { BEVEL_RUN_ID: waiting.runId, BEVEL_WORKFLOW: waiting.workflowPath, BEVEL_STEP: waiting.stepName, BEVEL_ENVIRONMENT: waiting.environment ?? '' }, stop.signal),
+      },
+      stop.signal,
+    );
+    log('stopped');
+    return 0;
+  }
+  if (/^(1|true|yes)$/i.test(env.EXECUTOR_IN_PLACE?.trim() ?? '')) {
+    const executor = createInPlaceExecutor({
+      queueName,
+      workspacesDir: path.join(tmpdir(), 'bevel-runs'),
+      withhold: ['EXECUTOR_TOKEN', 'EXECUTOR_TOKEN_FILE'],
+      log,
+      heartbeat: personalExecutorHooks.heartbeat,
+      cancellation: personalExecutorHooks.cancellation,
+    });
+    log('steps run in place: as processes of this executor, with no container engine');
+    await runPersonalExecutor({ apiUrl: `${platformUrl}/api`, token, name, kind, slots: Number(env.EXECUTOR_MAX_STEPS) || 1, executor, log, exitWhenIdleMs, run }, stop.signal);
+    log('stopped');
+    return 0;
+  }
+  const runOptions = (env.EXECUTOR_RUN_OPTIONS ?? '').split(/\s+/).filter(Boolean);
+  // The executor waits on each run container and removes it when it ends: one started detached would outlive
+  // its step, unwatched.
+  if (runOptions.some((option) => option === '-d' || option === '--detach' || option.startsWith('--detach='))) {
+    process.stderr.write('EXECUTOR_RUN_OPTIONS has --detach: the executor waits on every run container, so that option is refused\n');
+    return 2;
+  }
+  // The count, never the options: one may carry a value that should not be in a log.
+  if (runOptions.length > 0) log(`every run container here also gets ${runOptions.length} option(s) from EXECUTOR_RUN_OPTIONS`);
+  const executor = createExecutor({
+    driver: new DockerCliDriver(runOptions),
+    // What the platform pins a run to is its own queue for this executor; it rewrites the pin on the way back.
+    queueName,
+    harnessBundle: await harnessBundleOf(env, log),
+    deployment: await deploymentOf(env, log),
+    limits: limitsOf(env),
+    log,
+    heartbeat: personalExecutorHooks.heartbeat,
+    cancellation: personalExecutorHooks.cancellation,
+  });
+  await runPersonalExecutor({ apiUrl: `${platformUrl}/api`, token, name, kind, slots: Number(env.EXECUTOR_MAX_STEPS) || 2, executor, log, exitWhenIdleMs, run }, stop.signal);
+  log('stopped');
+  return 0;
+}
+
+/**
+ * The bundle run containers get: the one the engine mounted into THIS
+ * container, asked of the engine — a deployment platform may have created
+ * the volume under a name of its own, and the configured name would then
+ * reach run containers as an empty volume. Configured names are the fallback
+ * for an engine that cannot be asked.
+ */
+async function harnessBundleOf(env: NodeJS.ProcessEnv, log: (line: string) => void): Promise<{ kind: 'bind' | 'volume'; source: string }> {
+  const configured: { kind: 'bind' | 'volume'; source: string } = env.HARNESS_BUNDLE_PATH?.trim() ? { kind: 'bind', source: env.HARNESS_BUNDLE_PATH.trim() } : { kind: 'volume', source: env.HARNESS_BUNDLE_VOLUME?.trim() || 'bevel-harness' };
+  const bundleDir = env.HARNESS_BUNDLE_DIR?.trim() || '/harness';
+  const mounted = await discoverHarnessMount(bundleDir);
+  if (!mounted) log(`the engine did not say what is mounted at ${bundleDir}; run containers get the configured ${configured.kind} ${configured.source}`);
+  else if (mounted.source !== configured.source) log(`run containers get the ${mounted.kind} ${mounted.source} mounted at ${bundleDir}, not the configured ${configured.source}`);
+  return mounted ?? configured;
+}
+
+/** The deployment this executor serves: the configured one, else its compose project as the engine names it. */
+async function deploymentOf(env: NodeJS.ProcessEnv, log: (line: string) => void): Promise<string | null> {
+  const configured = env.EXECUTOR_DEPLOYMENT?.trim();
+  if (configured) return configured;
+  const discovered = await discoverDeployment();
+  log(discovered ? `serving deployment ${discovered}` : 'the engine did not name this executor\'s compose project; set EXECUTOR_DEPLOYMENT, or its workspace sweep stays off');
+  return discovered;
+}
+
+/**
+ * Run the operator's command in a shell, with `vars` added to this process's environment; rejects when it fails.
+ * It starts a worker elsewhere and returns: one that hangs is ended after two minutes, or when the executor stops.
+ */
+function shell(command: string, vars: Record<string, string>, stop: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, { shell: true, stdio: ['ignore', 'inherit', 'inherit'], env: { ...process.env, ...vars }, signal: AbortSignal.any([stop, AbortSignal.timeout(120_000)]), killSignal: 'SIGKILL' });
+    child.on('error', reject);
+    child.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`the command ended with ${code ?? 'a signal'}`))));
+  });
+}
+
+function limitsOf(env: NodeJS.ProcessEnv) {
+  return { memory: env.EXECUTOR_MEMORY?.trim() || '4g', cpus: env.EXECUTOR_CPUS?.trim() || '2', pids: Number(env.EXECUTOR_PIDS) || 512 };
+}

package/src/executor/personal.ts

@@ -0,0 +1,355 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { randomUUID } from 'node:crypto';
+import type { AgentStepOutcome, ContainerStepInput } from '../interpreter/contract.js';
+import type { Executor } from './activities.js';
+
+/**
+ * An executor's line to the platform. Every executor talks to the platform
+ * over HTTPS with a connection key its owner made by hand, and never to
+ * Temporal: no payload key, no certificate, no Temporal port reachable from
+ * outside. Two kinds, by name in one namespace:
+ *
+ * - a person's own (`personal`), run on their machine so the container steps
+ *   of their runs that name it run there — behind their VPN, inside their
+ *   network; the platform hands it only that person's runs;
+ * - one of the deployment's (`deployment`), on any server, whose key's owner
+ *   holds the Infrastructure Engineer role; the platform hands it anyone's
+ *   steps from the shared queue, and every step of a run that names it.
+ *
+ * Everything else — the image, the harness bundle, the workspace volume, the
+ * limits, the account-limit pause — is the ordinary executor's.
+ *
+ * The protocol, under the platform's `/api`:
+ *
+ *   POST /agents/executors/personal/poll            { name, kind, instance, run? } → 200 { job } | 204
+ *   POST /agents/executors/personal/waiting         { name, kind, instance } → { runs }
+ *   POST /agents/executors/personal/jobs/:id/alive  {}              → { cancelled }
+ *   POST /agents/executors/personal/jobs/:id/result PersonalJobResult
+ *   POST /agents/executors/personal/runs-open       { runIds }      → { open }
+ *
+ * `instance` is made once per executor process. Several processes of one
+ * owner may ask under one name: with `run`, a process takes that run's jobs
+ * alone, and `waiting` lists the runs with a step waiting that no process
+ * asks for (it takes nothing). That is how an executor in a CI pipeline
+ * works: one job looks and starts a worker job per run. A name held by
+ * another owner or kind is refused (409).
+ */
+
+export const PERSONAL_EXECUTOR_API = '/agents/executors/personal';
+
+/** What an executor process's `instance` looks like: a UUID fits. */
+export const EXECUTOR_INSTANCE = /^[A-Za-z0-9-]{8,64}$/;
+
+/** What the platform hands a personal executor: a step to run, or a finished run's workspace to remove. */
+export type PersonalJob = { id: string; kind: 'step'; input: ContainerStepInput } | { id: string; kind: 'release'; input: { runId: string } };
+
+/** What the executor answers for a job: the step's outcome, a release done, or why it could not. */
+export type PersonalJobResult = { outcome: AgentStepOutcome } | { done: true } | { error: string; pausedUntil?: number | null };
+
+type Fetch = (url: string, init: { method: string; headers: Record<string, string>; body?: string; signal?: AbortSignal }) => Promise<{ status: number; ok: boolean; json(): Promise<unknown>; text(): Promise<string> }>;
+
+export interface PersonalExecutorOptions {
+  /** The platform's API base, `https://<deployment>/api`. */
+  apiUrl: string;
+  /** A connection key its owner made by hand: a personal executor runs that person's steps. */
+  token: string;
+  /** This executor's name, unique on the deployment: what a run or trigger names it by. */
+  name: string;
+  /** A person's own executor, or one of the deployment's. */
+  kind: 'personal' | 'deployment';
+  /** Steps at once. */
+  slots: number;
+  /** The executor, built with {@link personalExecutorHooks}' heartbeat and cancellation. */
+  executor: Executor;
+  log: (line: string) => void;
+  fetch?: Fetch;
+  /** How long to wait before asking again after the platform did not answer. */
+  retryMs?: number;
+  /** How often the workspace volumes of runs that are over are swept. */
+  sweepIntervalMs?: number;
+  /** Least time between two "still alive" reports of one step. */
+  aliveEveryMs?: number;
+  /** How long one attempt to deliver a result may take before it is given up and tried again. */
+  resultTimeoutMs?: number;
+  /**
+   * Leave once no step is running and none came for this long: an executor
+   * inside a CI job, which should end when the work has. Judged between two
+   * asks, never during one, so no step is handed to an executor that left.
+   * Absent: run until stopped.
+   */
+  exitWhenIdleMs?: number;
+  /**
+   * Take the jobs of this run alone: a worker started for one run, which
+   * works it step after step and leaves (`exitWhenIdleMs`) when no step
+   * comes — the run waits at a gate, or is over.
+   */
+  run?: string;
+}
+
+interface JobScope {
+  heartbeat(): void;
+  signal: AbortSignal;
+}
+const scope = new AsyncLocalStorage<JobScope>();
+
+/**
+ * The heartbeat and cancellation an executor running under
+ * {@link runPersonalExecutor} is built with: each step's own, reported to
+ * the platform rather than to Temporal.
+ */
+export const personalExecutorHooks = {
+  heartbeat: (): void => scope.getStore()?.heartbeat(),
+  cancellation: (): AbortSignal | undefined => scope.getStore()?.signal,
+};
+
+/** Run until `stop` aborts: `slots` loops, each asking the platform for a step, running it and answering. */
+export async function runPersonalExecutor(options: PersonalExecutorOptions, stop: AbortSignal): Promise<void> {
+  const fetchImpl: Fetch = options.fetch ?? (globalThis.fetch as unknown as Fetch);
+  const retryMs = options.retryMs ?? 5_000;
+  const aliveEveryMs = options.aliveEveryMs ?? 10_000;
+  const resultTimeoutMs = options.resultTimeoutMs ?? 30_000;
+  /** The person this executor runs for, once a step has said so. */
+  let owner: string | null = null;
+  /** This process, the same on every poll of every slot: a second machine under the name is refused. */
+  const instance = randomUUID();
+  const base = `${options.apiUrl.replace(/\/+$/, '')}${PERSONAL_EXECUTOR_API}`;
+  const headers = { authorization: `Bearer ${options.token}`, 'content-type': 'application/json' };
+  const post = (path: string, body: unknown, signal?: AbortSignal) => fetchImpl(`${base}${path}`, { method: 'POST', headers, body: JSON.stringify(body), signal });
+  const wait = (ms: number) => new Promise<void>((resolve) => {
+    const t = setTimeout(resolve, ms);
+    stop.addEventListener('abort', () => { clearTimeout(t); resolve(); }, { once: true });
+  });
+
+  async function answer(job: PersonalJob, result: PersonalJobResult, stillAlive: () => void = () => undefined): Promise<void> {
+    // The result must land: the platform's step waits on it. Retried until it does, or the executor stops —
+    // saying "still alive" the whole time, or the platform gives the finished step up as silent; and no
+    // attempt may hang, or it would hold the delivery up without end.
+    const keepAlive = setInterval(stillAlive, Math.max(aliveEveryMs, 1));
+    keepAlive.unref?.();
+    try {
+      await deliver();
+    } finally {
+      clearInterval(keepAlive);
+    }
+
+    async function deliver(): Promise<void> {
+    for (;;) {
+      try {
+        const res = await post(`/jobs/${encodeURIComponent(job.id)}/result`, result, AbortSignal.timeout(resultTimeoutMs));
+        if (res.ok || res.status === 404) return; // 404: the platform gave the step up meanwhile; nothing waits on it
+        if (res.status === 408 || res.status === 429 || res.status >= 500) {
+          // Busy, restarting, or a proxy in between: the answer is still wanted.
+          if (stop.aborted) return;
+          options.log(`job ${job.id}: the platform could not take the result yet (${res.status}); trying again`);
+          await wait(retryMs);
+          continue;
+        }
+        // Any other refusal is about the answer itself, and the same answer would be refused again.
+        options.log(`job ${job.id}: the result was refused (${res.status}): ${await res.text()}`);
+        return;
+      } catch (err) {
+        if (stop.aborted) return;
+        options.log(`job ${job.id}: the result did not reach the platform (${errText(err)}); trying again`);
+        await wait(retryMs);
+      }
+    }
+    }
+  }
+
+  async function run(job: PersonalJob): Promise<void> {
+    // Whose executor this is, learnt from the first step it is given (every step is that person's run).
+    if (job.kind === 'step') owner ??= job.input.step?.actor?.userId ?? null;
+    if (job.kind === 'release') {
+      try {
+        await options.executor.releaseRun(job.input);
+        await answer(job, { done: true });
+      } catch (err) {
+        await answer(job, { error: errText(err) });
+      }
+      return;
+    }
+    // The step's cancellation: the platform says so in its answer to "still alive".
+    const cancelled = new AbortController();
+    let lastAlive = 0;
+    const heartbeat = () => {
+      const now = Date.now();
+      if (now - lastAlive < aliveEveryMs) return;
+      lastAlive = now;
+      void post(`/jobs/${encodeURIComponent(job.id)}/alive`, {})
+        .then(async (res) => {
+          if (res.status === 404) cancelled.abort(); // the platform no longer holds the step
+          else if (res.ok && ((await res.json()) as { cancelled?: boolean }).cancelled) cancelled.abort();
+        })
+        .catch((err) => options.log(`job ${job.id}: alive report failed: ${errText(err)}`));
+    };
+    let result: PersonalJobResult;
+    try {
+      const outcome = await scope.run({ heartbeat, signal: cancelled.signal }, () => options.executor.runContainerStep(job.input));
+      result = { outcome };
+    } catch (err) {
+      // The executor's account-limit pause names its end as the failure's detail; the platform waits for it.
+      const details = (err as { details?: unknown[] })?.details;
+      const type = (err as { type?: string })?.type;
+      result = { error: errText(err), ...(type === 'ExecutorPaused' && typeof details?.[0] === 'number' ? { pausedUntil: details[0] } : {}) };
+    }
+    await answer(job, result, heartbeat);
+  }
+
+  /** Steps running now, over every slot, and when the last one ended (or this executor started). */
+  let running = 0;
+  let lastWork = Date.now();
+  let left = false;
+  /** Whether it is time to leave; says so once. Asked only between two asks for work. */
+  const leaves = (): boolean => {
+    if (left) return true;
+    if (options.exitWhenIdleMs === undefined || running > 0 || Date.now() - lastWork < options.exitWhenIdleMs) return false;
+    left = true;
+    options.log(`no step for ${Math.round(options.exitWhenIdleMs / 1000)} s: leaving`);
+    return true;
+  };
+
+  async function loop(): Promise<void> {
+    while (!stop.aborted && !leaves()) {
+      let job: PersonalJob | null = null;
+      try {
+        const res = await post('/poll', { name: options.name, slots: options.slots, kind: options.kind, instance, ...(options.run ? { run: options.run } : {}) }, stop);
+        if (res.status === 204) continue;
+        if (!res.ok) {
+          options.log(`the platform refused the poll (${res.status}): ${(await res.text()).slice(0, 300)}`);
+          await wait(retryMs);
+          continue;
+        }
+        job = ((await res.json()) as { job?: PersonalJob }).job ?? null;
+      } catch (err) {
+        if (stop.aborted) return;
+        options.log(`the platform did not answer (${errText(err)}); asking again in ${Math.round(retryMs / 1000)} s`);
+        await wait(retryMs);
+        continue;
+      }
+      if (!job) continue;
+      running += 1;
+      try {
+        await run(job);
+      } finally {
+        running -= 1;
+        lastWork = Date.now();
+      }
+    }
+  }
+
+  // The workspace volumes of runs that ended while this machine was off: their release never came.
+  const runIsOpen = async (runId: string): Promise<boolean> => {
+    try {
+      const res = await post('/runs-open', { runIds: [runId] });
+      if (!res.ok) return true;
+      return ((await res.json()) as { open?: string[] }).open?.includes(runId) ?? true;
+    } catch {
+      return true; // unknown is kept: a volume removed under a live run loses its work
+    }
+  };
+  const sweep = setInterval(() => {
+    // A person's: only their volumes, and none before it knows the person (another person's executor may share the engine).
+    // The deployment's: only the volumes its deployment labelled (another deployment's may share the engine).
+    void options.executor.sweepVolumes(runIsOpen, options.kind === 'personal' ? owner : undefined).catch((err) => options.log(`volume sweep failed: ${errText(err)}`));
+  }, options.sweepIntervalMs ?? 60 * 60_000);
+  sweep.unref?.();
+  options.log(`${options.kind === 'personal' ? 'personal' : "the deployment's"} executor "${options.name}" asking ${options.apiUrl} for steps, up to ${options.slots} at once`);
+  try {
+    await Promise.all(Array.from({ length: Math.max(1, options.slots) }, () => loop()));
+  } finally {
+    clearInterval(sweep);
+  }
+}
+
+/** A run with a step waiting for an executor and no worker asking for it, as the platform lists them. */
+export interface WaitingRun {
+  runId: string;
+  workflowPath: string;
+  stepName: string;
+  /** The workflow's `environment:` as written; null for none. */
+  environment: string | null;
+}
+
+export interface WorkerStarterOptions {
+  apiUrl: string;
+  token: string;
+  name: string;
+  kind: 'personal' | 'deployment';
+  /** Start a worker for a run: a process elsewhere (a CI pipeline) that asks for that run's steps under the same name. */
+  startWorker: (run: WaitingRun) => Promise<void>;
+  log: (line: string) => void;
+  fetch?: Fetch;
+  /** How often to look for waiting runs. */
+  everyMs?: number;
+  /** How long a started worker is given to come up before another is started for the same run. */
+  workerStartMs?: number;
+  /** Leave once no run waited for this long. Absent: run until stopped. */
+  exitWhenIdleMs?: number;
+}
+
+/**
+ * An executor that runs no step itself: it looks for the runs with a step
+ * waiting under its name and starts a worker for each, until `stop` aborts
+ * or nothing waited for the idle time. Its looking keeps the name served, so
+ * the steps stay held while the workers come up.
+ */
+export async function runWorkerStarter(options: WorkerStarterOptions, stop: AbortSignal): Promise<void> {
+  const fetchImpl: Fetch = options.fetch ?? (globalThis.fetch as unknown as Fetch);
+  const everyMs = options.everyMs ?? 10_000;
+  const workerStartMs = options.workerStartMs ?? 5 * 60_000;
+  const instance = randomUUID();
+  const url = `${options.apiUrl.replace(/\/+$/, '')}${PERSONAL_EXECUTOR_API}/waiting`;
+  const headers = { authorization: `Bearer ${options.token}`, 'content-type': 'application/json' };
+  /** When a worker was last started for each run. */
+  const started = new Map<string, number>();
+  let lastWork = Date.now();
+  options.log(`executor "${options.name}" looking at ${options.apiUrl} for runs to start workers for`);
+  while (!stop.aborted) {
+    try {
+      const res = await fetchImpl(url, { method: 'POST', headers, body: JSON.stringify({ name: options.name, kind: options.kind, instance }), signal: stop });
+      if (!res.ok) {
+        options.log(`the platform refused the look (${res.status}): ${(await res.text()).slice(0, 300)}`);
+        // Not idle: nothing says no run waits. Only a look that was answered counts towards leaving.
+        lastWork = Date.now();
+      } else {
+        const runs = ((await res.json()) as { runs?: WaitingRun[] }).runs ?? [];
+        if (runs.length > 0) lastWork = Date.now();
+        for (const run of runs) {
+          const at = started.get(run.runId);
+          // A worker is on its way for it: the platform stops listing the run once that worker asks.
+          if (at !== undefined && Date.now() - at < workerStartMs) continue;
+          started.set(run.runId, Date.now());
+          try {
+            await options.startWorker(run);
+            options.log(`started a worker for run ${run.runId} (${run.stepName})`);
+          } catch (err) {
+            // Tried again at the next look, not after the wait a started worker is given.
+            started.delete(run.runId);
+            options.log(`a worker for run ${run.runId} could not be started: ${errText(err)}`);
+          }
+        }
+      }
+    } catch (err) {
+      if (stop.aborted) return;
+      options.log(`the platform did not answer (${errText(err)})`);
+      lastWork = Date.now();
+    }
+    if (options.exitWhenIdleMs !== undefined && Date.now() - lastWork >= options.exitWhenIdleMs) {
+      options.log(`no run waited for ${Math.round(options.exitWhenIdleMs / 1000)} s: leaving`);
+      return;
+    }
+    await new Promise<void>((resolve) => {
+      const done = () => {
+        clearTimeout(t);
+        stop.removeEventListener('abort', done);
+        resolve();
+      };
+      const t = setTimeout(done, everyMs);
+      stop.addEventListener('abort', done, { once: true });
+    });
+  }
+}
+
+function errText(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}

package/src/executor/slots.ts

@@ -0,0 +1,60 @@
+/**
+ * One limit for everything an executor runs at once. Its two Temporal workers
+ * (the shared queue and its own) each cap their activities on their own, so
+ * without this each queue could fill the host by itself; a step that arrives
+ * while every slot is taken waits here, and says so while it waits so the
+ * activity's heartbeat keeps the task alive. A waiter whose activity is
+ * cancelled (the run cancelled, the activity timed out) leaves the queue at
+ * once: a slot must never go to a step nobody waits for.
+ */
+export class Slots {
+  private free: number;
+  private readonly waiting: Array<{ grant: () => void; drop: () => void }> = [];
+
+  constructor(size: number) {
+    this.free = Math.max(1, size);
+  }
+
+  async run<T>(fn: () => Promise<T>, whileWaiting?: { every: number; tick: () => void }, signal?: AbortSignal): Promise<T> {
+    await this.take(whileWaiting, signal);
+    try {
+      return await fn();
+    } finally {
+      this.free += 1;
+      this.waiting.shift()?.grant();
+    }
+  }
+
+  private take(whileWaiting?: { every: number; tick: () => void }, signal?: AbortSignal): Promise<void> {
+    if (signal?.aborted) return Promise.reject(abortReason(signal));
+    if (this.free > 0) {
+      this.free -= 1;
+      return Promise.resolve();
+    }
+    return new Promise<void>((resolve, reject) => {
+      const timer = whileWaiting ? setInterval(whileWaiting.tick, whileWaiting.every) : null;
+      timer?.unref?.();
+      const waiter = {
+        grant: () => {
+          if (timer) clearInterval(timer);
+          signal?.removeEventListener('abort', onAbort);
+          this.free -= 1;
+          resolve();
+        },
+        drop: () => {
+          if (timer) clearInterval(timer);
+          const at = this.waiting.indexOf(waiter);
+          if (at !== -1) this.waiting.splice(at, 1);
+          reject(abortReason(signal!));
+        },
+      };
+      const onAbort = () => waiter.drop();
+      signal?.addEventListener('abort', onAbort, { once: true });
+      this.waiting.push(waiter);
+    });
+  }
+}
+
+function abortReason(signal: AbortSignal): unknown {
+  return signal.reason ?? new Error('cancelled while waiting for a slot');
+}