@wringer/core 0.1.0 → 0.1.2

package/src/run-supervisor.ts

@@ -0,0 +1,327 @@
+import { dirname } from 'node:path';
+import { match } from 'ts-pattern';
+import { createActor, waitFor } from 'xstate';
+import { buildRunHandle } from './build-run-handle.ts';
+import { buildShutDownSteps } from './build-shut-down-steps.ts';
+import type { StatePort } from './create-file-state.ts';
+import { getTarget } from './get-target.ts';
+import { defaultRefusalHint } from './make-fence.ts';
+import { pickPendingCleanups } from './pick-pending-cleanups.ts';
+import { pickSupervisorOperation } from './pick-supervisor-operation.ts';
+import { raceDeadline } from './race-deadline.ts';
+import type { LaunchHost } from './read-launch-host.ts';
+import type { Endpoints, RunEvent } from './run-events.ts';
+import { appStopGraceMs, startLaunch } from './start-launch.ts';
+import type { Launch } from './start-launch.ts';
+import { startServices } from './start-services.ts';
+import { supervisorMachine } from './supervisor-machine.ts';
+import type { ShutDownPlan } from './supervisor-machine.ts';
+import type { SupervisorRequest } from './supervisor-protocol.ts';
+import { toWringerError } from './to-wringer-error.ts';
+import type { ServiceInstance, TargetDefinition, Wringer } from './types.ts';
+import type { WringerError } from './wringer-error.ts';
+
+export interface SupervisorOptions {
+  readonly wringer: Wringer;
+  readonly launchHost: LaunchHost;
+  readonly state: StatePort;
+  readonly runID: string;
+  readonly target: string;
+  readonly readyTimeoutMs: number;
+
+  // The longest a stop waits for a startup step, or for one cleanup, that ignores the abort.
+  readonly stopWaitMs: number;
+  readonly now: () => Date;
+  readonly reportError: (message: string) => void;
+  readonly signal?: AbortSignal | undefined;
+}
+
+type Cleanup = readonly [string, () => Promise<void>];
+
+// The supervisor owns a run's app and every driver service. It resolves once the run has stopped,
+// by `stop` over the socket or by a crash of the app. Its lifecycle is the supervisor machine; the
+// functions here are the I/O the machine runs.
+export async function runSupervisor(options: SupervisorOptions): Promise<void> {
+  const target = getTarget(options.wringer, options.target);
+  const run = buildRunHandle(options.state.paths.root, options.runID);
+
+  const socket = await options.launchHost.createSocketFolder(run.paths.socket);
+
+  const services = new Map<string, ServiceInstance>();
+  const lifetime = new AbortController();
+  const serviceCommands = new AbortController();
+  const startupCommands = new AbortController();
+
+  const append = (event: RunEvent): Promise<void> =>
+    options.state.appendEvent(options.runID, event);
+
+  const at = (): string => options.now().toISOString();
+  let launch: Launch | undefined;
+
+  const stopping = new Set<string>();
+
+  // Each part a failed startup tried to stop and could not. The startup's error never reaches the
+  // stop that cancelled it, so the stop takes these from here.
+  const startupLeftovers: string[] = [];
+
+  // Settles once the app's stop has, past its cleanup wait too, so no service, such as a stream
+  // recorder, stops while the app can still log.
+  let appStopped: Promise<unknown> = Promise.resolve();
+
+  // Names each part whose stop failed. A part is marked before its stop starts, so two calls never
+  // stop the same part.
+  const stopStarted = async (): Promise<readonly string[]> => {
+    const current = launch;
+
+    const pending = pickPendingCleanups({
+      launched: current !== undefined,
+      serviceKeys: [...services.keys()],
+      stopping,
+    });
+
+    for (const key of [...pending.app, ...pending.services]) {
+      stopping.add(key);
+    }
+
+    const appCleanups = pending.app.flatMap((key): Cleanup[] =>
+      current === undefined
+        ? []
+        : [
+            [
+              key,
+              () => {
+                const stoppingApp = current.stop();
+
+                appStopped = Promise.allSettled([stoppingApp]);
+
+                return stoppingApp;
+              },
+            ],
+          ],
+    );
+
+    const serviceCleanups = pending.services.flatMap((key): Cleanup[] => {
+      const service = services.get(key);
+
+      return service === undefined ? [] : [[key, () => service.stop()]];
+    });
+
+    const appFailures = await runCleanups(appCleanups, options.stopWaitMs);
+
+    // The app's own stop ends within its limit, so this wait has a bound.
+    await appStopped;
+
+    const serviceFailures = await runCleanups(serviceCleanups, options.stopWaitMs);
+
+    return [...startupLeftovers.splice(0), ...appFailures, ...serviceFailures];
+  };
+
+  // Every cleanup runs, even after one fails, and the supervisor ends even when the final log write
+  // fails. A failed cleanup or log write goes to the stop's caller, as a part may still be up.
+  const shutDown = async (plan: ShutDownPlan): Promise<WringerError | null> => {
+    // The commands of a startup the stop gave up on end before the cleanups, so that startup settles
+    // early. A service that came up has left this signal, so its stop can still run a command.
+    if (plan.gaveUpOnStartup) {
+      startupCommands.abort();
+    }
+
+    const failures = await stopStarted();
+
+    const steps = buildShutDownSteps({ ...plan, failures, at: at() });
+
+    if (steps.abortCommands) {
+      serviceCommands.abort();
+    }
+
+    try {
+      for (const event of steps.events) {
+        await append(event);
+      }
+
+      await options.state.writeManifest(options.runID);
+    } catch (error) {
+      const failure = toWringerError(error);
+
+      // A stop that a crash or a signal began has no caller to take the failure.
+      options.reportError(`The run's final log write failed: ${failure.message}`);
+
+      return failure;
+    }
+
+    return steps.failure;
+  };
+
+  // The run's log is already closed, so a late part's failure goes to the supervisor's own log.
+  const cleanUpLate = async (): Promise<void> => {
+    for (const failure of await stopStarted()) {
+      options.reportError(`A late cleanup failed for ${failure}`);
+    }
+
+    serviceCommands.abort();
+  };
+
+  // A failed first write stops the run like any startup failure, so the socket still closes.
+  const startUp = async (): Promise<void> => {
+    await append({ type: 'supervisor-ready', at: at(), pid: process.pid, socket });
+
+    launch = await startTarget(target, {
+      launchHost: options.launchHost,
+      runDirectory: run.paths.directory,
+      appLog: run.childLog('app').path,
+      targetName: options.target,
+      readyTimeoutMs: options.readyTimeoutMs,
+      signal: lifetime.signal,
+      onCrash: (reason) => {
+        void stop(reason);
+      },
+      onStopFailed: (reason) => {
+        startupLeftovers.push(`app: ${reason}`);
+      },
+    });
+
+    lifetime.signal.throwIfAborted();
+
+    await append({ type: 'launch-ready', at: at(), endpoints: launch.endpoints });
+
+    await startServices({
+      wringer: options.wringer,
+      targetName: options.target,
+      target,
+      endpoints: launch.endpoints,
+      run,
+      hint: target.refusalHint ?? defaultRefusalHint,
+      signal: lifetime.signal,
+      commandSignal: serviceCommands.signal,
+      startupCommandSignal: startupCommands.signal,
+      started: services,
+      onReady: (key) => append({ type: 'service-ready', at: at(), service: key }),
+    });
+
+    lifetime.signal.throwIfAborted();
+
+    await append({ type: 'run-ready', at: at() });
+
+    await options.state.writeManifest(options.runID);
+  };
+
+  const lifecycle = createActor(supervisorMachine, {
+    input: {
+      stopWaitMs: options.stopWaitMs,
+      startUp,
+      abortStartup: () => {
+        lifetime.abort();
+      },
+      shutDown,
+      cleanUpLate,
+    },
+  });
+
+  const waitForClosed = () =>
+    waitFor(lifecycle, (snapshot) => snapshot.hasTag('closed'), { timeout: Infinity });
+
+  // A second stop, such as a `down` while a crash is still stopping the run, waits for the first.
+  const stop = async (crashReason: string | null): Promise<WringerError | null> => {
+    lifecycle.send({ type: 'stop', reason: crashReason });
+
+    const closed = await waitForClosed();
+
+    return closed.context.failure;
+  };
+
+  const answerRequest = (request: SupervisorRequest, signal: AbortSignal): Promise<unknown> =>
+    match(pickSupervisorOperation(request, services, options.now().getTime()))
+      .with({ kind: 'stop' }, async () => {
+        const failure = await stop(null);
+
+        if (failure !== null) {
+          throw failure;
+        }
+
+        return { pid: process.pid };
+      })
+      .with({ kind: 'status' }, () => Promise.resolve({ pid: process.pid }))
+      .with({ kind: 'call' }, (operation) => {
+        const timeouts = operation.cancelAfterMs.map((delayMs) => AbortSignal.timeout(delayMs));
+
+        return operation.call(AbortSignal.any([signal, ...timeouts]));
+      })
+      .with({ kind: 'refuse' }, (operation) => Promise.reject(operation.error))
+      .exhaustive();
+
+  const server = await options.launchHost.listenSupervisorSocket({
+    path: socket,
+    answerRequest,
+    onTransportError: (error) => {
+      options.reportError(`The supervisor socket failed: ${toWringerError(error).message}`);
+    },
+  });
+
+  // The startup writes the socket into the run's log, so it starts once the socket listens.
+  lifecycle.start();
+
+  const onSignal = (): void => {
+    void stop('the supervisor was signalled to stop');
+  };
+
+  options.signal?.addEventListener('abort', onSignal, { once: true });
+
+  // A signal that came while the socket opened fired before the listener, which never sees it.
+  if (options.signal?.aborted === true) {
+    onSignal();
+  }
+
+  await waitForClosed();
+
+  await server.close(options.stopWaitMs);
+  await options.launchHost.removeDirectory(dirname(socket));
+}
+
+// Each cleanup runs, even after one fails, and each failure names its part.
+async function runCleanups(cleanups: readonly Cleanup[], stopWaitMs: number): Promise<string[]> {
+  const settled = await Promise.allSettled(
+    cleanups.map(([, cleanup]) =>
+      raceDeadline(Promise.try(cleanup), AbortSignal.timeout(stopWaitMs)),
+    ),
+  );
+
+  return settled.flatMap((result, index) =>
+    result.status === 'rejected'
+      ? [`${cleanups[index]?.[0] ?? 'unknown'}: ${toWringerError(result.reason).message}`]
+      : [],
+  );
+}
+
+interface TargetStart {
+  readonly launchHost: LaunchHost;
+  readonly runDirectory: string;
+  readonly appLog: string;
+  readonly targetName: string;
+  readonly readyTimeoutMs: number;
+  readonly signal: AbortSignal;
+  readonly onCrash: (reason: string) => void;
+  readonly onStopFailed: (reason: string) => void;
+}
+
+async function startTarget(target: TargetDefinition, start: TargetStart): Promise<Launch> {
+  if (target.mode === 'launch') {
+    return startLaunch({
+      host: start.launchHost,
+      plan: target.launch({ runDirectory: start.runDirectory }),
+      logPath: start.appLog,
+      readyTimeoutMs: start.readyTimeoutMs,
+      signal: start.signal,
+      onCrash: start.onCrash,
+      onStopFailed: start.onStopFailed,
+      stopGraceMs: appStopGraceMs,
+    });
+  }
+
+  // The run owns no app of an attach target, so its stop is a no-op, and the services stop at once.
+  if (target.endpoints === undefined) {
+    return { endpoints: {}, stop: () => Promise.resolve() };
+  }
+
+  const endpoints: Endpoints = await target.endpoints({ target: start.targetName });
+
+  return { endpoints, stop: () => Promise.resolve() };
+}

package/src/settle-dispatch-answer.ts

@@ -0,0 +1,26 @@
+import { buildEnvelope } from './build-envelope.ts';
+import type { EnvelopeInput } from './build-envelope.ts';
+import type { Envelope } from './envelope.ts';
+import { exitCodeFor } from './error-codes.ts';
+import type { ExitCode } from './error-codes.ts';
+import type { WringerError } from './wringer-error.ts';
+
+export interface DispatchAnswer {
+  readonly envelope: Envelope;
+  readonly exitCode: ExitCode;
+}
+
+// A failed write of the run's log leaves the command out of the log, so the answer carries that
+// failure in place of the command's own outcome, and no data or truncated payload.
+export function settleDispatchAnswer(
+  input: EnvelopeInput,
+  logFailure: WringerError | null,
+): DispatchAnswer {
+  const final: EnvelopeInput =
+    logFailure === null ? input : { ...input, data: null, truncatedPath: null, error: logFailure };
+
+  return {
+    envelope: buildEnvelope(final),
+    exitCode: final.error === null ? 0 : exitCodeFor[final.error.code],
+  };
+}

package/src/settle-envelope.ts

@@ -0,0 +1,34 @@
+import type { EnvelopeInput } from './build-envelope.ts';
+import type { CappedData } from './cap-data.ts';
+import type { Evidence } from './envelope.ts';
+import type { EnvelopePayload } from './pick-envelope-payload.ts';
+
+export interface EnvelopeFacts {
+  readonly command: string;
+  readonly target: string | null;
+  readonly instance: string | null;
+  readonly feature: string | null;
+  readonly sha: string | null;
+  readonly durationMs: number;
+  readonly driveEvidence: readonly Evidence[];
+  readonly payload: EnvelopePayload;
+  readonly capped: CappedData;
+}
+
+// The envelope's data is what the cap left; a payload that failed to cap fails the envelope unless
+// an earlier failure already did.
+export function settleEnvelope(facts: EnvelopeFacts): EnvelopeInput {
+  return {
+    command: facts.command,
+    target: facts.target,
+    instance: facts.instance,
+    feature: facts.feature,
+    sha: facts.sha,
+    durationMs: facts.durationMs,
+    data: facts.capped.data,
+    evidence: [...facts.driveEvidence, ...facts.capped.evidence],
+    cursors: facts.payload.cursors,
+    truncatedPath: facts.capped.truncatedPath,
+    error: facts.payload.failure ?? facts.capped.failure,
+  };
+}

package/src/ssh.ts

@@ -0,0 +1,211 @@
+import { randomUUID } from 'node:crypto';
+import { findRemoteStopFailure } from './find-remote-stop-failure.ts';
+import { followProcess } from './follow-process.ts';
+import type { Host } from './host.ts';
+import { quoteShellWord } from './quote-shell-word.ts';
+import { runProcess } from './run-process.ts';
+import type { ProcessResult } from './run-process.ts';
+import { toWringerError } from './to-wringer-error.ts';
+import { WringerError } from './wringer-error.ts';
+
+export interface PrefixLayer {
+  readonly argv: readonly string[];
+
+  // `true` when this layer hands the rest of the command to a shell as one string, such as
+  // `sh -c` or `docker exec <container> sh -c`.
+  readonly shell: boolean;
+}
+
+export interface SSHOptions {
+  readonly destination: string;
+  readonly readOnlyUser?: string;
+  readonly prefix?: readonly PrefixLayer[];
+  readonly binary?: string;
+}
+
+// ssh exits 255 when it fails to connect or log in; any other code is the remote command's.
+const sshConnectionFailedExitCode = 255;
+const stopTimeoutMs = 10_000;
+
+interface RemoteIO {
+  readonly stdin?: string | undefined;
+  readonly signal?: AbortSignal | undefined;
+}
+
+export function ssh(options: SSHOptions): Host {
+  return {
+    kind: 'ssh',
+    run: async (request) => {
+      const user = request.access === 'read-only' ? options.readOnlyUser : undefined;
+      const prefix = options.prefix ?? [];
+      const groupFile = `/tmp/wringer-${randomUUID()}.pgid`;
+
+      const sendRemote = (argv: readonly string[], io: RemoteIO) =>
+        runProcess({
+          argv: buildSSHArgv({
+            binary: options.binary ?? 'ssh',
+            destination: options.destination,
+            user,
+            terminal: false,
+            remote: buildRemoteCommand(argv, prefix),
+          }),
+          ...io,
+        });
+
+      const stops: Promise<ProcessResult>[] = [];
+
+      const stopRemote = (): void => {
+        stops.push(
+          sendRemote(buildGroupStopArgv(groupFile), { signal: AbortSignal.timeout(stopTimeoutMs) }),
+        );
+      };
+
+      request.signal?.addEventListener('abort', stopRemote, { once: true });
+
+      const running = sendRemote(buildGroupedArgv(request.argv, groupFile), {
+        stdin: request.stdin,
+        signal: request.signal,
+      });
+
+      const [ran] = await Promise.allSettled([running]);
+
+      request.signal?.removeEventListener('abort', stopRemote);
+
+      const stopped = await Promise.allSettled(stops);
+
+      const stopFailure = findRemoteStopFailure(stopped);
+
+      // A stop that failed can leave the remote command running, so the caller hears of it.
+      if (ran.status === 'rejected') {
+        const failure = toWringerError(ran.reason);
+
+        throw stopFailure === null
+          ? failure
+          : new WringerError(failure.code, {
+              message: `${failure.message} The remote command may still run: ${stopFailure}`,
+              hint: `Check ${options.destination} for the command, and stop it by hand.`,
+            });
+      }
+
+      if (ran.value.exitCode === sshConnectionFailedExitCode) {
+        throw buildConnectionError(options.destination, ran.value.stderr);
+      }
+
+      return ran.value;
+    },
+    follow: async (request) => {
+      const user = request.access === 'read-only' ? options.readOnlyUser : undefined;
+
+      // Killing the local ssh leaves a plain remote command running, and an idle `tail -f` never
+      // notices. A forced terminal makes sshd hang it up; the terminal ends each line with `\r`,
+      // and merges stderr into stdout, so the remote shell drops stderr before the terminal.
+      const followed = await followProcess({
+        argv: buildSSHArgv({
+          binary: options.binary ?? 'ssh',
+          destination: options.destination,
+          user,
+          terminal: true,
+          remote: `${buildRemoteCommand(request.argv, options.prefix ?? [])} 2>/dev/null`,
+        }),
+        signal: request.signal,
+        onLine: (line) => {
+          request.onLine(line.replace(/\r$/u, ''));
+        },
+      });
+
+      // Only ssh's own diagnostics reach its stderr, since the terminal carries the remote's.
+      if (followed.exitCode === sshConnectionFailedExitCode && !request.signal.aborted) {
+        throw buildConnectionError(options.destination, followed.stderr);
+      }
+
+      return followed.exitCode;
+    },
+  };
+}
+
+interface SSHArgvInput {
+  readonly binary: string;
+  readonly destination: string;
+  readonly user: string | undefined;
+  readonly terminal: boolean;
+  readonly remote: string;
+}
+
+function buildSSHArgv(input: SSHArgvInput): readonly string[] {
+  const login = input.user === undefined ? [] : ['-l', input.user];
+  const terminal = input.terminal ? ['-tt'] : [];
+
+  return [
+    input.binary,
+    '-o',
+    'BatchMode=yes',
+    ...terminal,
+    ...login,
+    '--',
+    input.destination,
+    input.remote,
+  ];
+}
+
+// ssh hands its command to the remote login shell, so the outermost layer always parses through a
+// shell. Each prefix layer with `shell: true` parses its rest once more, so the rest is quoted once
+// more, from the innermost layer out.
+function buildRemoteCommand(argv: readonly string[], prefix: readonly PrefixLayer[]): string {
+  let words = argv;
+
+  for (const layer of prefix.toReversed()) {
+    words = layer.shell ? [...layer.argv, joinShellWords(words)] : [...layer.argv, ...words];
+  }
+
+  return joinShellWords(words);
+}
+
+function joinShellWords(words: readonly string[]): string {
+  return words.map((word) => quoteShellWord(word)).join(' ');
+}
+
+// An abort can land before the remote command records its group, so the stop waits 2 s for the
+// file. SIGINT goes first, so psql cancels its query on the server; SIGTERM and SIGKILL follow, each
+// 1 s after the one before, to what still runs. dash's `kill` refuses `--`.
+const groupStopScript = [
+  'i=0',
+  'while [ ! -f "$1" ] && [ "$i" -lt 20 ]; do sleep 0.1; i=$((i + 1)); done',
+  'group=$(cat "$1" 2>/dev/null) || exit 0',
+  'for signal in INT TERM KILL; do',
+  '  kill -"$signal" "-$group" 2>/dev/null || break',
+  '  i=0',
+  '  while kill -0 "-$group" 2>/dev/null && [ "$i" -lt 10 ]; do sleep 0.1; i=$((i + 1)); done',
+  'done',
+  'rm -f "$1"',
+].join('\n');
+
+function buildGroupStopArgv(groupFile: string): readonly string[] {
+  return ['sh', '-c', groupStopScript, 'wringer', groupFile];
+}
+
+// Killing the local ssh leaves the remote command running, so the command runs in a group of its
+// own under `setsid`, which needs no terminal. The group's own shell records it in `groupFile`, so
+// a stop that finds the file finds a group that exists. stdin reaches the command through fd 3.
+const groupedScript = [
+  'exec 3<&0',
+  `setsid sh -c 'echo "$$" > "$1"; exec sh -c "$2"' wringer "$1" "$2" <&3 3<&- & child=$!`,
+  'exec 3<&-',
+  'wait "$child"',
+  'status=$?',
+
+  // A background process can hold ssh's output open after the command exits, so a watcher that
+  // holds none of ssh's pipes keeps the file until the whole group has exited.
+  '(while kill -0 "-$child" 2>/dev/null; do sleep 0.1; done; rm -f "$1") </dev/null >/dev/null 2>&1 &',
+  'exit "$status"',
+].join('\n');
+
+function buildGroupedArgv(argv: readonly string[], groupFile: string): readonly string[] {
+  return ['sh', '-c', groupedScript, 'wringer', groupFile, joinShellWords(argv)];
+}
+
+function buildConnectionError(destination: string, stderr: string): WringerError {
+  return new WringerError('environment', {
+    message: `ssh could not reach ${destination}: ${stderr.trim()}`,
+    hint: 'Check that the host is up and that ssh logs in without a prompt.',
+  });
+}