@wringer/core 0.1.0 → 0.1.1

package/src/start-launch.ts

@@ -0,0 +1,327 @@
+import { dirname } from 'node:path';
+import { match } from 'ts-pattern';
+import type { Actor } from 'xstate';
+import { createActor, waitFor } from 'xstate';
+import type { AppStart } from './app-machine.ts';
+import { appMachine } from './app-machine.ts';
+import { buildSpawnError } from './build-spawn-error.ts';
+import { createLogWriter } from './create-log-writer.ts';
+import type { LogWriter } from './create-log-writer.ts';
+import { parseLaunchLine } from './parse-launch-line.ts';
+import { raceDeadline } from './race-deadline.ts';
+import type { AppProcess, AppSpawnOptions, LaunchHost } from './read-launch-host.ts';
+import { readStreamLineBatches } from './read-stream-line-batches.ts';
+import type { Endpoints } from './run-events.ts';
+import type { ProcessRequest } from './run-process.ts';
+import { toWringerError } from './to-wringer-error.ts';
+import type { LaunchPlan } from './types.ts';
+import { WringerError } from './wringer-error.ts';
+
+export interface LaunchRequest {
+  readonly host: LaunchHost;
+  readonly plan: LaunchPlan;
+  readonly logPath: string;
+  readonly readyTimeoutMs: number;
+  readonly signal: AbortSignal;
+  readonly onCrash: (reason: string) => void;
+
+  // Takes why the stop of an app that never got ready failed, as the app may still be up.
+  readonly onStopFailed: (reason: string) => void;
+
+  // How long the stop command, then the app after SIGTERM, each get before the next step.
+  readonly stopGraceMs: number;
+}
+
+export interface Launch {
+  readonly endpoints: Endpoints;
+
+  // Rejects when a part of the app may still be up, or when its output missed its log.
+  readonly stop: () => Promise<void>;
+}
+
+export const appStopGraceMs = 5000;
+
+// The app runs in a process group of its own, and every stop signals the whole group, so a server
+// that a shell or a script runner started stops with it. The app machine decides each move; this
+// function starts the process and feeds the machine what the process and its log do.
+export async function startLaunch(request: LaunchRequest): Promise<Launch> {
+  const cwd = request.plan.cwd ?? request.host.cwd;
+  const env = { ...request.host.env, ...request.plan.env };
+
+  await createLogFolder(request.host, request.logPath);
+
+  // The abort listener below never fires for a run that stopped already, so the app would start.
+  if (request.signal.aborted) {
+    throw new WringerError('environment', { message: 'The run stopped before the app was ready.' });
+  }
+
+  const child = spawnApp(request.host, [...request.host.guard, ...request.plan.command], {
+    cwd,
+    env,
+  });
+
+  const logs = Promise.withResolvers<unknown>();
+  const log = createLogWriter((text) => request.host.appendLog(request.logPath, text));
+
+  const app = createActor(appMachine, {
+    input: {
+      readyTimeoutMs: request.readyTimeoutMs,
+      reportCrash: request.onCrash,
+      killGroup: () => {
+        request.host.killProcessGroup(child.pid);
+      },
+      stopApp: () =>
+        stopApp({
+          host: request.host,
+          plan: request.plan,
+          cwd,
+          env,
+          child,
+          logged: logs.promise,
+          closeLog: log.close,
+          stopGraceMs: request.stopGraceMs,
+        }),
+    },
+  });
+
+  app.start();
+
+  const onAbort = (): void => {
+    app.send({ type: 'abort' });
+  };
+
+  request.signal.addEventListener('abort', onAbort, { once: true });
+
+  const pumpLog = async (
+    stream: ReadableStream<Uint8Array>,
+    onLine: (line: string) => void,
+  ): Promise<void> => {
+    try {
+      await pipeLines(log, stream, onLine);
+    } catch (error) {
+      app.send({ type: 'log.failed', reason: toWringerError(error).message });
+    }
+  };
+
+  logs.resolve(
+    Promise.all([
+      pumpLog(child.stdout, (line) => {
+        const endpoints = request.plan.ready(parseLaunchLine(line));
+
+        if (endpoints !== null) {
+          app.send({ type: 'ready', endpoints });
+        }
+      }),
+      pumpLog(child.stderr, () => {
+        // The log keeps stderr; only stdout carries the ready line.
+      }),
+    ]),
+  );
+
+  const watchExit = async (): Promise<void> => {
+    const exitCode = await child.exited;
+
+    app.send({ type: 'exited', exitCode });
+  };
+
+  void watchExit();
+
+  const started = await waitForStart(app, request.signal, onAbort);
+
+  const endpoints = match(started.start)
+    .with({ kind: 'ready' }, (ready) => ready.endpoints)
+    .with({ kind: 'failed' }, (failed) => {
+      throw buildStartFailure(failed.failure, pickStopFailure(started), request.onStopFailed);
+    })
+    .with({ kind: 'pending' }, () => {
+      throw new WringerError('internal', { message: 'The app settled before its start did.' });
+    })
+    .exhaustive();
+
+  return {
+    endpoints,
+    stop: async () => {
+      app.send({ type: 'stop' });
+
+      const done = await waitFor(app, (snapshot) => snapshot.hasTag('done'), {
+        timeout: Infinity,
+      });
+
+      const failure = pickStopFailure(done.context);
+
+      if (failure !== null) {
+        throw failure;
+      }
+    },
+  };
+}
+
+async function createLogFolder(host: LaunchHost, logPath: string): Promise<void> {
+  try {
+    await host.createDirectory(dirname(logPath));
+  } catch (error) {
+    throw new WringerError('environment', {
+      message: `The folder for the app log could not be created: ${String(error)}`,
+    });
+  }
+}
+
+function spawnApp(host: LaunchHost, argv: readonly string[], options: AppSpawnOptions): AppProcess {
+  try {
+    return host.spawnApp(argv, options);
+  } catch (error) {
+    throw buildSpawnError(argv.join(' '), error);
+  }
+}
+
+interface AppStop {
+  readonly host: LaunchHost;
+  readonly plan: LaunchPlan;
+  readonly cwd: string;
+  readonly env: Readonly<Record<string, string | undefined>>;
+  readonly child: AppProcess;
+  readonly logged: Promise<unknown>;
+  readonly closeLog: () => Promise<void>;
+  readonly stopGraceMs: number;
+}
+
+// A failed stop still closes the log, so a service that reads the log once the stop settles, such
+// as a stream recorder, reads its last line.
+async function stopApp(stop: AppStop): Promise<void> {
+  try {
+    await signalApp(stop);
+  } finally {
+    await stop.closeLog();
+  }
+}
+
+const logDrainMs = 2000;
+
+// The longest a stop takes: the stop command's grace, the app's grace before SIGKILL, then the
+// drain of its log.
+export const appStopLimitMs = 2 * appStopGraceMs + logDrainMs;
+
+// A hung stop command gives way to SIGTERM after the grace, and SIGKILL another grace later. The
+// stop ends once the app's last lines reach its log; a process outside the group can hold the
+// pipes open, so that wait has a bound.
+async function signalApp(stop: AppStop): Promise<void> {
+  if (stop.plan.stop !== undefined) {
+    const stopping = runStopCommand(stop.host, {
+      argv: stop.plan.stop,
+      cwd: stop.cwd,
+      env: stop.env,
+      signal: AbortSignal.timeout(stop.stopGraceMs),
+    });
+
+    await Promise.allSettled([raceDeadline(stopping, AbortSignal.timeout(stop.stopGraceMs))]);
+  }
+
+  stop.host.killProcessGroup(stop.child.pid, 'SIGTERM');
+
+  const forced = setTimeout(() => {
+    stop.host.killProcessGroup(stop.child.pid);
+  }, stop.stopGraceMs);
+
+  await stop.child.exited;
+
+  clearTimeout(forced);
+
+  stop.host.killProcessGroup(stop.child.pid);
+
+  const [drained] = await Promise.allSettled([
+    raceDeadline(stop.logged, AbortSignal.timeout(logDrainMs)),
+  ]);
+
+  if (drained.status === 'rejected') {
+    throw new WringerError('environment', {
+      message: `The app's output still had not reached its log ${logDrainMs} ms after it exited.`,
+      hint: 'A process the app started outside its process group may still hold its output open.',
+    });
+  }
+}
+
+// A failed stop command must not keep the app alive, so its failure is dropped: the SIGTERM and
+// SIGKILL that follow end the app whatever it did.
+async function runStopCommand(host: LaunchHost, request: ProcessRequest): Promise<void> {
+  try {
+    await host.runProcess(request);
+  } catch {
+    // Dropped on purpose; see above.
+  }
+}
+
+async function pipeLines(
+  log: LogWriter,
+  stream: ReadableStream<Uint8Array>,
+  onLine: (line: string) => void,
+): Promise<void> {
+  // One append per chunk: an app that prints many lines as it stops would otherwise pay one
+  // append per line, and miss the bound on the stop's wait for its log.
+  for await (const lines of readStreamLineBatches(stream)) {
+    await log.append(`${lines.join('\n')}\n`);
+
+    for (const line of lines) {
+      onLine(line);
+    }
+  }
+}
+
+interface StartedApp {
+  readonly start: AppStart;
+  readonly logFailure: string | null;
+  readonly stopFailure: WringerError | null;
+}
+
+async function waitForStart(
+  app: Actor<typeof appMachine>,
+  signal: AbortSignal,
+  onAbort: () => void,
+): Promise<StartedApp> {
+  try {
+    const started = await waitFor(app, (snapshot) => snapshot.hasTag('settled'), {
+      timeout: Infinity,
+    });
+
+    return started.context;
+  } finally {
+    signal.removeEventListener('abort', onAbort);
+  }
+}
+
+// The startup failure stays the error; a failed stop adds what may still hold the app, and goes to
+// the caller on its own too.
+function buildStartFailure(
+  startFailure: WringerError,
+  stopFailure: WringerError | null,
+  onStopFailed: (reason: string) => void,
+): WringerError {
+  if (stopFailure === null) {
+    return startFailure;
+  }
+
+  onStopFailed(stopFailure.message);
+
+  return new WringerError(startFailure.code, {
+    message: `${startFailure.message} ${stopFailure.message}`,
+  });
+}
+
+interface StopOutcome {
+  readonly logFailure: string | null;
+  readonly stopFailure: WringerError | null;
+}
+
+// A failed stop names what may still be up; with none, output that missed the log still fails it.
+function pickStopFailure(outcome: StopOutcome): WringerError | null {
+  if (outcome.stopFailure !== null) {
+    return outcome.stopFailure;
+  }
+
+  if (outcome.logFailure !== null) {
+    return new WringerError('environment', {
+      message: `The app's output could not be logged: ${outcome.logFailure}`,
+    });
+  }
+
+  return null;
+}

package/src/start-services.ts

@@ -0,0 +1,108 @@
+import { buildServiceKey } from './build-service-key.ts';
+import { collectDriverConfigs } from './collect-driver-configs.ts';
+import { findCollidingServiceKey } from './find-colliding-service-key.ts';
+import { makeDriverExec } from './make-fenced-exec.ts';
+import type { Endpoints } from './run-events.ts';
+import type { RunHandle, ServiceInstance, TargetDefinition, Wringer } from './types.ts';
+import { WringerError } from './wringer-error.ts';
+
+export interface ServicesRequest {
+  readonly wringer: Wringer;
+  readonly targetName: string;
+  readonly target: TargetDefinition;
+  readonly endpoints: Endpoints;
+  readonly run: RunHandle;
+  readonly hint: string;
+  readonly signal: AbortSignal;
+
+  // Ends the commands a service runs through its exec. It outlives `signal`, so a service's stop
+  // can still run a command after the startup was cancelled.
+  readonly commandSignal: AbortSignal;
+
+  // Also ends the commands a service runs while it starts, once the stop gives up on the startup.
+  readonly startupCommandSignal: AbortSignal;
+  readonly started: Map<string, ServiceInstance>;
+  readonly onReady: (key: string) => Promise<void>;
+}
+
+// Each service lands in `started` as soon as it is up, so a later failure still stops it.
+export async function startServices(request: ServicesRequest): Promise<void> {
+  const entries = collectDriverConfigs(request.wringer, request.target, {
+    target: request.targetName,
+    endpoints: request.endpoints,
+    run: request.run,
+  });
+
+  // A second service under one key would replace the first in `started`, so no stop reaches it.
+  const collision = findCollidingServiceKey(entries);
+
+  if (collision !== null) {
+    throw new WringerError('usage', {
+      message: `Two driver services share the key ${collision}.`,
+      hint: 'Rename a driver or a service, so each `<driver>/<service>` key names one service.',
+    });
+  }
+
+  for (const entry of entries) {
+    for (const service of entry.serviceNames) {
+      const commands = makeServiceCommands(request);
+
+      const exec = makeDriverExec({
+        host: request.target.host,
+        access: request.target.access,
+        hint: request.hint,
+        signal: commands.signal,
+      });
+
+      const instance = await entry.configured
+        .startService(service, {
+          target: request.targetName,
+          access: request.target.access,
+          endpoints: request.endpoints,
+          run: request.run,
+          host: request.target.host,
+          exec,
+          signal: request.signal,
+        })
+        .finally(commands.endStartup);
+
+      const key = buildServiceKey(entry.name, service);
+
+      // A service that came up after the abort still goes in `started`, so the stop reaches it.
+      request.started.set(key, instance);
+      request.signal.throwIfAborted();
+
+      await request.onReady(key);
+    }
+  }
+}
+
+interface ServiceCommands {
+  readonly signal: AbortSignal;
+
+  // Detaches the startup signal once the service's start settles.
+  readonly endStartup: () => void;
+}
+
+function makeServiceCommands(request: ServicesRequest): ServiceCommands {
+  const commands = new AbortController();
+
+  const end = (): void => {
+    commands.abort();
+  };
+
+  for (const signal of [request.commandSignal, request.startupCommandSignal]) {
+    signal.addEventListener('abort', end, { once: true });
+  }
+
+  if (request.commandSignal.aborted || request.startupCommandSignal.aborted) {
+    end();
+  }
+
+  return {
+    signal: commands.signal,
+    endStartup: () => {
+      request.startupCommandSignal.removeEventListener('abort', end);
+    },
+  };
+}

package/src/supervisor-machine.ts

@@ -0,0 +1,169 @@
+import { and, assign, fromPromise, not, raise, setup } from 'xstate';
+import { toWringerError } from './to-wringer-error.ts';
+import type { WringerError } from './wringer-error.ts';
+
+export interface ShutDownPlan {
+  readonly crashReason: string | null;
+
+  // True when the stop stopped waiting for a startup that ignored the abort.
+  readonly gaveUpOnStartup: boolean;
+}
+
+// The supervisor's I/O, which the machine runs as actors and actions.
+interface SupervisorMachineInput {
+  readonly stopWaitMs: number;
+  readonly startUp: () => Promise<void>;
+
+  // Tells the startup to give up, so a stop during startup ends it early.
+  readonly abortStartup: () => void;
+  readonly shutDown: (plan: ShutDownPlan) => Promise<WringerError | null>;
+
+  // Stops each part that came up after the stop stopped waiting for it.
+  readonly cleanUpLate: () => Promise<void>;
+}
+
+interface SupervisorContext extends SupervisorMachineInput {
+  readonly crashReason: string | null;
+  readonly gaveUpOnStartup: boolean;
+  readonly startupSettled: boolean;
+
+  // The failure of the final log write, which each stop's caller receives.
+  readonly failure: WringerError | null;
+}
+
+type SupervisorEvent =
+  | { readonly type: 'stop'; readonly reason: string | null }
+  | { readonly type: 'startup.done' }
+  | { readonly type: 'startup.failed'; readonly reason: string };
+
+const supervisorTypes: {
+  context?: SupervisorContext;
+  events?: SupervisorEvent;
+  input?: SupervisorMachineInput;
+} = {};
+
+interface RunInput<Input, Output> {
+  readonly run: (input: Input) => Promise<Output>;
+  readonly input: Input;
+}
+
+// The run is up once startup is done, and the first stop or crash names the reason. A stop during
+// startup waits a bounded time for the startup; a part that comes up after that still stops, once
+// the run's log is closed. Each state in which the run's log is closed carries the `closed` tag.
+export const supervisorMachine = setup({
+  types: supervisorTypes,
+  actors: {
+    startUp: fromPromise<null, RunInput<null, void>>(async (args) => {
+      await args.input.run(null);
+
+      return null;
+    }),
+    shutDown: fromPromise<WringerError | null, RunInput<ShutDownPlan, WringerError | null>>(
+      (args) => args.input.run(args.input.input),
+    ),
+    cleanUpLate: fromPromise<null, RunInput<null, void>>(async (args) => {
+      await args.input.run(null);
+
+      return null;
+    }),
+  },
+  delays: {
+    stopWait: (args) => args.context.stopWaitMs,
+  },
+  guards: {
+    gaveUpOnStartup: (args) => args.context.gaveUpOnStartup,
+    startupSettled: (args) => args.context.startupSettled,
+  },
+  actions: {
+    keepStopReason: assign({
+      crashReason: (args) => (args.event.type === 'stop' ? args.event.reason : null),
+    }),
+    keepStartupFailure: assign({
+      crashReason: (args) => (args.event.type === 'startup.failed' ? args.event.reason : null),
+    }),
+    abortStartup: (args) => {
+      args.context.abortStartup();
+    },
+    giveUpOnStartup: assign({ gaveUpOnStartup: true }),
+    settleStartup: assign({ startupSettled: true }),
+  },
+}).createMachine({
+  context: (args) => ({
+    ...args.input,
+    crashReason: null,
+    gaveUpOnStartup: false,
+    startupSettled: false,
+    failure: null,
+  }),
+  invoke: {
+    // oxlint-disable-next-line zgeoff/no-abbreviated-names -- xstate names this key `src`
+    src: 'startUp',
+    input: (args) => ({ run: args.context.startUp, input: null }),
+    onDone: { actions: ['settleStartup', raise({ type: 'startup.done' })] },
+    onError: {
+      actions: [
+        'settleStartup',
+        raise((args) => ({
+          type: 'startup.failed' as const,
+          reason: toWringerError(args.event.error).message,
+        })),
+      ],
+    },
+  },
+  initial: 'starting',
+  states: {
+    starting: {
+      on: {
+        stop: { target: 'waitingForStartup', actions: 'keepStopReason' },
+        'startup.done': 'running',
+        'startup.failed': { target: 'shuttingDown', actions: 'keepStartupFailure' },
+      },
+    },
+    running: {
+      on: { stop: { target: 'shuttingDown', actions: 'keepStopReason' } },
+    },
+    waitingForStartup: {
+      entry: 'abortStartup',
+      after: { stopWait: { target: 'shuttingDown', actions: 'giveUpOnStartup' } },
+      on: { 'startup.done': 'shuttingDown', 'startup.failed': 'shuttingDown' },
+    },
+    shuttingDown: {
+      entry: 'abortStartup',
+      invoke: {
+        // oxlint-disable-next-line zgeoff/no-abbreviated-names -- xstate names this key `src`
+        src: 'shutDown',
+        input: (args) => ({
+          run: args.context.shutDown,
+          input: {
+            crashReason: args.context.crashReason,
+            gaveUpOnStartup: args.context.gaveUpOnStartup,
+          },
+        }),
+        onDone: { target: 'closed', actions: assign({ failure: (args) => args.event.output }) },
+        onError: {
+          target: 'closed',
+          actions: assign({ failure: (args) => toWringerError(args.event.error) }),
+        },
+      },
+    },
+    closed: {
+      tags: ['closed'],
+      always: [
+        { guard: and(['gaveUpOnStartup', 'startupSettled']), target: 'cleaningUpLate' },
+        { guard: not('gaveUpOnStartup'), target: 'ended' },
+      ],
+      on: { 'startup.done': 'cleaningUpLate', 'startup.failed': 'cleaningUpLate' },
+    },
+    cleaningUpLate: {
+      tags: ['closed'],
+      invoke: {
+        // oxlint-disable-next-line zgeoff/no-abbreviated-names -- xstate names this key `src`
+        src: 'cleanUpLate',
+        input: (args) => ({ run: args.context.cleanUpLate, input: null }),
+        onDone: 'ended',
+        onError: 'ended',
+      },
+    },
+    ended: { tags: ['closed'], type: 'final' },
+  },
+});

package/src/supervisor-protocol.ts

@@ -0,0 +1,42 @@
+import { z } from 'zod';
+import { envelopeErrorSchema } from './envelope.ts';
+import { runIDSchema } from './run-events.ts';
+
+// One request per connection, one JSON line each way, so a crashed command never leaves a
+// half-read stream on the supervisor's socket.
+export const supervisorRequestSchema = z.strictObject({
+  service: z.string(),
+  method: z.string(),
+  args: z.unknown(),
+
+  // The caller's deadline, in ms since the epoch. A call to the supervisor itself has none.
+  deadlineAt: z.number().optional(),
+});
+
+export type SupervisorRequest = z.infer<typeof supervisorRequestSchema>;
+
+export const supervisorResponseSchema = z.discriminatedUnion('ok', [
+  z.strictObject({ ok: z.literal(true), result: z.unknown() }),
+  z.strictObject({ ok: z.literal(false), error: envelopeErrorSchema }),
+]);
+
+export type SupervisorResponse = z.infer<typeof supervisorResponseSchema>;
+
+export const supervisorServiceName = 'supervisor';
+export const supervisorCommandName = '__wringer-supervisor';
+
+// The arguments after the hidden command name, which the starter writes and the supervisor
+// process parses.
+export const supervisorArgsSchema = z
+  .tuple([runIDSchema, z.string().min(1), z.string().min(1)])
+  .transform(([runID, target, root]) => ({ runID, target, root }));
+
+export interface SupervisorArgvInput {
+  readonly runID: string;
+  readonly target: string;
+  readonly root: string;
+}
+
+export function buildSupervisorArgv(args: SupervisorArgvInput): readonly string[] {
+  return [supervisorCommandName, args.runID, args.target, args.root];
+}

package/src/to-wringer-error.ts

@@ -0,0 +1,20 @@
+import { WringerError } from './wringer-error.ts';
+
+// A thrown value that is not a WringerError is a bug in wringer or the repo's code: it becomes the
+// code `internal`, and its stack travels in the error's data to the run's log.
+export function toWringerError(error: unknown): WringerError {
+  if (error instanceof WringerError) {
+    return error;
+  }
+
+  if (error instanceof DOMException && error.name === 'TimeoutError') {
+    return new WringerError('timeout');
+  }
+
+  const stack = error instanceof Error ? (error.stack ?? error.message) : String(error);
+
+  return new WringerError('internal', {
+    message: `wringer failed with an internal error: ${error instanceof Error ? error.message : String(error)}`,
+    data: { stack },
+  });
+}