@pepys/ingest 0.4.2 → 0.4.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pepys/ingest",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "description": "pepys ingest: the receive routes, the fact log and the blob store",
   "license": "MIT",
   "repository": {
@@ -21,9 +21,9 @@
     "access": "public"
   },
   "dependencies": {
-    "@impz/client": "7.0.0",
-    "@pepys/kit": "0.4.2",
-    "@pepys/schema": "0.4.2",
+    "@impz/client": "7.2.0",
+    "@pepys/kit": "0.4.3",
+    "@pepys/schema": "0.4.3",
     "@s2-dev/streamstore": "0.30.0",
     "@zgeoff/std": "0.4.3",
     "tiny-invariant": "1.3.3",

package/src/fact-log/s2/find-append-outcome.ts

@@ -23,7 +23,7 @@ export function isUnknownStream(error: unknown): boolean {
   return error instanceof S2Error && error.status === 404 && error.code === 'stream_not_found';
 }
 
-function isDefiniteRefusal(error: unknown): boolean {
+export function isDefiniteRefusal(error: unknown): boolean {
   return (
     error instanceof S2Error &&
     error.origin === 'server' &&

package/src/serve/pick-start-step.ts

@@ -0,0 +1,23 @@
+import { S2Error } from '@s2-dev/streamstore';
+import { isDefiniteRefusal } from '../fact-log/s2/find-append-outcome.ts';
+
+export type StartStep = { kind: 'retry'; delayMs: number } | { kind: 'stop' };
+
+const firstDelayMs = 200;
+const maxDelayMs = 30_000;
+
+// the failures in a row after which the start ends, as the host's `up` does
+const startRetryLimit = 10;
+
+// What the start does after the fact log failed it `failures` times in a row. A failure that a later
+// try can pass, such as a refused connection or a 5xx, waits a delay that doubles from 200 ms up to
+// 30 s. A refusal of the request itself, such as a bad token, fails the same way each time.
+export function pickStartStep(error: unknown, failures: number): StartStep {
+  const canPass = error instanceof S2Error && !isDefiniteRefusal(error);
+
+  if (!canPass || failures >= startRetryLimit) {
+    return { kind: 'stop' };
+  }
+
+  return { kind: 'retry', delayMs: Math.min(firstDelayMs * 2 ** (failures - 1), maxDelayMs) };
+}

package/src/serve/run-start-with-retry.ts

@@ -0,0 +1,52 @@
+import type { Logger } from '@pepys/kit';
+import { pickStartStep } from './pick-start-step.ts';
+
+interface WaitOptions {
+  signal: AbortSignal;
+}
+
+interface StartRetry {
+  // `setTimeout` of `node:timers/promises`: settles after the delay, and rejects on an abort
+  wait: (delayMs: number, value: undefined, options: WaitOptions) => Promise<unknown>;
+
+  // aborts when ingest stops before its start ends
+  signal: AbortSignal;
+  logger: Logger;
+}
+
+export type StartRun<Value> =
+  | { kind: 'started'; value: Value }
+  | { kind: 'stopped' }
+  | { kind: 'failed'; error: unknown };
+
+// Runs the part of the start that reads and writes the fact log, and runs it again after each
+// failure that a later try can pass.
+export async function runStartWithRetry<Value>(
+  start: () => Promise<Value>,
+  retry: StartRetry,
+): Promise<StartRun<Value>> {
+  let failures = 0;
+
+  while (!retry.signal.aborted) {
+    try {
+      return { kind: 'started', value: await start() };
+    } catch (error: unknown) {
+      failures += 1;
+
+      const step = pickStartStep(error, failures);
+
+      if (step.kind === 'stop') {
+        return { kind: 'failed', error };
+      }
+
+      retry.logger.error(
+        `the fact log failed the start of ingest, which tries again in ${step.delayMs} ms`,
+        error,
+      );
+
+      await retry.wait(step.delayMs, undefined, { signal: retry.signal }).catch(() => {});
+    }
+  }
+
+  return { kind: 'stopped' };
+}

package/src/serve/serve-ingest.ts

@@ -1,3 +1,4 @@
+import { setTimeout as wait } from 'node:timers/promises';
 import { checkDeployment, createLogger, listReceiveRoutes } from '@pepys/kit';
 import type { Deployment, SourceDefinition, StreamName } from '@pepys/kit';
 import { makeShutdownHandler } from '@pepys/schema';
@@ -10,6 +11,7 @@ import { formatPullStart } from '../pull/cursors/format-pull-start.ts';
 import { readSavedCursors } from '../pull/cursors/read-saved-cursors.ts';
 import { createIngest } from '../receive/create-ingest.ts';
 import { readImpIdentitySenderProof } from '../sender-proof/imp-identity/read-imp-identity-sender-proof.ts';
+import { runStartWithRetry } from './run-start-with-retry.ts';
 import { startPullTimers } from './start-pull-timers.ts';
 
 interface ServeIngestOptions {
@@ -18,11 +20,14 @@ interface ServeIngestOptions {
 
   // false leaves SIGINT and SIGTERM to the caller, who stops ingest through the returned function
   exitOnSignal?: boolean;
+
+  // aborts a start that still waits for the fact log, for a caller that owns the stop signals
+  signal?: AbortSignal;
 }
 
-// The ingest process of a deployment. It exits 1 at start on a failed deployment check, on any
-// missing variable, or on a trust folder it cannot load, so no route ever runs with an empty secret.
-// A path whose trust documents or audience are unset starts, and refuses every delivery.
+// The ingest process of a deployment. It exits 1 at start on a failed deployment check, a missing
+// variable, or a trust folder it cannot load, so no route ever runs with an empty secret. A stop
+// before ingest listens ends the start, and the returned function then stops nothing.
 export async function serveIngest(options: ServeIngestOptions): Promise<() => Promise<void>> {
   const logger = createLogger(process.stderr);
   const checked = checkDeployment(options.deployment);
@@ -52,14 +57,62 @@ export async function serveIngest(options: ServeIngestOptions): Promise<() => Pr
     process.exit(1);
   }
 
+  const starting = new AbortController();
+
+  let stop = (): Promise<void> => {
+    starting.abort();
+
+    return Promise.resolve();
+  };
+
+  if (options.exitOnSignal !== false) {
+    const shutdown = makeShutdownHandler({
+      stop: () => stop(),
+      logError: (message, error) => {
+        logger.error(message, error);
+      },
+      exit: (code) => process.exit(code),
+    });
+
+    for (const signal of ['SIGINT', 'SIGTERM'] as const) {
+      process.on(signal, () => {
+        void shutdown();
+      });
+    }
+  }
+
   // The S2 adapter creates the basin and each missing stream, so no setup command creates streams.
-  const log = await createS2FactLog({
-    ...config.value.s2,
-    streams: [...collectStreams(sources), cursorStream],
-    logger,
-  });
+  const started = await runStartWithRetry(
+    async () => {
+      const log = await createS2FactLog({
+        ...config.value.s2,
+        streams: [...collectStreams(sources), cursorStream],
+        logger,
+      });
+
+      return { log, pulls: await readSavedCursors(log, collectTimedPulls(sources)) };
+    },
+    {
+      wait,
+      signal:
+        options.signal === undefined
+          ? starting.signal
+          : AbortSignal.any([starting.signal, options.signal]),
+      logger,
+    },
+  );
 
-  const pulls = await readSavedCursors(log, collectTimedPulls(sources));
+  if (started.kind === 'stopped') {
+    return () => Promise.resolve();
+  }
+
+  if (started.kind === 'failed') {
+    logger.error('the fact log failed the start of ingest', started.error);
+    process.exit(1);
+  }
+
+  const log = started.value.log;
+  const pulls = started.value.pulls;
 
   const ingest = createIngest({
     deployment: checked.value,
@@ -90,30 +143,12 @@ export async function serveIngest(options: ServeIngestOptions): Promise<() => Pr
 
   console.log(`ingest listening on ${server.hostname}:${server.port}`);
 
-  const stop = async (): Promise<void> => {
+  stop = async (): Promise<void> => {
     await stopPulls();
 
     await server.stop();
   };
 
-  if (options.exitOnSignal === false) {
-    return stop;
-  }
-
-  const shutdown = makeShutdownHandler({
-    stop,
-    logError: (message, error) => {
-      logger.error(message, error);
-    },
-    exit: (code) => process.exit(code),
-  });
-
-  for (const signal of ['SIGINT', 'SIGTERM'] as const) {
-    process.on(signal, () => {
-      void shutdown();
-    });
-  }
-
   return stop;
 }