@pepys/ingest 0.3.1 → 0.4.1

package/src/serve/serve-ingest.ts

@@ -1,9 +1,13 @@
-import { checkDeployment, listReceiveRoutes } from '@pepys/kit';
+import { checkDeployment, createLogger, listReceiveRoutes } from '@pepys/kit';
 import type { Deployment, SourceDefinition, StreamName } from '@pepys/kit';
 import { makeShutdownHandler } from '@pepys/schema';
 import { toTagged } from '@zgeoff/std';
 import { readIngestConfig } from '../config/read-ingest-config.ts';
 import { createS2FactLog } from '../fact-log/s2/create-s2-fact-log.ts';
+import { collectTimedPulls } from '../pull/collect-timed-pulls.ts';
+import { cursorStream } from '../pull/cursors/cursor-stream.ts';
+import { formatPullStart } from '../pull/cursors/format-pull-start.ts';
+import { readSavedCursors } from '../pull/cursors/read-saved-cursors.ts';
 import { createIngest } from '../receive/create-ingest.ts';
 import { readImpIdentitySenderProof } from '../sender-proof/imp-identity/read-imp-identity-sender-proof.ts';
 import { startPullTimers } from './start-pull-timers.ts';
@@ -11,16 +15,20 @@ import { startPullTimers } from './start-pull-timers.ts';
 interface ServeIngestOptions {
   deployment: Deployment;
   environment: Readonly<Record<string, string | undefined>>;
+
+  // false leaves SIGINT and SIGTERM to the caller, who stops ingest through the returned function
+  exitOnSignal?: boolean;
 }
 
 // The ingest process of a deployment. It exits 1 at start on a failed deployment check, on any
 // missing variable, or on a trust folder it cannot load, so no route ever runs with an empty secret.
 // A path whose trust documents or audience are unset starts, and refuses every delivery.
-export async function serveIngest(options: ServeIngestOptions): Promise<void> {
+export async function serveIngest(options: ServeIngestOptions): Promise<() => Promise<void>> {
+  const logger = createLogger(process.stderr);
   const checked = checkDeployment(options.deployment);
 
   if (!checked.ok) {
-    console.error('the deployment fails its check', JSON.stringify(checked.failure));
+    logger.error('the deployment fails its check', checked.failure);
     process.exit(1);
   }
 
@@ -28,7 +36,7 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
   const config = readIngestConfig(options.environment, collectSecretNames(sources));
 
   if (!config.ok) {
-    console.error(config.failure.join('\n'));
+    logger.error('the ingest config is incomplete', config.failure);
     process.exit(1);
   }
 
@@ -36,15 +44,22 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
     sources,
     environment: options.environment,
     now: () => Date.now(),
+    logger,
   });
 
   if (!senderProof.ok) {
-    console.error(senderProof.failure);
+    logger.error('the sender proof cannot start', senderProof.failure);
     process.exit(1);
   }
 
   // The S2 adapter creates the basin and each missing stream, so no setup command creates streams.
-  const log = await createS2FactLog({ ...config.value.s2, streams: collectStreams(sources) });
+  const log = await createS2FactLog({
+    ...config.value.s2,
+    streams: [...collectStreams(sources), cursorStream],
+    logger,
+  });
+
+  const pulls = await readSavedCursors(log, collectTimedPulls(sources));
 
   const ingest = createIngest({
     deployment: checked.value,
@@ -53,30 +68,42 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
     senderProof: senderProof.value,
     appendRetry: { maxAttempts: 5, retryDelayMs: 200 },
     now: () => Date.now(),
+    logger,
   });
 
   for (const route of listReceiveRoutes(options.deployment)) {
     console.log(`route ${route.route}`, JSON.stringify(route));
   }
 
-  const stopPulls = startPullTimers(sources, ingest.runPull);
-
+  // The server starts first, so a failed listen leaves no timer behind.
   const server = Bun.serve({
     hostname: config.value.host,
     port: config.value.port,
     fetch: ingest.handleRequest,
   });
 
+  for (const pull of pulls) {
+    console.log(formatPullStart(pull));
+  }
+
+  const stopPulls = startPullTimers(pulls, ingest.runPull, logger);
+
   console.log(`ingest listening on ${server.hostname}:${server.port}`);
 
-  const shutdown = makeShutdownHandler({
-    stop: () => {
-      stopPulls();
+  const stop = async (): Promise<void> => {
+    await stopPulls();
 
-      return server.stop();
-    },
+    await server.stop();
+  };
+
+  if (options.exitOnSignal === false) {
+    return stop;
+  }
+
+  const shutdown = makeShutdownHandler({
+    stop,
     logError: (message, error) => {
-      console.error(message, error);
+      logger.error(message, error);
     },
     exit: (code) => process.exit(code),
   });
@@ -86,6 +113,8 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
       void shutdown();
     });
   }
+
+  return stop;
 }
 
 // every receive path's secret, and every name a pull path's `env` lists

package/src/serve/start-pull-timers.ts

@@ -1,60 +1,68 @@
-import type { SourceDefinition } from '@pepys/kit';
+import type { Logger } from '@pepys/kit';
 import { match } from 'ts-pattern';
-import type { PullPathName, PullRun, PullRunner } from '../pull/make-pull-runner.ts';
+import type { ResumedPull } from '../pull/cursors/read-saved-cursors.ts';
+import type { PullRun, PullRunner } from '../pull/make-pull-runner.ts';
 
-const pullPathNames: readonly PullPathName[] = ['live', 'recovery', 'backfill', 'resync'];
-
-// One timer per pull path with an interval. The cursor lives in memory, so a restart starts each
-// pull from no cursor. A run never overlaps the one before it. Returns the function that stops them.
+// One timer per pull, whose first run starts from the pull's saved cursor. A timer starts no run
+// while its last one waits for its source or runs. Returns the function that stops the timers: it
+// aborts each run, waiting or in flight, and settles once every run has ended.
 export function startPullTimers(
-  sources: readonly SourceDefinition[],
+  pulls: readonly ResumedPull[],
   runPull: PullRunner,
-): () => void {
-  const timers = sources.flatMap((source) =>
-    pullPathNames.flatMap((pathName) => {
-      const path = source[pathName];
+  logger: Logger,
+): () => Promise<void> {
+  const controller = new AbortController();
+  const activeRuns = new Set<Promise<void>>();
+
+  const timers = pulls.map((pull) => {
+    let cursor = pull.cursor;
+    let isRunning = false;
 
-      if (path?.kind !== 'pull' || path.intervalMs === null) {
-        return [];
+    const runOnce = async (): Promise<void> => {
+      if (isRunning || controller.signal.aborted) {
+        return;
       }
 
-      let cursor: string | null = null;
-      let isRunning = false;
+      isRunning = true;
 
-      const runOnce = async (): Promise<void> => {
-        if (isRunning) {
-          return;
-        }
+      const ended = Promise.withResolvers<void>();
 
-        isRunning = true;
+      activeRuns.add(ended.promise);
 
-        try {
-          const run = await runPull(source.name, pathName, cursor);
+      try {
+        const run = await runPull(pull.source, pull.path, cursor, controller.signal);
 
-          cursor = pickNextCursor(run, cursor);
+        cursor = pickNextCursor(run, cursor);
 
-          if (run.kind !== 'stored') {
-            console.error(`the ${pathName} pull of ${source.name} did not store`, run);
-          }
-        } catch (error: unknown) {
-          console.error(`the ${pathName} pull of ${source.name} threw`, error);
-        } finally {
-          isRunning = false;
+        // a run that the stop cut short is no failure
+        if (run.kind !== 'stored' && !controller.signal.aborted) {
+          logger.error(`the ${pull.path} pull of ${pull.source} did not store`, run);
+        }
+      } catch (error: unknown) {
+        if (!controller.signal.aborted) {
+          logger.error(`the ${pull.path} pull of ${pull.source} threw`, error);
         }
-      };
+      } finally {
+        isRunning = false;
 
-      return [
-        setInterval(() => {
-          void runOnce();
-        }, path.intervalMs),
-      ];
-    }),
-  );
+        activeRuns.delete(ended.promise);
+        ended.resolve();
+      }
+    };
+
+    return setInterval(() => {
+      void runOnce();
+    }, pull.intervalMs);
+  });
 
-  return () => {
+  return async () => {
     for (const timer of timers) {
       clearInterval(timer);
     }
+
+    controller.abort();
+
+    await Promise.all(activeRuns);
   };
 }
 

package/src/{receive → utils}/find-batch-problem.ts

@@ -3,7 +3,10 @@ import type { Batch, SourceContext } from '@pepys/kit';
 
 // Every fact is checked before any is written: its stream must be one of the source's families, and
 // its envelope must parse.
-export function findBatchProblem(source: SourceContext, batch: Batch): string | undefined {
+export function findBatchProblem(
+  source: Pick<SourceContext, 'name' | 'families'>,
+  batch: Batch,
+): string | undefined {
   const streams = new Set<string>(source.families.map((family) => family.stream));
 
   for (const [index, pending] of batch.facts.entries()) {

package/src/utils/pick-batch-plan.ts

@@ -0,0 +1,37 @@
+import type { Batch, Result, SourceContext } from '@pepys/kit';
+import { findBatchProblem } from './find-batch-problem.ts';
+import { planBatch } from './plan-batch.ts';
+import type { BatchPlan } from './plan-batch.ts';
+
+export interface BatchRefusal {
+  kind: 'bad-batch' | 'has-blob' | 'too-large';
+  reason: string;
+}
+
+// The checks a batch passes before any of it is written, in order: every fact's stream and
+// envelope, no fact with a blob, then the cut into appends.
+export function pickBatchPlan(
+  source: Pick<SourceContext, 'name' | 'families'>,
+  batch: Batch,
+): Result<BatchPlan, BatchRefusal> {
+  const problem = findBatchProblem(source, batch);
+
+  if (problem !== undefined) {
+    return { ok: false, failure: { kind: 'bad-batch', reason: problem } };
+  }
+
+  if (batch.facts.some((pending) => pending.blobs.length > 0)) {
+    return { ok: false, failure: { kind: 'has-blob', reason: 'ingest has no blob store' } };
+  }
+
+  const plan = planBatch(batch);
+
+  if (!plan.ok) {
+    return {
+      ok: false,
+      failure: { kind: 'too-large', reason: 'one fact is larger than one append of the log takes' },
+    };
+  }
+
+  return plan;
+}

package/src/{receive → utils}/plan-batch.ts

@@ -1,5 +1,5 @@
 import type { Batch, FactEnvelope, Result, StreamName } from '@pepys/kit';
-import { planAppends } from '../fact-log/plan-appends.ts';
+import { planAppends } from './plan-appends.ts';
 
 export type BatchPlan = ReadonlyMap<StreamName, readonly (readonly FactEnvelope<unknown>[])[]>;
 

package/src/source-port/types.ts

@@ -1,64 +0,0 @@
-import type {
-  BlobHash,
-  ConsumerDeclaration,
-  EntityKind,
-  FactEnvelope,
-  LinkTypeDeclaration,
-  Result,
-  StreamName,
-} from '@pepys/kit';
-
-export interface FactFamily<Payload> {
-  stream: StreamName;
-  types: readonly string[];
-  api: string;
-  parsePayload: (raw: unknown) => Result<Payload, { reason: string }>;
-}
-
-// Provisional: source-retention
-export interface SourceDefinition {
-  name: string;
-  families: readonly FactFamily<unknown>[];
-  entityKinds: readonly EntityKind[];
-  linkTypes: readonly LinkTypeDeclaration[];
-  consumers: readonly ConsumerDeclaration[];
-  live: ReceivePath | PullPath;
-  recovery: PullPath;
-  backfill: PullPath;
-  resync: PullPath;
-}
-
-export interface ReceivePath {
-  kind: 'receive';
-  route: string;
-  receive: (request: Request) => Promise<Received>;
-}
-
-export type Received =
-  | { kind: 'refused'; status: 400 | 401 | 405 | 422; reason: string }
-  | { kind: 'ignored' }
-  | { kind: 'batch'; batch: Batch };
-
-export interface PullPath {
-  kind: 'pull';
-  name: string;
-  intervalMs: number | null;
-  pull: (cursor: string | null) => Promise<Pulled>;
-}
-
-// Provisional: source-retention
-export type Pulled =
-  | { kind: 'batch'; batch: Batch; cursor: string; isDone: boolean }
-  | { kind: 'expired' }
-  | { kind: 'unavailable'; retryAfterMs: number }
-  | { kind: 'rejected'; reason: string };
-
-export interface Batch {
-  facts: readonly PendingFact[];
-}
-
-export interface PendingFact {
-  stream: StreamName;
-  fact: FactEnvelope<unknown>;
-  blobs: readonly { hash: BlobHash; bytes: Uint8Array }[];
-}