@pepys/ingest 0.0.1 → 0.1.1

package/README.md

@@ -25,6 +25,8 @@ bun add @pepys/ingest
 ## Features
 
 - `serveIngest` and `createIngest`: the receive routes of a deployment.
+- The sender proof port, with the adapter that verifies a worker machine's identity token through
+  `@impz/client/identity`.
 - The fact log port, with its S2 and memory adapters.
 - The blob store port, with its Garage and local folder adapters.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pepys/ingest",
-  "version": "0.0.1",
+  "version": "0.1.1",
   "description": "pepys ingest: the receive routes, the fact log and the blob store",
   "license": "MIT",
   "repository": {
@@ -21,8 +21,9 @@
     "access": "public"
   },
   "dependencies": {
-    "@pepys/kit": "0.0.1",
-    "@pepys/schema": "0.0.1",
+    "@impz/client": "6.12.0",
+    "@pepys/kit": "0.1.1",
+    "@pepys/schema": "0.1.1",
     "@s2-dev/streamstore": "0.30.0",
     "@zgeoff/std": "0.4.3",
     "tiny-invariant": "1.3.3",

package/src/index.ts

@@ -11,18 +11,13 @@ export { planAppends } from './fact-log/plan-appends.ts';
 export { createS2FactLog } from './fact-log/s2/create-s2-fact-log.ts';
 export { createIngest } from './receive/create-ingest.ts';
 export type { Ingest, IngestOptions } from './receive/create-ingest.ts';
+export type { PullPathName, PullRun, PullRunner } from './pull/make-pull-runner.ts';
 export type { ResyncMarks } from './receive/resync-marks/create-resync-marks.ts';
 export { createFakeSenderProof } from './sender-proof/fake/create-fake-sender-proof.ts';
 export { createImpIdentitySenderProof } from './sender-proof/imp-identity/create-imp-identity-sender-proof.ts';
+export { readImpIdentitySenderProof } from './sender-proof/imp-identity/read-imp-identity-sender-proof.ts';
 export { readImpIdentityTrust } from './sender-proof/imp-identity/read-imp-identity-trust.ts';
-
-export type {
-  ImpIdentityTrust,
-  ImpIdentityVerifier,
-  VerifiedImp,
-  VerifyOptions,
-} from './sender-proof/imp-identity/types.ts';
-
+export type { ImpIdentityTrust } from './sender-proof/imp-identity/types.ts';
 export { noSenderProof } from './sender-proof/none/no-sender-proof.ts';
 export { serveIngest } from './serve/serve-ingest.ts';
 export type * from './source-port/types.ts';

package/src/pull/make-pull-runner.ts

@@ -0,0 +1,104 @@
+import type { CheckedDeployment, PullPath, Pulled, SourceDefinition } from '@pepys/kit';
+import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
+import { findBatchProblem } from '../receive/find-batch-problem.ts';
+import { planBatch } from '../receive/plan-batch.ts';
+import type { ResyncMarks } from '../receive/resync-marks/create-resync-marks.ts';
+import { writeBatch } from '../receive/write-batch.ts';
+import type { StreamTurns } from '../receive/write-batch.ts';
+
+export type PullPathName = 'live' | 'recovery' | 'backfill' | 'resync';
+
+// `stored`: the batch is in the log, and the next run starts from `cursor`. `not-stored`: an append
+// failed, so the next run starts from the same cursor. Any other answer is the pull's own.
+export type PullRun =
+  | { kind: 'stored'; stored: number; cursor: string; isDone: boolean }
+  | { kind: 'not-stored' }
+  | { kind: 'refused'; reason: string }
+  | Exclude<Pulled, { kind: 'batch' }>;
+
+export type PullRunner = (
+  source: string,
+  path: PullPathName,
+  cursor: string | null,
+) => Promise<PullRun>;
+
+interface PullRunnerDependencies {
+  deployment: CheckedDeployment;
+
+  // one value per name that a pull path's `env` lists
+  secrets: Readonly<Record<string, string>>;
+  append: BatchAppender;
+  turns: StreamTurns;
+  resyncMarks: ResyncMarks;
+}
+
+// One run of one pull path: the pull, the envelope check, the appends, and then, for a resync pull,
+// the removal of every mark it was given. A pull's fact is noted in no mark: a resynced fact ties
+// with the body it replaces, and would mark its entity again on every run.
+export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner {
+  return async (sourceName, pathName, cursor) => {
+    const source = dependencies.deployment.sources.find(
+      (candidate) => candidate.name === sourceName,
+    );
+
+    const path = source === undefined ? undefined : findPullPath(source, pathName);
+
+    if (source === undefined || path === undefined) {
+      return { kind: 'refused', reason: `the source ${sourceName} has no ${pathName} pull` };
+    }
+
+    const marks = pathName === 'resync' ? dependencies.resyncMarks.list(source.name) : [];
+    const env: Record<string, string> = {};
+
+    for (const name of path.env ?? []) {
+      env[name] = dependencies.secrets[name] ?? '';
+    }
+
+    const pulled = await path.pull(cursor, { env, marks });
+
+    if (pulled.kind !== 'batch') {
+      return pulled;
+    }
+
+    const problem = findBatchProblem(source, pulled.batch);
+
+    if (problem !== undefined) {
+      return { kind: 'refused', reason: problem };
+    }
+
+    if (pulled.batch.facts.some((pending) => pending.blobs.length > 0)) {
+      return { kind: 'refused', reason: 'ingest has no blob store' };
+    }
+
+    const plan = planBatch(pulled.batch);
+
+    if (!plan.ok) {
+      return { kind: 'refused', reason: 'one fact is larger than one append of the log takes' };
+    }
+
+    const outcome = await writeBatch(plan.value, dependencies);
+
+    if (outcome !== 'stored') {
+      return outcome === 'not-stored'
+        ? { kind: 'not-stored' }
+        : { kind: 'refused', reason: 'the log refused the batch' };
+    }
+
+    for (const mark of marks) {
+      dependencies.resyncMarks.remove(source.name, mark.entity);
+    }
+
+    return {
+      kind: 'stored',
+      stored: pulled.batch.facts.length,
+      cursor: pulled.cursor,
+      isDone: pulled.isDone,
+    };
+  };
+}
+
+function findPullPath(source: SourceDefinition, pathName: PullPathName): PullPath | undefined {
+  const path = source[pathName];
+
+  return path?.kind === 'pull' ? path : undefined;
+}

package/src/receive/create-ingest.ts

@@ -1,6 +1,8 @@
 import type { CheckedDeployment, SenderProof } from '@pepys/kit';
 import type { FactLog } from '../fact-log/fact-log.ts';
 import { makeBatchAppender } from '../fact-log/make-batch-appender.ts';
+import { makePullRunner } from '../pull/make-pull-runner.ts';
+import type { PullRunner } from '../pull/make-pull-runner.ts';
 import { buildFailureAnswer } from './answers/build-failure-answer.ts';
 import { makeIgnoredRouteHandler, makeReceiveHandler } from './make-receive-handler.ts';
 import { createRecentBodies } from './recent-bodies/create-recent-bodies.ts';
@@ -11,7 +13,8 @@ import type { StreamTurns } from './write-batch.ts';
 export interface IngestOptions {
   deployment: CheckedDeployment;
 
-  // one value per `check.secretEnv` of the deployment; a missing or empty one refuses every delivery
+  // One value per `check.secretEnv` and per name in a pull path's `env`. A missing or empty secret
+  // refuses every delivery.
   secrets: Readonly<Record<string, string>>;
   log: FactLog;
   senderProof: SenderProof;
@@ -25,6 +28,9 @@ export interface Ingest {
 
   // what a source's resync pull reads
   resyncMarks: ResyncMarks;
+
+  // one run of a pull path of the deployment, with its batch stored
+  runPull: PullRunner;
 }
 
 const recentBodyCapacity = 10_000;
@@ -86,5 +92,12 @@ export function createIngest(options: IngestOptions): Ingest {
       return answer;
     },
     resyncMarks: resync.marks,
+    runPull: makePullRunner({
+      deployment: options.deployment,
+      secrets: options.secrets,
+      append,
+      turns,
+      resyncMarks: resync.marks,
+    }),
   };
 }

package/src/receive/make-receive-handler.ts

@@ -137,7 +137,11 @@ export function makeReceiveHandler(
 
     dependencies.marks?.noteStored(
       digest,
-      facts.map((pending) => ({ entity: pending.fact.entity, at: pending.fact.at })),
+      facts.map((pending) => ({
+        entity: pending.fact.entity,
+        at: pending.fact.at,
+        locator: pending.locator,
+      })),
     );
 
     return buildSuccessAnswer(path, 'stored', facts.length);

package/src/receive/resync-marks/collect-tied-entities.ts

@@ -3,11 +3,13 @@ import type { EntityID } from '@pepys/kit';
 export interface LastBody {
   digest: string;
   at: string;
+  locator: string | undefined;
 }
 
-interface StoredFactTime {
+export interface StoredFactTime {
   entity: EntityID;
   at: string;
+  locator: string | undefined;
 }
 
 // A new body ties with the last body stored for an entity when both have the same time and differ.

package/src/receive/resync-marks/create-resync-marks.ts

@@ -1,25 +1,27 @@
-import type { EntityID } from '@pepys/kit';
+import type { EntityID, ResyncMark } from '@pepys/kit';
 import { collectReturnedEntities } from './collect-returned-entities.ts';
 import { collectTiedEntities } from './collect-tied-entities.ts';
-import type { LastBody } from './collect-tied-entities.ts';
+import type { LastBody, StoredFactTime } from './collect-tied-entities.ts';
 
 // The entities whose row may be behind the source, which a source's resync pull reads again. The
 // pull reads the marks, and removes each one only after its fact is stored, so a failed pull keeps
 // it. In memory only, so a restart loses them.
 export interface ResyncMarks {
-  list: (source: string) => readonly EntityID[];
+  list: (source: string) => readonly ResyncMark[];
   remove: (source: string, entity: EntityID) => void;
 }
 
 // What the receive route of one source tells the marks.
 export interface SourceMarks {
-  noteStored: (digest: string, facts: readonly { entity: EntityID; at: string }[]) => void;
+  noteStored: (digest: string, facts: readonly StoredFactTime[]) => void;
   noteCopy: (digest: string, entities: readonly EntityID[]) => void;
 }
 
 interface SourceState {
   lastBodies: Map<EntityID, LastBody>;
-  marks: Set<EntityID>;
+
+  // each marked entity, with the locator of the last fact stored for it
+  marks: Map<EntityID, string | undefined>;
 }
 
 export interface ResyncMarkStore {
@@ -42,7 +44,7 @@ export function createResyncMarks(options: ResyncMarksOptions): ResyncMarkStore
       return known;
     }
 
-    const state: SourceState = { lastBodies: new Map(), marks: new Set() };
+    const state: SourceState = { lastBodies: new Map(), marks: new Map() };
 
     sources.set(source, state);
 
@@ -51,7 +53,8 @@ export function createResyncMarks(options: ResyncMarksOptions): ResyncMarkStore
 
   return {
     marks: {
-      list: (source) => [...(sources.get(source)?.marks ?? [])],
+      list: (source) =>
+        [...(sources.get(source)?.marks ?? [])].map(([entity, locator]) => ({ entity, locator })),
       remove: (source, entity) => {
         sources.get(source)?.marks.delete(entity);
       },
@@ -61,41 +64,43 @@ export function createResyncMarks(options: ResyncMarksOptions): ResyncMarkStore
 
       return {
         noteStored: (digest, facts) => {
-          addMarks(state.marks, collectTiedEntities(state.lastBodies, digest, facts), options);
+          const tied = collectTiedEntities(state.lastBodies, digest, facts);
 
           for (const fact of facts) {
             state.lastBodies.delete(fact.entity);
-            state.lastBodies.set(fact.entity, { digest, at: fact.at });
+            state.lastBodies.set(fact.entity, { digest, at: fact.at, locator: fact.locator });
           }
 
+          addMarks(state, tied, options);
           trimToCapacity(state.lastBodies, options.capacity);
         },
         noteCopy: (digest, entities) => {
-          addMarks(
-            state.marks,
-            collectReturnedEntities(state.lastBodies, digest, entities),
-            options,
-          );
+          addMarks(state, collectReturnedEntities(state.lastBodies, digest, entities), options);
         },
       };
     },
   };
 }
 
+// The locator comes from the last body stored for the entity, which the caller has noted first.
 function addMarks(
-  marks: Set<EntityID>,
+  state: SourceState,
   entities: readonly EntityID[],
   options: ResyncMarksOptions,
 ): void {
   for (const entity of entities) {
-    marks.add(entity);
+    state.marks.delete(entity);
+    state.marks.set(entity, state.lastBodies.get(entity)?.locator);
   }
 
-  trimToCapacity(marks, options.capacity);
+  trimToCapacity(state.marks, options.capacity);
 }
 
 // drops the oldest entries first: a Map and a Set keep insertion order
-function trimToCapacity(entries: Map<EntityID, LastBody> | Set<EntityID>, capacity: number): void {
+function trimToCapacity(
+  entries: Map<EntityID, LastBody> | Map<EntityID, string | undefined>,
+  capacity: number,
+): void {
   for (const key of entries.keys()) {
     if (entries.size <= capacity) {
       return;

package/src/sender-proof/imp-identity/create-imp-identity-sender-proof.ts

@@ -1,45 +1,37 @@
-import type { Sender, SenderProof } from '@pepys/kit';
-import type { ImpIdentityTrust, ImpIdentityVerifier, VerifiedImp } from './types.ts';
+import type { verifyImpIdentity } from '@impz/client/identity';
+import type { SenderProof } from '@pepys/kit';
+import type { ImpIdentityTrust } from './types.ts';
 
-interface ImpIdentityOptions<Document> {
-  verify: ImpIdentityVerifier<Document>['verify'];
-  trust: ImpIdentityTrust<Document>;
+interface ImpIdentityOptions {
+  verify: typeof verifyImpIdentity;
+  trust: ImpIdentityTrust;
   now: () => number;
 }
 
-// The broker removes any `Imp-Identity` header a guest sends and adds its own: a compact JWS. A
-// header of any other shape, two joined headers included, never reaches the verifier.
-const compactToken = /^[\w-]+\.[\w-]+\.[\w-]+$/u;
-
-// Proves the sender from the signed identity token of a worker's machine. The issuer and the
-// subject come only from the verified token, and every failure, a throw included, proves nothing.
-// It never logs the token.
-export function createImpIdentitySenderProof<Document>(
-  options: ImpIdentityOptions<Document>,
-): SenderProof {
+// Proves the sender from the signed identity token in the `imp-identity` header. The issuer and
+// the subject come only from the verified token, and every failure, a throw included, proves
+// nothing. It never logs the token.
+export function createImpIdentitySenderProof(options: ImpIdentityOptions): SenderProof {
   return {
     prove: async (requirement, headers) => {
       const documents = options.trust.issuers.get(requirement.issuersEnv) ?? [];
       const audience = options.trust.audiences.get(requirement.audienceEnv);
-      const header = headers.get('imp-identity');
 
-      if (
-        documents.length === 0 ||
-        audience === undefined ||
-        header === null ||
-        !compactToken.test(header)
-      ) {
+      // The verifier throws on an empty list of documents, so a path with none refuses here.
+      if (documents.length === 0 || audience === undefined) {
         return null;
       }
 
       try {
-        const verified = await options.verify(header, {
+        const verified = await options.verify(headers.get('imp-identity'), {
           issuers: documents,
           audience,
           nowSeconds: Math.floor(options.now() / 1000),
         });
 
-        return verified === null ? null : toSender(verified);
+        return verified === null
+          ? null
+          : { issuer: verified.issuer, subject: verified.id, name: verified.name };
       } catch {
         console.error('the identity verifier failed, so the delivery proves no sender');
 
@@ -48,11 +40,3 @@ export function createImpIdentitySenderProof<Document>(
     },
   };
 }
-
-function toSender(verified: VerifiedImp): Sender | null {
-  if (verified.issuer === '' || verified.id === '') {
-    return null;
-  }
-
-  return { issuer: verified.issuer, subject: verified.id, name: verified.name };
-}

package/src/sender-proof/imp-identity/read-imp-identity-sender-proof.ts

@@ -0,0 +1,35 @@
+import { verifyImpIdentity } from '@impz/client/identity';
+import type { Result, SenderProof, SourceDefinition } from '@pepys/kit';
+import { createImpIdentitySenderProof } from './create-imp-identity-sender-proof.ts';
+import { readImpIdentityTrust } from './read-imp-identity-trust.ts';
+
+interface SenderProofReadOptions {
+  sources: readonly SourceDefinition[];
+  environment: Readonly<Record<string, string | undefined>>;
+  now: () => number;
+}
+
+// The sender proof that `serveIngest` runs: the trust of the deployment's environment and the
+// verifier of `@impz/client/identity`. It fails only when a set folder cannot be read or holds an
+// invalid document.
+export async function readImpIdentitySenderProof(
+  options: SenderProofReadOptions,
+): Promise<Result<SenderProof, string>> {
+  const trust = await readImpIdentityTrust({
+    sources: options.sources,
+    environment: options.environment,
+  });
+
+  if (!trust.ok) {
+    return trust;
+  }
+
+  return {
+    ok: true,
+    value: createImpIdentitySenderProof({
+      verify: verifyImpIdentity,
+      trust: trust.value,
+      now: options.now,
+    }),
+  };
+}

package/src/sender-proof/imp-identity/read-imp-identity-trust.ts

@@ -1,25 +1,26 @@
 import { readdir } from 'node:fs/promises';
 import { join } from 'node:path';
+import { loadIdentityIssuers } from '@impz/client/identity';
+import type { IdentityTrustDocument } from '@impz/client/identity';
 import type { Result, SenderRequirement, SourceDefinition } from '@pepys/kit';
-import type { ImpIdentityTrust, ImpIdentityVerifier } from './types.ts';
+import type { ImpIdentityTrust } from './types.ts';
 
-interface TrustReadOptions<Document> {
+interface TrustReadOptions {
   sources: readonly SourceDefinition[];
 
   // the environment of the ingest process; never a request
   environment: Readonly<Record<string, string | undefined>>;
-  loadIssuers: ImpIdentityVerifier<Document>['loadIssuers'];
 }
 
 // An unset or empty variable, or an empty folder, leaves its paths with no entry, so they refuse
-// every delivery. A folder that cannot be read, or a document the verifier refuses, fails the
+// every delivery. A folder that cannot be read, or a document that the loader refuses, fails the
 // start.
-export async function readImpIdentityTrust<Document>(
-  options: TrustReadOptions<Document>,
-): Promise<Result<ImpIdentityTrust<Document>, string>> {
+export async function readImpIdentityTrust(
+  options: TrustReadOptions,
+): Promise<Result<ImpIdentityTrust, string>> {
   const requirements = collectSenderRequirements(options.sources);
 
-  const issuers = new Map<string, readonly Document[]>();
+  const issuers = new Map<string, readonly IdentityTrustDocument[]>();
   const audiences = new Map<string, string>();
 
   for (const name of new Set(requirements.map((requirement) => requirement.audienceEnv))) {
@@ -48,9 +49,9 @@ export async function readImpIdentityTrust<Document>(
     }
 
     try {
-      issuers.set(name, options.loadIssuers(documents.value));
+      issuers.set(name, loadIdentityIssuers(documents.value));
     } catch (error) {
-      const reason = error instanceof Error ? error.message : 'the verifier gave no reason';
+      const reason = error instanceof Error ? error.message : 'the loader gave no reason';
 
       return { ok: false, failure: `${name}: the trust documents are invalid: ${reason}` };
     }

package/src/sender-proof/imp-identity/types.ts

@@ -1,38 +1,9 @@
-// The imp that a verified identity token names. The shape of the verifier's answer in the
-// `@impz/client/identity` entry of impz.
-export interface VerifiedImp {
-  // the token's `iss`: the impd that signed it
-  readonly issuer: string;
-
-  // the token's `sub`: the imp's ID, unique on one impd only
-  readonly id: string;
-
-  // the imp's name, for logs; a name is free again after a destroy, so it is no key
-  readonly name: string;
-}
+import type { IdentityTrustDocument } from '@impz/client/identity';
 
 // What the deployment trusts, read from its environment at start: the trust documents of each
 // `issuersEnv` and the audience of each `audienceEnv` that its paths name. A name with no entry
 // refuses every delivery on its paths.
-export interface ImpIdentityTrust<Document> {
-  readonly issuers: ReadonlyMap<string, readonly Document[]>;
+export interface ImpIdentityTrust {
+  readonly issuers: ReadonlyMap<string, readonly IdentityTrustDocument[]>;
   readonly audiences: ReadonlyMap<string, string>;
 }
-
-export interface VerifyOptions<Document> {
-  readonly issuers: readonly Document[];
-  readonly audience: string;
-  readonly nowSeconds: number;
-}
-
-// The verifier of `@impz/client/identity`, which owns the token format and the trust document
-// format. `Document` stays opaque to ingest.
-export interface ImpIdentityVerifier<Document> {
-  // Answers null for a token to refuse: a bad signature, an issuer with no trust document, another
-  // audience, or a token out of its time. Throws on an empty or doubled list of documents.
-  verify: (header: string | null, options: VerifyOptions<Document>) => Promise<VerifiedImp | null>;
-
-  // Parses the documents a deployment pinned. Throws on an invalid document, on none, and on two
-  // with one issuer.
-  loadIssuers: (documents: readonly unknown[]) => Document[];
-}

package/src/serve/serve-ingest.ts

@@ -5,15 +5,17 @@ import { toTagged } from '@zgeoff/std';
 import { readIngestConfig } from '../config/read-ingest-config.ts';
 import { createS2FactLog } from '../fact-log/s2/create-s2-fact-log.ts';
 import { createIngest } from '../receive/create-ingest.ts';
-import { noSenderProof } from '../sender-proof/none/no-sender-proof.ts';
+import { readImpIdentitySenderProof } from '../sender-proof/imp-identity/read-imp-identity-sender-proof.ts';
+import { startPullTimers } from './start-pull-timers.ts';
 
 interface ServeIngestOptions {
   deployment: Deployment;
   environment: Readonly<Record<string, string | undefined>>;
 }
 
-// The ingest process of a deployment. It exits 1 at start on a failed deployment check or on any
-// missing variable, so no route ever runs with an empty secret.
+// The ingest process of a deployment. It exits 1 at start on a failed deployment check, on any
+// missing variable, or on a trust folder it cannot load, so no route ever runs with an empty secret.
+// A path whose trust documents or audience are unset starts, and refuses every delivery.
 export async function serveIngest(options: ServeIngestOptions): Promise<void> {
   const checked = checkDeployment(options.deployment);
 
@@ -30,15 +32,25 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
     process.exit(1);
   }
 
+  const senderProof = await readImpIdentitySenderProof({
+    sources,
+    environment: options.environment,
+    now: () => Date.now(),
+  });
+
+  if (!senderProof.ok) {
+    console.error(senderProof.failure);
+    process.exit(1);
+  }
+
   // The S2 adapter creates the basin and each missing stream, so no setup command creates streams.
   const log = await createS2FactLog({ ...config.value.s2, streams: collectStreams(sources) });
 
-  // No adapter verifies a sender yet, so a path that needs one refuses every delivery.
   const ingest = createIngest({
     deployment: checked.value,
     secrets: config.value.secrets,
     log,
-    senderProof: noSenderProof,
+    senderProof: senderProof.value,
     appendRetry: { maxAttempts: 5, retryDelayMs: 200 },
     now: () => Date.now(),
   });
@@ -47,6 +59,8 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
     console.log(`route ${route.route}`, JSON.stringify(route));
   }
 
+  const stopPulls = startPullTimers(sources, ingest.runPull);
+
   const server = Bun.serve({
     hostname: config.value.host,
     port: config.value.port,
@@ -56,7 +70,11 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
   console.log(`ingest listening on ${server.hostname}:${server.port}`);
 
   const shutdown = makeShutdownHandler({
-    stop: () => server.stop(),
+    stop: () => {
+      stopPulls();
+
+      return server.stop();
+    },
     logError: (message, error) => {
       console.error(message, error);
     },
@@ -70,10 +88,14 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
   }
 }
 
+// every receive path's secret, and every name a pull path's `env` lists
 function collectSecretNames(sources: readonly SourceDefinition[]): string[] {
-  const names = sources.flatMap((source) =>
-    source.live?.kind === 'receive' ? [source.live.check.secretEnv] : [],
-  );
+  const names = sources.flatMap((source) => [
+    ...(source.live?.kind === 'receive' ? [source.live.check.secretEnv] : []),
+    ...[source.live, source.recovery, source.backfill, source.resync].flatMap((path) =>
+      path?.kind === 'pull' ? (path.env ?? []) : [],
+    ),
+  ]);
 
   return [...new Set(names)];
 }

package/src/serve/start-pull-timers.ts

@@ -0,0 +1,69 @@
+import type { SourceDefinition } from '@pepys/kit';
+import { match } from 'ts-pattern';
+import type { PullPathName, PullRun, PullRunner } from '../pull/make-pull-runner.ts';
+
+const pullPathNames: readonly PullPathName[] = ['live', 'recovery', 'backfill', 'resync'];
+
+// One timer per pull path with an interval. The cursor lives in memory, so a restart starts each
+// pull from no cursor. A run never overlaps the one before it. Returns the function that stops them.
+export function startPullTimers(
+  sources: readonly SourceDefinition[],
+  runPull: PullRunner,
+): () => void {
+  const timers = sources.flatMap((source) =>
+    pullPathNames.flatMap((pathName) => {
+      const path = source[pathName];
+
+      if (path?.kind !== 'pull' || path.intervalMs === null) {
+        return [];
+      }
+
+      let cursor: string | null = null;
+      let isRunning = false;
+
+      const runOnce = async (): Promise<void> => {
+        if (isRunning) {
+          return;
+        }
+
+        isRunning = true;
+
+        try {
+          const run = await runPull(source.name, pathName, cursor);
+
+          cursor = pickNextCursor(run, cursor);
+
+          if (run.kind !== 'stored') {
+            console.error(`the ${pathName} pull of ${source.name} did not store`, run);
+          }
+        } catch (error: unknown) {
+          console.error(`the ${pathName} pull of ${source.name} threw`, error);
+        } finally {
+          isRunning = false;
+        }
+      };
+
+      return [
+        setInterval(() => {
+          void runOnce();
+        }, path.intervalMs),
+      ];
+    }),
+  );
+
+  return () => {
+    for (const timer of timers) {
+      clearInterval(timer);
+    }
+  };
+}
+
+// An expired cursor starts the pull again from none; any other failure keeps the cursor.
+function pickNextCursor(run: PullRun, cursor: string | null): string | null {
+  return match(run)
+    .with({ kind: 'stored' }, (stored) => stored.cursor)
+    .with({ kind: 'expired' }, () => null)
+    .with({ kind: 'not-stored' }, { kind: 'refused' }, { kind: 'unavailable' }, () => cursor)
+    .with({ kind: 'rejected' }, () => cursor)
+    .exhaustive();
+}