@pepys/ingest 0.3.1 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pepys/ingest",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "pepys ingest: the receive routes, the fact log and the blob store",
   "license": "MIT",
   "repository": {
@@ -22,8 +22,8 @@
   },
   "dependencies": {
     "@impz/client": "6.12.0",
-    "@pepys/kit": "0.3.1",
-    "@pepys/schema": "0.3.1",
+    "@pepys/kit": "0.4.0",
+    "@pepys/schema": "0.4.0",
     "@s2-dev/streamstore": "0.30.0",
     "@zgeoff/std": "0.4.3",
     "tiny-invariant": "1.3.3",

package/src/fact-log/make-batch-appender.ts

@@ -1,11 +1,14 @@
-import type { FactEnvelope, Result, StreamName, StreamOffset } from '@pepys/kit';
+import type { FactEnvelope, Logger, Result, StreamName, StreamOffset } from '@pepys/kit';
 import { toTagged } from '@zgeoff/std';
-import { P, match } from 'ts-pattern';
 import type { FactLog, StoredFact } from './fact-log.ts';
-import { pickSettlement } from './pick-settlement.ts';
+import { pickAttemptStep } from './pick-attempt-step.ts';
+import type { AttemptResult } from './pick-attempt-step.ts';
+import { pickRetryStep } from './pick-retry-step.ts';
+import { pickSettledAttempt } from './pick-settled-attempt.ts';
 
 interface BatchAppenderOptions {
   log: FactLog;
+  logger: Logger;
   maxAttempts: number;
   retryDelayMs: number;
 }
@@ -31,100 +34,59 @@ export function makeBatchAppender(options: BatchAppenderOptions): BatchAppender
     for (let attempt = 1; ; attempt += 1) {
       const result = await runAttempt(options, stream, facts, pendingTail);
 
-      if (result.kind === 'stored') {
-        return { ok: true, value: result.stored };
-      }
+      const step = pickRetryStep(result, attempt, options);
 
-      if (result.kind === 'refused') {
-        return { ok: false, failure: result.failure };
+      if (step.kind === 'answer') {
+        return step.answer;
       }
 
-      pendingTail = result.pendingTail;
-
-      if (attempt >= options.maxAttempts) {
-        return { ok: false, failure: { kind: 'not-stored', attempts: attempt } };
-      }
+      pendingTail = step.pendingTail;
 
-      await Bun.sleep(options.retryDelayMs * attempt);
+      await Bun.sleep(step.delayMs);
     }
   };
 }
 
-interface Stored {
-  firstOffset: StreamOffset;
-  count: number;
-}
-
-type StopFailure = Extract<AppendFailure, { kind: 'refused' }>;
-
-// What one attempt leaves: the answer, or the tail that earlier attempts with an unknown outcome
-// aimed at, which a later attempt can still land on. `null` when no such attempt is open.
-type AttemptResult =
-  | { kind: 'stored'; stored: Stored }
-  | { kind: 'refused'; failure: StopFailure }
-  | { kind: 'again'; pendingTail: StreamOffset | null };
-
+// Every attempt that is still open aimed at the same tail: a new tail comes only after the offsets
+// there are known to hold another writer's records.
 async function runAttempt(
-  options: BatchAppenderOptions,
+  options: Pick<BatchAppenderOptions, 'log' | 'logger'>,
   stream: StreamName,
   facts: readonly FactEnvelope<unknown>[],
   pendingTail: StreamOffset | null,
 ): Promise<AttemptResult> {
-  // Every attempt that is still open aimed at the same tail: a new tail comes only after the
-  // offsets there are known to hold another writer's records.
-  const read =
+  const aim =
     pendingTail === null
-      ? await readTail(options.log, stream)
-      : { ok: true as const, value: pendingTail };
+      ? await readAimedTail(options, stream)
+      : { kind: 'aim' as const, tail: pendingTail };
 
-  if (read === null) {
-    return { kind: 'again', pendingTail };
+  if (aim.kind !== 'aim') {
+    return aim;
   }
 
-  if (!read.ok) {
-    return { kind: 'refused', failure: read.failure };
-  }
+  const outcome = await options.log.appendBatch(stream, facts, aim.tail);
 
-  const aimed = read.value;
-
-  const outcome = await options.log.appendBatch(stream, facts, aimed);
-
-  return match(outcome)
-    .with({ kind: 'acknowledged' }, (acknowledged) => ({
-      kind: 'stored' as const,
-      stored: { firstOffset: acknowledged.firstOffset, count: acknowledged.count },
-    }))
-    .with({ kind: 'rejected', reason: 'tail-mismatch' }, () =>
-      pendingTail === null
-        ? { kind: 'again' as const, pendingTail }
-        : settleAttempt(options, stream, facts, aimed),
-    )
-    .with(
-      { kind: 'rejected', reason: P.union('unknown-stream', 'invalid-envelope', 'too-large') },
-      (rejected) => ({
-        kind: 'refused' as const,
-        failure: { kind: 'refused' as const, reason: rejected.reason },
-      }),
-    )
-    .with({ kind: 'unknown' }, () => settleAttempt(options, stream, facts, aimed))
-    .exhaustive();
+  const step = pickAttemptStep(outcome, aim.tail, pendingTail);
+
+  return step.kind === 'settle' ? settleAttempt(options, stream, facts, step.tail) : step;
 }
 
-// A failed read stored nothing, so the next attempt repeats it. Returns `null` for a failed read,
-// and the failure for a stream the log lacks. An error the append throws is a refusal the log gives
-// no outcome for, such as a bad token, and a retry cannot change it.
-async function readTail(
-  log: FactLog,
+// A failed read stored nothing, so the next attempt repeats it. An error the append throws is a
+// refusal the log gives no outcome for, such as a bad token, and a retry cannot change it.
+async function readAimedTail(
+  options: Pick<BatchAppenderOptions, 'log' | 'logger'>,
   stream: StreamName,
-): Promise<Result<StreamOffset, StopFailure> | null> {
+): Promise<{ kind: 'aim'; tail: StreamOffset } | AttemptResult> {
   try {
-    const tail = await log.readTail(stream);
+    const tail = await options.log.readTail(stream);
 
-    return tail.ok ? tail : { ok: false, failure: { kind: 'refused', reason: 'unknown-stream' } };
+    return tail.ok
+      ? { kind: 'aim', tail: tail.value }
+      : { kind: 'refused', failure: { kind: 'refused', reason: 'unknown-stream' } };
   } catch (error: unknown) {
-    console.error('a read before an append failed', error);
+    options.logger.error('a read before an append failed', error);
 
-    return null;
+    return { kind: 'again', pendingTail: null };
   }
 }
 
@@ -132,38 +94,22 @@ async function readTail(
 // with the run's, in order. A search for the first ID anywhere after the tail would take another
 // writer's copy of it for the whole run.
 async function settleAttempt(
-  options: BatchAppenderOptions,
+  options: Pick<BatchAppenderOptions, 'log' | 'logger'>,
   stream: StreamName,
   facts: readonly FactEnvelope<unknown>[],
   tail: StreamOffset,
 ): Promise<AttemptResult> {
   const stored = await readRun(options.log, stream, tail, facts.length).catch((error: unknown) => {
-    console.error('a read that settles an append failed', error);
+    options.logger.error('a read that settles an append failed', error);
 
     return null;
   });
 
-  if (stored === null) {
-    return { kind: 'again', pendingTail: tail };
-  }
-
-  if (!stored.ok) {
-    return { kind: 'refused', failure: { kind: 'refused', reason: 'unknown-stream' } };
-  }
-
-  return match(
-    pickSettlement(
-      facts.map((fact) => fact.id),
-      stored.value,
-    ),
-  )
-    .with('landed', () => ({
-      kind: 'stored' as const,
-      stored: { firstOffset: tail, count: facts.length },
-    }))
-    .with('not-landed', () => ({ kind: 'again' as const, pendingTail: tail }))
-    .with('taken', () => ({ kind: 'again' as const, pendingTail: null }))
-    .exhaustive();
+  return pickSettledAttempt(
+    stored,
+    facts.map((fact) => fact.id),
+    tail,
+  );
 }
 
 // A read can end before `count` records, so it repeats from the next offset until it has them all

package/src/fact-log/pick-attempt-step.ts

@@ -0,0 +1,53 @@
+import type { StreamOffset } from '@pepys/kit';
+import { P, match } from 'ts-pattern';
+import type { AppendBatchOutcome } from './fact-log.ts';
+
+export interface StoredRun {
+  firstOffset: StreamOffset;
+  count: number;
+}
+
+// a reason the log gives that no retry changes
+export interface StopFailure {
+  kind: 'refused';
+  reason: 'unknown-stream' | 'invalid-envelope' | 'too-large';
+}
+
+// What one attempt leaves: the answer, or the tail that earlier attempts with an unknown outcome
+// aimed at, which a later attempt can still land on. `null` when no such attempt is open.
+export type AttemptResult =
+  | { kind: 'stored'; stored: StoredRun }
+  | { kind: 'refused'; failure: StopFailure }
+  | { kind: 'again'; pendingTail: StreamOffset | null };
+
+// `settle`: read the offsets from `tail` to learn whether the run is there.
+export type AttemptStep = AttemptResult | { kind: 'settle'; tail: StreamOffset };
+
+// What follows an append that aimed at `aimed`. A tail mismatch with no attempt open means another
+// writer moved the tail, so the next attempt reads it again. With an attempt open, the mismatch may
+// be that attempt landing late, so the offsets it aimed at are read.
+export function pickAttemptStep(
+  outcome: AppendBatchOutcome,
+  aimed: StreamOffset,
+  pendingTail: StreamOffset | null,
+): AttemptStep {
+  return match(outcome)
+    .with({ kind: 'acknowledged' }, (acknowledged) => ({
+      kind: 'stored' as const,
+      stored: { firstOffset: acknowledged.firstOffset, count: acknowledged.count },
+    }))
+    .with({ kind: 'rejected', reason: 'tail-mismatch' }, () =>
+      pendingTail === null
+        ? { kind: 'again' as const, pendingTail }
+        : { kind: 'settle' as const, tail: aimed },
+    )
+    .with(
+      { kind: 'rejected', reason: P.union('unknown-stream', 'invalid-envelope', 'too-large') },
+      (rejected) => ({
+        kind: 'refused' as const,
+        failure: { kind: 'refused' as const, reason: rejected.reason },
+      }),
+    )
+    .with({ kind: 'unknown' }, () => ({ kind: 'settle' as const, tail: aimed }))
+    .exhaustive();
+}

package/src/fact-log/pick-retry-step.ts

@@ -0,0 +1,47 @@
+import type { Result, StreamOffset } from '@pepys/kit';
+import { match } from 'ts-pattern';
+import type { AttemptResult, StopFailure, StoredRun } from './pick-attempt-step.ts';
+
+interface RetryLimits {
+  maxAttempts: number;
+  retryDelayMs: number;
+}
+
+export type RetryStep =
+  | {
+      kind: 'answer';
+      answer: Result<StoredRun, StopFailure | { kind: 'not-stored'; attempts: number }>;
+    }
+  | { kind: 'retry'; pendingTail: StreamOffset | null; delayMs: number };
+
+// The wait before attempt `attempt + 1` grows by `retryDelayMs` with each attempt.
+export function pickRetryStep(
+  result: AttemptResult,
+  attempt: number,
+  limits: RetryLimits,
+): RetryStep {
+  return match(result)
+    .with({ kind: 'stored' }, (stored) => ({
+      kind: 'answer' as const,
+      answer: { ok: true as const, value: stored.stored },
+    }))
+    .with({ kind: 'refused' }, (refused) => ({
+      kind: 'answer' as const,
+      answer: { ok: false as const, failure: refused.failure },
+    }))
+    .with({ kind: 'again' }, (again): RetryStep => {
+      if (attempt >= limits.maxAttempts) {
+        return {
+          kind: 'answer',
+          answer: { ok: false, failure: { kind: 'not-stored', attempts: attempt } },
+        };
+      }
+
+      return {
+        kind: 'retry',
+        pendingTail: again.pendingTail,
+        delayMs: limits.retryDelayMs * attempt,
+      };
+    })
+    .exhaustive();
+}

package/src/fact-log/pick-settled-attempt.ts

@@ -0,0 +1,30 @@
+import type { FactID, Result, StreamOffset } from '@pepys/kit';
+import { match } from 'ts-pattern';
+import type { StoredFact } from './fact-log.ts';
+import type { AttemptResult } from './pick-attempt-step.ts';
+import { pickSettlement } from './pick-settlement.ts';
+
+// `stored` is the read of the offsets `[tail, tail + n)` that an attempt aimed at, or `null` when
+// the read failed, which leaves the attempt open.
+export function pickSettledAttempt(
+  stored: Result<readonly StoredFact[], { reason: 'unknown-stream' }> | null,
+  ids: readonly FactID[],
+  tail: StreamOffset,
+): AttemptResult {
+  if (stored === null) {
+    return { kind: 'again', pendingTail: tail };
+  }
+
+  if (!stored.ok) {
+    return { kind: 'refused', failure: { kind: 'refused', reason: 'unknown-stream' } };
+  }
+
+  return match(pickSettlement(ids, stored.value))
+    .with('landed', () => ({
+      kind: 'stored' as const,
+      stored: { firstOffset: tail, count: ids.length },
+    }))
+    .with('not-landed', () => ({ kind: 'again' as const, pendingTail: tail }))
+    .with('taken', () => ({ kind: 'again' as const, pendingTail: null }))
+    .exhaustive();
+}

package/src/fact-log/s2/create-s2-fact-log.ts

@@ -1,4 +1,5 @@
-import type { FactEnvelope, Result, StreamName, StreamOffset } from '@pepys/kit';
+import { createLogger } from '@pepys/kit';
+import type { FactEnvelope, Logger, Result, StreamName, StreamOffset } from '@pepys/kit';
 import {
   AppendInput,
   AppendRecord,
@@ -18,6 +19,9 @@ interface S2FactLogParameters {
   accessToken: string;
   basin: string;
   streams: readonly StreamName[];
+
+  // defaults to a logger that writes to stderr
+  logger?: Logger | undefined;
 }
 
 // Creates the basin and the streams when they are missing. A stream that exists keeps its records
@@ -29,6 +33,8 @@ export async function createS2FactLog(parameters: S2FactLogParameters): Promise<
     requestTimeoutMillis: 5000,
   };
 
+  const logger = parameters.logger ?? createLogger(process.stderr);
+
   const client = new S2(options);
 
   // A client retry could store a record twice before the writer knows; the writer owns retries.
@@ -45,7 +51,7 @@ export async function createS2FactLog(parameters: S2FactLogParameters): Promise<
 
   return {
     appendBatch: (stream, facts, expectedTail) =>
-      appendBatch(appendBasin, stream, facts, expectedTail),
+      appendBatch({ basin: appendBasin, logger }, stream, facts, expectedTail),
     read: (stream, from, count) => readFacts(basin, stream, from, count),
     readTail: (stream) => readTail(basin, stream),
   };
@@ -78,8 +84,13 @@ async function createInfiniteStream(basin: S2Basin, stream: StreamName): Promise
   }
 }
 
+interface S2Writer {
+  basin: S2Basin;
+  logger: Logger;
+}
+
 async function appendBatch(
-  basin: S2Basin,
+  writer: S2Writer,
   stream: StreamName,
   facts: readonly FactEnvelope<unknown>[],
   expectedTail: StreamOffset,
@@ -97,7 +108,7 @@ async function appendBatch(
   );
 
   try {
-    const ack = await basin.stream(stream).append(input);
+    const ack = await writer.basin.stream(stream).append(input);
 
     return {
       kind: 'acknowledged',
@@ -112,7 +123,7 @@ async function appendBatch(
     }
 
     if (outcome.kind === 'unknown') {
-      console.error('an s2 append has an unknown outcome', error);
+      writer.logger.error('an s2 append has an unknown outcome', error);
     }
 
     return outcome;

package/src/index.ts

@@ -20,4 +20,3 @@ export { readImpIdentityTrust } from './sender-proof/imp-identity/read-imp-ident
 export type { ImpIdentityTrust } from './sender-proof/imp-identity/types.ts';
 export { noSenderProof } from './sender-proof/none/no-sender-proof.ts';
 export { serveIngest } from './serve/serve-ingest.ts';
-export type * from './source-port/types.ts';

package/src/pull/make-pull-runner.ts

@@ -1,7 +1,6 @@
-import type { CheckedDeployment, PullPath, Pulled, SourceDefinition } from '@pepys/kit';
+import type { CheckedDeployment, Logger, PullPath, Pulled, SourceDefinition } from '@pepys/kit';
 import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
-import { findBatchProblem } from '../receive/find-batch-problem.ts';
-import { planBatch } from '../receive/plan-batch.ts';
+import { pickBatchPlan } from '../receive/pick-batch-plan.ts';
 import type { ResyncMarks } from '../receive/resync-marks/create-resync-marks.ts';
 import { writeBatch } from '../receive/write-batch.ts';
 import type { StreamTurns } from '../receive/write-batch.ts';
@@ -20,6 +19,7 @@ export type PullRunner = (
   source: string,
   path: PullPathName,
   cursor: string | null,
+  signal: AbortSignal,
 ) => Promise<PullRun>;
 
 interface PullRunnerDependencies {
@@ -31,13 +31,14 @@ interface PullRunnerDependencies {
   turns: StreamTurns;
   resyncMarks: ResyncMarks;
   now: () => number;
+  logger: Logger;
 }
 
 // For a resync pull, removes each mark it was given that no tie set again since. A pull's fact is
 // noted in no mark: a resynced fact ties with the body it replaces, and would mark its entity again
 // on every run.
 export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner {
-  return async (sourceName, pathName, cursor) => {
+  return async (sourceName, pathName, cursor, signal) => {
     const source = dependencies.deployment.sources.find(
       (candidate) => candidate.name === sourceName,
     );
@@ -55,26 +56,22 @@ export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner
       env[name] = dependencies.secrets[name] ?? '';
     }
 
-    const pulled = await path.pull(cursor, { env, marks, now: dependencies.now });
+    const pulled = await path.pull(cursor, {
+      env,
+      marks,
+      now: dependencies.now,
+      log: dependencies.logger,
+      signal,
+    });
 
     if (pulled.kind !== 'batch') {
       return pulled;
     }
 
-    const problem = findBatchProblem(source, pulled.batch);
-
-    if (problem !== undefined) {
-      return { kind: 'refused', reason: problem };
-    }
-
-    if (pulled.batch.facts.some((pending) => pending.blobs.length > 0)) {
-      return { kind: 'refused', reason: 'ingest has no blob store' };
-    }
-
-    const plan = planBatch(pulled.batch);
+    const plan = pickBatchPlan(source, pulled.batch);
 
     if (!plan.ok) {
-      return { kind: 'refused', reason: 'one fact is larger than one append of the log takes' };
+      return { kind: 'refused', reason: plan.failure.reason };
     }
 
     const outcome = await writeBatch(plan.value, dependencies);

package/src/receive/create-ingest.ts

@@ -1,4 +1,5 @@
-import type { CheckedDeployment, SenderProof } from '@pepys/kit';
+import { createLogger } from '@pepys/kit';
+import type { CheckedDeployment, Logger, SenderProof } from '@pepys/kit';
 import type { FactLog } from '../fact-log/fact-log.ts';
 import { makeBatchAppender } from '../fact-log/make-batch-appender.ts';
 import { makePullRunner } from '../pull/make-pull-runner.ts';
@@ -20,6 +21,9 @@ export interface IngestOptions {
   senderProof: SenderProof;
   appendRetry: { maxAttempts: number; retryDelayMs: number };
   now: () => number;
+
+  // Defaults to a logger that writes to stderr. A source's `read` and `pull` get it as `log`.
+  logger?: Logger | undefined;
 }
 
 export interface Ingest {
@@ -38,7 +42,8 @@ const recentBodyMaxAgeMs = 3_600_000;
 
 // The receive routes of a deployment over a given log, as one fetch function with no listener.
 export function createIngest(options: IngestOptions): Ingest {
-  const append = makeBatchAppender({ log: options.log, ...options.appendRetry });
+  const logger = options.logger ?? createLogger(process.stderr);
+  const append = makeBatchAppender({ log: options.log, logger, ...options.appendRetry });
 
   const turns: StreamTurns = new Map();
 
@@ -60,6 +65,7 @@ export function createIngest(options: IngestOptions): Ingest {
       makeReceiveHandler(source, path, {
         secret,
         senderProof: options.senderProof,
+        logger,
         append,
         turns,
         recentBodies: createRecentBodies({
@@ -74,7 +80,7 @@ export function createIngest(options: IngestOptions): Ingest {
     for (const route of path.ignoredRoutes ?? []) {
       handlers.set(
         route,
-        makeIgnoredRouteHandler(path, { secret, senderProof: options.senderProof }),
+        makeIgnoredRouteHandler(path, { secret, senderProof: options.senderProof, logger }),
       );
     }
   }
@@ -99,6 +105,7 @@ export function createIngest(options: IngestOptions): Ingest {
       turns,
       resyncMarks: resync.marks,
       now: options.now,
+      logger,
     }),
   };
 }

package/src/receive/find-batch-problem.ts

@@ -3,7 +3,10 @@ import type { Batch, SourceContext } from '@pepys/kit';
 
 // Every fact is checked before any is written: its stream must be one of the source's families, and
 // its envelope must parse.
-export function findBatchProblem(source: SourceContext, batch: Batch): string | undefined {
+export function findBatchProblem(
+  source: Pick<SourceContext, 'name' | 'families'>,
+  batch: Batch,
+): string | undefined {
   const streams = new Set<string>(source.families.map((family) => family.stream));
 
   for (const [index, pending] of batch.facts.entries()) {

package/src/receive/make-receive-handler.ts

@@ -1,6 +1,7 @@
 import { createHash } from 'node:crypto';
 import { isDeliveryAuthentic } from '@pepys/kit';
 import type {
+  Logger,
   ReceivePath,
   Received,
   Sender,
@@ -12,8 +13,7 @@ import { match } from 'ts-pattern';
 import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
 import { buildFailureAnswer, buildRetryAnswer } from './answers/build-failure-answer.ts';
 import { buildSuccessAnswer } from './answers/build-success-answer.ts';
-import { findBatchProblem } from './find-batch-problem.ts';
-import { planBatch } from './plan-batch.ts';
+import { pickReceivedStep } from './pick-received-step.ts';
 import { readDecodedBody } from './read-decoded-body.ts';
 import { readSentBody } from './read-sent-body.ts';
 import type { RecentBodies } from './recent-bodies/create-recent-bodies.ts';
@@ -25,6 +25,7 @@ export interface ReceiveDependencies {
   // the value of the path's `check.secretEnv`; an empty secret passes no delivery
   secret: string;
   senderProof: SenderProof;
+  logger: Logger;
   append: BatchAppender;
   turns: StreamTurns;
   recentBodies: RecentBodies;
@@ -77,51 +78,33 @@ export function makeReceiveHandler(
       return pickDecodeFailure(body);
     }
 
-    const received = readSourceDelivery(source, path, {
+    const received = readSourceDelivery(source, path, dependencies.logger, {
       headers: request.headers,
       body,
       ...(checked.sender === undefined ? {} : { sender: checked.sender }),
     });
 
-    if (received === 'threw') {
-      return buildFailureAnswer(500, 'the source could not read the delivery');
-    }
+    const step = pickReceivedStep(source, received, factLimit);
 
-    if (received.kind === 'refused') {
-      return buildFailureAnswer(received.status, received.reason);
+    if (step.kind === 'refuse') {
+      return buildFailureAnswer(step.status, step.reason);
     }
 
-    if (received.kind === 'ignored' || received.batch.facts.length === 0) {
+    if (step.kind === 'ignore') {
       return buildSuccessAnswer(path, 'ignored', 0);
     }
 
-    const facts = received.batch.facts;
-
-    if (facts.length > factLimit) {
-      return buildFailureAnswer(413, `a batch holds ${factLimit} facts at most`);
-    }
-
-    const problem = findBatchProblem(source, received.batch);
-
-    if (problem !== undefined) {
-      return buildFailureAnswer(422, problem);
-    }
-
-    if (facts.some((pending) => pending.blobs.length > 0)) {
-      console.error(
+    if (step.kind === 'has-blob') {
+      dependencies.logger.error(
         `the source ${source.name} read a fact with a blob, and ingest has no blob store`,
       );
 
       return buildFailureAnswer(500, 'ingest has no blob store');
     }
 
-    const plan = planBatch(received.batch);
-
-    if (!plan.ok) {
-      return buildFailureAnswer(413, 'one fact is larger than one append of the log takes');
-    }
+    const facts = step.facts;
 
-    const outcome = await writeBatch(plan.value, dependencies);
+    const outcome = await writeBatch(step.plan, dependencies);
 
     if (outcome === 'not-stored') {
       return buildRetryAnswer();
@@ -152,7 +135,7 @@ export function makeReceiveHandler(
 // the ignored answer. It reads no body further and stores nothing.
 export function makeIgnoredRouteHandler(
   path: ReceivePath,
-  dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
+  dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof' | 'logger'>,
 ): Handler {
   return async (request) => {
     const checked = await readCheckedDelivery(request, path, dependencies);
@@ -170,7 +153,7 @@ type CheckedDelivery =
 async function readCheckedDelivery(
   request: Request,
   path: ReceivePath,
-  dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
+  dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof' | 'logger'>,
 ): Promise<CheckedDelivery> {
   if (request.method !== 'POST') {
     return { kind: 'answer', answer: buildFailureAnswer(405, 'method not allowed') };
@@ -190,7 +173,7 @@ async function readCheckedDelivery(
     return { kind: 'checked', sent, sender: undefined };
   }
 
-  const sender = await readProvenSender(path.sender, request.headers, dependencies.senderProof);
+  const sender = await readProvenSender(path.sender, request.headers, dependencies);
 
   if (sender === null) {
     return { kind: 'answer', answer: buildFailureAnswer(401, 'not authorised') };
@@ -203,14 +186,14 @@ async function readCheckedDelivery(
 async function readProvenSender(
   requirement: SenderRequirement,
   headers: Headers,
-  senderProof: SenderProof,
+  dependencies: Pick<ReceiveDependencies, 'senderProof' | 'logger'>,
 ): Promise<Sender | null> {
   try {
-    const sender = await senderProof.prove(requirement, headers);
+    const sender = await dependencies.senderProof.prove(requirement, headers);
 
     return sender;
   } catch {
-    console.error('the sender proof threw, so the delivery proves no sender');
+    dependencies.logger.error('the sender proof threw, so the delivery proves no sender');
 
     return null;
   }
@@ -233,12 +216,13 @@ function pickDecodeFailure(failure: 'unsupported' | 'too-large' | 'corrupt'): Re
 function readSourceDelivery(
   source: SourceDefinition,
   path: ReceivePath,
+  logger: Logger,
   delivery: Parameters<ReceivePath['read']>[0],
 ): Received | 'threw' {
   try {
-    return path.read(delivery, { name: source.name, families: source.families });
+    return path.read(delivery, { name: source.name, families: source.families, log: logger });
   } catch (error: unknown) {
-    console.error(`the read of the source ${source.name} threw`, error);
+    logger.error(`the read of the source ${source.name} threw`, error);
 
     return 'threw';
   }

package/src/receive/pick-batch-plan.ts

@@ -0,0 +1,37 @@
+import type { Batch, Result, SourceContext } from '@pepys/kit';
+import { findBatchProblem } from './find-batch-problem.ts';
+import { planBatch } from './plan-batch.ts';
+import type { BatchPlan } from './plan-batch.ts';
+
+export interface BatchRefusal {
+  kind: 'bad-batch' | 'has-blob' | 'too-large';
+  reason: string;
+}
+
+// The checks a batch passes before any of it is written, in order: every fact's stream and
+// envelope, no fact with a blob, then the cut into appends.
+export function pickBatchPlan(
+  source: Pick<SourceContext, 'name' | 'families'>,
+  batch: Batch,
+): Result<BatchPlan, BatchRefusal> {
+  const problem = findBatchProblem(source, batch);
+
+  if (problem !== undefined) {
+    return { ok: false, failure: { kind: 'bad-batch', reason: problem } };
+  }
+
+  if (batch.facts.some((pending) => pending.blobs.length > 0)) {
+    return { ok: false, failure: { kind: 'has-blob', reason: 'ingest has no blob store' } };
+  }
+
+  const plan = planBatch(batch);
+
+  if (!plan.ok) {
+    return {
+      ok: false,
+      failure: { kind: 'too-large', reason: 'one fact is larger than one append of the log takes' },
+    };
+  }
+
+  return plan;
+}

package/src/receive/pick-received-step.ts

@@ -0,0 +1,56 @@
+import type { PendingFact, Received, SourceContext } from '@pepys/kit';
+import { match } from 'ts-pattern';
+import { pickBatchPlan } from './pick-batch-plan.ts';
+import type { BatchPlan } from './plan-batch.ts';
+
+// `has-blob`: the source read a fact with a blob, a fault of the deployment that ingest answers 500.
+export type ReceivedStep =
+  | { kind: 'refuse'; status: number; reason: string }
+  | { kind: 'has-blob' }
+  | { kind: 'ignore' }
+  | { kind: 'write'; facts: readonly PendingFact[]; plan: BatchPlan };
+
+// What follows the source's read of a delivery, `threw` when the read threw.
+export function pickReceivedStep(
+  source: Pick<SourceContext, 'name' | 'families'>,
+  received: Received | 'threw',
+  factLimit: number,
+): ReceivedStep {
+  if (received === 'threw') {
+    return { kind: 'refuse', status: 500, reason: 'the source could not read the delivery' };
+  }
+
+  if (received.kind === 'refused') {
+    return { kind: 'refuse', status: received.status, reason: received.reason };
+  }
+
+  if (received.kind === 'ignored' || received.batch.facts.length === 0) {
+    return { kind: 'ignore' };
+  }
+
+  const facts = received.batch.facts;
+
+  if (facts.length > factLimit) {
+    return { kind: 'refuse', status: 413, reason: `a batch holds ${factLimit} facts at most` };
+  }
+
+  const plan = pickBatchPlan(source, received.batch);
+
+  if (!plan.ok) {
+    return match(plan.failure.kind)
+      .with('bad-batch', () => ({
+        kind: 'refuse' as const,
+        status: 422,
+        reason: plan.failure.reason,
+      }))
+      .with('has-blob', () => ({ kind: 'has-blob' as const }))
+      .with('too-large', () => ({
+        kind: 'refuse' as const,
+        status: 413,
+        reason: plan.failure.reason,
+      }))
+      .exhaustive();
+  }
+
+  return { kind: 'write', facts, plan: plan.value };
+}

package/src/receive/write-batch.ts

@@ -1,4 +1,4 @@
-import type { StreamName } from '@pepys/kit';
+import type { Logger, StreamName } from '@pepys/kit';
 import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
 import type { BatchPlan } from './plan-batch.ts';
 
@@ -11,6 +11,7 @@ export type StreamTurns = Map<StreamName, Promise<void>>;
 interface WriteBatchDependencies {
   append: BatchAppender;
   turns: StreamTurns;
+  logger: Logger;
 }
 
 // The streams append at the same time, and the appends of one stream one after the other. Every
@@ -21,12 +22,12 @@ export async function writeBatch(
 ): Promise<BatchOutcome> {
   const settled = await Promise.all(
     [...plan].map(([stream, runs]) =>
-      withStreamTurn(dependencies.turns, stream, async () => {
+      withStreamTurn(dependencies, stream, async () => {
         for (const run of runs) {
           const appended = await dependencies.append(stream, run);
 
           if (!appended.ok) {
-            console.error(`an append to ${stream} failed`, appended.failure);
+            dependencies.logger.error(`an append to ${stream} failed`, appended.failure);
 
             return appended.failure.kind;
           }
@@ -42,10 +43,11 @@ export async function writeBatch(
 
 // A throw is a refusal the log gives no outcome for, such as a bad access token.
 async function withStreamTurn(
-  turns: StreamTurns,
+  dependencies: Pick<WriteBatchDependencies, 'turns' | 'logger'>,
   stream: StreamName,
   work: () => Promise<BatchOutcome>,
 ): Promise<BatchOutcome> {
+  const turns = dependencies.turns;
   const previous = turns.get(stream) ?? Promise.resolve();
   const done = Promise.withResolvers<void>();
 
@@ -56,7 +58,7 @@ async function withStreamTurn(
 
     return await work();
   } catch (error: unknown) {
-    console.error(`an append to ${stream} threw`, error);
+    dependencies.logger.error(`an append to ${stream} threw`, error);
 
     return 'refused';
   } finally {

package/src/sender-proof/imp-identity/create-imp-identity-sender-proof.ts

@@ -1,17 +1,23 @@
 import type { verifyImpIdentity } from '@impz/client/identity';
-import type { SenderProof } from '@pepys/kit';
+import { createLogger } from '@pepys/kit';
+import type { Logger, SenderProof } from '@pepys/kit';
 import type { ImpIdentityTrust } from './types.ts';
 
 interface ImpIdentityOptions {
   verify: typeof verifyImpIdentity;
   trust: ImpIdentityTrust;
   now: () => number;
+
+  // defaults to a logger that writes to stderr
+  logger?: Logger | undefined;
 }
 
 // Proves the sender from the signed identity token in the `imp-identity` header. The issuer and
 // the subject come only from the verified token, and every failure, a throw included, proves
 // nothing. It never logs the token.
 export function createImpIdentitySenderProof(options: ImpIdentityOptions): SenderProof {
+  const logger = options.logger ?? createLogger(process.stderr);
+
   return {
     prove: async (requirement, headers) => {
       const documents = options.trust.issuers.get(requirement.issuersEnv) ?? [];
@@ -33,7 +39,7 @@ export function createImpIdentitySenderProof(options: ImpIdentityOptions): Sende
           ? null
           : { issuer: verified.issuer, subject: verified.id, name: verified.name };
       } catch {
-        console.error('the identity verifier failed, so the delivery proves no sender');
+        logger.error('the identity verifier failed, so the delivery proves no sender');
 
         return null;
       }

package/src/sender-proof/imp-identity/read-imp-identity-sender-proof.ts

@@ -1,5 +1,5 @@
 import { verifyImpIdentity } from '@impz/client/identity';
-import type { Result, SenderProof, SourceDefinition } from '@pepys/kit';
+import type { Logger, Result, SenderProof, SourceDefinition } from '@pepys/kit';
 import { createImpIdentitySenderProof } from './create-imp-identity-sender-proof.ts';
 import { readImpIdentityTrust } from './read-imp-identity-trust.ts';
 
@@ -7,6 +7,9 @@ interface SenderProofReadOptions {
   sources: readonly SourceDefinition[];
   environment: Readonly<Record<string, string | undefined>>;
   now: () => number;
+
+  // defaults to a logger that writes to stderr
+  logger?: Logger | undefined;
 }
 
 // The sender proof that `serveIngest` runs: the trust of the deployment's environment and the
@@ -30,6 +33,7 @@ export async function readImpIdentitySenderProof(
       verify: verifyImpIdentity,
       trust: trust.value,
       now: options.now,
+      logger: options.logger,
     }),
   };
 }

package/src/serve/serve-ingest.ts

@@ -1,4 +1,4 @@
-import { checkDeployment, listReceiveRoutes } from '@pepys/kit';
+import { checkDeployment, createLogger, listReceiveRoutes } from '@pepys/kit';
 import type { Deployment, SourceDefinition, StreamName } from '@pepys/kit';
 import { makeShutdownHandler } from '@pepys/schema';
 import { toTagged } from '@zgeoff/std';
@@ -11,16 +11,20 @@ import { startPullTimers } from './start-pull-timers.ts';
 interface ServeIngestOptions {
   deployment: Deployment;
   environment: Readonly<Record<string, string | undefined>>;
+
+  // false leaves SIGINT and SIGTERM to the caller, who stops ingest through the returned function
+  exitOnSignal?: boolean;
 }
 
 // The ingest process of a deployment. It exits 1 at start on a failed deployment check, on any
 // missing variable, or on a trust folder it cannot load, so no route ever runs with an empty secret.
 // A path whose trust documents or audience are unset starts, and refuses every delivery.
-export async function serveIngest(options: ServeIngestOptions): Promise<void> {
+export async function serveIngest(options: ServeIngestOptions): Promise<() => Promise<void>> {
+  const logger = createLogger(process.stderr);
   const checked = checkDeployment(options.deployment);
 
   if (!checked.ok) {
-    console.error('the deployment fails its check', JSON.stringify(checked.failure));
+    logger.error('the deployment fails its check', checked.failure);
     process.exit(1);
   }
 
@@ -28,7 +32,7 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
   const config = readIngestConfig(options.environment, collectSecretNames(sources));
 
   if (!config.ok) {
-    console.error(config.failure.join('\n'));
+    logger.error('the ingest config is incomplete', config.failure);
     process.exit(1);
   }
 
@@ -36,15 +40,20 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
     sources,
     environment: options.environment,
     now: () => Date.now(),
+    logger,
   });
 
   if (!senderProof.ok) {
-    console.error(senderProof.failure);
+    logger.error('the sender proof cannot start', senderProof.failure);
     process.exit(1);
   }
 
   // The S2 adapter creates the basin and each missing stream, so no setup command creates streams.
-  const log = await createS2FactLog({ ...config.value.s2, streams: collectStreams(sources) });
+  const log = await createS2FactLog({
+    ...config.value.s2,
+    streams: collectStreams(sources),
+    logger,
+  });
 
   const ingest = createIngest({
     deployment: checked.value,
@@ -53,30 +62,38 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
     senderProof: senderProof.value,
     appendRetry: { maxAttempts: 5, retryDelayMs: 200 },
     now: () => Date.now(),
+    logger,
   });
 
   for (const route of listReceiveRoutes(options.deployment)) {
     console.log(`route ${route.route}`, JSON.stringify(route));
   }
 
-  const stopPulls = startPullTimers(sources, ingest.runPull);
-
+  // The server starts first, so a failed listen leaves no timer behind.
   const server = Bun.serve({
     hostname: config.value.host,
     port: config.value.port,
     fetch: ingest.handleRequest,
   });
 
+  const stopPulls = startPullTimers(sources, ingest.runPull, logger);
+
   console.log(`ingest listening on ${server.hostname}:${server.port}`);
 
-  const shutdown = makeShutdownHandler({
-    stop: () => {
-      stopPulls();
+  const stop = async (): Promise<void> => {
+    await stopPulls();
 
-      return server.stop();
-    },
+    await server.stop();
+  };
+
+  if (options.exitOnSignal === false) {
+    return stop;
+  }
+
+  const shutdown = makeShutdownHandler({
+    stop,
     logError: (message, error) => {
-      console.error(message, error);
+      logger.error(message, error);
     },
     exit: (code) => process.exit(code),
   });
@@ -86,6 +103,8 @@ export async function serveIngest(options: ServeIngestOptions): Promise<void> {
       void shutdown();
     });
   }
+
+  return stop;
 }
 
 // every receive path's secret, and every name a pull path's `env` lists

package/src/serve/start-pull-timers.ts

@@ -1,15 +1,20 @@
-import type { SourceDefinition } from '@pepys/kit';
+import type { Logger, SourceDefinition } from '@pepys/kit';
 import { match } from 'ts-pattern';
 import type { PullPathName, PullRun, PullRunner } from '../pull/make-pull-runner.ts';
 
 const pullPathNames: readonly PullPathName[] = ['live', 'recovery', 'backfill', 'resync'];
 
 // One timer per pull path with an interval. The cursor lives in memory, so a restart starts each
-// pull from no cursor. A run never overlaps the one before it. Returns the function that stops them.
+// pull from no cursor. A run never overlaps the one before it. Returns the function that stops them:
+// it aborts each run in flight and settles once every run has ended.
 export function startPullTimers(
   sources: readonly SourceDefinition[],
   runPull: PullRunner,
-): () => void {
+  logger: Logger,
+): () => Promise<void> {
+  const controller = new AbortController();
+  const activeRuns = new Set<Promise<void>>();
+
   const timers = sources.flatMap((source) =>
     pullPathNames.flatMap((pathName) => {
       const path = source[pathName];
@@ -22,24 +27,34 @@ export function startPullTimers(
       let isRunning = false;
 
       const runOnce = async (): Promise<void> => {
-        if (isRunning) {
+        if (isRunning || controller.signal.aborted) {
           return;
         }
 
         isRunning = true;
 
+        const ended = Promise.withResolvers<void>();
+
+        activeRuns.add(ended.promise);
+
         try {
-          const run = await runPull(source.name, pathName, cursor);
+          const run = await runPull(source.name, pathName, cursor, controller.signal);
 
           cursor = pickNextCursor(run, cursor);
 
-          if (run.kind !== 'stored') {
-            console.error(`the ${pathName} pull of ${source.name} did not store`, run);
+          // a run that the stop cut short is no failure
+          if (run.kind !== 'stored' && !controller.signal.aborted) {
+            logger.error(`the ${pathName} pull of ${source.name} did not store`, run);
           }
         } catch (error: unknown) {
-          console.error(`the ${pathName} pull of ${source.name} threw`, error);
+          if (!controller.signal.aborted) {
+            logger.error(`the ${pathName} pull of ${source.name} threw`, error);
+          }
         } finally {
           isRunning = false;
+
+          activeRuns.delete(ended.promise);
+          ended.resolve();
         }
       };
 
@@ -51,10 +66,14 @@ export function startPullTimers(
     }),
   );
 
-  return () => {
+  return async () => {
     for (const timer of timers) {
       clearInterval(timer);
     }
+
+    controller.abort();
+
+    await Promise.all(activeRuns);
   };
 }
 

package/src/source-port/types.ts

@@ -1,64 +0,0 @@
-import type {
-  BlobHash,
-  ConsumerDeclaration,
-  EntityKind,
-  FactEnvelope,
-  LinkTypeDeclaration,
-  Result,
-  StreamName,
-} from '@pepys/kit';
-
-export interface FactFamily<Payload> {
-  stream: StreamName;
-  types: readonly string[];
-  api: string;
-  parsePayload: (raw: unknown) => Result<Payload, { reason: string }>;
-}
-
-// Provisional: source-retention
-export interface SourceDefinition {
-  name: string;
-  families: readonly FactFamily<unknown>[];
-  entityKinds: readonly EntityKind[];
-  linkTypes: readonly LinkTypeDeclaration[];
-  consumers: readonly ConsumerDeclaration[];
-  live: ReceivePath | PullPath;
-  recovery: PullPath;
-  backfill: PullPath;
-  resync: PullPath;
-}
-
-export interface ReceivePath {
-  kind: 'receive';
-  route: string;
-  receive: (request: Request) => Promise<Received>;
-}
-
-export type Received =
-  | { kind: 'refused'; status: 400 | 401 | 405 | 422; reason: string }
-  | { kind: 'ignored' }
-  | { kind: 'batch'; batch: Batch };
-
-export interface PullPath {
-  kind: 'pull';
-  name: string;
-  intervalMs: number | null;
-  pull: (cursor: string | null) => Promise<Pulled>;
-}
-
-// Provisional: source-retention
-export type Pulled =
-  | { kind: 'batch'; batch: Batch; cursor: string; isDone: boolean }
-  | { kind: 'expired' }
-  | { kind: 'unavailable'; retryAfterMs: number }
-  | { kind: 'rejected'; reason: string };
-
-export interface Batch {
-  facts: readonly PendingFact[];
-}
-
-export interface PendingFact {
-  stream: StreamName;
-  fact: FactEnvelope<unknown>;
-  blobs: readonly { hash: BlobHash; bytes: Uint8Array }[];
-}