@pepys/ingest 0.3.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/package.json +3 -3
  2. package/src/fact-log/make-batch-appender.ts +43 -128
  3. package/src/fact-log/memory/create-memory-fact-log.ts +1 -1
  4. package/src/fact-log/pick-attempt-step.ts +53 -0
  5. package/src/fact-log/pick-retry-step.ts +47 -0
  6. package/src/fact-log/pick-settled-attempt.ts +30 -0
  7. package/src/fact-log/read-run.ts +34 -0
  8. package/src/fact-log/s2/create-s2-fact-log.ts +17 -6
  9. package/src/index.ts +3 -3
  10. package/src/pull/collect-timed-pulls.ts +23 -0
  11. package/src/pull/cursors/build-cursor-entity.ts +8 -0
  12. package/src/pull/cursors/build-cursor-record.ts +32 -0
  13. package/src/pull/cursors/cursor-stream.ts +6 -0
  14. package/src/pull/cursors/format-pull-start.ts +13 -0
  15. package/src/pull/cursors/read-saved-cursors.ts +55 -0
  16. package/src/pull/make-pull-runner.ts +98 -51
  17. package/src/pull/with-source-lock.ts +76 -0
  18. package/src/receive/answers/build-failure-answer.ts +2 -1
  19. package/src/receive/create-ingest.ts +10 -3
  20. package/src/receive/make-receive-handler.ts +21 -37
  21. package/src/receive/pick-received-step.ts +56 -0
  22. package/src/receive/write-batch.ts +8 -6
  23. package/src/sender-proof/imp-identity/create-imp-identity-sender-proof.ts +8 -2
  24. package/src/sender-proof/imp-identity/read-imp-identity-sender-proof.ts +5 -1
  25. package/src/serve/serve-ingest.ts +43 -14
  26. package/src/serve/start-pull-timers.ts +47 -39
  27. package/src/{receive → utils}/find-batch-problem.ts +4 -1
  28. package/src/utils/pick-batch-plan.ts +37 -0
  29. package/src/{receive → utils}/plan-batch.ts +1 -1
  30. package/src/source-port/types.ts +0 -64
  31. /package/src/{fact-log → utils}/check-append-batch.ts +0 -0
  32. /package/src/{fact-log → utils}/parse-stored-body.ts +0 -0
  33. /package/src/{fact-log → utils}/plan-appends.ts +0 -0
@@ -1,15 +1,19 @@
1
- import type { CheckedDeployment, PullPath, Pulled, SourceDefinition } from '@pepys/kit';
1
+ import type { CheckedDeployment, Logger, PullPath, Pulled, SourceDefinition } from '@pepys/kit';
2
2
  import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
3
- import { findBatchProblem } from '../receive/find-batch-problem.ts';
4
- import { planBatch } from '../receive/plan-batch.ts';
5
3
  import type { ResyncMarks } from '../receive/resync-marks/create-resync-marks.ts';
6
4
  import { writeBatch } from '../receive/write-batch.ts';
7
5
  import type { StreamTurns } from '../receive/write-batch.ts';
6
+ import { pickBatchPlan } from '../utils/pick-batch-plan.ts';
7
+ import { buildCursorRecord } from './cursors/build-cursor-record.ts';
8
+ import { cursorStream } from './cursors/cursor-stream.ts';
9
+ import { withSourceLock } from './with-source-lock.ts';
10
+ import type { SourceLocks } from './with-source-lock.ts';
8
11
 
9
12
  export type PullPathName = 'live' | 'recovery' | 'backfill' | 'resync';
10
13
 
11
- // `stored`: the batch is in the log, and the next run starts from `cursor`. `not-stored`: an append
12
- // failed, so the next run starts from the same cursor. Any other answer is the pull's own.
14
+ // `stored`: the batch and then the cursor record are in the log, and the next run starts from
15
+ // `cursor`. `not-stored`: an append failed, so the next run starts from the same cursor. Any other
16
+ // answer is the pull's own.
13
17
  export type PullRun =
14
18
  | { kind: 'stored'; stored: number; cursor: string; isDone: boolean }
15
19
  | { kind: 'not-stored' }
@@ -20,6 +24,7 @@ export type PullRunner = (
20
24
  source: string,
21
25
  path: PullPathName,
22
26
  cursor: string | null,
27
+ signal: AbortSignal,
23
28
  ) => Promise<PullRun>;
24
29
 
25
30
  interface PullRunnerDependencies {
@@ -31,13 +36,16 @@ interface PullRunnerDependencies {
31
36
  turns: StreamTurns;
32
37
  resyncMarks: ResyncMarks;
33
38
  now: () => number;
39
+ logger: Logger;
34
40
  }
35
41
 
36
- // For a resync pull, removes each mark it was given that no tie set again since. A pull's fact is
37
- // noted in no mark: a resynced fact ties with the body it replaces, and would mark its entity again
38
- // on every run.
42
+ // The pull paths of one source run one after the other, in the order they were asked for, and the
43
+ // pull paths of two sources run at the same time. A run that waits for its source ends as refused
44
+ // when its signal aborts, and its pull never starts.
39
45
  export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner {
40
- return async (sourceName, pathName, cursor) => {
46
+ const locks: SourceLocks = new Map();
47
+
48
+ return async (sourceName, pathName, cursor, signal) => {
41
49
  const source = dependencies.deployment.sources.find(
42
50
  (candidate) => candidate.name === sourceName,
43
51
  );
@@ -48,58 +56,97 @@ export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner
48
56
  return { kind: 'refused', reason: `the source ${sourceName} has no ${pathName} pull` };
49
57
  }
50
58
 
51
- const marks = pathName === 'resync' ? dependencies.resyncMarks.list(source.name) : [];
52
- const env: Record<string, string> = {};
53
-
54
- for (const name of path.env ?? []) {
55
- env[name] = dependencies.secrets[name] ?? '';
56
- }
57
-
58
- const pulled = await path.pull(cursor, { env, marks, now: dependencies.now });
59
-
60
- if (pulled.kind !== 'batch') {
61
- return pulled;
62
- }
59
+ const run = await withSourceLock(locks, source.name, signal, () =>
60
+ runPull(dependencies, { source, path, pathName, cursor, signal }),
61
+ );
63
62
 
64
- const problem = findBatchProblem(source, pulled.batch);
63
+ return run === 'stopped'
64
+ ? { kind: 'refused', reason: `the ${pathName} pull of ${sourceName} stopped before its turn` }
65
+ : run;
66
+ };
67
+ }
65
68
 
66
- if (problem !== undefined) {
67
- return { kind: 'refused', reason: problem };
68
- }
69
+ function findPullPath(source: SourceDefinition, pathName: PullPathName): PullPath | undefined {
70
+ const path = source[pathName];
69
71
 
70
- if (pulled.batch.facts.some((pending) => pending.blobs.length > 0)) {
71
- return { kind: 'refused', reason: 'ingest has no blob store' };
72
- }
72
+ return path?.kind === 'pull' ? path : undefined;
73
+ }
73
74
 
74
- const plan = planBatch(pulled.batch);
75
+ interface PullRequest {
76
+ source: SourceDefinition;
77
+ path: PullPath;
78
+ pathName: PullPathName;
79
+ cursor: string | null;
80
+ signal: AbortSignal;
81
+ }
75
82
 
76
- if (!plan.ok) {
77
- return { kind: 'refused', reason: 'one fact is larger than one append of the log takes' };
78
- }
83
+ // For a resync pull, removes each mark it was given that no tie set again since. A pull's fact is
84
+ // noted in no mark: a resynced fact ties with the body it replaces, and would mark its entity again
85
+ // on every run.
86
+ async function runPull(
87
+ dependencies: PullRunnerDependencies,
88
+ request: PullRequest,
89
+ ): Promise<PullRun> {
90
+ const source = request.source;
91
+ const marks = request.pathName === 'resync' ? dependencies.resyncMarks.list(source.name) : [];
92
+ const env: Record<string, string> = {};
93
+
94
+ for (const name of request.path.env ?? []) {
95
+ env[name] = dependencies.secrets[name] ?? '';
96
+ }
97
+
98
+ const pulled = await request.path.pull(request.cursor, {
99
+ env,
100
+ marks,
101
+ now: dependencies.now,
102
+ log: dependencies.logger,
103
+ signal: request.signal,
104
+ });
105
+
106
+ if (pulled.kind !== 'batch') {
107
+ return pulled;
108
+ }
109
+
110
+ const plan = pickBatchPlan(source, pulled.batch);
111
+
112
+ if (!plan.ok) {
113
+ return { kind: 'refused', reason: plan.failure.reason };
114
+ }
115
+
116
+ const outcome = await writeBatch(plan.value, dependencies);
117
+
118
+ if (outcome !== 'stored') {
119
+ return outcome === 'not-stored'
120
+ ? { kind: 'not-stored' }
121
+ : { kind: 'refused', reason: 'the log refused the batch' };
122
+ }
123
+
124
+ // A run that leaves the cursor where it was has nothing to save.
125
+ if (pulled.cursor !== request.cursor) {
126
+ const record = buildCursorRecord({
127
+ source: source.name,
128
+ path: request.pathName,
129
+ cursor: pulled.cursor,
130
+ nowMs: dependencies.now(),
131
+ });
79
132
 
80
- const outcome = await writeBatch(plan.value, dependencies);
133
+ const saved = await writeBatch(new Map([[cursorStream, [[record]]]]), dependencies);
81
134
 
82
- if (outcome !== 'stored') {
83
- return outcome === 'not-stored'
135
+ if (saved !== 'stored') {
136
+ return saved === 'not-stored'
84
137
  ? { kind: 'not-stored' }
85
- : { kind: 'refused', reason: 'the log refused the batch' };
138
+ : { kind: 'refused', reason: 'the log refused the cursor record' };
86
139
  }
140
+ }
87
141
 
88
- for (const mark of marks) {
89
- dependencies.resyncMarks.remove(source.name, mark);
90
- }
142
+ for (const mark of marks) {
143
+ dependencies.resyncMarks.remove(source.name, mark);
144
+ }
91
145
 
92
- return {
93
- kind: 'stored',
94
- stored: pulled.batch.facts.length,
95
- cursor: pulled.cursor,
96
- isDone: pulled.isDone,
97
- };
146
+ return {
147
+ kind: 'stored',
148
+ stored: pulled.batch.facts.length,
149
+ cursor: pulled.cursor,
150
+ isDone: pulled.isDone,
98
151
  };
99
152
  }
100
-
101
- function findPullPath(source: SourceDefinition, pathName: PullPathName): PullPath | undefined {
102
- const path = source[pathName];
103
-
104
- return path?.kind === 'pull' ? path : undefined;
105
- }
@@ -0,0 +1,76 @@
1
+ // One chain of turns per source, for the whole process: the promise that settles when every run
2
+ // that asked for the source's lock so far has ended.
3
+ export type SourceLocks = Map<string, Promise<void>>;
4
+
5
+ // Runs `work` once every earlier run of the source has ended, so the runs of one source go one
6
+ // after the other in the order they asked. Returns `stopped`, and never calls `work`, when the
7
+ // signal aborts before the turn comes.
8
+ export async function withSourceLock<Value>(
9
+ locks: SourceLocks,
10
+ source: string,
11
+ signal: AbortSignal,
12
+ work: () => Promise<Value>,
13
+ ): Promise<Value | 'stopped'> {
14
+ const previous = locks.get(source) ?? Promise.resolve();
15
+ const done = Promise.withResolvers<void>();
16
+
17
+ // A run that stops while it waits ends early, and the runs behind it still wait for the runs
18
+ // ahead of it.
19
+ const turn = waitForBoth(previous, done.promise);
20
+
21
+ locks.set(source, turn);
22
+
23
+ try {
24
+ const waited = await waitForTurn(previous, signal);
25
+
26
+ return waited === 'stopped' ? 'stopped' : await work();
27
+ } finally {
28
+ done.resolve();
29
+
30
+ // A run that stops while it waits ends before the runs ahead of it, so the entry stays until
31
+ // those runs end too.
32
+ void removeEndedTurn(locks, source, turn);
33
+ }
34
+ }
35
+
36
+ async function waitForBoth(first: Promise<void>, second: Promise<void>): Promise<void> {
37
+ await first;
38
+ await second;
39
+ }
40
+
41
+ async function waitForTurn(
42
+ previous: Promise<void>,
43
+ signal: AbortSignal,
44
+ ): Promise<'turn' | 'stopped'> {
45
+ if (signal.aborted) {
46
+ return 'stopped';
47
+ }
48
+
49
+ const stopped = Promise.withResolvers<void>();
50
+
51
+ const stop = (): void => {
52
+ stopped.resolve();
53
+ };
54
+
55
+ signal.addEventListener('abort', stop);
56
+
57
+ try {
58
+ await Promise.race([previous, stopped.promise]);
59
+
60
+ return signal.aborted ? 'stopped' : 'turn';
61
+ } finally {
62
+ signal.removeEventListener('abort', stop);
63
+ }
64
+ }
65
+
66
+ async function removeEndedTurn(
67
+ locks: SourceLocks,
68
+ source: string,
69
+ turn: Promise<void>,
70
+ ): Promise<void> {
71
+ await turn;
72
+
73
+ if (locks.get(source) === turn) {
74
+ locks.delete(source);
75
+ }
76
+ }
@@ -3,7 +3,8 @@ export function buildFailureAnswer(status: number, reason: string): Response {
3
3
  return Response.json({ error: reason }, { status });
4
4
  }
5
5
 
6
- // 503 means "send the same delivery again, unchanged": every sender of v1 does that.
6
+ // 503 means "send the same delivery again, unchanged". A sender that never sends a batch twice,
7
+ // such as Codex's exporter, loses the batch.
7
8
  export function buildRetryAnswer(): Response {
8
9
  return Response.json(
9
10
  { error: 'the batch is not stored, send it again' },
@@ -1,4 +1,5 @@
1
- import type { CheckedDeployment, SenderProof } from '@pepys/kit';
1
+ import { createLogger } from '@pepys/kit';
2
+ import type { CheckedDeployment, Logger, SenderProof } from '@pepys/kit';
2
3
  import type { FactLog } from '../fact-log/fact-log.ts';
3
4
  import { makeBatchAppender } from '../fact-log/make-batch-appender.ts';
4
5
  import { makePullRunner } from '../pull/make-pull-runner.ts';
@@ -20,6 +21,9 @@ export interface IngestOptions {
20
21
  senderProof: SenderProof;
21
22
  appendRetry: { maxAttempts: number; retryDelayMs: number };
22
23
  now: () => number;
24
+
25
+ // Defaults to a logger that writes to stderr. A source's `read` and `pull` get it as `log`.
26
+ logger?: Logger | undefined;
23
27
  }
24
28
 
25
29
  export interface Ingest {
@@ -38,7 +42,8 @@ const recentBodyMaxAgeMs = 3_600_000;
38
42
 
39
43
  // The receive routes of a deployment over a given log, as one fetch function with no listener.
40
44
  export function createIngest(options: IngestOptions): Ingest {
41
- const append = makeBatchAppender({ log: options.log, ...options.appendRetry });
45
+ const logger = options.logger ?? createLogger(process.stderr);
46
+ const append = makeBatchAppender({ log: options.log, logger, ...options.appendRetry });
42
47
 
43
48
  const turns: StreamTurns = new Map();
44
49
 
@@ -60,6 +65,7 @@ export function createIngest(options: IngestOptions): Ingest {
60
65
  makeReceiveHandler(source, path, {
61
66
  secret,
62
67
  senderProof: options.senderProof,
68
+ logger,
63
69
  append,
64
70
  turns,
65
71
  recentBodies: createRecentBodies({
@@ -74,7 +80,7 @@ export function createIngest(options: IngestOptions): Ingest {
74
80
  for (const route of path.ignoredRoutes ?? []) {
75
81
  handlers.set(
76
82
  route,
77
- makeIgnoredRouteHandler(path, { secret, senderProof: options.senderProof }),
83
+ makeIgnoredRouteHandler(path, { secret, senderProof: options.senderProof, logger }),
78
84
  );
79
85
  }
80
86
  }
@@ -99,6 +105,7 @@ export function createIngest(options: IngestOptions): Ingest {
99
105
  turns,
100
106
  resyncMarks: resync.marks,
101
107
  now: options.now,
108
+ logger,
102
109
  }),
103
110
  };
104
111
  }
@@ -1,6 +1,7 @@
1
1
  import { createHash } from 'node:crypto';
2
2
  import { isDeliveryAuthentic } from '@pepys/kit';
3
3
  import type {
4
+ Logger,
4
5
  ReceivePath,
5
6
  Received,
6
7
  Sender,
@@ -12,8 +13,7 @@ import { match } from 'ts-pattern';
12
13
  import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
13
14
  import { buildFailureAnswer, buildRetryAnswer } from './answers/build-failure-answer.ts';
14
15
  import { buildSuccessAnswer } from './answers/build-success-answer.ts';
15
- import { findBatchProblem } from './find-batch-problem.ts';
16
- import { planBatch } from './plan-batch.ts';
16
+ import { pickReceivedStep } from './pick-received-step.ts';
17
17
  import { readDecodedBody } from './read-decoded-body.ts';
18
18
  import { readSentBody } from './read-sent-body.ts';
19
19
  import type { RecentBodies } from './recent-bodies/create-recent-bodies.ts';
@@ -25,6 +25,7 @@ export interface ReceiveDependencies {
25
25
  // the value of the path's `check.secretEnv`; an empty secret passes no delivery
26
26
  secret: string;
27
27
  senderProof: SenderProof;
28
+ logger: Logger;
28
29
  append: BatchAppender;
29
30
  turns: StreamTurns;
30
31
  recentBodies: RecentBodies;
@@ -77,51 +78,33 @@ export function makeReceiveHandler(
77
78
  return pickDecodeFailure(body);
78
79
  }
79
80
 
80
- const received = readSourceDelivery(source, path, {
81
+ const received = readSourceDelivery(source, path, dependencies.logger, {
81
82
  headers: request.headers,
82
83
  body,
83
84
  ...(checked.sender === undefined ? {} : { sender: checked.sender }),
84
85
  });
85
86
 
86
- if (received === 'threw') {
87
- return buildFailureAnswer(500, 'the source could not read the delivery');
88
- }
87
+ const step = pickReceivedStep(source, received, factLimit);
89
88
 
90
- if (received.kind === 'refused') {
91
- return buildFailureAnswer(received.status, received.reason);
89
+ if (step.kind === 'refuse') {
90
+ return buildFailureAnswer(step.status, step.reason);
92
91
  }
93
92
 
94
- if (received.kind === 'ignored' || received.batch.facts.length === 0) {
93
+ if (step.kind === 'ignore') {
95
94
  return buildSuccessAnswer(path, 'ignored', 0);
96
95
  }
97
96
 
98
- const facts = received.batch.facts;
99
-
100
- if (facts.length > factLimit) {
101
- return buildFailureAnswer(413, `a batch holds ${factLimit} facts at most`);
102
- }
103
-
104
- const problem = findBatchProblem(source, received.batch);
105
-
106
- if (problem !== undefined) {
107
- return buildFailureAnswer(422, problem);
108
- }
109
-
110
- if (facts.some((pending) => pending.blobs.length > 0)) {
111
- console.error(
97
+ if (step.kind === 'has-blob') {
98
+ dependencies.logger.error(
112
99
  `the source ${source.name} read a fact with a blob, and ingest has no blob store`,
113
100
  );
114
101
 
115
102
  return buildFailureAnswer(500, 'ingest has no blob store');
116
103
  }
117
104
 
118
- const plan = planBatch(received.batch);
119
-
120
- if (!plan.ok) {
121
- return buildFailureAnswer(413, 'one fact is larger than one append of the log takes');
122
- }
105
+ const facts = step.facts;
123
106
 
124
- const outcome = await writeBatch(plan.value, dependencies);
107
+ const outcome = await writeBatch(step.plan, dependencies);
125
108
 
126
109
  if (outcome === 'not-stored') {
127
110
  return buildRetryAnswer();
@@ -152,7 +135,7 @@ export function makeReceiveHandler(
152
135
  // the ignored answer. It reads no body further and stores nothing.
153
136
  export function makeIgnoredRouteHandler(
154
137
  path: ReceivePath,
155
- dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
138
+ dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof' | 'logger'>,
156
139
  ): Handler {
157
140
  return async (request) => {
158
141
  const checked = await readCheckedDelivery(request, path, dependencies);
@@ -170,7 +153,7 @@ type CheckedDelivery =
170
153
  async function readCheckedDelivery(
171
154
  request: Request,
172
155
  path: ReceivePath,
173
- dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
156
+ dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof' | 'logger'>,
174
157
  ): Promise<CheckedDelivery> {
175
158
  if (request.method !== 'POST') {
176
159
  return { kind: 'answer', answer: buildFailureAnswer(405, 'method not allowed') };
@@ -190,7 +173,7 @@ async function readCheckedDelivery(
190
173
  return { kind: 'checked', sent, sender: undefined };
191
174
  }
192
175
 
193
- const sender = await readProvenSender(path.sender, request.headers, dependencies.senderProof);
176
+ const sender = await readProvenSender(path.sender, request.headers, dependencies);
194
177
 
195
178
  if (sender === null) {
196
179
  return { kind: 'answer', answer: buildFailureAnswer(401, 'not authorised') };
@@ -203,14 +186,14 @@ async function readCheckedDelivery(
203
186
  async function readProvenSender(
204
187
  requirement: SenderRequirement,
205
188
  headers: Headers,
206
- senderProof: SenderProof,
189
+ dependencies: Pick<ReceiveDependencies, 'senderProof' | 'logger'>,
207
190
  ): Promise<Sender | null> {
208
191
  try {
209
- const sender = await senderProof.prove(requirement, headers);
192
+ const sender = await dependencies.senderProof.prove(requirement, headers);
210
193
 
211
194
  return sender;
212
195
  } catch {
213
- console.error('the sender proof threw, so the delivery proves no sender');
196
+ dependencies.logger.error('the sender proof threw, so the delivery proves no sender');
214
197
 
215
198
  return null;
216
199
  }
@@ -233,12 +216,13 @@ function pickDecodeFailure(failure: 'unsupported' | 'too-large' | 'corrupt'): Re
233
216
  function readSourceDelivery(
234
217
  source: SourceDefinition,
235
218
  path: ReceivePath,
219
+ logger: Logger,
236
220
  delivery: Parameters<ReceivePath['read']>[0],
237
221
  ): Received | 'threw' {
238
222
  try {
239
- return path.read(delivery, { name: source.name, families: source.families });
223
+ return path.read(delivery, { name: source.name, families: source.families, log: logger });
240
224
  } catch (error: unknown) {
241
- console.error(`the read of the source ${source.name} threw`, error);
225
+ logger.error(`the read of the source ${source.name} threw`, error);
242
226
 
243
227
  return 'threw';
244
228
  }
@@ -0,0 +1,56 @@
1
+ import type { PendingFact, Received, SourceContext } from '@pepys/kit';
2
+ import { match } from 'ts-pattern';
3
+ import { pickBatchPlan } from '../utils/pick-batch-plan.ts';
4
+ import type { BatchPlan } from '../utils/plan-batch.ts';
5
+
6
+ // `has-blob`: the source read a fact with a blob, a fault of the deployment that ingest answers 500.
7
+ export type ReceivedStep =
8
+ | { kind: 'refuse'; status: number; reason: string }
9
+ | { kind: 'has-blob' }
10
+ | { kind: 'ignore' }
11
+ | { kind: 'write'; facts: readonly PendingFact[]; plan: BatchPlan };
12
+
13
+ // What follows the source's read of a delivery, `threw` when the read threw.
14
+ export function pickReceivedStep(
15
+ source: Pick<SourceContext, 'name' | 'families'>,
16
+ received: Received | 'threw',
17
+ factLimit: number,
18
+ ): ReceivedStep {
19
+ if (received === 'threw') {
20
+ return { kind: 'refuse', status: 500, reason: 'the source could not read the delivery' };
21
+ }
22
+
23
+ if (received.kind === 'refused') {
24
+ return { kind: 'refuse', status: received.status, reason: received.reason };
25
+ }
26
+
27
+ if (received.kind === 'ignored' || received.batch.facts.length === 0) {
28
+ return { kind: 'ignore' };
29
+ }
30
+
31
+ const facts = received.batch.facts;
32
+
33
+ if (facts.length > factLimit) {
34
+ return { kind: 'refuse', status: 413, reason: `a batch holds ${factLimit} facts at most` };
35
+ }
36
+
37
+ const plan = pickBatchPlan(source, received.batch);
38
+
39
+ if (!plan.ok) {
40
+ return match(plan.failure.kind)
41
+ .with('bad-batch', () => ({
42
+ kind: 'refuse' as const,
43
+ status: 422,
44
+ reason: plan.failure.reason,
45
+ }))
46
+ .with('has-blob', () => ({ kind: 'has-blob' as const }))
47
+ .with('too-large', () => ({
48
+ kind: 'refuse' as const,
49
+ status: 413,
50
+ reason: plan.failure.reason,
51
+ }))
52
+ .exhaustive();
53
+ }
54
+
55
+ return { kind: 'write', facts, plan: plan.value };
56
+ }
@@ -1,6 +1,6 @@
1
- import type { StreamName } from '@pepys/kit';
1
+ import type { Logger, StreamName } from '@pepys/kit';
2
2
  import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
3
- import type { BatchPlan } from './plan-batch.ts';
3
+ import type { BatchPlan } from '../utils/plan-batch.ts';
4
4
 
5
5
  export type BatchOutcome = 'stored' | 'refused' | 'not-stored';
6
6
 
@@ -11,6 +11,7 @@ export type StreamTurns = Map<StreamName, Promise<void>>;
11
11
  interface WriteBatchDependencies {
12
12
  append: BatchAppender;
13
13
  turns: StreamTurns;
14
+ logger: Logger;
14
15
  }
15
16
 
16
17
  // The streams append at the same time, and the appends of one stream one after the other. Every
@@ -21,12 +22,12 @@ export async function writeBatch(
21
22
  ): Promise<BatchOutcome> {
22
23
  const settled = await Promise.all(
23
24
  [...plan].map(([stream, runs]) =>
24
- withStreamTurn(dependencies.turns, stream, async () => {
25
+ withStreamTurn(dependencies, stream, async () => {
25
26
  for (const run of runs) {
26
27
  const appended = await dependencies.append(stream, run);
27
28
 
28
29
  if (!appended.ok) {
29
- console.error(`an append to ${stream} failed`, appended.failure);
30
+ dependencies.logger.error(`an append to ${stream} failed`, appended.failure);
30
31
 
31
32
  return appended.failure.kind;
32
33
  }
@@ -42,10 +43,11 @@ export async function writeBatch(
42
43
 
43
44
  // A throw is a refusal the log gives no outcome for, such as a bad access token.
44
45
  async function withStreamTurn(
45
- turns: StreamTurns,
46
+ dependencies: Pick<WriteBatchDependencies, 'turns' | 'logger'>,
46
47
  stream: StreamName,
47
48
  work: () => Promise<BatchOutcome>,
48
49
  ): Promise<BatchOutcome> {
50
+ const turns = dependencies.turns;
49
51
  const previous = turns.get(stream) ?? Promise.resolve();
50
52
  const done = Promise.withResolvers<void>();
51
53
 
@@ -56,7 +58,7 @@ async function withStreamTurn(
56
58
 
57
59
  return await work();
58
60
  } catch (error: unknown) {
59
- console.error(`an append to ${stream} threw`, error);
61
+ dependencies.logger.error(`an append to ${stream} threw`, error);
60
62
 
61
63
  return 'refused';
62
64
  } finally {
@@ -1,17 +1,23 @@
1
1
  import type { verifyImpIdentity } from '@impz/client/identity';
2
- import type { SenderProof } from '@pepys/kit';
2
+ import { createLogger } from '@pepys/kit';
3
+ import type { Logger, SenderProof } from '@pepys/kit';
3
4
  import type { ImpIdentityTrust } from './types.ts';
4
5
 
5
6
  interface ImpIdentityOptions {
6
7
  verify: typeof verifyImpIdentity;
7
8
  trust: ImpIdentityTrust;
8
9
  now: () => number;
10
+
11
+ // defaults to a logger that writes to stderr
12
+ logger?: Logger | undefined;
9
13
  }
10
14
 
11
15
  // Proves the sender from the signed identity token in the `imp-identity` header. The issuer and
12
16
  // the subject come only from the verified token, and every failure, a throw included, proves
13
17
  // nothing. It never logs the token.
14
18
  export function createImpIdentitySenderProof(options: ImpIdentityOptions): SenderProof {
19
+ const logger = options.logger ?? createLogger(process.stderr);
20
+
15
21
  return {
16
22
  prove: async (requirement, headers) => {
17
23
  const documents = options.trust.issuers.get(requirement.issuersEnv) ?? [];
@@ -33,7 +39,7 @@ export function createImpIdentitySenderProof(options: ImpIdentityOptions): Sende
33
39
  ? null
34
40
  : { issuer: verified.issuer, subject: verified.id, name: verified.name };
35
41
  } catch {
36
- console.error('the identity verifier failed, so the delivery proves no sender');
42
+ logger.error('the identity verifier failed, so the delivery proves no sender');
37
43
 
38
44
  return null;
39
45
  }
@@ -1,5 +1,5 @@
1
1
  import { verifyImpIdentity } from '@impz/client/identity';
2
- import type { Result, SenderProof, SourceDefinition } from '@pepys/kit';
2
+ import type { Logger, Result, SenderProof, SourceDefinition } from '@pepys/kit';
3
3
  import { createImpIdentitySenderProof } from './create-imp-identity-sender-proof.ts';
4
4
  import { readImpIdentityTrust } from './read-imp-identity-trust.ts';
5
5
 
@@ -7,6 +7,9 @@ interface SenderProofReadOptions {
7
7
  sources: readonly SourceDefinition[];
8
8
  environment: Readonly<Record<string, string | undefined>>;
9
9
  now: () => number;
10
+
11
+ // defaults to a logger that writes to stderr
12
+ logger?: Logger | undefined;
10
13
  }
11
14
 
12
15
  // The sender proof that `serveIngest` runs: the trust of the deployment's environment and the
@@ -30,6 +33,7 @@ export async function readImpIdentitySenderProof(
30
33
  verify: verifyImpIdentity,
31
34
  trust: trust.value,
32
35
  now: options.now,
36
+ logger: options.logger,
33
37
  }),
34
38
  };
35
39
  }