@pepys/ingest 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/fact-log/make-batch-appender.ts +43 -128
- package/src/fact-log/memory/create-memory-fact-log.ts +1 -1
- package/src/fact-log/pick-attempt-step.ts +53 -0
- package/src/fact-log/pick-retry-step.ts +47 -0
- package/src/fact-log/pick-settled-attempt.ts +30 -0
- package/src/fact-log/read-run.ts +34 -0
- package/src/fact-log/s2/create-s2-fact-log.ts +17 -6
- package/src/index.ts +3 -3
- package/src/pull/collect-timed-pulls.ts +23 -0
- package/src/pull/cursors/build-cursor-entity.ts +8 -0
- package/src/pull/cursors/build-cursor-record.ts +32 -0
- package/src/pull/cursors/cursor-stream.ts +6 -0
- package/src/pull/cursors/format-pull-start.ts +13 -0
- package/src/pull/cursors/read-saved-cursors.ts +55 -0
- package/src/pull/make-pull-runner.ts +98 -51
- package/src/pull/with-source-lock.ts +76 -0
- package/src/receive/answers/build-failure-answer.ts +2 -1
- package/src/receive/create-ingest.ts +10 -3
- package/src/receive/make-receive-handler.ts +21 -37
- package/src/receive/pick-received-step.ts +56 -0
- package/src/receive/write-batch.ts +8 -6
- package/src/sender-proof/imp-identity/create-imp-identity-sender-proof.ts +8 -2
- package/src/sender-proof/imp-identity/read-imp-identity-sender-proof.ts +5 -1
- package/src/serve/serve-ingest.ts +43 -14
- package/src/serve/start-pull-timers.ts +47 -39
- package/src/{receive → utils}/find-batch-problem.ts +4 -1
- package/src/utils/pick-batch-plan.ts +37 -0
- package/src/{receive → utils}/plan-batch.ts +1 -1
- package/src/source-port/types.ts +0 -64
- /package/src/{fact-log → utils}/check-append-batch.ts +0 -0
- /package/src/{fact-log → utils}/parse-stored-body.ts +0 -0
- /package/src/{fact-log → utils}/plan-appends.ts +0 -0
|
@@ -1,15 +1,19 @@
|
|
|
1
|
-
import type { CheckedDeployment, PullPath, Pulled, SourceDefinition } from '@pepys/kit';
|
|
1
|
+
import type { CheckedDeployment, Logger, PullPath, Pulled, SourceDefinition } from '@pepys/kit';
|
|
2
2
|
import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
|
|
3
|
-
import { findBatchProblem } from '../receive/find-batch-problem.ts';
|
|
4
|
-
import { planBatch } from '../receive/plan-batch.ts';
|
|
5
3
|
import type { ResyncMarks } from '../receive/resync-marks/create-resync-marks.ts';
|
|
6
4
|
import { writeBatch } from '../receive/write-batch.ts';
|
|
7
5
|
import type { StreamTurns } from '../receive/write-batch.ts';
|
|
6
|
+
import { pickBatchPlan } from '../utils/pick-batch-plan.ts';
|
|
7
|
+
import { buildCursorRecord } from './cursors/build-cursor-record.ts';
|
|
8
|
+
import { cursorStream } from './cursors/cursor-stream.ts';
|
|
9
|
+
import { withSourceLock } from './with-source-lock.ts';
|
|
10
|
+
import type { SourceLocks } from './with-source-lock.ts';
|
|
8
11
|
|
|
9
12
|
export type PullPathName = 'live' | 'recovery' | 'backfill' | 'resync';
|
|
10
13
|
|
|
11
|
-
// `stored`: the batch
|
|
12
|
-
// failed, so the next run starts from the same cursor. Any other
|
|
14
|
+
// `stored`: the batch and then the cursor record are in the log, and the next run starts from
|
|
15
|
+
// `cursor`. `not-stored`: an append failed, so the next run starts from the same cursor. Any other
|
|
16
|
+
// answer is the pull's own.
|
|
13
17
|
export type PullRun =
|
|
14
18
|
| { kind: 'stored'; stored: number; cursor: string; isDone: boolean }
|
|
15
19
|
| { kind: 'not-stored' }
|
|
@@ -20,6 +24,7 @@ export type PullRunner = (
|
|
|
20
24
|
source: string,
|
|
21
25
|
path: PullPathName,
|
|
22
26
|
cursor: string | null,
|
|
27
|
+
signal: AbortSignal,
|
|
23
28
|
) => Promise<PullRun>;
|
|
24
29
|
|
|
25
30
|
interface PullRunnerDependencies {
|
|
@@ -31,13 +36,16 @@ interface PullRunnerDependencies {
|
|
|
31
36
|
turns: StreamTurns;
|
|
32
37
|
resyncMarks: ResyncMarks;
|
|
33
38
|
now: () => number;
|
|
39
|
+
logger: Logger;
|
|
34
40
|
}
|
|
35
41
|
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
42
|
+
// The pull paths of one source run one after the other, in the order they were asked for, and the
|
|
43
|
+
// pull paths of two sources run at the same time. A run that waits for its source ends as refused
|
|
44
|
+
// when its signal aborts, and its pull never starts.
|
|
39
45
|
export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner {
|
|
40
|
-
|
|
46
|
+
const locks: SourceLocks = new Map();
|
|
47
|
+
|
|
48
|
+
return async (sourceName, pathName, cursor, signal) => {
|
|
41
49
|
const source = dependencies.deployment.sources.find(
|
|
42
50
|
(candidate) => candidate.name === sourceName,
|
|
43
51
|
);
|
|
@@ -48,58 +56,97 @@ export function makePullRunner(dependencies: PullRunnerDependencies): PullRunner
|
|
|
48
56
|
return { kind: 'refused', reason: `the source ${sourceName} has no ${pathName} pull` };
|
|
49
57
|
}
|
|
50
58
|
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
for (const name of path.env ?? []) {
|
|
55
|
-
env[name] = dependencies.secrets[name] ?? '';
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
const pulled = await path.pull(cursor, { env, marks, now: dependencies.now });
|
|
59
|
-
|
|
60
|
-
if (pulled.kind !== 'batch') {
|
|
61
|
-
return pulled;
|
|
62
|
-
}
|
|
59
|
+
const run = await withSourceLock(locks, source.name, signal, () =>
|
|
60
|
+
runPull(dependencies, { source, path, pathName, cursor, signal }),
|
|
61
|
+
);
|
|
63
62
|
|
|
64
|
-
|
|
63
|
+
return run === 'stopped'
|
|
64
|
+
? { kind: 'refused', reason: `the ${pathName} pull of ${sourceName} stopped before its turn` }
|
|
65
|
+
: run;
|
|
66
|
+
};
|
|
67
|
+
}
|
|
65
68
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
}
|
|
69
|
+
function findPullPath(source: SourceDefinition, pathName: PullPathName): PullPath | undefined {
|
|
70
|
+
const path = source[pathName];
|
|
69
71
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
}
|
|
72
|
+
return path?.kind === 'pull' ? path : undefined;
|
|
73
|
+
}
|
|
73
74
|
|
|
74
|
-
|
|
75
|
+
interface PullRequest {
|
|
76
|
+
source: SourceDefinition;
|
|
77
|
+
path: PullPath;
|
|
78
|
+
pathName: PullPathName;
|
|
79
|
+
cursor: string | null;
|
|
80
|
+
signal: AbortSignal;
|
|
81
|
+
}
|
|
75
82
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
83
|
+
// For a resync pull, removes each mark it was given that no tie set again since. A pull's fact is
|
|
84
|
+
// noted in no mark: a resynced fact ties with the body it replaces, and would mark its entity again
|
|
85
|
+
// on every run.
|
|
86
|
+
async function runPull(
|
|
87
|
+
dependencies: PullRunnerDependencies,
|
|
88
|
+
request: PullRequest,
|
|
89
|
+
): Promise<PullRun> {
|
|
90
|
+
const source = request.source;
|
|
91
|
+
const marks = request.pathName === 'resync' ? dependencies.resyncMarks.list(source.name) : [];
|
|
92
|
+
const env: Record<string, string> = {};
|
|
93
|
+
|
|
94
|
+
for (const name of request.path.env ?? []) {
|
|
95
|
+
env[name] = dependencies.secrets[name] ?? '';
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const pulled = await request.path.pull(request.cursor, {
|
|
99
|
+
env,
|
|
100
|
+
marks,
|
|
101
|
+
now: dependencies.now,
|
|
102
|
+
log: dependencies.logger,
|
|
103
|
+
signal: request.signal,
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
if (pulled.kind !== 'batch') {
|
|
107
|
+
return pulled;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const plan = pickBatchPlan(source, pulled.batch);
|
|
111
|
+
|
|
112
|
+
if (!plan.ok) {
|
|
113
|
+
return { kind: 'refused', reason: plan.failure.reason };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const outcome = await writeBatch(plan.value, dependencies);
|
|
117
|
+
|
|
118
|
+
if (outcome !== 'stored') {
|
|
119
|
+
return outcome === 'not-stored'
|
|
120
|
+
? { kind: 'not-stored' }
|
|
121
|
+
: { kind: 'refused', reason: 'the log refused the batch' };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// A run that leaves the cursor where it was has nothing to save.
|
|
125
|
+
if (pulled.cursor !== request.cursor) {
|
|
126
|
+
const record = buildCursorRecord({
|
|
127
|
+
source: source.name,
|
|
128
|
+
path: request.pathName,
|
|
129
|
+
cursor: pulled.cursor,
|
|
130
|
+
nowMs: dependencies.now(),
|
|
131
|
+
});
|
|
79
132
|
|
|
80
|
-
const
|
|
133
|
+
const saved = await writeBatch(new Map([[cursorStream, [[record]]]]), dependencies);
|
|
81
134
|
|
|
82
|
-
if (
|
|
83
|
-
return
|
|
135
|
+
if (saved !== 'stored') {
|
|
136
|
+
return saved === 'not-stored'
|
|
84
137
|
? { kind: 'not-stored' }
|
|
85
|
-
: { kind: 'refused', reason: 'the log refused the
|
|
138
|
+
: { kind: 'refused', reason: 'the log refused the cursor record' };
|
|
86
139
|
}
|
|
140
|
+
}
|
|
87
141
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
142
|
+
for (const mark of marks) {
|
|
143
|
+
dependencies.resyncMarks.remove(source.name, mark);
|
|
144
|
+
}
|
|
91
145
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
};
|
|
146
|
+
return {
|
|
147
|
+
kind: 'stored',
|
|
148
|
+
stored: pulled.batch.facts.length,
|
|
149
|
+
cursor: pulled.cursor,
|
|
150
|
+
isDone: pulled.isDone,
|
|
98
151
|
};
|
|
99
152
|
}
|
|
100
|
-
|
|
101
|
-
function findPullPath(source: SourceDefinition, pathName: PullPathName): PullPath | undefined {
|
|
102
|
-
const path = source[pathName];
|
|
103
|
-
|
|
104
|
-
return path?.kind === 'pull' ? path : undefined;
|
|
105
|
-
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// One chain of turns per source, for the whole process: the promise that settles when every run
|
|
2
|
+
// that asked for the source's lock so far has ended.
|
|
3
|
+
export type SourceLocks = Map<string, Promise<void>>;
|
|
4
|
+
|
|
5
|
+
// Runs `work` once every earlier run of the source has ended, so the runs of one source go one
|
|
6
|
+
// after the other in the order they asked. Returns `stopped`, and never calls `work`, when the
|
|
7
|
+
// signal aborts before the turn comes.
|
|
8
|
+
export async function withSourceLock<Value>(
|
|
9
|
+
locks: SourceLocks,
|
|
10
|
+
source: string,
|
|
11
|
+
signal: AbortSignal,
|
|
12
|
+
work: () => Promise<Value>,
|
|
13
|
+
): Promise<Value | 'stopped'> {
|
|
14
|
+
const previous = locks.get(source) ?? Promise.resolve();
|
|
15
|
+
const done = Promise.withResolvers<void>();
|
|
16
|
+
|
|
17
|
+
// A run that stops while it waits ends early, and the runs behind it still wait for the runs
|
|
18
|
+
// ahead of it.
|
|
19
|
+
const turn = waitForBoth(previous, done.promise);
|
|
20
|
+
|
|
21
|
+
locks.set(source, turn);
|
|
22
|
+
|
|
23
|
+
try {
|
|
24
|
+
const waited = await waitForTurn(previous, signal);
|
|
25
|
+
|
|
26
|
+
return waited === 'stopped' ? 'stopped' : await work();
|
|
27
|
+
} finally {
|
|
28
|
+
done.resolve();
|
|
29
|
+
|
|
30
|
+
// A run that stops while it waits ends before the runs ahead of it, so the entry stays until
|
|
31
|
+
// those runs end too.
|
|
32
|
+
void removeEndedTurn(locks, source, turn);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async function waitForBoth(first: Promise<void>, second: Promise<void>): Promise<void> {
|
|
37
|
+
await first;
|
|
38
|
+
await second;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function waitForTurn(
|
|
42
|
+
previous: Promise<void>,
|
|
43
|
+
signal: AbortSignal,
|
|
44
|
+
): Promise<'turn' | 'stopped'> {
|
|
45
|
+
if (signal.aborted) {
|
|
46
|
+
return 'stopped';
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const stopped = Promise.withResolvers<void>();
|
|
50
|
+
|
|
51
|
+
const stop = (): void => {
|
|
52
|
+
stopped.resolve();
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
signal.addEventListener('abort', stop);
|
|
56
|
+
|
|
57
|
+
try {
|
|
58
|
+
await Promise.race([previous, stopped.promise]);
|
|
59
|
+
|
|
60
|
+
return signal.aborted ? 'stopped' : 'turn';
|
|
61
|
+
} finally {
|
|
62
|
+
signal.removeEventListener('abort', stop);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async function removeEndedTurn(
|
|
67
|
+
locks: SourceLocks,
|
|
68
|
+
source: string,
|
|
69
|
+
turn: Promise<void>,
|
|
70
|
+
): Promise<void> {
|
|
71
|
+
await turn;
|
|
72
|
+
|
|
73
|
+
if (locks.get(source) === turn) {
|
|
74
|
+
locks.delete(source);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
@@ -3,7 +3,8 @@ export function buildFailureAnswer(status: number, reason: string): Response {
|
|
|
3
3
|
return Response.json({ error: reason }, { status });
|
|
4
4
|
}
|
|
5
5
|
|
|
6
|
-
// 503 means "send the same delivery again, unchanged"
|
|
6
|
+
// 503 means "send the same delivery again, unchanged". A sender that never sends a batch twice,
|
|
7
|
+
// such as Codex's exporter, loses the batch.
|
|
7
8
|
export function buildRetryAnswer(): Response {
|
|
8
9
|
return Response.json(
|
|
9
10
|
{ error: 'the batch is not stored, send it again' },
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { createLogger } from '@pepys/kit';
|
|
2
|
+
import type { CheckedDeployment, Logger, SenderProof } from '@pepys/kit';
|
|
2
3
|
import type { FactLog } from '../fact-log/fact-log.ts';
|
|
3
4
|
import { makeBatchAppender } from '../fact-log/make-batch-appender.ts';
|
|
4
5
|
import { makePullRunner } from '../pull/make-pull-runner.ts';
|
|
@@ -20,6 +21,9 @@ export interface IngestOptions {
|
|
|
20
21
|
senderProof: SenderProof;
|
|
21
22
|
appendRetry: { maxAttempts: number; retryDelayMs: number };
|
|
22
23
|
now: () => number;
|
|
24
|
+
|
|
25
|
+
// Defaults to a logger that writes to stderr. A source's `read` and `pull` get it as `log`.
|
|
26
|
+
logger?: Logger | undefined;
|
|
23
27
|
}
|
|
24
28
|
|
|
25
29
|
export interface Ingest {
|
|
@@ -38,7 +42,8 @@ const recentBodyMaxAgeMs = 3_600_000;
|
|
|
38
42
|
|
|
39
43
|
// The receive routes of a deployment over a given log, as one fetch function with no listener.
|
|
40
44
|
export function createIngest(options: IngestOptions): Ingest {
|
|
41
|
-
const
|
|
45
|
+
const logger = options.logger ?? createLogger(process.stderr);
|
|
46
|
+
const append = makeBatchAppender({ log: options.log, logger, ...options.appendRetry });
|
|
42
47
|
|
|
43
48
|
const turns: StreamTurns = new Map();
|
|
44
49
|
|
|
@@ -60,6 +65,7 @@ export function createIngest(options: IngestOptions): Ingest {
|
|
|
60
65
|
makeReceiveHandler(source, path, {
|
|
61
66
|
secret,
|
|
62
67
|
senderProof: options.senderProof,
|
|
68
|
+
logger,
|
|
63
69
|
append,
|
|
64
70
|
turns,
|
|
65
71
|
recentBodies: createRecentBodies({
|
|
@@ -74,7 +80,7 @@ export function createIngest(options: IngestOptions): Ingest {
|
|
|
74
80
|
for (const route of path.ignoredRoutes ?? []) {
|
|
75
81
|
handlers.set(
|
|
76
82
|
route,
|
|
77
|
-
makeIgnoredRouteHandler(path, { secret, senderProof: options.senderProof }),
|
|
83
|
+
makeIgnoredRouteHandler(path, { secret, senderProof: options.senderProof, logger }),
|
|
78
84
|
);
|
|
79
85
|
}
|
|
80
86
|
}
|
|
@@ -99,6 +105,7 @@ export function createIngest(options: IngestOptions): Ingest {
|
|
|
99
105
|
turns,
|
|
100
106
|
resyncMarks: resync.marks,
|
|
101
107
|
now: options.now,
|
|
108
|
+
logger,
|
|
102
109
|
}),
|
|
103
110
|
};
|
|
104
111
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { isDeliveryAuthentic } from '@pepys/kit';
|
|
3
3
|
import type {
|
|
4
|
+
Logger,
|
|
4
5
|
ReceivePath,
|
|
5
6
|
Received,
|
|
6
7
|
Sender,
|
|
@@ -12,8 +13,7 @@ import { match } from 'ts-pattern';
|
|
|
12
13
|
import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
|
|
13
14
|
import { buildFailureAnswer, buildRetryAnswer } from './answers/build-failure-answer.ts';
|
|
14
15
|
import { buildSuccessAnswer } from './answers/build-success-answer.ts';
|
|
15
|
-
import {
|
|
16
|
-
import { planBatch } from './plan-batch.ts';
|
|
16
|
+
import { pickReceivedStep } from './pick-received-step.ts';
|
|
17
17
|
import { readDecodedBody } from './read-decoded-body.ts';
|
|
18
18
|
import { readSentBody } from './read-sent-body.ts';
|
|
19
19
|
import type { RecentBodies } from './recent-bodies/create-recent-bodies.ts';
|
|
@@ -25,6 +25,7 @@ export interface ReceiveDependencies {
|
|
|
25
25
|
// the value of the path's `check.secretEnv`; an empty secret passes no delivery
|
|
26
26
|
secret: string;
|
|
27
27
|
senderProof: SenderProof;
|
|
28
|
+
logger: Logger;
|
|
28
29
|
append: BatchAppender;
|
|
29
30
|
turns: StreamTurns;
|
|
30
31
|
recentBodies: RecentBodies;
|
|
@@ -77,51 +78,33 @@ export function makeReceiveHandler(
|
|
|
77
78
|
return pickDecodeFailure(body);
|
|
78
79
|
}
|
|
79
80
|
|
|
80
|
-
const received = readSourceDelivery(source, path, {
|
|
81
|
+
const received = readSourceDelivery(source, path, dependencies.logger, {
|
|
81
82
|
headers: request.headers,
|
|
82
83
|
body,
|
|
83
84
|
...(checked.sender === undefined ? {} : { sender: checked.sender }),
|
|
84
85
|
});
|
|
85
86
|
|
|
86
|
-
|
|
87
|
-
return buildFailureAnswer(500, 'the source could not read the delivery');
|
|
88
|
-
}
|
|
87
|
+
const step = pickReceivedStep(source, received, factLimit);
|
|
89
88
|
|
|
90
|
-
if (
|
|
91
|
-
return buildFailureAnswer(
|
|
89
|
+
if (step.kind === 'refuse') {
|
|
90
|
+
return buildFailureAnswer(step.status, step.reason);
|
|
92
91
|
}
|
|
93
92
|
|
|
94
|
-
if (
|
|
93
|
+
if (step.kind === 'ignore') {
|
|
95
94
|
return buildSuccessAnswer(path, 'ignored', 0);
|
|
96
95
|
}
|
|
97
96
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
if (facts.length > factLimit) {
|
|
101
|
-
return buildFailureAnswer(413, `a batch holds ${factLimit} facts at most`);
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
const problem = findBatchProblem(source, received.batch);
|
|
105
|
-
|
|
106
|
-
if (problem !== undefined) {
|
|
107
|
-
return buildFailureAnswer(422, problem);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
if (facts.some((pending) => pending.blobs.length > 0)) {
|
|
111
|
-
console.error(
|
|
97
|
+
if (step.kind === 'has-blob') {
|
|
98
|
+
dependencies.logger.error(
|
|
112
99
|
`the source ${source.name} read a fact with a blob, and ingest has no blob store`,
|
|
113
100
|
);
|
|
114
101
|
|
|
115
102
|
return buildFailureAnswer(500, 'ingest has no blob store');
|
|
116
103
|
}
|
|
117
104
|
|
|
118
|
-
const
|
|
119
|
-
|
|
120
|
-
if (!plan.ok) {
|
|
121
|
-
return buildFailureAnswer(413, 'one fact is larger than one append of the log takes');
|
|
122
|
-
}
|
|
105
|
+
const facts = step.facts;
|
|
123
106
|
|
|
124
|
-
const outcome = await writeBatch(plan
|
|
107
|
+
const outcome = await writeBatch(step.plan, dependencies);
|
|
125
108
|
|
|
126
109
|
if (outcome === 'not-stored') {
|
|
127
110
|
return buildRetryAnswer();
|
|
@@ -152,7 +135,7 @@ export function makeReceiveHandler(
|
|
|
152
135
|
// the ignored answer. It reads no body further and stores nothing.
|
|
153
136
|
export function makeIgnoredRouteHandler(
|
|
154
137
|
path: ReceivePath,
|
|
155
|
-
dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
|
|
138
|
+
dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof' | 'logger'>,
|
|
156
139
|
): Handler {
|
|
157
140
|
return async (request) => {
|
|
158
141
|
const checked = await readCheckedDelivery(request, path, dependencies);
|
|
@@ -170,7 +153,7 @@ type CheckedDelivery =
|
|
|
170
153
|
async function readCheckedDelivery(
|
|
171
154
|
request: Request,
|
|
172
155
|
path: ReceivePath,
|
|
173
|
-
dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof'>,
|
|
156
|
+
dependencies: Pick<ReceiveDependencies, 'secret' | 'senderProof' | 'logger'>,
|
|
174
157
|
): Promise<CheckedDelivery> {
|
|
175
158
|
if (request.method !== 'POST') {
|
|
176
159
|
return { kind: 'answer', answer: buildFailureAnswer(405, 'method not allowed') };
|
|
@@ -190,7 +173,7 @@ async function readCheckedDelivery(
|
|
|
190
173
|
return { kind: 'checked', sent, sender: undefined };
|
|
191
174
|
}
|
|
192
175
|
|
|
193
|
-
const sender = await readProvenSender(path.sender, request.headers, dependencies
|
|
176
|
+
const sender = await readProvenSender(path.sender, request.headers, dependencies);
|
|
194
177
|
|
|
195
178
|
if (sender === null) {
|
|
196
179
|
return { kind: 'answer', answer: buildFailureAnswer(401, 'not authorised') };
|
|
@@ -203,14 +186,14 @@ async function readCheckedDelivery(
|
|
|
203
186
|
async function readProvenSender(
|
|
204
187
|
requirement: SenderRequirement,
|
|
205
188
|
headers: Headers,
|
|
206
|
-
|
|
189
|
+
dependencies: Pick<ReceiveDependencies, 'senderProof' | 'logger'>,
|
|
207
190
|
): Promise<Sender | null> {
|
|
208
191
|
try {
|
|
209
|
-
const sender = await senderProof.prove(requirement, headers);
|
|
192
|
+
const sender = await dependencies.senderProof.prove(requirement, headers);
|
|
210
193
|
|
|
211
194
|
return sender;
|
|
212
195
|
} catch {
|
|
213
|
-
|
|
196
|
+
dependencies.logger.error('the sender proof threw, so the delivery proves no sender');
|
|
214
197
|
|
|
215
198
|
return null;
|
|
216
199
|
}
|
|
@@ -233,12 +216,13 @@ function pickDecodeFailure(failure: 'unsupported' | 'too-large' | 'corrupt'): Re
|
|
|
233
216
|
function readSourceDelivery(
|
|
234
217
|
source: SourceDefinition,
|
|
235
218
|
path: ReceivePath,
|
|
219
|
+
logger: Logger,
|
|
236
220
|
delivery: Parameters<ReceivePath['read']>[0],
|
|
237
221
|
): Received | 'threw' {
|
|
238
222
|
try {
|
|
239
|
-
return path.read(delivery, { name: source.name, families: source.families });
|
|
223
|
+
return path.read(delivery, { name: source.name, families: source.families, log: logger });
|
|
240
224
|
} catch (error: unknown) {
|
|
241
|
-
|
|
225
|
+
logger.error(`the read of the source ${source.name} threw`, error);
|
|
242
226
|
|
|
243
227
|
return 'threw';
|
|
244
228
|
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { PendingFact, Received, SourceContext } from '@pepys/kit';
|
|
2
|
+
import { match } from 'ts-pattern';
|
|
3
|
+
import { pickBatchPlan } from '../utils/pick-batch-plan.ts';
|
|
4
|
+
import type { BatchPlan } from '../utils/plan-batch.ts';
|
|
5
|
+
|
|
6
|
+
// `has-blob`: the source read a fact with a blob, a fault of the deployment that ingest answers 500.
|
|
7
|
+
export type ReceivedStep =
|
|
8
|
+
| { kind: 'refuse'; status: number; reason: string }
|
|
9
|
+
| { kind: 'has-blob' }
|
|
10
|
+
| { kind: 'ignore' }
|
|
11
|
+
| { kind: 'write'; facts: readonly PendingFact[]; plan: BatchPlan };
|
|
12
|
+
|
|
13
|
+
// What follows the source's read of a delivery, `threw` when the read threw.
|
|
14
|
+
export function pickReceivedStep(
|
|
15
|
+
source: Pick<SourceContext, 'name' | 'families'>,
|
|
16
|
+
received: Received | 'threw',
|
|
17
|
+
factLimit: number,
|
|
18
|
+
): ReceivedStep {
|
|
19
|
+
if (received === 'threw') {
|
|
20
|
+
return { kind: 'refuse', status: 500, reason: 'the source could not read the delivery' };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (received.kind === 'refused') {
|
|
24
|
+
return { kind: 'refuse', status: received.status, reason: received.reason };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (received.kind === 'ignored' || received.batch.facts.length === 0) {
|
|
28
|
+
return { kind: 'ignore' };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const facts = received.batch.facts;
|
|
32
|
+
|
|
33
|
+
if (facts.length > factLimit) {
|
|
34
|
+
return { kind: 'refuse', status: 413, reason: `a batch holds ${factLimit} facts at most` };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const plan = pickBatchPlan(source, received.batch);
|
|
38
|
+
|
|
39
|
+
if (!plan.ok) {
|
|
40
|
+
return match(plan.failure.kind)
|
|
41
|
+
.with('bad-batch', () => ({
|
|
42
|
+
kind: 'refuse' as const,
|
|
43
|
+
status: 422,
|
|
44
|
+
reason: plan.failure.reason,
|
|
45
|
+
}))
|
|
46
|
+
.with('has-blob', () => ({ kind: 'has-blob' as const }))
|
|
47
|
+
.with('too-large', () => ({
|
|
48
|
+
kind: 'refuse' as const,
|
|
49
|
+
status: 413,
|
|
50
|
+
reason: plan.failure.reason,
|
|
51
|
+
}))
|
|
52
|
+
.exhaustive();
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return { kind: 'write', facts, plan: plan.value };
|
|
56
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { StreamName } from '@pepys/kit';
|
|
1
|
+
import type { Logger, StreamName } from '@pepys/kit';
|
|
2
2
|
import type { BatchAppender } from '../fact-log/make-batch-appender.ts';
|
|
3
|
-
import type { BatchPlan } from '
|
|
3
|
+
import type { BatchPlan } from '../utils/plan-batch.ts';
|
|
4
4
|
|
|
5
5
|
export type BatchOutcome = 'stored' | 'refused' | 'not-stored';
|
|
6
6
|
|
|
@@ -11,6 +11,7 @@ export type StreamTurns = Map<StreamName, Promise<void>>;
|
|
|
11
11
|
interface WriteBatchDependencies {
|
|
12
12
|
append: BatchAppender;
|
|
13
13
|
turns: StreamTurns;
|
|
14
|
+
logger: Logger;
|
|
14
15
|
}
|
|
15
16
|
|
|
16
17
|
// The streams append at the same time, and the appends of one stream one after the other. Every
|
|
@@ -21,12 +22,12 @@ export async function writeBatch(
|
|
|
21
22
|
): Promise<BatchOutcome> {
|
|
22
23
|
const settled = await Promise.all(
|
|
23
24
|
[...plan].map(([stream, runs]) =>
|
|
24
|
-
withStreamTurn(dependencies
|
|
25
|
+
withStreamTurn(dependencies, stream, async () => {
|
|
25
26
|
for (const run of runs) {
|
|
26
27
|
const appended = await dependencies.append(stream, run);
|
|
27
28
|
|
|
28
29
|
if (!appended.ok) {
|
|
29
|
-
|
|
30
|
+
dependencies.logger.error(`an append to ${stream} failed`, appended.failure);
|
|
30
31
|
|
|
31
32
|
return appended.failure.kind;
|
|
32
33
|
}
|
|
@@ -42,10 +43,11 @@ export async function writeBatch(
|
|
|
42
43
|
|
|
43
44
|
// A throw is a refusal the log gives no outcome for, such as a bad access token.
|
|
44
45
|
async function withStreamTurn(
|
|
45
|
-
|
|
46
|
+
dependencies: Pick<WriteBatchDependencies, 'turns' | 'logger'>,
|
|
46
47
|
stream: StreamName,
|
|
47
48
|
work: () => Promise<BatchOutcome>,
|
|
48
49
|
): Promise<BatchOutcome> {
|
|
50
|
+
const turns = dependencies.turns;
|
|
49
51
|
const previous = turns.get(stream) ?? Promise.resolve();
|
|
50
52
|
const done = Promise.withResolvers<void>();
|
|
51
53
|
|
|
@@ -56,7 +58,7 @@ async function withStreamTurn(
|
|
|
56
58
|
|
|
57
59
|
return await work();
|
|
58
60
|
} catch (error: unknown) {
|
|
59
|
-
|
|
61
|
+
dependencies.logger.error(`an append to ${stream} threw`, error);
|
|
60
62
|
|
|
61
63
|
return 'refused';
|
|
62
64
|
} finally {
|
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
import type { verifyImpIdentity } from '@impz/client/identity';
|
|
2
|
-
import
|
|
2
|
+
import { createLogger } from '@pepys/kit';
|
|
3
|
+
import type { Logger, SenderProof } from '@pepys/kit';
|
|
3
4
|
import type { ImpIdentityTrust } from './types.ts';
|
|
4
5
|
|
|
5
6
|
interface ImpIdentityOptions {
|
|
6
7
|
verify: typeof verifyImpIdentity;
|
|
7
8
|
trust: ImpIdentityTrust;
|
|
8
9
|
now: () => number;
|
|
10
|
+
|
|
11
|
+
// defaults to a logger that writes to stderr
|
|
12
|
+
logger?: Logger | undefined;
|
|
9
13
|
}
|
|
10
14
|
|
|
11
15
|
// Proves the sender from the signed identity token in the `imp-identity` header. The issuer and
|
|
12
16
|
// the subject come only from the verified token, and every failure, a throw included, proves
|
|
13
17
|
// nothing. It never logs the token.
|
|
14
18
|
export function createImpIdentitySenderProof(options: ImpIdentityOptions): SenderProof {
|
|
19
|
+
const logger = options.logger ?? createLogger(process.stderr);
|
|
20
|
+
|
|
15
21
|
return {
|
|
16
22
|
prove: async (requirement, headers) => {
|
|
17
23
|
const documents = options.trust.issuers.get(requirement.issuersEnv) ?? [];
|
|
@@ -33,7 +39,7 @@ export function createImpIdentitySenderProof(options: ImpIdentityOptions): Sende
|
|
|
33
39
|
? null
|
|
34
40
|
: { issuer: verified.issuer, subject: verified.id, name: verified.name };
|
|
35
41
|
} catch {
|
|
36
|
-
|
|
42
|
+
logger.error('the identity verifier failed, so the delivery proves no sender');
|
|
37
43
|
|
|
38
44
|
return null;
|
|
39
45
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { verifyImpIdentity } from '@impz/client/identity';
|
|
2
|
-
import type { Result, SenderProof, SourceDefinition } from '@pepys/kit';
|
|
2
|
+
import type { Logger, Result, SenderProof, SourceDefinition } from '@pepys/kit';
|
|
3
3
|
import { createImpIdentitySenderProof } from './create-imp-identity-sender-proof.ts';
|
|
4
4
|
import { readImpIdentityTrust } from './read-imp-identity-trust.ts';
|
|
5
5
|
|
|
@@ -7,6 +7,9 @@ interface SenderProofReadOptions {
|
|
|
7
7
|
sources: readonly SourceDefinition[];
|
|
8
8
|
environment: Readonly<Record<string, string | undefined>>;
|
|
9
9
|
now: () => number;
|
|
10
|
+
|
|
11
|
+
// defaults to a logger that writes to stderr
|
|
12
|
+
logger?: Logger | undefined;
|
|
10
13
|
}
|
|
11
14
|
|
|
12
15
|
// The sender proof that `serveIngest` runs: the trust of the deployment's environment and the
|
|
@@ -30,6 +33,7 @@ export async function readImpIdentitySenderProof(
|
|
|
30
33
|
verify: verifyImpIdentity,
|
|
31
34
|
trust: trust.value,
|
|
32
35
|
now: options.now,
|
|
36
|
+
logger: options.logger,
|
|
33
37
|
}),
|
|
34
38
|
};
|
|
35
39
|
}
|