@pithy-sh/audit 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/dist/accessHistory.d.ts +315 -0
- package/dist/accessHistory.d.ts.map +1 -0
- package/dist/accessHistory.js +299 -0
- package/dist/actions.d.ts +15 -1
- package/dist/actions.d.ts.map +1 -1
- package/dist/actions.js +16 -2
- package/dist/capability.d.ts +14 -0
- package/dist/capability.d.ts.map +1 -1
- package/dist/capability.js +124 -12
- package/dist/chain/append.d.ts +43 -0
- package/dist/chain/append.d.ts.map +1 -0
- package/dist/chain/append.js +110 -0
- package/dist/chain/checkpoint.d.ts +70 -0
- package/dist/chain/checkpoint.d.ts.map +1 -0
- package/dist/chain/checkpoint.js +133 -0
- package/dist/chain/lockCheck.d.ts +15 -0
- package/dist/chain/lockCheck.d.ts.map +1 -0
- package/dist/chain/lockCheck.js +75 -0
- package/dist/chain/prune.d.ts +84 -0
- package/dist/chain/prune.d.ts.map +1 -0
- package/dist/chain/prune.js +231 -0
- package/dist/chain/rows.d.ts +74 -0
- package/dist/chain/rows.d.ts.map +1 -0
- package/dist/chain/rows.js +133 -0
- package/dist/chain/tombstone.d.ts +21 -0
- package/dist/chain/tombstone.d.ts.map +1 -0
- package/dist/chain/tombstone.js +98 -0
- package/dist/chain/verify.d.ts +63 -0
- package/dist/chain/verify.d.ts.map +1 -0
- package/dist/chain/verify.js +220 -0
- package/dist/citations.d.ts +17 -0
- package/dist/citations.d.ts.map +1 -0
- package/dist/citations.js +48 -0
- package/dist/cli/emitFromCLI.d.ts +5 -0
- package/dist/cli/emitFromCLI.d.ts.map +1 -1
- package/dist/cli/emitFromCLI.js +5 -0
- package/dist/data/auditEvent.d.ts +4 -0
- package/dist/data/auditEvent.d.ts.map +1 -1
- package/dist/data/auditEvent.js +7 -3
- package/dist/data/sinkState.d.ts +23 -0
- package/dist/data/sinkState.d.ts.map +1 -0
- package/dist/data/sinkState.js +25 -0
- package/dist/data/tables.d.ts +14 -1
- package/dist/data/tables.d.ts.map +1 -1
- package/dist/data/tables.js +5 -1
- package/dist/http/guards.d.ts +39 -6
- package/dist/http/guards.d.ts.map +1 -1
- package/dist/http/guards.js +101 -17
- package/dist/http/responses.d.ts +503 -0
- package/dist/http/responses.d.ts.map +1 -1
- package/dist/http/responses.js +157 -5
- package/dist/http/routes.d.ts +25 -3
- package/dist/http/routes.d.ts.map +1 -1
- package/dist/http/routes.js +321 -15
- package/dist/http/schemas.d.ts +67 -0
- package/dist/http/schemas.d.ts.map +1 -1
- package/dist/http/schemas.js +88 -4
- package/dist/http/views.d.ts +25 -1
- package/dist/http/views.d.ts.map +1 -1
- package/dist/http/views.js +107 -2
- package/dist/migrations/0003_call_index.d.ts +21 -0
- package/dist/migrations/0003_call_index.d.ts.map +1 -0
- package/dist/migrations/0003_call_index.js +33 -0
- package/dist/migrations/0004_request_index.d.ts +16 -0
- package/dist/migrations/0004_request_index.d.ts.map +1 -0
- package/dist/migrations/0004_request_index.js +24 -0
- package/dist/migrations/0005_chain.d.ts +24 -0
- package/dist/migrations/0005_chain.d.ts.map +1 -0
- package/dist/migrations/0005_chain.js +43 -0
- package/dist/migrations/0006_sinks.d.ts +11 -0
- package/dist/migrations/0006_sinks.d.ts.map +1 -0
- package/dist/migrations/0006_sinks.js +20 -0
- package/dist/provision/resolveAuditConfig.d.ts +31 -0
- package/dist/provision/resolveAuditConfig.d.ts.map +1 -0
- package/dist/provision/resolveAuditConfig.js +43 -0
- package/dist/purge/contribution.d.ts +21 -0
- package/dist/purge/contribution.d.ts.map +1 -0
- package/dist/purge/contribution.js +41 -0
- package/dist/reach.d.ts +250 -0
- package/dist/reach.d.ts.map +1 -0
- package/dist/reach.js +244 -0
- package/dist/recorder.d.ts +11 -0
- package/dist/recorder.d.ts.map +1 -1
- package/dist/recorder.js +23 -5
- package/dist/routePattern.d.ts +60 -0
- package/dist/routePattern.d.ts.map +1 -0
- package/dist/routePattern.js +130 -0
- package/dist/seeds/example.d.ts.map +1 -1
- package/dist/seeds/example.js +23 -6
- package/dist/ship/consumer.d.ts +45 -0
- package/dist/ship/consumer.d.ts.map +1 -0
- package/dist/ship/consumer.js +141 -0
- package/dist/ship/event.d.ts +59 -0
- package/dist/ship/event.d.ts.map +1 -0
- package/dist/ship/event.js +77 -0
- package/dist/ship/health.d.ts +33 -0
- package/dist/ship/health.d.ts.map +1 -0
- package/dist/ship/health.js +80 -0
- package/dist/ship/profiles.d.ts +34 -0
- package/dist/ship/profiles.d.ts.map +1 -0
- package/dist/ship/profiles.js +62 -0
- package/dist/ship/queue.d.ts +57 -0
- package/dist/ship/queue.d.ts.map +1 -0
- package/dist/ship/queue.js +70 -0
- package/dist/ship/secrets.d.ts +7 -0
- package/dist/ship/secrets.d.ts.map +1 -0
- package/dist/ship/secrets.js +40 -0
- package/dist/ship/shipper.d.ts +15 -0
- package/dist/ship/shipper.d.ts.map +1 -0
- package/dist/ship/shipper.js +27 -0
- package/dist/ship/sinks.d.ts +54 -0
- package/dist/ship/sinks.d.ts.map +1 -0
- package/dist/ship/sinks.js +45 -0
- package/dist/ship/state.d.ts +27 -0
- package/dist/ship/state.d.ts.map +1 -0
- package/dist/ship/state.js +29 -0
- package/dist/usage.d.ts +159 -0
- package/dist/usage.d.ts.map +1 -0
- package/dist/usage.js +87 -0
- package/dist/version.generated.d.ts +1 -1
- package/dist/version.generated.js +1 -1
- package/dist/workflows/prune.d.ts +73 -0
- package/dist/workflows/prune.d.ts.map +1 -0
- package/dist/workflows/prune.js +128 -0
- package/dist/workflows/retryPolicy.d.ts +10 -0
- package/dist/workflows/retryPolicy.d.ts.map +1 -0
- package/dist/workflows/retryPolicy.js +14 -0
- package/dist/workflows/specs.d.ts +70 -0
- package/dist/workflows/specs.d.ts.map +1 -0
- package/dist/workflows/specs.js +70 -0
- package/dist/workflows/worker.d.ts +32 -0
- package/dist/workflows/worker.d.ts.map +1 -0
- package/dist/workflows/worker.js +41 -0
- package/package.json +5 -2
- package/pithy.manifest.json +36 -4
- package/src/accessHistory.ts +360 -0
- package/src/actions.ts +15 -1
- package/src/capability.ts +154 -12
- package/src/chain/append.ts +161 -0
- package/src/chain/checkpoint.ts +172 -0
- package/src/chain/lockCheck.ts +72 -0
- package/src/chain/prune.ts +340 -0
- package/src/chain/rows.ts +211 -0
- package/src/chain/tombstone.ts +104 -0
- package/src/chain/verify.ts +366 -0
- package/src/citations.ts +62 -0
- package/src/cli/emitFromCLI.ts +5 -0
- package/src/cloudflare-test.d.ts +1 -0
- package/src/data/auditEvent.ts +31 -1
- package/src/data/sinkState.ts +41 -0
- package/src/data/tables.ts +7 -1
- package/src/http/guards.ts +90 -6
- package/src/http/responses.ts +318 -3
- package/src/http/routes.ts +379 -7
- package/src/http/schemas.ts +158 -2
- package/src/http/views.ts +123 -1
- package/src/migrations/0003_call_index.ts +35 -0
- package/src/migrations/0004_request_index.ts +26 -0
- package/src/migrations/0005_chain.ts +56 -0
- package/src/migrations/0006_sinks.ts +28 -0
- package/src/provision/resolveAuditConfig.ts +60 -0
- package/src/purge/contribution.ts +39 -0
- package/src/reach.ts +454 -0
- package/src/recorder.ts +64 -42
- package/src/routePattern.ts +141 -0
- package/src/seeds/example.ts +13 -0
- package/src/ship/consumer.ts +169 -0
- package/src/ship/event.ts +86 -0
- package/src/ship/health.ts +92 -0
- package/src/ship/profiles.ts +99 -0
- package/src/ship/queue.ts +108 -0
- package/src/ship/secrets.ts +42 -0
- package/src/ship/shipper.ts +32 -0
- package/src/ship/sinks.ts +82 -0
- package/src/ship/state.ts +61 -0
- package/src/usage.ts +176 -0
- package/src/version.generated.ts +1 -1
- package/src/workflows/prune.ts +196 -0
- package/src/workflows/retryPolicy.ts +14 -0
- package/src/workflows/specs.ts +92 -0
- package/src/workflows/worker.ts +65 -0
- package/src/workflows/wrangler.jsonc +40 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
import { chainedEventOf, readChainHead, readChainLink, readChainRows, whereChain } from "./rows.js";
|
|
4
|
+
import { readCheckpoints } from "./checkpoint.js";
|
|
5
|
+
import { readTombstones, tombstoneEndingAt, writeTombstone } from "./tombstone.js";
|
|
6
|
+
import { sql } from "kysely";
|
|
7
|
+
import { AUDIT_TOMBSTONE_FORMAT, chainVerifier } from "@pithy-sh/core/src/audit/chain";
|
|
8
|
+
import { chunkByBoundParameters } from "@pithy-sh/core/src/data/boundParameters";
|
|
9
|
+
import { MS_PER_DAY, auditWindowDays } from "@pithy-sh/core/src/retention/policy";
|
|
10
|
+
//#region src/chain/prune.ts
|
|
11
|
+
/**
|
|
12
|
+
* **The retention prune: audit events past their window, removed without breaking the chain.**
|
|
13
|
+
*
|
|
14
|
+
* The Worker's `retention.audit` block says how long each action is kept. An event past its window is removed, and
|
|
15
|
+
* because every chained event names the one before it, a removed run leaves a tombstone in the checkpoint bucket
|
|
16
|
+
* first: the run's positions, the link before it, and the hashes at either end. A verify bridges the gap with it and
|
|
17
|
+
* reads the chain as whole. Without a checkpoint bucket nothing chained is pruned, since nothing could vouch for the
|
|
18
|
+
* gap.
|
|
19
|
+
*
|
|
20
|
+
* - **Runs, not single events.** Windows differ by action, so an event kept longer can sit between two that have
|
|
21
|
+
* passed theirs. Each maximal run of consecutive positions past their windows is pruned on its own, and the event
|
|
22
|
+
* kept between two runs stays where it is.
|
|
23
|
+
* - **A run is verified before it goes.** Its links, its hashes and the link before it are checked as a verify would,
|
|
24
|
+
* and a run that does not verify is left in place and counted as refused: the prune never removes the evidence of a
|
|
25
|
+
* break. A run whose positions hold any row it did not read, a second row at one position among them, is refused
|
|
26
|
+
* too, and the delete names the ids of the rows it verified, never a range of positions.
|
|
27
|
+
* - **The head is never pruned.** The next append links to it, so the last event of every chain stays.
|
|
28
|
+
* - **Rows from before the chain** carry no link, and are removed on their own window with nothing to tombstone.
|
|
29
|
+
* - **A page reads only rows past their own windows.** An event kept longer than the shortest window, a floored
|
|
30
|
+
* control-plane call under a short default, is never read, so however many of them sit at the front of a chain a
|
|
31
|
+
* page still reaches the events behind them.
|
|
32
|
+
*/
|
|
33
|
+
/** How many rows one prune step reads. */
|
|
34
|
+
const AUDIT_PRUNE_PAGE = 500;
|
|
35
|
+
/** The shortest window any action can have under this policy, in days, or null when nothing is ever pruned. */
|
|
36
|
+
function shortestWindowDays(retention) {
|
|
37
|
+
const days = [...retention.audit.rules.map((rule) => rule.days), ...retention.audit.defaultDays === null ? [] : [retention.audit.defaultDays]];
|
|
38
|
+
return days.length === 0 ? null : Math.min(...days);
|
|
39
|
+
}
|
|
40
|
+
/** Whether an event with this action, occurring at `occurredAt` ms, is past its window at `now`. */
|
|
41
|
+
function pastWindow(retention, action, occurredAt, now) {
|
|
42
|
+
const days = auditWindowDays(retention, action);
|
|
43
|
+
return days !== null && occurredAt < now.getTime() - days * MS_PER_DAY;
|
|
44
|
+
}
|
|
45
|
+
/** The actions grouped by the instant each is past its window at `now`. An action kept forever is in no group. */
|
|
46
|
+
function expiryCutoffs(retention, actions, now) {
|
|
47
|
+
const groups = /* @__PURE__ */ new Map();
|
|
48
|
+
for (const action of new Set(actions)) {
|
|
49
|
+
const days = auditWindowDays(retention, action);
|
|
50
|
+
if (days === null) continue;
|
|
51
|
+
const before = now.getTime() - days * MS_PER_DAY;
|
|
52
|
+
groups.set(before, [...groups.get(before) ?? [], action]);
|
|
53
|
+
}
|
|
54
|
+
return [...groups.entries()].sort(([a], [b]) => a - b).map(([before, grouped]) => ({
|
|
55
|
+
before,
|
|
56
|
+
actions: grouped.sort()
|
|
57
|
+
}));
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* The rows past their own windows, as one condition. Each group's actions travel as one JSON array, so the bound
|
|
61
|
+
* parameters stay two per window however many actions the trail holds.
|
|
62
|
+
*/
|
|
63
|
+
function pastCutoffs(cutoffs) {
|
|
64
|
+
return sql`(${sql.join(cutoffs.map((cutoff) => sql`(${sql.ref("action")} in (select value from json_each(${JSON.stringify(cutoff.actions)})) and ${sql.ref("occurredAt")} < ${cutoff.before})`), sql` or `)})`;
|
|
65
|
+
}
|
|
66
|
+
/** Every maximal run of consecutive positions among `rows` that are all past their windows. */
|
|
67
|
+
function expiredRuns(rows, retention, now) {
|
|
68
|
+
const runs = [];
|
|
69
|
+
let current = [];
|
|
70
|
+
for (const row of rows) {
|
|
71
|
+
const expired = pastWindow(retention, row.action, row.occurredAt, now);
|
|
72
|
+
const last = current.at(-1);
|
|
73
|
+
const follows = last !== void 0 && row.chainSeq === (last.chainSeq ?? 0) + 1;
|
|
74
|
+
if (!expired || last && !follows) {
|
|
75
|
+
if (current.length > 0) runs.push(current);
|
|
76
|
+
current = [];
|
|
77
|
+
}
|
|
78
|
+
if (expired) current.push(row);
|
|
79
|
+
}
|
|
80
|
+
if (current.length > 0) runs.push(current);
|
|
81
|
+
return runs;
|
|
82
|
+
}
|
|
83
|
+
/** The link before a run at `fromSeq`: null at the chain's start, else the stored event or a tombstone ending there. */
|
|
84
|
+
async function linkBeforeRun(db, tenant, fromSeq, tombstones) {
|
|
85
|
+
if (fromSeq === 1) return null;
|
|
86
|
+
const stored = await readChainLink(db, tenant, fromSeq - 1);
|
|
87
|
+
if (stored) return stored;
|
|
88
|
+
const run = tombstoneEndingAt(tombstones, fromSeq - 1);
|
|
89
|
+
return run ? {
|
|
90
|
+
seq: run.throughSeq,
|
|
91
|
+
hash: run.last
|
|
92
|
+
} : void 0;
|
|
93
|
+
}
|
|
94
|
+
/** Whether a run is whole: every link and hash, from the link before it, against the checkpoints covering it. */
|
|
95
|
+
async function runVerifies(bucket, tenant, run, prior) {
|
|
96
|
+
const from = run[0]?.chainSeq ?? 0;
|
|
97
|
+
const through = run.at(-1)?.chainSeq ?? 0;
|
|
98
|
+
const checkpoints = await readCheckpoints(bucket, tenant, {
|
|
99
|
+
fromCount: from,
|
|
100
|
+
throughCount: through
|
|
101
|
+
});
|
|
102
|
+
const verifier = chainVerifier({
|
|
103
|
+
tenant,
|
|
104
|
+
prior,
|
|
105
|
+
checkpoints
|
|
106
|
+
});
|
|
107
|
+
for (const row of run) if (await verifier.push(await chainedEventOf(row, { detail: false }))) return false;
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
/** How many rows of this chain sit at positions `fromSeq` through `throughSeq`. */
|
|
111
|
+
async function rowsAt(db, tenant, fromSeq, throughSeq) {
|
|
112
|
+
const row = await whereChain(db.selectFrom("pithyAuditEvents").select((eb) => eb.fn.countAll().as("count")).where("chainSeq", ">=", fromSeq).where("chainSeq", "<=", throughSeq), tenant).executeTakeFirst();
|
|
113
|
+
return Number(row?.count ?? 0);
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Prune one page of one chain: read up to `limit` chained rows after `afterSeq` that are old enough for the shortest
|
|
117
|
+
* window, and remove every run among them that is past its windows and verifies, tombstone first.
|
|
118
|
+
*/
|
|
119
|
+
async function pruneChainPage(db, bucket, options) {
|
|
120
|
+
const { tenant, retention, now } = options;
|
|
121
|
+
const shortest = shortestWindowDays(retention);
|
|
122
|
+
const head = await readChainHead(db, tenant);
|
|
123
|
+
if (shortest === null || !head || head.seq <= 1) return {
|
|
124
|
+
pruned: 0,
|
|
125
|
+
refused: 0,
|
|
126
|
+
next: null
|
|
127
|
+
};
|
|
128
|
+
const occurredBefore = now.getTime() - shortest * MS_PER_DAY;
|
|
129
|
+
const cutoffs = expiryCutoffs(retention, (await whereChain(db.selectFrom("pithyAuditEvents").select("action").distinct().where("chainSeq", ">", options.afterSeq).where("chainSeq", "<=", head.seq - 1).where("occurredAt", "<", occurredBefore), tenant).execute()).map((row) => row.action), now);
|
|
130
|
+
if (cutoffs.length === 0) return {
|
|
131
|
+
pruned: 0,
|
|
132
|
+
refused: 0,
|
|
133
|
+
next: null
|
|
134
|
+
};
|
|
135
|
+
const rows = await readChainRows(db, {
|
|
136
|
+
tenant,
|
|
137
|
+
afterSeq: options.afterSeq,
|
|
138
|
+
limit: options.limit,
|
|
139
|
+
throughSeq: head.seq - 1,
|
|
140
|
+
occurredBefore,
|
|
141
|
+
matching: pastCutoffs(cutoffs)
|
|
142
|
+
});
|
|
143
|
+
const tombstones = await readTombstones(bucket, tenant);
|
|
144
|
+
let pruned = 0;
|
|
145
|
+
let refused = 0;
|
|
146
|
+
for (const run of expiredRuns(rows, retention, now)) {
|
|
147
|
+
const first = run[0];
|
|
148
|
+
const last = run.at(-1);
|
|
149
|
+
const fromSeq = first.chainSeq;
|
|
150
|
+
if (await rowsAt(db, tenant, fromSeq, last.chainSeq) !== run.length) {
|
|
151
|
+
refused++;
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
const prior = await linkBeforeRun(db, tenant, fromSeq, tombstones);
|
|
155
|
+
if (prior === void 0 || !await runVerifies(bucket, tenant, run, prior)) {
|
|
156
|
+
refused++;
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
const stored = await writeTombstone(bucket, {
|
|
160
|
+
format: AUDIT_TOMBSTONE_FORMAT,
|
|
161
|
+
tenant,
|
|
162
|
+
fromSeq,
|
|
163
|
+
throughSeq: last.chainSeq,
|
|
164
|
+
count: run.length,
|
|
165
|
+
prior: prior?.hash ?? null,
|
|
166
|
+
first: first.chainHash,
|
|
167
|
+
last: last.chainHash,
|
|
168
|
+
prunedAt: now.toISOString()
|
|
169
|
+
});
|
|
170
|
+
const end = run.find((row) => row.chainSeq === stored.throughSeq);
|
|
171
|
+
const named = run.filter((row) => row.chainSeq >= stored.fromSeq && row.chainSeq <= stored.throughSeq);
|
|
172
|
+
if (!end || end.chainHash !== stored.last || first.chainHash !== stored.first || named.length !== stored.count || await rowsAt(db, tenant, stored.fromSeq, stored.throughSeq) !== stored.count) {
|
|
173
|
+
refused++;
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
const fixed = tenant === null ? 2 : 3;
|
|
177
|
+
for (const ids of chunkByBoundParameters(named.map((row) => row.id), fixed)) await whereChain(db.deleteFrom("pithyAuditEvents").where("id", "in", ids).where("chainSeq", ">=", stored.fromSeq).where("chainSeq", "<=", stored.throughSeq), tenant).execute();
|
|
178
|
+
tombstones.push(stored);
|
|
179
|
+
pruned += stored.count;
|
|
180
|
+
}
|
|
181
|
+
const lastRead = rows.at(-1);
|
|
182
|
+
return {
|
|
183
|
+
pruned,
|
|
184
|
+
refused,
|
|
185
|
+
next: rows.length === options.limit && lastRead ? lastRead.chainSeq : null
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
/** The chains the trail holds: each tenant with a chained event, and null for the untenanted chain. */
|
|
189
|
+
async function chainTenants(db) {
|
|
190
|
+
return (await db.selectFrom("pithyAuditEvents").select("tenant").distinct().where("chainSeq", "is not", null).execute()).map((row) => row.tenant ?? null).sort((a, b) => (a ?? "").localeCompare(b ?? ""));
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Prune one page of the rows that carry no chain — written before it began, or by an older recorder since. Each is
|
|
194
|
+
* removed on its own window, unless a hold stands on its tenant's trail, and nothing is tombstoned: no chain names it.
|
|
195
|
+
*/
|
|
196
|
+
async function pruneUnchainedPage(db, options) {
|
|
197
|
+
const { retention, now } = options;
|
|
198
|
+
const shortest = shortestWindowDays(retention);
|
|
199
|
+
if (shortest === null) return {
|
|
200
|
+
deleted: 0,
|
|
201
|
+
next: null
|
|
202
|
+
};
|
|
203
|
+
const occurredBefore = now.getTime() - shortest * MS_PER_DAY;
|
|
204
|
+
const cutoffs = expiryCutoffs(retention, (await db.selectFrom("pithyAuditEvents").select("action").distinct().where("chainSeq", "is", null).where("id", ">", options.afterId).where("occurredAt", "<", occurredBefore).execute()).map((row) => row.action), now);
|
|
205
|
+
if (cutoffs.length === 0) return {
|
|
206
|
+
deleted: 0,
|
|
207
|
+
next: null
|
|
208
|
+
};
|
|
209
|
+
const rows = await db.selectFrom("pithyAuditEvents").select([
|
|
210
|
+
"id",
|
|
211
|
+
"action",
|
|
212
|
+
"occurredAt",
|
|
213
|
+
"tenant"
|
|
214
|
+
]).where("chainSeq", "is", null).where("id", ">", options.afterId).where("occurredAt", "<", occurredBefore).where(pastCutoffs(cutoffs)).orderBy("id", "asc").limit(options.limit).execute();
|
|
215
|
+
const holds = /* @__PURE__ */ new Map();
|
|
216
|
+
const doomed = [];
|
|
217
|
+
for (const row of rows) {
|
|
218
|
+
if (!pastWindow(retention, row.action, row.occurredAt, now)) continue;
|
|
219
|
+
const tenant = row.tenant ?? null;
|
|
220
|
+
if (!holds.has(tenant)) holds.set(tenant, await options.held(tenant));
|
|
221
|
+
if (!holds.get(tenant)) doomed.push(row.id);
|
|
222
|
+
}
|
|
223
|
+
if (doomed.length > 0) await db.deleteFrom("pithyAuditEvents").where("id", "in", doomed).execute();
|
|
224
|
+
const lastRead = rows.at(-1);
|
|
225
|
+
return {
|
|
226
|
+
deleted: doomed.length,
|
|
227
|
+
next: rows.length === options.limit && lastRead ? lastRead.id : null
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
//#endregion
|
|
231
|
+
export { AUDIT_PRUNE_PAGE, chainTenants, expiredRuns, expiryCutoffs, pastWindow, pruneChainPage, pruneUnchainedPage, shortestWindowDays };
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
import { type ChainedEvent, type ChainedEventFields, type ChainLink } from "@pithy-sh/core/src/audit/chain";
|
|
4
|
+
import type { Expression, SqlBool } from "kysely";
|
|
5
|
+
import type { AuditDatabase } from "../data/tables";
|
|
6
|
+
/** One stored row of a chain, raw. */
|
|
7
|
+
export interface ChainRow {
|
|
8
|
+
id: number;
|
|
9
|
+
eventId: string;
|
|
10
|
+
occurredAt: number;
|
|
11
|
+
action: string;
|
|
12
|
+
outcome: string;
|
|
13
|
+
severity: string;
|
|
14
|
+
actorType: string;
|
|
15
|
+
actorId: string | null;
|
|
16
|
+
actorKind: string | null;
|
|
17
|
+
sessionId: string | null;
|
|
18
|
+
resourceType: string | null;
|
|
19
|
+
resourceId: string | null;
|
|
20
|
+
requestId: string | null;
|
|
21
|
+
tenant: string | null;
|
|
22
|
+
project: string | null;
|
|
23
|
+
environment: string | null;
|
|
24
|
+
worker: string | null;
|
|
25
|
+
version: string | null;
|
|
26
|
+
ip: string | null;
|
|
27
|
+
userAgent: string | null;
|
|
28
|
+
metadata: string | null;
|
|
29
|
+
chainSeq: number | null;
|
|
30
|
+
chainPrev: string | null;
|
|
31
|
+
chainHash: string | null;
|
|
32
|
+
chainSalt: string | null;
|
|
33
|
+
}
|
|
34
|
+
/** A query builder narrowed to one chain: one tenant's, or the untenanted one. */
|
|
35
|
+
export declare function whereChain<Q extends {
|
|
36
|
+
where: (...args: any[]) => Q;
|
|
37
|
+
}>(query: Q, tenant: string | null): Q;
|
|
38
|
+
/** The chain's head — its last position and that event's hash — or null when it holds no event. */
|
|
39
|
+
export declare function readChainHead(db: AuditDatabase, tenant: string | null): Promise<ChainLink | null>;
|
|
40
|
+
/** The link at one position, or null when no event holds it. */
|
|
41
|
+
export declare function readChainLink(db: AuditDatabase, tenant: string | null, seq: number): Promise<ChainLink | null>;
|
|
42
|
+
/**
|
|
43
|
+
* A page of one chain in chain order — by position, then id — never past `throughSeq`. It resumes after
|
|
44
|
+
* `afterSeq`, or, given `afterId`, after that one row: a page that resumed from a bare position would skip a second
|
|
45
|
+
* row holding the position the last page ended on, and that second row is exactly the insert a walk must see.
|
|
46
|
+
*/
|
|
47
|
+
export declare function readChainRows(db: AuditDatabase, options: {
|
|
48
|
+
tenant: string | null;
|
|
49
|
+
afterSeq: number;
|
|
50
|
+
afterId?: number;
|
|
51
|
+
limit: number;
|
|
52
|
+
throughSeq?: number;
|
|
53
|
+
/** Only rows that occurred before this instant, in stored ms: what the retention prune asks for. */
|
|
54
|
+
occurredBefore?: number;
|
|
55
|
+
/** Only rows this condition holds for: the retention prune's rows past their own windows. */
|
|
56
|
+
matching?: Expression<SqlBool>;
|
|
57
|
+
}): Promise<ChainRow[]>;
|
|
58
|
+
/** The hashed fields of a stored row, its personal values committed under its salt. */
|
|
59
|
+
export declare function chainFieldsOf(row: Omit<ChainRow, "id" | "chainSeq" | "chainPrev" | "chainHash">, salt: string): Promise<ChainedEventFields>;
|
|
60
|
+
/**
|
|
61
|
+
* One stored row as an export carries it. Its commitments are recomputed from the values it holds now, so a value
|
|
62
|
+
* edited in place changes its commitment and the hash no longer matches. The values and the salt travel only with
|
|
63
|
+
* `detail`; without it the commitments are all a reader gets.
|
|
64
|
+
*/
|
|
65
|
+
export declare function chainedEventOf(row: ChainRow, options: {
|
|
66
|
+
detail: boolean;
|
|
67
|
+
}): Promise<ChainedEvent>;
|
|
68
|
+
/**
|
|
69
|
+
* The stored rows for a set of event ids, raw, in no particular order (#824). What the shipping consumer builds each
|
|
70
|
+
* delivery from: the row as stored now, read at delivery rather than carried on the queue, so no personal value
|
|
71
|
+
* ever sits in a queue message. An id with no row is simply absent from the answer.
|
|
72
|
+
*/
|
|
73
|
+
export declare function readRowsByEventId(db: AuditDatabase, eventIds: readonly string[]): Promise<ChainRow[]>;
|
|
74
|
+
//# sourceMappingURL=rows.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rows.d.ts","sourceRoot":"","sources":["../../src/chain/rows.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,kBAAkB,EAAE,KAAK,SAAS,EAAc,MAAM,gCAAgC,CAAC;AACxH,OAAO,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AA0CpD,sCAAsC;AACtC,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,kFAAkF;AAElF,wBAAgB,UAAU,CAAC,CAAC,SAAS;IAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,CAAC,CAAA;CAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,CAAC,CAEzG;AAED,mGAAmG;AACnG,wBAAsB,aAAa,CAAC,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAWvG;AAYD,gEAAgE;AAChE,wBAAsB,aAAa,CAAC,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAMpH;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CACjC,EAAE,EAAE,aAAa,EACjB,OAAO,EAAE;IACP,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oGAAoG;IACpG,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6FAA6F;IAC7F,QAAQ,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;CAChC,GACA,OAAO,CAAC,QAAQ,EAAE,CAAC,CAiBrB;AAED,uFAAuF;AACvF,wBAAsB,aAAa,CACjC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC,EAClE,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,kBAAkB,CAAC,CAuB7B;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE;IAAE,MAAM,EAAE,OAAO,CAAA;CAAE,GAAG,OAAO,CAAC,YAAY,CAAC,CAUvG;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAO3G"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
import { commitment } from "@pithy-sh/core/src/audit/chain";
|
|
4
|
+
//#region src/chain/rows.ts
|
|
5
|
+
/**
|
|
6
|
+
* **Reading a tenant's chain out of `pithy_audit_events` (#822).**
|
|
7
|
+
*
|
|
8
|
+
* One chain per tenant, and one for the events with no tenant. Every read here names which, through
|
|
9
|
+
* {@link whereChain}, because `tenant = null` matches nothing in SQL and the untenanted chain would read as empty.
|
|
10
|
+
*
|
|
11
|
+
* Rows are read raw — the `z.input` side, `occurredAt` as its stored ms and `metadata` as its stored JSON text —
|
|
12
|
+
* because the hash covers the bytes as stored. Decoding the metadata and encoding it again would hash a value the
|
|
13
|
+
* row does not hold the first time the two serializations differ.
|
|
14
|
+
*/
|
|
15
|
+
/** The columns a chain read selects: every hashed field, the link, and the personal values with their salt. */
|
|
16
|
+
const CHAIN_COLUMNS = [
|
|
17
|
+
"id",
|
|
18
|
+
"eventId",
|
|
19
|
+
"occurredAt",
|
|
20
|
+
"action",
|
|
21
|
+
"outcome",
|
|
22
|
+
"severity",
|
|
23
|
+
"actorType",
|
|
24
|
+
"actorId",
|
|
25
|
+
"actorKind",
|
|
26
|
+
"sessionId",
|
|
27
|
+
"resourceType",
|
|
28
|
+
"resourceId",
|
|
29
|
+
"requestId",
|
|
30
|
+
"tenant",
|
|
31
|
+
"project",
|
|
32
|
+
"environment",
|
|
33
|
+
"worker",
|
|
34
|
+
"version",
|
|
35
|
+
"ip",
|
|
36
|
+
"userAgent",
|
|
37
|
+
"metadata",
|
|
38
|
+
"chainSeq",
|
|
39
|
+
"chainPrev",
|
|
40
|
+
"chainHash",
|
|
41
|
+
"chainSalt"
|
|
42
|
+
];
|
|
43
|
+
/** A query builder narrowed to one chain: one tenant's, or the untenanted one. */
|
|
44
|
+
function whereChain(query, tenant) {
|
|
45
|
+
return tenant === null ? query.where("tenant", "is", null) : query.where("tenant", "=", tenant);
|
|
46
|
+
}
|
|
47
|
+
/** The chain's head — its last position and that event's hash — or null when it holds no event. */
|
|
48
|
+
async function readChainHead(db, tenant) {
|
|
49
|
+
return linkOf(await whereChain(db.selectFrom("pithyAuditEvents").select(["chainSeq", "chainHash"]).where("chainSeq", "is not", null).orderBy("chainSeq", "desc").limit(1), tenant).executeTakeFirst());
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* A selected link, or null. The type check is not decoration: SQLite reads an unknown double-quoted identifier as a
|
|
53
|
+
* string literal, so on a database without `audit_0005_chain` the column comes back as the text `chain_seq`.
|
|
54
|
+
*/
|
|
55
|
+
function linkOf(row) {
|
|
56
|
+
return typeof row?.chainSeq === "number" && typeof row.chainHash === "string" ? {
|
|
57
|
+
seq: row.chainSeq,
|
|
58
|
+
hash: row.chainHash
|
|
59
|
+
} : null;
|
|
60
|
+
}
|
|
61
|
+
/** The link at one position, or null when no event holds it. */
|
|
62
|
+
async function readChainLink(db, tenant, seq) {
|
|
63
|
+
return linkOf(await whereChain(db.selectFrom("pithyAuditEvents").select(["chainSeq", "chainHash"]).where("chainSeq", "=", seq).limit(1), tenant).executeTakeFirst());
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* A page of one chain in chain order — by position, then id — never past `throughSeq`. It resumes after
|
|
67
|
+
* `afterSeq`, or, given `afterId`, after that one row: a page that resumed from a bare position would skip a second
|
|
68
|
+
* row holding the position the last page ended on, and that second row is exactly the insert a walk must see.
|
|
69
|
+
*/
|
|
70
|
+
async function readChainRows(db, options) {
|
|
71
|
+
const { afterSeq, afterId } = options;
|
|
72
|
+
let query = db.selectFrom("pithyAuditEvents").select([...CHAIN_COLUMNS]).where((eb) => afterId === void 0 ? eb("chainSeq", ">", afterSeq) : eb.or([eb("chainSeq", ">", afterSeq), eb.and([eb("chainSeq", "=", afterSeq), eb("id", ">", afterId)])])).orderBy("chainSeq", "asc").orderBy("id", "asc").limit(options.limit);
|
|
73
|
+
if (options.throughSeq !== void 0) query = query.where("chainSeq", "<=", options.throughSeq);
|
|
74
|
+
if (options.occurredBefore !== void 0) query = query.where("occurredAt", "<", options.occurredBefore);
|
|
75
|
+
if (options.matching !== void 0) query = query.where(options.matching);
|
|
76
|
+
return await whereChain(query, options.tenant).execute();
|
|
77
|
+
}
|
|
78
|
+
/** The hashed fields of a stored row, its personal values committed under its salt. */
|
|
79
|
+
async function chainFieldsOf(row, salt) {
|
|
80
|
+
return {
|
|
81
|
+
eventId: row.eventId,
|
|
82
|
+
occurredAt: row.occurredAt,
|
|
83
|
+
action: row.action,
|
|
84
|
+
outcome: row.outcome,
|
|
85
|
+
severity: row.severity,
|
|
86
|
+
actorType: row.actorType,
|
|
87
|
+
actorId: row.actorId,
|
|
88
|
+
actorKind: row.actorKind,
|
|
89
|
+
sessionId: row.sessionId,
|
|
90
|
+
resourceType: row.resourceType,
|
|
91
|
+
resourceId: row.resourceId,
|
|
92
|
+
requestId: row.requestId,
|
|
93
|
+
tenant: row.tenant,
|
|
94
|
+
project: row.project,
|
|
95
|
+
environment: row.environment,
|
|
96
|
+
worker: row.worker,
|
|
97
|
+
version: row.version,
|
|
98
|
+
ipCommitment: await commitment(salt, row.ip),
|
|
99
|
+
userAgentCommitment: await commitment(salt, row.userAgent),
|
|
100
|
+
metadataCommitment: await commitment(salt, row.metadata)
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* One stored row as an export carries it. Its commitments are recomputed from the values it holds now, so a value
|
|
105
|
+
* edited in place changes its commitment and the hash no longer matches. The values and the salt travel only with
|
|
106
|
+
* `detail`; without it the commitments are all a reader gets.
|
|
107
|
+
*/
|
|
108
|
+
async function chainedEventOf(row, options) {
|
|
109
|
+
const salt = row.chainSalt ?? "";
|
|
110
|
+
return {
|
|
111
|
+
...await chainFieldsOf(row, salt),
|
|
112
|
+
seq: row.chainSeq ?? 0,
|
|
113
|
+
prev: row.chainPrev,
|
|
114
|
+
hash: row.chainHash ?? "",
|
|
115
|
+
...options.detail ? {
|
|
116
|
+
salt,
|
|
117
|
+
ip: row.ip,
|
|
118
|
+
userAgent: row.userAgent,
|
|
119
|
+
metadata: row.metadata
|
|
120
|
+
} : {}
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* The stored rows for a set of event ids, raw, in no particular order (#824). What the shipping consumer builds each
|
|
125
|
+
* delivery from: the row as stored now, read at delivery rather than carried on the queue, so no personal value
|
|
126
|
+
* ever sits in a queue message. An id with no row is simply absent from the answer.
|
|
127
|
+
*/
|
|
128
|
+
async function readRowsByEventId(db, eventIds) {
|
|
129
|
+
if (eventIds.length === 0) return [];
|
|
130
|
+
return await db.selectFrom("pithyAuditEvents").select([...CHAIN_COLUMNS]).where("eventId", "in", [...eventIds]).execute();
|
|
131
|
+
}
|
|
132
|
+
//#endregion
|
|
133
|
+
export { chainFieldsOf, chainedEventOf, readChainHead, readChainLink, readChainRows, readRowsByEventId, whereChain };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
import type { R2Bucket } from "@cloudflare/workers-types";
|
|
4
|
+
import { AuditTombstone } from "@pithy-sh/core/src/audit/chain";
|
|
5
|
+
/** The key prefix of one chain's tombstones. */
|
|
6
|
+
export declare function tombstonePrefix(tenant: string | null): string;
|
|
7
|
+
/** The key of one run's tombstone: its chain's prefix and its first position. */
|
|
8
|
+
export declare function tombstoneKey(tenant: string | null, fromSeq: number): string;
|
|
9
|
+
/**
|
|
10
|
+
* Write a run's tombstone, or hand back the one already written at its key. What the prune deletes is the run the
|
|
11
|
+
* stored tombstone names, never the one it meant to write, so a retry cannot delete past what the bucket records.
|
|
12
|
+
*/
|
|
13
|
+
export declare function writeTombstone(bucket: R2Bucket, tombstone: AuditTombstone): Promise<AuditTombstone>;
|
|
14
|
+
/**
|
|
15
|
+
* Every tombstone of one chain, in chain order. An object that does not parse, or names another chain or position
|
|
16
|
+
* than its key, is skipped: a tombstone bridges a gap only where it says it does.
|
|
17
|
+
*/
|
|
18
|
+
export declare function readTombstones(bucket: R2Bucket, tenant: string | null): Promise<AuditTombstone[]>;
|
|
19
|
+
/** The tombstone whose run ends at `throughSeq`, or null: the link a range starting after a pruned run begins from. */
|
|
20
|
+
export declare function tombstoneEndingAt(tombstones: readonly AuditTombstone[], throughSeq: number): AuditTombstone | null;
|
|
21
|
+
//# sourceMappingURL=tombstone.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tombstone.d.ts","sourceRoot":"","sources":["../../src/chain/tombstone.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAiBhE,gDAAgD;AAChD,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAE7D;AAED,iFAAiF;AACjF,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAE3E;AAqBD;;;GAGG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAkBzG;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAoBvG;AAED,uHAAuH;AACvH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAElH"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
import { checkpointPrefix } from "./checkpoint.js";
|
|
4
|
+
import { InternalError } from "@pithy-sh/core/src/error/pithyError";
|
|
5
|
+
import { AuditTombstone } from "@pithy-sh/core/src/audit/chain";
|
|
6
|
+
//#region src/chain/tombstone.ts
|
|
7
|
+
/**
|
|
8
|
+
* **Tombstones: the runs of a chain the retention prune removed, kept beside its checkpoints.**
|
|
9
|
+
*
|
|
10
|
+
* One object per pruned run, under the chain's checkpoint prefix, so the bucket's object-lock rule covers it as it
|
|
11
|
+
* covers the checkpoints. The key carries the run's first position and the write is conditional on the key being
|
|
12
|
+
* new, so a run is tombstoned once and a prune retried after a failed delete finds the tombstone it already wrote.
|
|
13
|
+
*
|
|
14
|
+
* A checkpoint listing skips these keys: a checkpoint's key is a bare count, and a tombstone's is under `tombstones/`.
|
|
15
|
+
*/
|
|
16
|
+
/** The zero-padded width of a position in a key, so keys list in chain order. */
|
|
17
|
+
const SEQ_WIDTH = 15;
|
|
18
|
+
/** The key prefix of one chain's tombstones. */
|
|
19
|
+
function tombstonePrefix(tenant) {
|
|
20
|
+
return `${checkpointPrefix(tenant)}tombstones/`;
|
|
21
|
+
}
|
|
22
|
+
/** The key of one run's tombstone: its chain's prefix and its first position. */
|
|
23
|
+
function tombstoneKey(tenant, fromSeq) {
|
|
24
|
+
return `${tombstonePrefix(tenant)}${String(fromSeq).padStart(SEQ_WIDTH, "0")}.json`;
|
|
25
|
+
}
|
|
26
|
+
/** The first position a key names, or null when the key is not a tombstone's. */
|
|
27
|
+
function fromSeqOf(key, prefix) {
|
|
28
|
+
const match = /^(\d+)\.json$/.exec(key.slice(prefix.length));
|
|
29
|
+
return match ? Number(match[1]) : null;
|
|
30
|
+
}
|
|
31
|
+
/** A stored tombstone, or null when the object does not parse or names another chain or position than its key. */
|
|
32
|
+
async function parseTombstone(text, tenant, fromSeq) {
|
|
33
|
+
let body;
|
|
34
|
+
try {
|
|
35
|
+
body = JSON.parse(text);
|
|
36
|
+
} catch {
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
const parsed = AuditTombstone.safeParse(body);
|
|
40
|
+
if (!parsed.success || parsed.data.tenant !== tenant || parsed.data.fromSeq !== fromSeq) return null;
|
|
41
|
+
return parsed.data;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Write a run's tombstone, or hand back the one already written at its key. What the prune deletes is the run the
|
|
45
|
+
* stored tombstone names, never the one it meant to write, so a retry cannot delete past what the bucket records.
|
|
46
|
+
*/
|
|
47
|
+
async function writeTombstone(bucket, tombstone) {
|
|
48
|
+
const key = tombstoneKey(tombstone.tenant, tombstone.fromSeq);
|
|
49
|
+
if (await bucket.put(key, JSON.stringify(AuditTombstone.parse(tombstone)), {
|
|
50
|
+
onlyIf: { etagDoesNotMatch: "*" },
|
|
51
|
+
httpMetadata: { contentType: "application/json" }
|
|
52
|
+
})) return tombstone;
|
|
53
|
+
const existing = await bucket.get(key);
|
|
54
|
+
const stored = existing ? await parseTombstone(await existing.text(), tombstone.tenant, tombstone.fromSeq) : null;
|
|
55
|
+
if (!stored) throw new InternalError({
|
|
56
|
+
message: "The audit prune found a tombstone it cannot read, and pruned nothing there.",
|
|
57
|
+
action: "Inspect the object in the checkpoint bucket; the prune leaves that run in place until it reads.",
|
|
58
|
+
detail: `tombstone ${key} exists but does not parse as one`
|
|
59
|
+
});
|
|
60
|
+
return stored;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Every tombstone of one chain, in chain order. An object that does not parse, or names another chain or position
|
|
64
|
+
* than its key, is skipped: a tombstone bridges a gap only where it says it does.
|
|
65
|
+
*/
|
|
66
|
+
async function readTombstones(bucket, tenant) {
|
|
67
|
+
const prefix = tombstonePrefix(tenant);
|
|
68
|
+
const keys = [];
|
|
69
|
+
let cursor;
|
|
70
|
+
do {
|
|
71
|
+
const page = await bucket.list({
|
|
72
|
+
prefix,
|
|
73
|
+
...cursor ? { cursor } : {}
|
|
74
|
+
});
|
|
75
|
+
for (const object of page.objects) {
|
|
76
|
+
const fromSeq = fromSeqOf(object.key, prefix);
|
|
77
|
+
if (fromSeq !== null) keys.push({
|
|
78
|
+
key: object.key,
|
|
79
|
+
fromSeq
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
cursor = page.truncated ? page.cursor : void 0;
|
|
83
|
+
} while (cursor);
|
|
84
|
+
const tombstones = [];
|
|
85
|
+
for (const { key, fromSeq } of keys.sort((a, b) => a.fromSeq - b.fromSeq)) {
|
|
86
|
+
const object = await bucket.get(key);
|
|
87
|
+
if (!object) continue;
|
|
88
|
+
const parsed = await parseTombstone(await object.text(), tenant, fromSeq);
|
|
89
|
+
if (parsed) tombstones.push(parsed);
|
|
90
|
+
}
|
|
91
|
+
return tombstones;
|
|
92
|
+
}
|
|
93
|
+
/** The tombstone whose run ends at `throughSeq`, or null: the link a range starting after a pruned run begins from. */
|
|
94
|
+
function tombstoneEndingAt(tombstones, throughSeq) {
|
|
95
|
+
return tombstones.find((tombstone) => tombstone.throughSeq === throughSeq) ?? null;
|
|
96
|
+
}
|
|
97
|
+
//#endregion
|
|
98
|
+
export { readTombstones, tombstoneEndingAt, tombstoneKey, tombstonePrefix, writeTombstone };
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
import type { R2Bucket } from "@cloudflare/workers-types";
|
|
4
|
+
import { type AuditChainExport, type ChainBreak, type ChainLink } from "@pithy-sh/core/src/audit/chain";
|
|
5
|
+
import type { AuditDatabase } from "../data/tables";
|
|
6
|
+
/** One tenant's chain over a window of time. */
|
|
7
|
+
export interface ChainRangeQuery {
|
|
8
|
+
/** The tenant whose chain is read, or null for the untenanted chain. */
|
|
9
|
+
tenant: string | null;
|
|
10
|
+
/** The window's inclusive start, or omitted for the chain's start. */
|
|
11
|
+
since?: Date;
|
|
12
|
+
/** The window's inclusive end, or omitted for the chain's head. */
|
|
13
|
+
until?: Date;
|
|
14
|
+
}
|
|
15
|
+
/** What the verify read found. */
|
|
16
|
+
export interface ChainRangeVerification {
|
|
17
|
+
/** Whether the range is whole. */
|
|
18
|
+
intact: boolean;
|
|
19
|
+
/** How many chained events were checked before the first break, or all of them. */
|
|
20
|
+
checked: number;
|
|
21
|
+
/** How many checkpoints the walk was held to. */
|
|
22
|
+
checkpoints: number;
|
|
23
|
+
/** How many of the tenant's events in the window were written before the chain, and so are not in it. */
|
|
24
|
+
beforeChain: number;
|
|
25
|
+
/** How many rows in the range carry no chain though written after it began: an older recorder's, not a break. */
|
|
26
|
+
outsideChain: number;
|
|
27
|
+
/** How many events the walk crossed as pruned, each run bridged by its tombstone. */
|
|
28
|
+
pruned: number;
|
|
29
|
+
/** The first break, or null. */
|
|
30
|
+
break: ChainBreak | null;
|
|
31
|
+
/** The last position the walk reached intact, or null when it reached none. */
|
|
32
|
+
head: ChainLink | null;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Walk a range: every link, every hash, and the checkpoints covering it. `pageSize` is how many rows one read
|
|
36
|
+
* takes, and is a test's to set, so a page boundary can be put where a test needs it.
|
|
37
|
+
*/
|
|
38
|
+
export declare function verifyChainRange(db: AuditDatabase, bucket: R2Bucket | null, query: ChainRangeQuery, options?: {
|
|
39
|
+
pageSize?: number;
|
|
40
|
+
}): Promise<ChainRangeVerification>;
|
|
41
|
+
/** One page of an export. */
|
|
42
|
+
export interface ChainExportOptions {
|
|
43
|
+
/** Whether the personal values and their salt travel with the commitments. */
|
|
44
|
+
detail: boolean;
|
|
45
|
+
/** Where the previous page ended — its last position and row id — from its `nextCursor`. */
|
|
46
|
+
cursor?: string;
|
|
47
|
+
/** How many events one page carries. */
|
|
48
|
+
limit: number;
|
|
49
|
+
/** When the export is taken. */
|
|
50
|
+
now: Date;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* One page of a range as an export document: its events in canonical form with their hashes, the link before
|
|
54
|
+
* the page, and the checkpoints covering it. Each page verifies on its own from its `prior`, so an auditor can
|
|
55
|
+
* check a page without the ones before it.
|
|
56
|
+
*/
|
|
57
|
+
export declare function exportChainRange(db: AuditDatabase, bucket: R2Bucket | null, query: ChainRangeQuery, options: ChainExportOptions): Promise<AuditChainExport>;
|
|
58
|
+
/** A `nextCursor` read back, or null when it is not one — a malformed cursor is a first page. */
|
|
59
|
+
export declare function parseExportCursor(cursor: string | undefined): {
|
|
60
|
+
seq: number;
|
|
61
|
+
id: number;
|
|
62
|
+
} | null;
|
|
63
|
+
//# sourceMappingURL=verify.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../../src/chain/verify.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAGL,KAAK,gBAAgB,EAGrB,KAAK,UAAU,EACf,KAAK,SAAS,EAEf,MAAM,gCAAgC,CAAC;AAExC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAiBpD,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC9B,wEAAwE;IACxE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,sEAAsE;IACtE,KAAK,CAAC,EAAE,IAAI,CAAC;IACb,mEAAmE;IACnE,KAAK,CAAC,EAAE,IAAI,CAAC;CACd;AAkHD,kCAAkC;AAClC,MAAM,WAAW,sBAAsB;IACrC,kCAAkC;IAClC,MAAM,EAAE,OAAO,CAAC;IAChB,mFAAmF;IACnF,OAAO,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,yGAAyG;IACzG,WAAW,EAAE,MAAM,CAAC;IACpB,iHAAiH;IACjH,YAAY,EAAE,MAAM,CAAC;IACrB,qFAAqF;IACrF,MAAM,EAAE,MAAM,CAAC;IACf,gCAAgC;IAChC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACzB,+EAA+E;IAC/E,IAAI,EAAE,SAAS,GAAG,IAAI,CAAC;CACxB;AAkBD;;;GAGG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,aAAa,EACjB,MAAM,EAAE,QAAQ,GAAG,IAAI,EACvB,KAAK,EAAE,eAAe,EACtB,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,OAAO,CAAC,sBAAsB,CAAC,CAMjC;AA0ED,6BAA6B;AAC7B,MAAM,WAAW,kBAAkB;IACjC,8EAA8E;IAC9E,MAAM,EAAE,OAAO,CAAC;IAChB,4FAA4F;IAC5F,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,gCAAgC;IAChC,GAAG,EAAE,IAAI,CAAC;CACX;AAED;;;;GAIG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,aAAa,EACjB,MAAM,EAAE,QAAQ,GAAG,IAAI,EACvB,KAAK,EAAE,eAAe,EACtB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,gBAAgB,CAAC,CAoD3B;AAOD,iGAAiG;AACjG,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAIhG"}
|