@pithy-sh/audit 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/dist/accessHistory.d.ts +315 -0
- package/dist/accessHistory.d.ts.map +1 -0
- package/dist/accessHistory.js +299 -0
- package/dist/actions.d.ts +15 -1
- package/dist/actions.d.ts.map +1 -1
- package/dist/actions.js +16 -2
- package/dist/capability.d.ts +14 -0
- package/dist/capability.d.ts.map +1 -1
- package/dist/capability.js +124 -12
- package/dist/chain/append.d.ts +43 -0
- package/dist/chain/append.d.ts.map +1 -0
- package/dist/chain/append.js +110 -0
- package/dist/chain/checkpoint.d.ts +70 -0
- package/dist/chain/checkpoint.d.ts.map +1 -0
- package/dist/chain/checkpoint.js +133 -0
- package/dist/chain/lockCheck.d.ts +15 -0
- package/dist/chain/lockCheck.d.ts.map +1 -0
- package/dist/chain/lockCheck.js +75 -0
- package/dist/chain/prune.d.ts +84 -0
- package/dist/chain/prune.d.ts.map +1 -0
- package/dist/chain/prune.js +231 -0
- package/dist/chain/rows.d.ts +74 -0
- package/dist/chain/rows.d.ts.map +1 -0
- package/dist/chain/rows.js +133 -0
- package/dist/chain/tombstone.d.ts +21 -0
- package/dist/chain/tombstone.d.ts.map +1 -0
- package/dist/chain/tombstone.js +98 -0
- package/dist/chain/verify.d.ts +63 -0
- package/dist/chain/verify.d.ts.map +1 -0
- package/dist/chain/verify.js +220 -0
- package/dist/citations.d.ts +17 -0
- package/dist/citations.d.ts.map +1 -0
- package/dist/citations.js +48 -0
- package/dist/cli/emitFromCLI.d.ts +5 -0
- package/dist/cli/emitFromCLI.d.ts.map +1 -1
- package/dist/cli/emitFromCLI.js +5 -0
- package/dist/data/auditEvent.d.ts +4 -0
- package/dist/data/auditEvent.d.ts.map +1 -1
- package/dist/data/auditEvent.js +7 -3
- package/dist/data/sinkState.d.ts +23 -0
- package/dist/data/sinkState.d.ts.map +1 -0
- package/dist/data/sinkState.js +25 -0
- package/dist/data/tables.d.ts +14 -1
- package/dist/data/tables.d.ts.map +1 -1
- package/dist/data/tables.js +5 -1
- package/dist/http/guards.d.ts +39 -6
- package/dist/http/guards.d.ts.map +1 -1
- package/dist/http/guards.js +101 -17
- package/dist/http/responses.d.ts +503 -0
- package/dist/http/responses.d.ts.map +1 -1
- package/dist/http/responses.js +157 -5
- package/dist/http/routes.d.ts +25 -3
- package/dist/http/routes.d.ts.map +1 -1
- package/dist/http/routes.js +321 -15
- package/dist/http/schemas.d.ts +67 -0
- package/dist/http/schemas.d.ts.map +1 -1
- package/dist/http/schemas.js +88 -4
- package/dist/http/views.d.ts +25 -1
- package/dist/http/views.d.ts.map +1 -1
- package/dist/http/views.js +107 -2
- package/dist/migrations/0003_call_index.d.ts +21 -0
- package/dist/migrations/0003_call_index.d.ts.map +1 -0
- package/dist/migrations/0003_call_index.js +33 -0
- package/dist/migrations/0004_request_index.d.ts +16 -0
- package/dist/migrations/0004_request_index.d.ts.map +1 -0
- package/dist/migrations/0004_request_index.js +24 -0
- package/dist/migrations/0005_chain.d.ts +24 -0
- package/dist/migrations/0005_chain.d.ts.map +1 -0
- package/dist/migrations/0005_chain.js +43 -0
- package/dist/migrations/0006_sinks.d.ts +11 -0
- package/dist/migrations/0006_sinks.d.ts.map +1 -0
- package/dist/migrations/0006_sinks.js +20 -0
- package/dist/provision/resolveAuditConfig.d.ts +31 -0
- package/dist/provision/resolveAuditConfig.d.ts.map +1 -0
- package/dist/provision/resolveAuditConfig.js +43 -0
- package/dist/purge/contribution.d.ts +21 -0
- package/dist/purge/contribution.d.ts.map +1 -0
- package/dist/purge/contribution.js +41 -0
- package/dist/reach.d.ts +250 -0
- package/dist/reach.d.ts.map +1 -0
- package/dist/reach.js +244 -0
- package/dist/recorder.d.ts +11 -0
- package/dist/recorder.d.ts.map +1 -1
- package/dist/recorder.js +23 -5
- package/dist/routePattern.d.ts +60 -0
- package/dist/routePattern.d.ts.map +1 -0
- package/dist/routePattern.js +130 -0
- package/dist/seeds/example.d.ts.map +1 -1
- package/dist/seeds/example.js +23 -6
- package/dist/ship/consumer.d.ts +45 -0
- package/dist/ship/consumer.d.ts.map +1 -0
- package/dist/ship/consumer.js +141 -0
- package/dist/ship/event.d.ts +59 -0
- package/dist/ship/event.d.ts.map +1 -0
- package/dist/ship/event.js +77 -0
- package/dist/ship/health.d.ts +33 -0
- package/dist/ship/health.d.ts.map +1 -0
- package/dist/ship/health.js +80 -0
- package/dist/ship/profiles.d.ts +34 -0
- package/dist/ship/profiles.d.ts.map +1 -0
- package/dist/ship/profiles.js +62 -0
- package/dist/ship/queue.d.ts +57 -0
- package/dist/ship/queue.d.ts.map +1 -0
- package/dist/ship/queue.js +70 -0
- package/dist/ship/secrets.d.ts +7 -0
- package/dist/ship/secrets.d.ts.map +1 -0
- package/dist/ship/secrets.js +40 -0
- package/dist/ship/shipper.d.ts +15 -0
- package/dist/ship/shipper.d.ts.map +1 -0
- package/dist/ship/shipper.js +27 -0
- package/dist/ship/sinks.d.ts +54 -0
- package/dist/ship/sinks.d.ts.map +1 -0
- package/dist/ship/sinks.js +45 -0
- package/dist/ship/state.d.ts +27 -0
- package/dist/ship/state.d.ts.map +1 -0
- package/dist/ship/state.js +29 -0
- package/dist/usage.d.ts +159 -0
- package/dist/usage.d.ts.map +1 -0
- package/dist/usage.js +87 -0
- package/dist/version.generated.d.ts +1 -1
- package/dist/version.generated.js +1 -1
- package/dist/workflows/prune.d.ts +73 -0
- package/dist/workflows/prune.d.ts.map +1 -0
- package/dist/workflows/prune.js +128 -0
- package/dist/workflows/retryPolicy.d.ts +10 -0
- package/dist/workflows/retryPolicy.d.ts.map +1 -0
- package/dist/workflows/retryPolicy.js +14 -0
- package/dist/workflows/specs.d.ts +70 -0
- package/dist/workflows/specs.d.ts.map +1 -0
- package/dist/workflows/specs.js +70 -0
- package/dist/workflows/worker.d.ts +32 -0
- package/dist/workflows/worker.d.ts.map +1 -0
- package/dist/workflows/worker.js +41 -0
- package/package.json +5 -2
- package/pithy.manifest.json +36 -4
- package/src/accessHistory.ts +360 -0
- package/src/actions.ts +15 -1
- package/src/capability.ts +154 -12
- package/src/chain/append.ts +161 -0
- package/src/chain/checkpoint.ts +172 -0
- package/src/chain/lockCheck.ts +72 -0
- package/src/chain/prune.ts +340 -0
- package/src/chain/rows.ts +211 -0
- package/src/chain/tombstone.ts +104 -0
- package/src/chain/verify.ts +366 -0
- package/src/citations.ts +62 -0
- package/src/cli/emitFromCLI.ts +5 -0
- package/src/cloudflare-test.d.ts +1 -0
- package/src/data/auditEvent.ts +31 -1
- package/src/data/sinkState.ts +41 -0
- package/src/data/tables.ts +7 -1
- package/src/http/guards.ts +90 -6
- package/src/http/responses.ts +318 -3
- package/src/http/routes.ts +379 -7
- package/src/http/schemas.ts +158 -2
- package/src/http/views.ts +123 -1
- package/src/migrations/0003_call_index.ts +35 -0
- package/src/migrations/0004_request_index.ts +26 -0
- package/src/migrations/0005_chain.ts +56 -0
- package/src/migrations/0006_sinks.ts +28 -0
- package/src/provision/resolveAuditConfig.ts +60 -0
- package/src/purge/contribution.ts +39 -0
- package/src/reach.ts +454 -0
- package/src/recorder.ts +64 -42
- package/src/routePattern.ts +141 -0
- package/src/seeds/example.ts +13 -0
- package/src/ship/consumer.ts +169 -0
- package/src/ship/event.ts +86 -0
- package/src/ship/health.ts +92 -0
- package/src/ship/profiles.ts +99 -0
- package/src/ship/queue.ts +108 -0
- package/src/ship/secrets.ts +42 -0
- package/src/ship/shipper.ts +32 -0
- package/src/ship/sinks.ts +82 -0
- package/src/ship/state.ts +61 -0
- package/src/usage.ts +176 -0
- package/src/version.generated.ts +1 -1
- package/src/workflows/prune.ts +196 -0
- package/src/workflows/retryPolicy.ts +14 -0
- package/src/workflows/specs.ts +92 -0
- package/src/workflows/worker.ts +65 -0
- package/src/workflows/wrangler.jsonc +40 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
|
|
4
|
+
import type { AdminRoute } from "@pithy-sh/core/src/controlPlane/discovery/adminRoute";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* A declared admin route, in the form the reach statement matches a called path against (#828).
|
|
9
|
+
*
|
|
10
|
+
* The gate records the path as called — `/admin/users/u_8f2`, not `/admin/users/:userId` — so grouping a credential's
|
|
11
|
+
* calls by what they did means matching each one back to the route that served it. That match runs in SQL, inside the
|
|
12
|
+
* one statement, so the routes go in as one JSON parameter of these objects and SQLite compares each call against
|
|
13
|
+
* them with `GLOB`.
|
|
14
|
+
*
|
|
15
|
+
* **A glob and a slash count, together.** A parameter becomes `*`, and `*` in a glob also matches `/`, so on its own
|
|
16
|
+
* `/admin/users/*` would claim `/admin/users/u_8f2/sessions`. Requiring the called path to have exactly as many
|
|
17
|
+
* slashes as the pattern closes that: the pattern's literal slashes already account for every one, so no `*` can
|
|
18
|
+
* swallow another. A route ending in Hono's `*` wildcard matches any depth, and carries no count.
|
|
19
|
+
*/
|
|
20
|
+
export const ReachPattern = z
|
|
21
|
+
.object({
|
|
22
|
+
method: z.string().describe("The HTTP method the route answers on."),
|
|
23
|
+
pattern: z.string().describe("The route as declared, with the capability's `basePath` applied."),
|
|
24
|
+
glob: z.string().describe("The pattern as an SQLite `GLOB`: each parameter a `*`, each metacharacter escaped."),
|
|
25
|
+
slashes: z
|
|
26
|
+
.number()
|
|
27
|
+
.int()
|
|
28
|
+
.nullable()
|
|
29
|
+
.describe("How many `/` a called path must hold to be this route, or null for a wildcard route of any depth."),
|
|
30
|
+
params: z.number().int().describe("How many named parameters the route declares — the subjects it can name."),
|
|
31
|
+
rank: z
|
|
32
|
+
.number()
|
|
33
|
+
.int()
|
|
34
|
+
.describe("Which route wins when two match one path: the lower rank, so `/users/search` beats `/users/:id`."),
|
|
35
|
+
})
|
|
36
|
+
.describe("One declared admin route, ready for the reach statement to match called paths against.");
|
|
37
|
+
export type ReachPattern = z.infer<typeof ReachPattern>;
|
|
38
|
+
|
|
39
|
+
/** A segment that names a parameter, with Hono's optional `{regex}` and `?` after the name. */
|
|
40
|
+
const PARAM_SEGMENT = /^:([A-Za-z0-9_]+)(\{.*\})?\??$/;
|
|
41
|
+
|
|
42
|
+
/** One literal segment, with every `GLOB` metacharacter wrapped in a one-character class so it matches itself. */
|
|
43
|
+
function escapeGlob(segment: string): string {
|
|
44
|
+
return segment.replace(/[*?[]/g, (char) => `[${char}]`);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** How specific a route is: literal segments first, then fewer parameters, and a wildcard last. */
|
|
48
|
+
function specificity(path: string): [number, number, number] {
|
|
49
|
+
const segments = path.split("/").slice(1);
|
|
50
|
+
const params = segments.filter((segment) => PARAM_SEGMENT.test(segment)).length;
|
|
51
|
+
const wildcard = segments.includes("*") ? 1 : 0;
|
|
52
|
+
return [segments.length - params - wildcard, -params, -wildcard];
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Every declared route as a {@link ReachPattern}, most specific first, one per method and path.
|
|
57
|
+
*
|
|
58
|
+
* Ranked the way a router prefers a static segment to a parameter, so a call to `/admin/users/search` is the search
|
|
59
|
+
* route and not a read of a user called `search`. Two routes of equal specificity cannot both match one path — they
|
|
60
|
+
* differ in a literal segment — so the order between them never decides anything.
|
|
61
|
+
*/
|
|
62
|
+
export function reachPatterns(routes: readonly Pick<AdminRoute, "method" | "path">[]): ReachPattern[] {
|
|
63
|
+
const unique = new Map<string, Pick<AdminRoute, "method" | "path">>();
|
|
64
|
+
for (const route of routes) unique.set(`${route.method} ${route.path}`, route);
|
|
65
|
+
const ordered = [...unique.values()].sort((a, b) => {
|
|
66
|
+
const [x, y] = [specificity(a.path), specificity(b.path)];
|
|
67
|
+
for (let i = 0; i < x.length; i++) {
|
|
68
|
+
const difference = (y[i] as number) - (x[i] as number);
|
|
69
|
+
if (difference !== 0) return difference;
|
|
70
|
+
}
|
|
71
|
+
return 0;
|
|
72
|
+
});
|
|
73
|
+
return ordered.map((route, rank) => {
|
|
74
|
+
const segments = route.path.split("/");
|
|
75
|
+
const wildcard = segments.includes("*");
|
|
76
|
+
return {
|
|
77
|
+
method: route.method,
|
|
78
|
+
pattern: route.path,
|
|
79
|
+
glob: segments
|
|
80
|
+
.map((segment) => (segment === "*" || PARAM_SEGMENT.test(segment) ? "*" : escapeGlob(segment)))
|
|
81
|
+
.join("/"),
|
|
82
|
+
slashes: wildcard ? null : segments.length - 1,
|
|
83
|
+
params: segments.filter((segment) => PARAM_SEGMENT.test(segment)).length,
|
|
84
|
+
rank,
|
|
85
|
+
};
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** One segment of a called path, decoded, or as called when it is not valid percent-encoding. */
|
|
90
|
+
function decodeSegment(segment: string): string {
|
|
91
|
+
try {
|
|
92
|
+
return decodeURIComponent(segment);
|
|
93
|
+
} catch {
|
|
94
|
+
return segment;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* The value each of `pattern`'s parameters took in the called `path` — the subject ids a call touched.
|
|
100
|
+
*
|
|
101
|
+
* Read in the Worker rather than in SQL, and only for the rows a `subjects=true` read returns: the statement has
|
|
102
|
+
* already narrowed them to one route, and splitting a path on `/` is not something SQLite does without a recursive
|
|
103
|
+
* CTE per row. A path with a different number of segments names nothing.
|
|
104
|
+
*/
|
|
105
|
+
export function routeParams(pattern: string, path: string): Record<string, string> {
|
|
106
|
+
const declared = pattern.split("/");
|
|
107
|
+
const called = path.split("/");
|
|
108
|
+
if (declared.length !== called.length) return {};
|
|
109
|
+
const params: Record<string, string> = {};
|
|
110
|
+
declared.forEach((segment, index) => {
|
|
111
|
+
const name = PARAM_SEGMENT.exec(segment)?.[1];
|
|
112
|
+
if (name !== undefined) params[name] = decodeSegment(called[index] as string);
|
|
113
|
+
});
|
|
114
|
+
return params;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** A parameter name that names an id: `id` itself, or anything ending in `Id`. */
|
|
118
|
+
const ID_PARAM = /^id$|Id$/;
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* The subject one call to `pattern` acted on, read from the called `path`.
|
|
122
|
+
*
|
|
123
|
+
* **The subject is the last parameter that names an id**, not the last parameter: on
|
|
124
|
+
* `/accounts/:userId/:currency/transactions` the user is whose data was read, and the currency only narrows what was
|
|
125
|
+
* read of them. A route none of whose parameters names an id falls back to its last one.
|
|
126
|
+
*
|
|
127
|
+
* **`key` tells subjects apart**: the values of the subject parameter and of every parameter before it. The same id
|
|
128
|
+
* under another subject type or another organization is another subject — `/entitlements/user/s_1` and
|
|
129
|
+
* `/entitlements/organization/s_1` are two holders — while the parameters after it are not part of who it is, so one
|
|
130
|
+
* user's ledger read in two currencies is one subject. Both are null when the route declares no parameter, or the path
|
|
131
|
+
* does not fit it.
|
|
132
|
+
*/
|
|
133
|
+
export function routeSubject(pattern: string, path: string): { subjectId: string | null; key: string | null } {
|
|
134
|
+
const params = routeParams(pattern, path);
|
|
135
|
+
const names = Object.keys(params);
|
|
136
|
+
if (names.length === 0) return { subjectId: null, key: null };
|
|
137
|
+
const ids = names.filter((name) => ID_PARAM.test(name));
|
|
138
|
+
const subject = (ids[ids.length - 1] ?? names[names.length - 1]) as string;
|
|
139
|
+
const qualifiers = names.slice(0, names.indexOf(subject) + 1);
|
|
140
|
+
return { subjectId: params[subject] ?? null, key: JSON.stringify(qualifiers.map((name) => params[name])) };
|
|
141
|
+
}
|
package/src/seeds/example.ts
CHANGED
|
@@ -32,6 +32,13 @@ const minutesAgo = (minutes: number): Date => new Date(Date.now() - minutes * 60
|
|
|
32
32
|
*/
|
|
33
33
|
const EXAMPLE_ORIGIN = { project: "example-app", environment: null, worker: "api", version: null } as const;
|
|
34
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Example rows are seeded straight into the table, outside the recorder, so they carry no chain (#822): they are
|
|
37
|
+
* marked as before the chain, exactly as a row written before the upgrade is. A seed is a dev fixture, never
|
|
38
|
+
* evidence, and inventing links for it would teach the verify read to trust rows nobody recorded.
|
|
39
|
+
*/
|
|
40
|
+
const BEFORE_THE_CHAIN = { chainSeq: null, chainPrev: null, chainHash: null, chainSalt: null } as const;
|
|
41
|
+
|
|
35
42
|
/**
|
|
36
43
|
* Two fictional tenants, so a dev database can show what the tenant filter is for.
|
|
37
44
|
*
|
|
@@ -83,6 +90,7 @@ export const auditExampleSeed: SeedSet = defineSeed({
|
|
|
83
90
|
metadata: { method: "magic_link" },
|
|
84
91
|
tenant: EXAMPLE_TENANT_ACME,
|
|
85
92
|
...EXAMPLE_ORIGIN,
|
|
93
|
+
...BEFORE_THE_CHAIN,
|
|
86
94
|
},
|
|
87
95
|
{
|
|
88
96
|
id: 2,
|
|
@@ -103,6 +111,7 @@ export const auditExampleSeed: SeedSet = defineSeed({
|
|
|
103
111
|
metadata: { reason: "expired_link" },
|
|
104
112
|
tenant: EXAMPLE_TENANT_GLOBEX,
|
|
105
113
|
...EXAMPLE_ORIGIN,
|
|
114
|
+
...BEFORE_THE_CHAIN,
|
|
106
115
|
},
|
|
107
116
|
{
|
|
108
117
|
id: 3,
|
|
@@ -127,6 +136,7 @@ export const auditExampleSeed: SeedSet = defineSeed({
|
|
|
127
136
|
metadata: { plan: "pro", grantedBy: "system" },
|
|
128
137
|
tenant: EXAMPLE_TENANT_ACME,
|
|
129
138
|
...EXAMPLE_ORIGIN,
|
|
139
|
+
...BEFORE_THE_CHAIN,
|
|
130
140
|
},
|
|
131
141
|
{
|
|
132
142
|
id: 4,
|
|
@@ -148,6 +158,7 @@ export const auditExampleSeed: SeedSet = defineSeed({
|
|
|
148
158
|
// The same actor as event 1, in the other tenant — what `actorId` alone cannot tell apart.
|
|
149
159
|
tenant: EXAMPLE_TENANT_GLOBEX,
|
|
150
160
|
...EXAMPLE_ORIGIN,
|
|
161
|
+
...BEFORE_THE_CHAIN,
|
|
151
162
|
},
|
|
152
163
|
{
|
|
153
164
|
id: 5,
|
|
@@ -168,6 +179,7 @@ export const auditExampleSeed: SeedSet = defineSeed({
|
|
|
168
179
|
metadata: { familyId: "example-family-grace", action: "family_revoked" },
|
|
169
180
|
tenant: EXAMPLE_TENANT_GLOBEX,
|
|
170
181
|
...EXAMPLE_ORIGIN,
|
|
182
|
+
...BEFORE_THE_CHAIN,
|
|
171
183
|
},
|
|
172
184
|
{
|
|
173
185
|
id: 6,
|
|
@@ -189,6 +201,7 @@ export const auditExampleSeed: SeedSet = defineSeed({
|
|
|
189
201
|
// No tenant, on purpose: the shape every row in a single-tenant app has.
|
|
190
202
|
tenant: null,
|
|
191
203
|
...EXAMPLE_ORIGIN,
|
|
204
|
+
...BEFORE_THE_CHAIN,
|
|
192
205
|
},
|
|
193
206
|
]),
|
|
194
207
|
],
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
|
|
4
|
+
import type { Logger } from "@pithy-sh/core/src/logger/logger";
|
|
5
|
+
import type { CapabilityQueueHandler, QueueMessage } from "@pithy-sh/core/src/queue/consumer";
|
|
6
|
+
import { readRowsByEventId } from "../chain/rows";
|
|
7
|
+
import type { AuditDatabase } from "../data/tables";
|
|
8
|
+
import { shippedEventOf } from "./event";
|
|
9
|
+
import { sinkDelivery } from "./profiles";
|
|
10
|
+
import { AUDIT_SHIP_MAX_RETRIES, isFinalAttempt, retryDelaySeconds, ShipMessage } from "./queue";
|
|
11
|
+
import type { AuditSinks } from "./sinks";
|
|
12
|
+
import { recordSinkOutcome } from "./state";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* **The shipping consumer (#824): one request per sink per batch, at least once.**
|
|
16
|
+
*
|
|
17
|
+
* Each sink's share of a batch is read back from the trail, built into the sink's own format, and posted once. A 2xx
|
|
18
|
+
* acknowledges every message in it; anything else — a refusal, a timeout, a token that will not resolve — retries
|
|
19
|
+
* every one of them after a delay that doubles from 30 seconds to an hour, so a failing sink backs off instead of
|
|
20
|
+
* being hammered. A message whose last retry fails goes to the dead-letter queue, and the sink's state counts it.
|
|
21
|
+
*
|
|
22
|
+
* A sink fails on its own. One sink down costs that sink's messages a retry and leaves every other sink's delivered:
|
|
23
|
+
* a message names one sink, so nothing a healthy sink took is ever sent to it again.
|
|
24
|
+
*
|
|
25
|
+
* **Two messages are acknowledged without a delivery**, because no retry could ever deliver them: one naming a sink
|
|
26
|
+
* the config no longer declares, and one naming an event with no row. Both are logged.
|
|
27
|
+
*
|
|
28
|
+
* **What the log never carries:** a token, a response body, or any value from the event. The sink's name, the
|
|
29
|
+
* status and the count are enough to act on, and the receiver's response is the receiver's.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
/** How long one delivery may take before it counts as a failure, in ms. */
|
|
33
|
+
export const SHIP_TIMEOUT_MS = 10_000;
|
|
34
|
+
|
|
35
|
+
/** What the consumer needs, every piece injectable so a test runs it against recorded requests. */
|
|
36
|
+
export interface ShipConsumerOptions {
|
|
37
|
+
/** The declared sinks. */
|
|
38
|
+
sinks: AuditSinks;
|
|
39
|
+
/** The audit database for this invocation's env. Throws when the binding is absent. */
|
|
40
|
+
database: (env: Record<string, unknown>) => AuditDatabase;
|
|
41
|
+
/** A sink's token, read through `@pithy-sh/secrets`. Throws when it is not set. */
|
|
42
|
+
token: (env: Record<string, unknown>, secret: string) => Promise<string>;
|
|
43
|
+
/** The network. Defaults to the global `fetch`. */
|
|
44
|
+
fetch?: (url: string, init: RequestInit) => Promise<Response>;
|
|
45
|
+
/** The clock. */
|
|
46
|
+
now?: () => Date;
|
|
47
|
+
/** The retries the queue's consumer entry declares. Defaults to {@link AUDIT_SHIP_MAX_RETRIES}. */
|
|
48
|
+
maxRetries?: number;
|
|
49
|
+
/** Where a skipped message or a failed delivery is said. */
|
|
50
|
+
log?: Pick<Logger, "warn">;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Why a delivery failed, as one line with nothing from the receiver's body in it. */
|
|
54
|
+
function failureOf(error: unknown): string {
|
|
55
|
+
if (error instanceof Error) return error.name === "TimeoutError" ? "timed out" : error.message;
|
|
56
|
+
return String(error);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Deliver one sink's messages, and settle every one of them. */
|
|
60
|
+
async function deliver(
|
|
61
|
+
options: ShipConsumerOptions,
|
|
62
|
+
env: Record<string, unknown>,
|
|
63
|
+
db: AuditDatabase,
|
|
64
|
+
sinkName: string,
|
|
65
|
+
messages: readonly { message: QueueMessage; eventId: string }[],
|
|
66
|
+
): Promise<void> {
|
|
67
|
+
const sink = options.sinks[sinkName];
|
|
68
|
+
const log = options.log;
|
|
69
|
+
if (sink === undefined) {
|
|
70
|
+
log?.warn("audit shipment dropped", { reason: "sink no longer declared", sink: sinkName, count: messages.length });
|
|
71
|
+
for (const { message } of messages) message.ack();
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const rows = new Map(
|
|
76
|
+
(await readRowsByEventId(db, [...new Set(messages.map((m) => m.eventId))])).map((r) => [r.eventId, r]),
|
|
77
|
+
);
|
|
78
|
+
const deliverable: { message: QueueMessage; eventId: string }[] = [];
|
|
79
|
+
for (const entry of messages) {
|
|
80
|
+
if (rows.has(entry.eventId)) deliverable.push(entry);
|
|
81
|
+
else {
|
|
82
|
+
log?.warn("audit shipment dropped", { reason: "event not in the trail", sink: sinkName, eventId: entry.eventId });
|
|
83
|
+
entry.message.ack();
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
if (deliverable.length === 0) return;
|
|
87
|
+
|
|
88
|
+
// One event per row, however many messages named it: a redelivery beside its original ships it once.
|
|
89
|
+
const unique = [...new Set(deliverable.map((entry) => entry.eventId))];
|
|
90
|
+
const now = options.now ?? (() => new Date());
|
|
91
|
+
const maxRetries = options.maxRetries ?? AUDIT_SHIP_MAX_RETRIES;
|
|
92
|
+
let failure: string | null = null;
|
|
93
|
+
try {
|
|
94
|
+
const events = await Promise.all(
|
|
95
|
+
unique.map((eventId) => shippedEventOf(rows.get(eventId) as NonNullable<ReturnType<typeof rows.get>>, sink)),
|
|
96
|
+
);
|
|
97
|
+
const token = await options.token(env, sink.secret);
|
|
98
|
+
const request = sinkDelivery(sink, token, events);
|
|
99
|
+
const response = await (options.fetch ?? fetch)(request.url, {
|
|
100
|
+
...request.init,
|
|
101
|
+
// Never followed: a redirect would carry the token and the batch to a host nobody declared, or down to plain
|
|
102
|
+
// HTTP, past the https-only check the sink's URL passed at config load. A 3xx is a refusal, retried like one.
|
|
103
|
+
redirect: "manual",
|
|
104
|
+
signal: AbortSignal.timeout(SHIP_TIMEOUT_MS),
|
|
105
|
+
});
|
|
106
|
+
// The body is the receiver's and is never read; canceling it frees the connection.
|
|
107
|
+
await response.body?.cancel().catch(() => {});
|
|
108
|
+
if (!response.ok) failure = `answered ${response.status}`;
|
|
109
|
+
} catch (error) {
|
|
110
|
+
failure = failureOf(error);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (failure === null) {
|
|
114
|
+
for (const { message } of deliverable) message.ack();
|
|
115
|
+
await recordSinkOutcome(db, sinkName, {
|
|
116
|
+
delivered: true,
|
|
117
|
+
at: now(),
|
|
118
|
+
retryingDelta: -deliverable.filter(({ message }) => message.attempts > 1).length,
|
|
119
|
+
deadLettered: 0,
|
|
120
|
+
});
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
let retryingDelta = 0;
|
|
125
|
+
let deadLettered = 0;
|
|
126
|
+
for (const { message } of deliverable) {
|
|
127
|
+
const final = isFinalAttempt(message.attempts, maxRetries);
|
|
128
|
+
if (final) {
|
|
129
|
+
deadLettered += 1;
|
|
130
|
+
if (message.attempts > 1) retryingDelta -= 1;
|
|
131
|
+
} else if (message.attempts === 1) {
|
|
132
|
+
retryingDelta += 1;
|
|
133
|
+
}
|
|
134
|
+
// A final attempt is retried too: that is what sends it to the dead-letter queue.
|
|
135
|
+
message.retry({ delaySeconds: retryDelaySeconds(message.attempts) });
|
|
136
|
+
}
|
|
137
|
+
log?.warn("audit shipment failed", { sink: sinkName, failure, count: deliverable.length, deadLettered });
|
|
138
|
+
await recordSinkOutcome(db, sinkName, { delivered: false, at: now(), retryingDelta, deadLettered });
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** The capability's queue handler: every message of a batch that names audit, by sink. */
|
|
142
|
+
export function auditShipConsumer(options: ShipConsumerOptions): CapabilityQueueHandler {
|
|
143
|
+
return async (messages, env) => {
|
|
144
|
+
const bySink = new Map<string, { message: QueueMessage; eventId: string }[]>();
|
|
145
|
+
for (const message of messages) {
|
|
146
|
+
const parsed = ShipMessage.safeParse(message.body);
|
|
147
|
+
if (!parsed.success) {
|
|
148
|
+
options.log?.warn("audit shipment dropped", { reason: "not a shipment", messageId: message.id });
|
|
149
|
+
message.ack();
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const list = bySink.get(parsed.data.sink) ?? [];
|
|
153
|
+
list.push({ message, eventId: parsed.data.eventId });
|
|
154
|
+
bySink.set(parsed.data.sink, list);
|
|
155
|
+
}
|
|
156
|
+
if (bySink.size === 0) return;
|
|
157
|
+
const db = options.database(env);
|
|
158
|
+
// Each sink settles its own messages; one that throws while recording leaves the others delivered.
|
|
159
|
+
const failures: unknown[] = [];
|
|
160
|
+
for (const [sink, list] of bySink) {
|
|
161
|
+
try {
|
|
162
|
+
await deliver(options, env, db, sink, list);
|
|
163
|
+
} catch (error) {
|
|
164
|
+
failures.push(error);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if (failures[0] !== undefined) throw failures[0];
|
|
168
|
+
};
|
|
169
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
|
|
4
|
+
import { AUDIT_CHAIN_FORMAT, ChainedEventFields, Sha256Hex } from "@pithy-sh/core/src/audit/chain";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
import { type ChainRow, chainFieldsOf } from "../chain/rows";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* **One event as a sink receives it (#824)** — the chain's own canonical form, so a receiver can check it.
|
|
10
|
+
*
|
|
11
|
+
* Every field the event's chain hash covers, its position, the hash before it and its own hash. A receiver
|
|
12
|
+
* de-duplicates a redelivery on `eventId`, which a redelivery carries unchanged, and spots a gap where a tenant's
|
|
13
|
+
* `seq` skips or a `prev` names no hash it holds. With the values and the salt — only for a sink declared
|
|
14
|
+
* `detail: true` — it can recompute the commitments too; without them it holds the commitments, which verify
|
|
15
|
+
* against the hash and disclose nothing. It is the same serialization `pithy audit verify` checks, so what a
|
|
16
|
+
* receiver holds checks against an export of the same range.
|
|
17
|
+
*
|
|
18
|
+
* **Not every skip is a loss.** An event the CLI records takes a position in the chain and is never shipped (see
|
|
19
|
+
* `emitFromCLI`), and a sink with an `actions` filter skips what it does not take.
|
|
20
|
+
*
|
|
21
|
+
* **A row from before the chain**, or written by an older recorder, carries no position and no hash, and its
|
|
22
|
+
* commitments are null: they were never made, and an unsalted hash of an IP address would be reversible.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** The format string of a shipped event. A new shape is a new string. */
|
|
26
|
+
export const AUDIT_SHIP_FORMAT = "pithy-audit-ship/1";
|
|
27
|
+
|
|
28
|
+
export const ShippedAuditEvent = ChainedEventFields.extend({
|
|
29
|
+
format: z.literal(AUDIT_SHIP_FORMAT).describe("The shipped-event format."),
|
|
30
|
+
chain: z.literal(AUDIT_CHAIN_FORMAT).describe("The canonical form `hash` was computed over."),
|
|
31
|
+
seq: z
|
|
32
|
+
.number()
|
|
33
|
+
.int()
|
|
34
|
+
.min(1)
|
|
35
|
+
.nullable()
|
|
36
|
+
.describe("The 1-based position in the tenant's chain, or null for an event outside the chain."),
|
|
37
|
+
prev: Sha256Hex.nullable().describe("The hash of the event before it, or null for a chain's first event."),
|
|
38
|
+
hash: Sha256Hex.nullable().describe("This event's chain hash, or null for an event outside the chain."),
|
|
39
|
+
salt: z
|
|
40
|
+
.string()
|
|
41
|
+
.optional()
|
|
42
|
+
.describe("The per-event salt the commitments were made under. Only for a sink declared `detail: true`."),
|
|
43
|
+
ip: z.string().nullable().optional().describe("The client IP. Only for a sink declared `detail: true`."),
|
|
44
|
+
userAgent: z.string().nullable().optional().describe("The user-agent. Only for a sink declared `detail: true`."),
|
|
45
|
+
metadata: z
|
|
46
|
+
.string()
|
|
47
|
+
.nullable()
|
|
48
|
+
.optional()
|
|
49
|
+
.describe("The metadata's stored JSON text, byte for byte. Only for a sink declared `detail: true`."),
|
|
50
|
+
}).describe("One audit event as a sink receives it: the chain's canonical form, personal values only with detail.");
|
|
51
|
+
export type ShippedAuditEvent = z.infer<typeof ShippedAuditEvent>;
|
|
52
|
+
|
|
53
|
+
/** One stored row as a sink receives it. `detail` decides whether the personal values travel. */
|
|
54
|
+
export async function shippedEventOf(row: ChainRow, options: { detail: boolean }): Promise<ShippedAuditEvent> {
|
|
55
|
+
const link = chainLinkOf(row);
|
|
56
|
+
const fields = await chainFieldsOf(row, link?.salt ?? "");
|
|
57
|
+
return {
|
|
58
|
+
format: AUDIT_SHIP_FORMAT,
|
|
59
|
+
chain: AUDIT_CHAIN_FORMAT,
|
|
60
|
+
...fields,
|
|
61
|
+
...(link ? {} : { ipCommitment: null, userAgentCommitment: null, metadataCommitment: null }),
|
|
62
|
+
seq: link?.seq ?? null,
|
|
63
|
+
prev: link?.prev ?? null,
|
|
64
|
+
hash: link?.hash ?? null,
|
|
65
|
+
...(options.detail
|
|
66
|
+
? {
|
|
67
|
+
...(link ? { salt: link.salt } : {}),
|
|
68
|
+
ip: row.ip,
|
|
69
|
+
userAgent: row.userAgent,
|
|
70
|
+
metadata: row.metadata,
|
|
71
|
+
}
|
|
72
|
+
: {}),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* A row's place in the chain, or null for a row outside it. The type checks are not decoration, the same as the
|
|
78
|
+
* reader's: SQLite reads an unknown double-quoted identifier as a string literal, so a database the app reached
|
|
79
|
+
* before `audit_0005_chain` ran hands back the text `chain_seq` for the position and `chain_salt` for the salt, and
|
|
80
|
+
* the recorder writes there unchained on purpose. A null check would ship those names as a position, a hash and a salt.
|
|
81
|
+
*/
|
|
82
|
+
function chainLinkOf(row: ChainRow): { seq: number; prev: string | null; hash: string; salt: string } | null {
|
|
83
|
+
const { chainSeq, chainPrev, chainHash, chainSalt } = row as Record<keyof ChainRow, unknown>;
|
|
84
|
+
if (typeof chainSeq !== "number" || typeof chainHash !== "string" || typeof chainSalt !== "string") return null;
|
|
85
|
+
return { seq: chainSeq, prev: typeof chainPrev === "string" ? chainPrev : null, hash: chainHash, salt: chainSalt };
|
|
86
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
|
|
4
|
+
import type { PithyHonoEnv } from "@pithy-sh/core/src/capability/capability";
|
|
5
|
+
import { type CapabilityHealth, defineCapabilityHealth } from "@pithy-sh/core/src/controlPlane/discovery/health";
|
|
6
|
+
import type { Context } from "hono";
|
|
7
|
+
import type { AuditSinkStateRow } from "../data/sinkState";
|
|
8
|
+
import type { AuditDatabase } from "../data/tables";
|
|
9
|
+
import { AUDIT_TRAIL_READ_SCOPE } from "../http/guards";
|
|
10
|
+
import type { AuditSinks } from "./sinks";
|
|
11
|
+
import { readSinkStates } from "./state";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* **Shipping's health, on audit's manifest entry (#824)** — whether the trail is reaching its sinks.
|
|
15
|
+
*
|
|
16
|
+
* Three keys, all behind `audit:events:read`, the scope the listing already requires: a backlog, a dead-letter
|
|
17
|
+
* count, and a state. Each is read off `pithy_audit_sinks` by primary key, one row per declared sink, so the bound
|
|
18
|
+
* is the config's sink count and never the trail's size. Declared only when a sink is: a Worker that ships nothing
|
|
19
|
+
* has nothing to report and reports nothing.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/** Events waiting on a retry, across every sink. */
|
|
23
|
+
export const AUDIT_SHIP_BACKLOG = "auditShipBacklog";
|
|
24
|
+
|
|
25
|
+
/** Events given up on, across every sink: their last retry failed and they went to the dead-letter queue. */
|
|
26
|
+
export const AUDIT_SHIP_DEAD_LETTERED = "auditShipDeadLettered";
|
|
27
|
+
|
|
28
|
+
/** How delivery stands, across every sink. */
|
|
29
|
+
export const AUDIT_SHIP_DELIVERY = "auditShipDelivery";
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The values {@link AUDIT_SHIP_DELIVERY} takes. `failing` when any sink's last attempt failed; `delivering` when
|
|
33
|
+
* every sink that has been attempted last took a delivery; `idle` when no sink has been attempted yet.
|
|
34
|
+
*/
|
|
35
|
+
export const SHIP_DELIVERY_STATES = ["delivering", "failing", "idle"] as const;
|
|
36
|
+
export type ShipDeliveryState = (typeof SHIP_DELIVERY_STATES)[number];
|
|
37
|
+
|
|
38
|
+
/** How delivery stands, from the declared sinks' rows. */
|
|
39
|
+
export function shipDelivery(states: readonly AuditSinkStateRow[]): ShipDeliveryState {
|
|
40
|
+
const attempted = states.filter((state) => state.lastDeliveredAt !== null || state.lastFailedAt !== null);
|
|
41
|
+
if (attempted.length === 0) return "idle";
|
|
42
|
+
const failing = attempted.some(
|
|
43
|
+
(state) =>
|
|
44
|
+
state.lastFailedAt !== null && (state.lastDeliveredAt === null || state.lastFailedAt > state.lastDeliveredAt),
|
|
45
|
+
);
|
|
46
|
+
return failing ? "failing" : "delivering";
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Declare shipping's health keys over the declared sinks. */
|
|
50
|
+
export function shipHealth(sinks: AuditSinks, database: (c: Context<PithyHonoEnv>) => AuditDatabase): CapabilityHealth {
|
|
51
|
+
const names = Object.keys(sinks);
|
|
52
|
+
return defineCapabilityHealth({
|
|
53
|
+
keys: [
|
|
54
|
+
{
|
|
55
|
+
key: AUDIT_SHIP_BACKLOG,
|
|
56
|
+
kind: "count",
|
|
57
|
+
states: null,
|
|
58
|
+
scope: AUDIT_TRAIL_READ_SCOPE,
|
|
59
|
+
// A primary-key lookup per declared sink on `pithy_audit_sinks`. The bound is the config, not the trail.
|
|
60
|
+
cost: "indexed",
|
|
61
|
+
nominal: { atMost: 0 },
|
|
62
|
+
summary: "Audit events waiting on a retry to a sink that refused or could not be reached.",
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
key: AUDIT_SHIP_DEAD_LETTERED,
|
|
66
|
+
kind: "count",
|
|
67
|
+
states: null,
|
|
68
|
+
scope: AUDIT_TRAIL_READ_SCOPE,
|
|
69
|
+
cost: "indexed",
|
|
70
|
+
nominal: { atMost: 0 },
|
|
71
|
+
summary: "Audit events a sink never took: their retries ran out and they went to the dead-letter queue.",
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
key: AUDIT_SHIP_DELIVERY,
|
|
75
|
+
kind: "state",
|
|
76
|
+
states: [...SHIP_DELIVERY_STATES],
|
|
77
|
+
scope: AUDIT_TRAIL_READ_SCOPE,
|
|
78
|
+
cost: "indexed",
|
|
79
|
+
nominal: ["delivering", "idle"],
|
|
80
|
+
summary: "Whether every sink took its last delivery, any sink is failing, or nothing has been shipped yet.",
|
|
81
|
+
},
|
|
82
|
+
],
|
|
83
|
+
read: async (c) => {
|
|
84
|
+
const states = await readSinkStates(database(c), names);
|
|
85
|
+
return {
|
|
86
|
+
[AUDIT_SHIP_BACKLOG]: states.reduce((sum, state) => sum + state.retrying, 0),
|
|
87
|
+
[AUDIT_SHIP_DEAD_LETTERED]: states.reduce((sum, state) => sum + state.deadLettered, 0),
|
|
88
|
+
[AUDIT_SHIP_DELIVERY]: shipDelivery(states),
|
|
89
|
+
};
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: FSL-1.1-MIT
|
|
3
|
+
|
|
4
|
+
import type { ShippedAuditEvent } from "./event";
|
|
5
|
+
import { AUDIT_SHIP_FORMAT } from "./event";
|
|
6
|
+
import type { AuditSink, AuditSinkProfile } from "./sinks";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* **One delivery, in the receiver's own documented format (#824).**
|
|
10
|
+
*
|
|
11
|
+
* - **`splunk-hec`** — Splunk's HTTP Event Collector: one JSON object per event, concatenated, with
|
|
12
|
+
* `Authorization: Splunk <token>`. Each carries `time` in epoch seconds, `source` and `sourcetype`, and the event
|
|
13
|
+
* under `event`. https://docs.splunk.com/Documentation/Splunk/latest/Data/FormateventsforHTTPEventCollector
|
|
14
|
+
* - **`datadog-logs`** — the Datadog Logs intake (`/api/v2/logs`): a JSON array, with `DD-API-KEY: <key>`. Each log
|
|
15
|
+
* carries `ddsource`, `service`, `ddtags`, a one-line `message`, and the event under `audit`.
|
|
16
|
+
* https://docs.datadoghq.com/api/latest/logs/#send-logs
|
|
17
|
+
* - **`webhook`** — a JSON document `{ format, events }` with `Authorization: Bearer <token>`, for Elastic, Sumo Logic,
|
|
18
|
+
* or anything else that takes HTTPS. `format` is `pithy-audit-ship/1`.
|
|
19
|
+
*
|
|
20
|
+
* Every profile carries the whole shipped event, so the chain hash and the event id reach every receiver.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** What one delivery sends: where, and the request. */
|
|
24
|
+
export interface SinkDelivery {
|
|
25
|
+
url: string;
|
|
26
|
+
init: { method: "POST"; headers: Record<string, string>; body: string };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Splunk's `source`, Datadog's `ddsource`: what the events are, at a glance, in the receiver's own search. */
|
|
30
|
+
export const SHIP_SOURCE = "pithy";
|
|
31
|
+
|
|
32
|
+
/** Splunk's `sourcetype` for the trail. */
|
|
33
|
+
export const SPLUNK_SOURCETYPE = "pithy:audit";
|
|
34
|
+
|
|
35
|
+
/** A Datadog tag value: its own allowed characters, lowercased. */
|
|
36
|
+
function tag(value: string): string {
|
|
37
|
+
return value.toLowerCase().replace(/[^a-z0-9_./:-]/g, "_");
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function body(profile: AuditSinkProfile, events: readonly ShippedAuditEvent[]): string {
|
|
41
|
+
switch (profile) {
|
|
42
|
+
case "splunk-hec":
|
|
43
|
+
return events
|
|
44
|
+
.map((event) =>
|
|
45
|
+
JSON.stringify({
|
|
46
|
+
time: event.occurredAt / 1000,
|
|
47
|
+
source: SHIP_SOURCE,
|
|
48
|
+
sourcetype: SPLUNK_SOURCETYPE,
|
|
49
|
+
...(event.worker === null ? {} : { host: event.worker }),
|
|
50
|
+
event,
|
|
51
|
+
}),
|
|
52
|
+
)
|
|
53
|
+
.join("\n");
|
|
54
|
+
case "datadog-logs":
|
|
55
|
+
return JSON.stringify(
|
|
56
|
+
events.map((event) => ({
|
|
57
|
+
ddsource: SHIP_SOURCE,
|
|
58
|
+
service: event.project ?? SHIP_SOURCE,
|
|
59
|
+
...(event.worker === null ? {} : { hostname: event.worker }),
|
|
60
|
+
ddtags: [
|
|
61
|
+
...(event.environment === null ? [] : [`env:${tag(event.environment)}`]),
|
|
62
|
+
`action:${tag(event.action)}`,
|
|
63
|
+
`outcome:${tag(event.outcome)}`,
|
|
64
|
+
].join(","),
|
|
65
|
+
message: `${event.action} ${event.outcome}`,
|
|
66
|
+
audit: event,
|
|
67
|
+
})),
|
|
68
|
+
);
|
|
69
|
+
case "webhook":
|
|
70
|
+
return JSON.stringify({ format: AUDIT_SHIP_FORMAT, events });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function authorization(profile: AuditSinkProfile, token: string): Record<string, string> {
|
|
75
|
+
switch (profile) {
|
|
76
|
+
case "splunk-hec":
|
|
77
|
+
return { authorization: `Splunk ${token}` };
|
|
78
|
+
case "datadog-logs":
|
|
79
|
+
return { "dd-api-key": token };
|
|
80
|
+
case "webhook":
|
|
81
|
+
return { authorization: `Bearer ${token}` };
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** The request that delivers these events to this sink. */
|
|
86
|
+
export function sinkDelivery(
|
|
87
|
+
sink: Pick<AuditSink, "profile" | "url">,
|
|
88
|
+
token: string,
|
|
89
|
+
events: readonly ShippedAuditEvent[],
|
|
90
|
+
): SinkDelivery {
|
|
91
|
+
return {
|
|
92
|
+
url: sink.url,
|
|
93
|
+
init: {
|
|
94
|
+
method: "POST",
|
|
95
|
+
headers: { "content-type": "application/json", ...authorization(sink.profile, token) },
|
|
96
|
+
body: body(sink.profile, events),
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|