@coffre/conformance 0.10.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/main.js +93 -26
- package/package.json +4 -4
- package/src/checks/audit.ts +23 -8
- package/src/checks/checkpoints.ts +4 -3
- package/src/checks/readiness.ts +4 -2
- package/src/harness.ts +45 -13
package/dist/main.js
CHANGED
|
@@ -453,7 +453,25 @@ set: (member, access) => call("PATCH /access/:member", { member }, access) },
|
|
|
453
453
|
},
|
|
454
454
|
audit: {
|
|
455
455
|
list: (query = {}) => call("GET /audit", {}, query),
|
|
456
|
-
|
|
456
|
+
/**
|
|
457
|
+
* Whether the log holds, from the newest checkpoint; or, with `from`, a
|
|
458
|
+
* stretch of the walk from the first entry: `start`, then each `next`
|
|
459
|
+
* until it is null (`verifyWhole`).
|
|
460
|
+
*/
|
|
461
|
+
verify: (query = {}) => call("GET /audit/verification", {}, query),
|
|
462
|
+
/**
|
|
463
|
+
* The walk from the first entry to the head, a stretch a request, each
|
|
464
|
+
* as long as the server walks in one: the first stretch that fails, or
|
|
465
|
+
* the last. `onStretch` sees each verdict as it comes.
|
|
466
|
+
*/
|
|
467
|
+
verifyWhole: async (onStretch = () => {}) => {
|
|
468
|
+
for (let from = "start";;) {
|
|
469
|
+
const verdict = await call("GET /audit/verification", {}, { from });
|
|
470
|
+
onStretch(verdict);
|
|
471
|
+
if (!verdict.ok || verdict.next === void 0 || verdict.next === null) return verdict;
|
|
472
|
+
from = verdict.next;
|
|
473
|
+
}
|
|
474
|
+
},
|
|
457
475
|
/** What an escrowed key is checked against: public material, for admins and owners of the whole instance. */
|
|
458
476
|
keys: () => call("GET /audit/keys", {})
|
|
459
477
|
}
|
|
@@ -629,6 +647,16 @@ async function boot(kind, at, options) {
|
|
|
629
647
|
children.push(entry);
|
|
630
648
|
return entry;
|
|
631
649
|
}
|
|
650
|
+
/** Stop one of the daemons on purpose: its exit fails nothing. */
|
|
651
|
+
async function halt(entry) {
|
|
652
|
+
entry.daemon = false;
|
|
653
|
+
if (entry.exited) return;
|
|
654
|
+
const exited = new Promise((done) => entry.child.once("exit", done));
|
|
655
|
+
process.kill(-entry.child.pid, "SIGTERM");
|
|
656
|
+
const killed = setTimeout(() => process.kill(-entry.child.pid, "SIGKILL"), 1e4);
|
|
657
|
+
await exited;
|
|
658
|
+
clearTimeout(killed);
|
|
659
|
+
}
|
|
632
660
|
async function run(command, args, env = {}, input) {
|
|
633
661
|
const entry = start(args[0] ?? command, command, args, env, false, input);
|
|
634
662
|
if (await new Promise((done) => {
|
|
@@ -689,7 +717,7 @@ async function boot(kind, at, options) {
|
|
|
689
717
|
], {}, `${owner}\n`);
|
|
690
718
|
await run(bin("vite"), ["build", "app"]);
|
|
691
719
|
const state = join(scratch, "state");
|
|
692
|
-
start("wrangler", bin("wrangler"), [
|
|
720
|
+
const startWrangler = () => start("wrangler", bin("wrangler"), [
|
|
693
721
|
"dev",
|
|
694
722
|
...[
|
|
695
723
|
"-c",
|
|
@@ -729,6 +757,7 @@ async function boot(kind, at, options) {
|
|
|
729
757
|
CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_VAULT_HYPERDRIVE: vaultRuntime,
|
|
730
758
|
WRANGLER_SEND_METRICS: "false"
|
|
731
759
|
});
|
|
760
|
+
let wrangler = startWrangler();
|
|
732
761
|
return {
|
|
733
762
|
kind,
|
|
734
763
|
origin,
|
|
@@ -739,6 +768,12 @@ async function boot(kind, at, options) {
|
|
|
739
768
|
const response = await fetch(`${origin}/cdn-cgi/local/scheduled?cron=*/5+*+*+*+*`);
|
|
740
769
|
if (!response.ok && !options.allowFailure) throw new Failure(`the scheduled handler answered ${response.status}`, `${await response.text()}\n${logs()}`);
|
|
741
770
|
},
|
|
771
|
+
async restart() {
|
|
772
|
+
await halt(wrangler);
|
|
773
|
+
wrangler = startWrangler();
|
|
774
|
+
await until(`${origin}/livez`, async () => (await fetch(`${origin}/livez`)).ok, 120, alive);
|
|
775
|
+
},
|
|
776
|
+
serving: () => ({ wrangler: wrangler.child.pid }),
|
|
742
777
|
database: () => postgres(owner),
|
|
743
778
|
runtime: () => postgres(runtime),
|
|
744
779
|
vaultRuntime: () => postgres(vaultRuntime),
|
|
@@ -752,14 +787,19 @@ async function boot(kind, at, options) {
|
|
|
752
787
|
const database = join(scratch, "coffre.db");
|
|
753
788
|
const socket = join(scratch, "vault.sock");
|
|
754
789
|
await run(bin("coffre-server"), ["migrate", `file:${database}`]);
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
790
|
+
const startVault = async () => {
|
|
791
|
+
rmSync(socket, { force: true });
|
|
792
|
+
const started = start("vault", process.execPath, ["src/vault.ts"], {
|
|
793
|
+
VAULT_SOCKET: socket,
|
|
794
|
+
DATABASE_URL: `file:${database}`,
|
|
795
|
+
VAULT_KEY_ID: KEYS.VAULT_KEY_ID,
|
|
796
|
+
VAULT_KEY: KEYS.VAULT_KEY,
|
|
797
|
+
...OWNER_VARS
|
|
798
|
+
});
|
|
799
|
+
await until("the vault socket", async () => existsSync(socket), 30, alive);
|
|
800
|
+
return started;
|
|
801
|
+
};
|
|
802
|
+
let vault = await startVault();
|
|
763
803
|
await run(bin("vite"), ["build", "app"]);
|
|
764
804
|
const startServer = () => start("server", bin("srvx"), [
|
|
765
805
|
"--prod",
|
|
@@ -782,13 +822,21 @@ async function boot(kind, at, options) {
|
|
|
782
822
|
owner: OWNER,
|
|
783
823
|
beforeFirstBeat: false,
|
|
784
824
|
async scheduled() {
|
|
785
|
-
|
|
786
|
-
server
|
|
787
|
-
|
|
788
|
-
|
|
825
|
+
await halt(server);
|
|
826
|
+
server = startServer();
|
|
827
|
+
await until("the restarted server", async () => (await fetch(`${origin}/livez`)).ok, 30, alive);
|
|
828
|
+
},
|
|
829
|
+
async restart() {
|
|
830
|
+
await halt(server);
|
|
831
|
+
await halt(vault);
|
|
832
|
+
vault = await startVault();
|
|
789
833
|
server = startServer();
|
|
790
834
|
await until("the restarted server", async () => (await fetch(`${origin}/livez`)).ok, 30, alive);
|
|
791
835
|
},
|
|
836
|
+
serving: () => ({
|
|
837
|
+
vault: vault.child.pid,
|
|
838
|
+
server: server.child.pid
|
|
839
|
+
}),
|
|
792
840
|
database: async () => sqlite(database),
|
|
793
841
|
runtime: null,
|
|
794
842
|
vaultRuntime: null,
|
|
@@ -2386,10 +2434,21 @@ async function revealAudited(deployment, { admin, reader }, canaries, purpose) {
|
|
|
2386
2434
|
});
|
|
2387
2435
|
return `${read.length} values read by the vault for ${purpose}, in one entry under ${operationId}`;
|
|
2388
2436
|
}
|
|
2389
|
-
/**
|
|
2437
|
+
/**
|
|
2438
|
+
* Verification must reach the stored head, not just some intact prefix:
|
|
2439
|
+
* the page's, from the newest checkpoint, and the walk from the first
|
|
2440
|
+
* entry, a stretch a request, which ends where it does. And the audit page
|
|
2441
|
+
* is sent with its verdict, rendered by the server, never a badge to come.
|
|
2442
|
+
*/
|
|
2390
2443
|
async function verification(deployment, { admin }) {
|
|
2391
|
-
const
|
|
2392
|
-
expect(
|
|
2444
|
+
const anchored = await admin.api.audit.verify();
|
|
2445
|
+
expect(anchored.ok && anchored.remaining === 0, "the log does not verify from its newest checkpoint", anchored);
|
|
2446
|
+
const result = await admin.api.audit.verifyWhole();
|
|
2447
|
+
expect(result.ok && result.remaining === 0, "the full log does not verify to its end", result);
|
|
2448
|
+
expect(result.through !== null && anchored.through !== null && result.through >= anchored.through, "the walk from the first entry stopped short", {
|
|
2449
|
+
anchored,
|
|
2450
|
+
result
|
|
2451
|
+
});
|
|
2393
2452
|
await using(deployment.database(), async (sql) => {
|
|
2394
2453
|
const [head] = await sql.query("SELECT next_seq, head_hash FROM audit_chain_head");
|
|
2395
2454
|
const [last] = await sql.query("SELECT seq, hash FROM audit_log ORDER BY seq DESC LIMIT 1");
|
|
@@ -2402,7 +2461,10 @@ async function verification(deployment, { admin }) {
|
|
|
2402
2461
|
});
|
|
2403
2462
|
expect(BigInt(result.entries) === BigInt(head.next_seq), "verification did not count every entry", result);
|
|
2404
2463
|
});
|
|
2405
|
-
|
|
2464
|
+
const page = await admin.browser.fetch("/audit");
|
|
2465
|
+
const html = await page.text();
|
|
2466
|
+
expect(page.status === 200 && html.includes("chain-seal") && !html.includes("Verifying"), "the audit page was not sent with its verification result", { status: page.status });
|
|
2467
|
+
return `${result.entries} entries verified through the head at ${result.through}, and the audit page sent with its result`;
|
|
2406
2468
|
}
|
|
2407
2469
|
/**
|
|
2408
2470
|
* Each heartbeat has the vault sign the log up to its last entry, in an
|
|
@@ -2551,9 +2613,13 @@ async function forgedVaultEntry(deployment, { admin }) {
|
|
|
2551
2613
|
});
|
|
2552
2614
|
return "a publicly chained vault entry with a made-up MAC was refused at its sequence";
|
|
2553
2615
|
}
|
|
2554
|
-
/**
|
|
2616
|
+
/**
|
|
2617
|
+
* Rewrite either author's first entry in place and require a fault at that
|
|
2618
|
+
* exact sequence: from the walk from the first entry, since the page's
|
|
2619
|
+
* check goes on from the newest checkpoint and reads nothing before it.
|
|
2620
|
+
*/
|
|
2555
2621
|
async function rewrittenEntry(deployment, { admin }, author) {
|
|
2556
|
-
expect((await admin.api.audit.
|
|
2622
|
+
expect((await admin.api.audit.verifyWhole()).ok, "the log does not verify before the rewrite");
|
|
2557
2623
|
let seq = 0;
|
|
2558
2624
|
await using(deployment.database(), (sql) => appendOnlyLifted(sql, async () => {
|
|
2559
2625
|
const [row] = await query(sql, "SELECT seq, actor FROM audit_log WHERE author = $1 ORDER BY seq LIMIT 1", [author]);
|
|
@@ -2561,13 +2627,13 @@ async function rewrittenEntry(deployment, { admin }, author) {
|
|
|
2561
2627
|
seq = Number(row.seq);
|
|
2562
2628
|
await query(sql, "UPDATE audit_log SET actor = $1 WHERE seq = $2", ["user:nobody@conformance.example", row.seq]);
|
|
2563
2629
|
try {
|
|
2564
|
-
const result = await admin.api.audit.
|
|
2630
|
+
const result = await admin.api.audit.verifyWhole();
|
|
2565
2631
|
expect(!result.ok && result.failedAtSeq === Number(row.seq), "a rewritten entry was not caught at its sequence", result);
|
|
2566
2632
|
} finally {
|
|
2567
2633
|
await query(sql, "UPDATE audit_log SET actor = $1 WHERE seq = $2", [row.actor, row.seq]);
|
|
2568
2634
|
}
|
|
2569
2635
|
}));
|
|
2570
|
-
expect((await admin.api.audit.
|
|
2636
|
+
expect((await admin.api.audit.verifyWhole()).ok, "the restored entry did not verify");
|
|
2571
2637
|
return `entry ${seq}, the ${author}'s, rewritten: caught there, then put back`;
|
|
2572
2638
|
}
|
|
2573
2639
|
/** Remove an unreferenced entry, then put its exact bytes back. */
|
|
@@ -2629,7 +2695,7 @@ async function everyAuditEntry(api) {
|
|
|
2629
2695
|
//#region src/checks/checkpoints.ts
|
|
2630
2696
|
/** Leave one old signature invalid, reseal the chain, and make later checkpoints valid. */
|
|
2631
2697
|
async function earlierCheckpoint(deployment, { admin }) {
|
|
2632
|
-
expect((await admin.api.audit.
|
|
2698
|
+
expect((await admin.api.audit.verifyWhole()).ok, "the log does not verify before checkpoint tampering");
|
|
2633
2699
|
await using(deployment.database(), (sql) => appendOnlyLifted(sql, async () => {
|
|
2634
2700
|
const rows = await sql.query("SELECT * FROM audit_log ORDER BY seq");
|
|
2635
2701
|
const signed = rows.filter((row) => row.author === "vault" && row.action === "audit.checkpoint" && row.decision === "allow");
|
|
@@ -2673,7 +2739,7 @@ async function earlierCheckpoint(deployment, { admin }) {
|
|
|
2673
2739
|
head_hash: previous
|
|
2674
2740
|
}, "only_row");
|
|
2675
2741
|
});
|
|
2676
|
-
const result = await admin.api.audit.
|
|
2742
|
+
const result = await admin.api.audit.verifyWhole();
|
|
2677
2743
|
expect(!result.ok && result.author === "vault" && result.failedAtSeq === Number(signed[0].seq) && result.reason === "a checkpoint the vault did not sign", "a valid later checkpoint hid an invalid earlier one", result);
|
|
2678
2744
|
} finally {
|
|
2679
2745
|
await transaction(sql, async () => {
|
|
@@ -2685,7 +2751,7 @@ async function earlierCheckpoint(deployment, { admin }) {
|
|
|
2685
2751
|
seed.fill(0);
|
|
2686
2752
|
}
|
|
2687
2753
|
}));
|
|
2688
|
-
expect((await admin.api.audit.
|
|
2754
|
+
expect((await admin.api.audit.verifyWhole()).ok, "restoring the signed prefixes did not restore verification");
|
|
2689
2755
|
return "an invalid earlier checkpoint was caught despite valid MACs and a valid later checkpoint";
|
|
2690
2756
|
}
|
|
2691
2757
|
//#endregion
|
|
@@ -3197,12 +3263,13 @@ async function middleCut(deployment, { admin }) {
|
|
|
3197
3263
|
const [heartbeat] = await query(sql, `SELECT h.seq FROM audit_log h JOIN audit_log c ON c.seq = h.seq + 1
|
|
3198
3264
|
WHERE h.action = 'audit.heartbeat' AND h.seq > 0 AND c.author = 'vault' AND c.action = 'audit.checkpoint'
|
|
3199
3265
|
AND c.decision = 'allow' AND c.seq < $1 AND NOT EXISTS
|
|
3200
|
-
(SELECT 1 FROM audit_log linked WHERE linked.related_seq IN (h.seq, c.seq)) ORDER BY h.seq LIMIT 1`, [JSON.parse(latest.metadata).seq]);
|
|
3266
|
+
(SELECT 1 FROM audit_log linked WHERE linked.related_seq IN (h.seq, c.seq) AND linked.seq <> c.seq) ORDER BY h.seq LIMIT 1`, [JSON.parse(latest.metadata).seq]);
|
|
3201
3267
|
expect(heartbeat !== void 0, "no older heartbeat and checkpoint to cut");
|
|
3202
3268
|
seq = Number(heartbeat.seq);
|
|
3203
3269
|
const kept = await query(sql, "SELECT * FROM audit_log WHERE seq >= $1 AND seq <= $2 ORDER BY seq", [seq, seq + 1]);
|
|
3204
3270
|
expect(kept.length === 2, "the middle cut needs two adjacent entries");
|
|
3205
3271
|
await appendOnlyLifted(sql, async () => {
|
|
3272
|
+
await query(sql, "UPDATE audit_log SET related_seq = NULL WHERE seq = $1", [seq + 1]);
|
|
3206
3273
|
await query(sql, "DELETE FROM audit_log WHERE seq >= $1 AND seq <= $2", [seq, seq + 1]);
|
|
3207
3274
|
try {
|
|
3208
3275
|
await deployment.scheduled({ allowFailure: true });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@coffre/conformance",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.1",
|
|
4
4
|
"description": "Checks a coffre deployment from outside: what it must never do, whatever code it runs.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -31,8 +31,8 @@
|
|
|
31
31
|
"dependencies": {
|
|
32
32
|
"jose": "6.2.3",
|
|
33
33
|
"pg": "8.16.3",
|
|
34
|
-
"@coffre/cli": "0.10.
|
|
35
|
-
"@coffre/db": "0.10.
|
|
34
|
+
"@coffre/cli": "0.10.1",
|
|
35
|
+
"@coffre/db": "0.10.1"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
38
|
"@types/node": "26.1.1",
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"tsdown": "0.23.0",
|
|
42
42
|
"typescript": "5.9.3",
|
|
43
43
|
"vite": "8.1.5",
|
|
44
|
-
"@coffre/client": "0.10.
|
|
44
|
+
"@coffre/client": "0.10.1"
|
|
45
45
|
},
|
|
46
46
|
"scripts": {
|
|
47
47
|
"build": "tsdown",
|
package/src/checks/audit.ts
CHANGED
|
@@ -54,10 +54,18 @@ export async function revealAudited(deployment: Deployment, { admin, reader }: P
|
|
|
54
54
|
return `${read.length} values read by the vault for ${purpose}, in one entry under ${operationId}`;
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* Verification must reach the stored head, not just some intact prefix:
|
|
59
|
+
* the page's, from the newest checkpoint, and the walk from the first
|
|
60
|
+
* entry, a stretch a request, which ends where it does. And the audit page
|
|
61
|
+
* is sent with its verdict, rendered by the server, never a badge to come.
|
|
62
|
+
*/
|
|
58
63
|
export async function verification(deployment: Deployment, { admin }: People): Promise<string> {
|
|
59
|
-
const
|
|
60
|
-
expect(
|
|
64
|
+
const anchored = await admin.api.audit.verify();
|
|
65
|
+
expect(anchored.ok && anchored.remaining === 0, 'the log does not verify from its newest checkpoint', anchored);
|
|
66
|
+
const result = await admin.api.audit.verifyWhole();
|
|
67
|
+
expect(result.ok && result.remaining === 0, 'the full log does not verify to its end', result);
|
|
68
|
+
expect(result.through !== null && anchored.through !== null && result.through >= anchored.through, 'the walk from the first entry stopped short', { anchored, result });
|
|
61
69
|
await using(deployment.database(), async (sql) => {
|
|
62
70
|
const [head] = await sql.query<{ next_seq: string | number; head_hash: Uint8Array }>('SELECT next_seq, head_hash FROM audit_chain_head');
|
|
63
71
|
const [last] = await sql.query<{ seq: string | number; hash: Uint8Array }>('SELECT seq, hash FROM audit_log ORDER BY seq DESC LIMIT 1');
|
|
@@ -67,7 +75,10 @@ export async function verification(deployment: Deployment, { admin }: People): P
|
|
|
67
75
|
expect(result.through !== null && BigInt(result.through) === BigInt(last.seq), 'verification stopped before the head', { result, head: head.next_seq });
|
|
68
76
|
expect(BigInt(result.entries) === BigInt(head.next_seq), 'verification did not count every entry', result);
|
|
69
77
|
});
|
|
70
|
-
|
|
78
|
+
const page = await admin.browser.fetch('/audit');
|
|
79
|
+
const html = await page.text();
|
|
80
|
+
expect(page.status === 200 && html.includes('chain-seal') && !html.includes('Verifying'), 'the audit page was not sent with its verification result', { status: page.status });
|
|
81
|
+
return `${result.entries} entries verified through the head at ${result.through}, and the audit page sent with its result`;
|
|
71
82
|
}
|
|
72
83
|
|
|
73
84
|
/**
|
|
@@ -203,9 +214,13 @@ export async function forgedVaultEntry(deployment: Deployment, { admin }: People
|
|
|
203
214
|
return 'a publicly chained vault entry with a made-up MAC was refused at its sequence';
|
|
204
215
|
}
|
|
205
216
|
|
|
206
|
-
/**
|
|
217
|
+
/**
|
|
218
|
+
* Rewrite either author's first entry in place and require a fault at that
|
|
219
|
+
* exact sequence: from the walk from the first entry, since the page's
|
|
220
|
+
* check goes on from the newest checkpoint and reads nothing before it.
|
|
221
|
+
*/
|
|
207
222
|
export async function rewrittenEntry(deployment: Deployment, { admin }: People, author: 'app' | 'vault'): Promise<string> {
|
|
208
|
-
expect((await admin.api.audit.
|
|
223
|
+
expect((await admin.api.audit.verifyWhole()).ok, 'the log does not verify before the rewrite');
|
|
209
224
|
let seq = 0;
|
|
210
225
|
await using(deployment.database(), (sql) => appendOnlyLifted(sql, async () => {
|
|
211
226
|
const [row] = await query<{ seq: string | number; actor: string }>(sql,
|
|
@@ -214,13 +229,13 @@ export async function rewrittenEntry(deployment: Deployment, { admin }: People,
|
|
|
214
229
|
seq = Number(row.seq);
|
|
215
230
|
await query(sql, 'UPDATE audit_log SET actor = $1 WHERE seq = $2', ['user:nobody@conformance.example', row.seq]);
|
|
216
231
|
try {
|
|
217
|
-
const result = await admin.api.audit.
|
|
232
|
+
const result = await admin.api.audit.verifyWhole();
|
|
218
233
|
expect(!result.ok && result.failedAtSeq === Number(row.seq), 'a rewritten entry was not caught at its sequence', result);
|
|
219
234
|
} finally {
|
|
220
235
|
await query(sql, 'UPDATE audit_log SET actor = $1 WHERE seq = $2', [row.actor, row.seq]);
|
|
221
236
|
}
|
|
222
237
|
}));
|
|
223
|
-
expect((await admin.api.audit.
|
|
238
|
+
expect((await admin.api.audit.verifyWhole()).ok, 'the restored entry did not verify');
|
|
224
239
|
return `entry ${seq}, the ${author}'s, rewritten: caught there, then put back`;
|
|
225
240
|
}
|
|
226
241
|
|
|
@@ -11,7 +11,7 @@ import { appendOnlyLifted, entryFields, restoreRow, transaction } from './storag
|
|
|
11
11
|
|
|
12
12
|
/** Leave one old signature invalid, reseal the chain, and make later checkpoints valid. */
|
|
13
13
|
export async function earlierCheckpoint(deployment: Deployment, { admin }: People): Promise<string> {
|
|
14
|
-
expect((await admin.api.audit.
|
|
14
|
+
expect((await admin.api.audit.verifyWhole()).ok, 'the log does not verify before checkpoint tampering');
|
|
15
15
|
await using(deployment.database(), (sql) => appendOnlyLifted(sql, async () => {
|
|
16
16
|
const rows = await sql.query('SELECT * FROM audit_log ORDER BY seq');
|
|
17
17
|
const signed = rows.filter((row) => row.author === 'vault' && row.action === 'audit.checkpoint' && row.decision === 'allow');
|
|
@@ -49,7 +49,8 @@ export async function earlierCheckpoint(deployment: Deployment, { admin }: Peopl
|
|
|
49
49
|
}
|
|
50
50
|
await restoreRow(sql, 'audit_chain_head', { ...head, head_hash: previous }, 'only_row');
|
|
51
51
|
});
|
|
52
|
-
|
|
52
|
+
// The page's check goes on from the newest checkpoint, valid: the walk from the first entry reads them all.
|
|
53
|
+
const result = await admin.api.audit.verifyWhole();
|
|
53
54
|
expect(!result.ok && result.author === 'vault' && result.failedAtSeq === Number(signed[0].seq)
|
|
54
55
|
&& result.reason === 'a checkpoint the vault did not sign', 'a valid later checkpoint hid an invalid earlier one', result);
|
|
55
56
|
} finally {
|
|
@@ -60,6 +61,6 @@ export async function earlierCheckpoint(deployment: Deployment, { admin }: Peopl
|
|
|
60
61
|
keys.app.fill(0); keys.vault.fill(0); seed.fill(0);
|
|
61
62
|
}
|
|
62
63
|
}));
|
|
63
|
-
expect((await admin.api.audit.
|
|
64
|
+
expect((await admin.api.audit.verifyWhole()).ok, 'restoring the signed prefixes did not restore verification');
|
|
64
65
|
return 'an invalid earlier checkpoint was caught despite valid MACs and a valid later checkpoint';
|
|
65
66
|
}
|
package/src/checks/readiness.ts
CHANGED
|
@@ -55,16 +55,18 @@ export async function middleCut(deployment: Deployment, { admin }: People): Prom
|
|
|
55
55
|
);
|
|
56
56
|
expect(latest !== undefined, 'no signed prefix to cut from');
|
|
57
57
|
const signed = JSON.parse(latest.metadata) as { seq: number };
|
|
58
|
-
// An old heartbeat and its checkpoint carry no access change or linked write
|
|
58
|
+
// An old heartbeat and its checkpoint carry no access change or linked write: nothing names them but the
|
|
59
|
+
// checkpoint, which names the heartbeat as the last entry it signs.
|
|
59
60
|
const [heartbeat] = await query<{ seq: string | number }>(sql, `SELECT h.seq FROM audit_log h JOIN audit_log c ON c.seq = h.seq + 1
|
|
60
61
|
WHERE h.action = 'audit.heartbeat' AND h.seq > 0 AND c.author = 'vault' AND c.action = 'audit.checkpoint'
|
|
61
62
|
AND c.decision = 'allow' AND c.seq < $1 AND NOT EXISTS
|
|
62
|
-
(SELECT 1 FROM audit_log linked WHERE linked.related_seq IN (h.seq, c.seq)) ORDER BY h.seq LIMIT 1`, [signed.seq]);
|
|
63
|
+
(SELECT 1 FROM audit_log linked WHERE linked.related_seq IN (h.seq, c.seq) AND linked.seq <> c.seq) ORDER BY h.seq LIMIT 1`, [signed.seq]);
|
|
63
64
|
expect(heartbeat !== undefined, 'no older heartbeat and checkpoint to cut');
|
|
64
65
|
seq = Number(heartbeat.seq);
|
|
65
66
|
const kept = await query(sql, 'SELECT * FROM audit_log WHERE seq >= $1 AND seq <= $2 ORDER BY seq', [seq, seq + 1]);
|
|
66
67
|
expect(kept.length === 2, 'the middle cut needs two adjacent entries');
|
|
67
68
|
await appendOnlyLifted(sql, async () => {
|
|
69
|
+
await query(sql, 'UPDATE audit_log SET related_seq = NULL WHERE seq = $1', [seq + 1]);
|
|
68
70
|
await query(sql, 'DELETE FROM audit_log WHERE seq >= $1 AND seq <= $2', [seq, seq + 1]);
|
|
69
71
|
try {
|
|
70
72
|
// Do not call full verification first: the scheduled job must find the hole itself.
|
package/src/harness.ts
CHANGED
|
@@ -46,6 +46,10 @@ export type Deployment = {
|
|
|
46
46
|
beforeFirstBeat: boolean;
|
|
47
47
|
/** What the Cron trigger does every five minutes. */
|
|
48
48
|
scheduled(options?: { allowFailure?: boolean }): Promise<void>;
|
|
49
|
+
/** Stop what serves the deployment and start it again on the same data: new isolates on Workers, new processes on Node. */
|
|
50
|
+
restart(): Promise<void>;
|
|
51
|
+
/** The process group of each process serving it, by name, for whoever measures them: `wrangler` on Workers; `vault` and `server` on Node. */
|
|
52
|
+
serving(): Record<string, number>;
|
|
49
53
|
/** The app's database, as its owner. */
|
|
50
54
|
database(): Promise<Sql>;
|
|
51
55
|
/** The app's database as the login the app runs as, on an engine that has logins. */
|
|
@@ -143,6 +147,17 @@ export async function boot(kind: Kind, at: string, options: HarnessOptions): Pro
|
|
|
143
147
|
return entry;
|
|
144
148
|
}
|
|
145
149
|
|
|
150
|
+
/** Stop one of the daemons on purpose: its exit fails nothing. */
|
|
151
|
+
async function halt(entry: Child): Promise<void> {
|
|
152
|
+
entry.daemon = false;
|
|
153
|
+
if (entry.exited) return;
|
|
154
|
+
const exited = new Promise((done) => entry.child.once('exit', done));
|
|
155
|
+
process.kill(-entry.child.pid!, 'SIGTERM');
|
|
156
|
+
const killed = setTimeout(() => process.kill(-entry.child.pid!, 'SIGKILL'), 10_000);
|
|
157
|
+
await exited;
|
|
158
|
+
clearTimeout(killed);
|
|
159
|
+
}
|
|
160
|
+
|
|
146
161
|
async function run(command: string, args: string[], env: Record<string, string> = {}, input?: string): Promise<void> {
|
|
147
162
|
const entry = start(args[0] ?? command, command, args, env, false, input);
|
|
148
163
|
const code = await new Promise((done) => {
|
|
@@ -216,7 +231,7 @@ export async function boot(kind: Kind, at: string, options: HarnessOptions): Pro
|
|
|
216
231
|
// The app as it deploys: its Start app, built by its own Vite, which wrangler runs unbundled.
|
|
217
232
|
await run(bin('vite'), ['build', 'app']);
|
|
218
233
|
const state = join(scratch, 'state');
|
|
219
|
-
start('wrangler', bin('wrangler'), [
|
|
234
|
+
const startWrangler = () => start('wrangler', bin('wrangler'), [
|
|
220
235
|
'dev',
|
|
221
236
|
...['-c', drainingConfig(join(dir, 'app/dist/server')), '-c', 'vault/wrangler.jsonc'],
|
|
222
237
|
...['--ip', '127.0.0.1', '--port', String(port), '--inspector-port', String(port + 2)],
|
|
@@ -239,6 +254,7 @@ export async function boot(kind: Kind, at: string, options: HarnessOptions): Pro
|
|
|
239
254
|
CLOUDFLARE_HYPERDRIVE_LOCAL_CONNECTION_STRING_VAULT_HYPERDRIVE: vaultRuntime,
|
|
240
255
|
WRANGLER_SEND_METRICS: 'false',
|
|
241
256
|
});
|
|
257
|
+
let wrangler = startWrangler();
|
|
242
258
|
return {
|
|
243
259
|
kind,
|
|
244
260
|
origin,
|
|
@@ -249,6 +265,12 @@ export async function boot(kind: Kind, at: string, options: HarnessOptions): Pro
|
|
|
249
265
|
const response = await fetch(`${origin}/cdn-cgi/local/scheduled?cron=*/5+*+*+*+*`);
|
|
250
266
|
if (!response.ok && !options.allowFailure) throw new Failure(`the scheduled handler answered ${response.status}`, `${await response.text()}\n${logs()}`);
|
|
251
267
|
},
|
|
268
|
+
async restart() {
|
|
269
|
+
await halt(wrangler);
|
|
270
|
+
wrangler = startWrangler();
|
|
271
|
+
await until(`${origin}/livez`, async () => (await fetch(`${origin}/livez`)).ok, 120, alive);
|
|
272
|
+
},
|
|
273
|
+
serving: () => ({ wrangler: wrangler.child.pid! }),
|
|
252
274
|
database: () => postgres(owner),
|
|
253
275
|
runtime: () => postgres(runtime),
|
|
254
276
|
vaultRuntime: () => postgres(vaultRuntime),
|
|
@@ -263,14 +285,19 @@ export async function boot(kind: Kind, at: string, options: HarnessOptions): Pro
|
|
|
263
285
|
const database = join(scratch, 'coffre.db');
|
|
264
286
|
const socket = join(scratch, 'vault.sock');
|
|
265
287
|
await run(bin('coffre-server'), ['migrate', `file:${database}`]);
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
288
|
+
const startVault = async () => {
|
|
289
|
+
rmSync(socket, { force: true });
|
|
290
|
+
const started = start('vault', process.execPath, ['src/vault.ts'], {
|
|
291
|
+
VAULT_SOCKET: socket,
|
|
292
|
+
DATABASE_URL: `file:${database}`,
|
|
293
|
+
VAULT_KEY_ID: KEYS.VAULT_KEY_ID,
|
|
294
|
+
VAULT_KEY: KEYS.VAULT_KEY,
|
|
295
|
+
...OWNER_VARS,
|
|
296
|
+
});
|
|
297
|
+
await until('the vault socket', async () => existsSync(socket), 30, alive);
|
|
298
|
+
return started;
|
|
299
|
+
};
|
|
300
|
+
let vault = await startVault();
|
|
274
301
|
// The pages as they run: the deployment's Start app, built by its own Vite.
|
|
275
302
|
await run(bin('vite'), ['build', 'app']);
|
|
276
303
|
// As `pnpm start` runs it: the app's build, under srvx.
|
|
@@ -294,13 +321,18 @@ export async function boot(kind: Kind, at: string, options: HarnessOptions): Pro
|
|
|
294
321
|
// `serve` runs the job as it starts, and every five minutes after:
|
|
295
322
|
// so restart the server.
|
|
296
323
|
async scheduled() {
|
|
297
|
-
|
|
298
|
-
server
|
|
299
|
-
|
|
300
|
-
|
|
324
|
+
await halt(server);
|
|
325
|
+
server = startServer();
|
|
326
|
+
await until('the restarted server', async () => (await fetch(`${origin}/livez`)).ok, 30, alive);
|
|
327
|
+
},
|
|
328
|
+
async restart() {
|
|
329
|
+
await halt(server);
|
|
330
|
+
await halt(vault);
|
|
331
|
+
vault = await startVault();
|
|
301
332
|
server = startServer();
|
|
302
333
|
await until('the restarted server', async () => (await fetch(`${origin}/livez`)).ok, 30, alive);
|
|
303
334
|
},
|
|
335
|
+
serving: () => ({ vault: vault.child.pid!, server: server.child.pid! }),
|
|
304
336
|
database: async () => sqlite(database),
|
|
305
337
|
runtime: null,
|
|
306
338
|
vaultRuntime: null,
|