@checkstack/script-packages-backend 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +273 -0
  2. package/drizzle/0000_flashy_squadron_supreme.sql +63 -0
  3. package/drizzle/0001_flawless_drax.sql +15 -0
  4. package/drizzle/meta/0000_snapshot.json +395 -0
  5. package/drizzle/meta/0001_snapshot.json +491 -0
  6. package/drizzle/meta/_journal.json +20 -0
  7. package/drizzle.config.ts +7 -0
  8. package/package.json +32 -0
  9. package/src/atomic-symlink.test.ts +47 -0
  10. package/src/atomic-symlink.ts +66 -0
  11. package/src/blob-gc-runner.test.ts +120 -0
  12. package/src/blob-gc-runner.ts +139 -0
  13. package/src/blob-gc.test.ts +182 -0
  14. package/src/blob-gc.ts +161 -0
  15. package/src/blob-hash.test.ts +70 -0
  16. package/src/blob-hash.ts +56 -0
  17. package/src/blob-store-registry.test.ts +78 -0
  18. package/src/blob-store-registry.ts +75 -0
  19. package/src/blob-store.ts +51 -0
  20. package/src/cache-archive.test.ts +164 -0
  21. package/src/cache-archive.ts +192 -0
  22. package/src/cache-layout.ts +64 -0
  23. package/src/data-dir.test.ts +41 -0
  24. package/src/data-dir.ts +42 -0
  25. package/src/e2e-install-reconcile.test.ts +121 -0
  26. package/src/hooks.ts +20 -0
  27. package/src/index.ts +594 -0
  28. package/src/install-controller.test.ts +257 -0
  29. package/src/install-controller.ts +144 -0
  30. package/src/install-service.test.ts +104 -0
  31. package/src/install-service.ts +116 -0
  32. package/src/install-state-store.ts +131 -0
  33. package/src/lockfile.test.ts +60 -0
  34. package/src/lockfile.ts +0 -0
  35. package/src/npmrc.test.ts +48 -0
  36. package/src/npmrc.ts +42 -0
  37. package/src/package-types.test.ts +293 -0
  38. package/src/package-types.ts +408 -0
  39. package/src/parse-bun-lock.test.ts +62 -0
  40. package/src/parse-bun-lock.ts +59 -0
  41. package/src/reconcile-diff.test.ts +41 -0
  42. package/src/reconcile-diff.ts +26 -0
  43. package/src/reconcile-fs.ts +199 -0
  44. package/src/reconciler.test.ts +289 -0
  45. package/src/reconciler.ts +81 -0
  46. package/src/registry-client.test.ts +314 -0
  47. package/src/registry-client.ts +0 -0
  48. package/src/registry-request-config.ts +63 -0
  49. package/src/registry-token.test.ts +124 -0
  50. package/src/registry-token.ts +104 -0
  51. package/src/resolution-root.test.ts +82 -0
  52. package/src/resolution-root.ts +127 -0
  53. package/src/resolver.test.ts +133 -0
  54. package/src/resolver.ts +132 -0
  55. package/src/router.ts +273 -0
  56. package/src/schema.ts +166 -0
  57. package/src/size-cap.test.ts +32 -0
  58. package/src/size-cap.ts +40 -0
  59. package/src/storage-migration.test.ts +318 -0
  60. package/src/storage-migration.ts +213 -0
  61. package/src/stores.ts +533 -0
  62. package/src/tree-gc.test.ts +184 -0
  63. package/src/tree-gc.ts +160 -0
  64. package/src/tree-retirement.ts +81 -0
  65. package/src/type-acquisition-route.ts +178 -0
  66. package/tsconfig.json +23 -0
@@ -0,0 +1,120 @@
1
+ import { describe, expect, test } from "bun:test";
2
+ import type { ManifestEntry } from "@checkstack/script-packages-common";
3
+ import { createBlobGcTrigger, type BlobGcRunnerDeps } from "./blob-gc-runner";
4
+ import type { InstallerLock } from "./install-state-store";
5
+ import type { BlobStore } from "./blob-store";
6
+ import { createBlobStoreRegistry } from "./blob-store-registry";
7
+ import type { GcBlob } from "./blob-gc";
8
+
9
+ function entry(integrity: string): ManifestEntry {
10
+ return { name: integrity, version: "1.0.0", integrity };
11
+ }
12
+
13
+ function fakeStore(id: string, deletes: string[]): BlobStore {
14
+ return {
15
+ id,
16
+ put: async () => {},
17
+ get: async () => undefined,
18
+ has: async () => true,
19
+ delete: async ({ integrity }) => void deletes.push(`${id}:${integrity}`),
20
+ list: async () => [],
21
+ };
22
+ }
23
+
24
+ function lockThatGrants(calls: string[]): InstallerLock {
25
+ return {
26
+ tryInstallerLock: async () => ({
27
+ release: async () => void calls.push("released"),
28
+ }),
29
+ };
30
+ }
31
+
32
+ const OLD = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000);
33
+
34
+ function makeDeps(over: Partial<BlobGcRunnerDeps> = {}): {
35
+ deps: BlobGcRunnerDeps;
36
+ storeDeletes: string[];
37
+ lockCalls: string[];
38
+ pruned: number[];
39
+ removed: string[];
40
+ } {
41
+ const storeDeletes: string[] = [];
42
+ const lockCalls: string[] = [];
43
+ const pruned: number[] = [];
44
+ const removed: string[] = [];
45
+ const registry = createBlobStoreRegistry();
46
+ registry.register(fakeStore("postgres", storeDeletes));
47
+ registry.register(fakeStore("s3", storeDeletes));
48
+
49
+ const deps: BlobGcRunnerDeps = {
50
+ installerLock: lockThatGrants(lockCalls),
51
+ blobStores: registry,
52
+ loadCurrent: async () => ({ lockfileHash: "CUR", manifest: [entry("a")] }),
53
+ recentHistory: async () => [[entry("a")]],
54
+ pruneHistory: async (keep) => void pruned.push(keep),
55
+ listBlobs: async (): Promise<GcBlob[]> => [
56
+ { integrity: "a", backend: "postgres", sizeBytes: 10, createdAt: OLD },
57
+ {
58
+ integrity: "orphan",
59
+ backend: "s3",
60
+ sizeBytes: 99,
61
+ createdAt: OLD,
62
+ },
63
+ ],
64
+ removeBlobRow: async (i) => void removed.push(i),
65
+ isBusy: async () => false,
66
+ recordRun: async () => {},
67
+ ...over,
68
+ };
69
+ return { deps, storeDeletes, lockCalls, pruned, removed };
70
+ }
71
+
72
+ describe("createBlobGcTrigger", () => {
73
+ test("deletes orphan bytes via its recorded backend, drops the index row, releases the lock", async () => {
74
+ const { deps, storeDeletes, lockCalls, removed } = makeDeps();
75
+ const out = await createBlobGcTrigger(deps)();
76
+ expect(out.ran).toBe(true);
77
+ expect(out.deleted).toBe(1);
78
+ expect(out.bytesReclaimed).toBe(99);
79
+ // routed to s3 (the orphan's backend), referenced "a" untouched.
80
+ expect(storeDeletes).toEqual(["s3:orphan"]);
81
+ expect(removed).toEqual(["orphan"]);
82
+ expect(lockCalls).toContain("released");
83
+ });
84
+
85
+ test("refuses + releases nothing when the installer lock is held", async () => {
86
+ const { deps, storeDeletes } = makeDeps({
87
+ installerLock: { tryInstallerLock: async () => null },
88
+ });
89
+ const out = await createBlobGcTrigger(deps)();
90
+ expect(out.ran).toBe(false);
91
+ expect(out.reason).toMatch(/installer lock/i);
92
+ expect(storeDeletes).toHaveLength(0);
93
+ });
94
+
95
+ test("prunes lockfile history to the retained window and releases the lock even on failure", async () => {
96
+ const { deps, pruned, lockCalls } = makeDeps({
97
+ listBlobs: async () => {
98
+ throw new Error("db down");
99
+ },
100
+ retainPrevious: 2,
101
+ });
102
+ const out = await createBlobGcTrigger(deps)();
103
+ expect(out.ran).toBe(false);
104
+ expect(out.reason).toBe("db down");
105
+ // pruneHistory(keep = retainPrevious + 1) still ran; lock released.
106
+ expect(pruned).toEqual([3]);
107
+ expect(lockCalls).toContain("released");
108
+ });
109
+
110
+ test("unions the live install-state manifest into the retained set", async () => {
111
+ // History is empty, but the current manifest references "a" → keep it.
112
+ const { deps, storeDeletes } = makeDeps({
113
+ recentHistory: async () => [],
114
+ loadCurrent: async () => ({ lockfileHash: "CUR", manifest: [entry("a")] }),
115
+ });
116
+ const out = await createBlobGcTrigger(deps)();
117
+ expect(storeDeletes).toEqual(["s3:orphan"]); // "a" retained via current
118
+ expect(out.deleted).toBe(1);
119
+ });
120
+ });
@@ -0,0 +1,139 @@
1
+ import type {
2
+ BlobGcSummary,
3
+ ManifestEntry,
4
+ } from "@checkstack/script-packages-common";
5
+ import {
6
+ DEFAULT_BLOB_GC_GRACE_MS,
7
+ DEFAULT_BLOB_GC_RETAIN_PREVIOUS,
8
+ } from "@checkstack/script-packages-common";
9
+ import { extractErrorMessage } from "@checkstack/common";
10
+ import type { InstallerLock } from "./install-state-store";
11
+ import type { BlobStoreRegistry } from "./blob-store-registry";
12
+ import { runBlobGc, type GcBlob } from "./blob-gc";
13
+
14
+ /**
15
+ * Build a `triggerBlobGc()` callable that wires the pure {@link runBlobGc}
16
+ * orchestration to the real stores + blob backends, and is shared by the
17
+ * admin `gcBlobs` RPC and the scheduled recurring job (so the safety logic
18
+ * lives in exactly one place).
19
+ *
20
+ * Mutual exclusion: the trigger takes the installer-election advisory lock
21
+ * for the whole pass, so a concurrent install / migration (which contend for
22
+ * the same lock) cannot run while GC does, and vice versa. If the lock is
23
+ * held, GC refuses cleanly (the install / migration wins; GC retries on its
24
+ * next scheduled run). On top of the lock, {@link runBlobGc} re-checks
25
+ * `isBusy()`.
26
+ *
27
+ * Retained set: the current desired manifest PLUS the previous N hashes'
28
+ * manifests, from lockfile history. History older than the retention window
29
+ * is pruned after a successful pass.
30
+ */
31
+ export interface BlobGcRunnerDeps {
32
+ installerLock: InstallerLock;
33
+ blobStores: BlobStoreRegistry;
34
+ /** Current desired manifest + hash (from install state). */
35
+ loadCurrent(): Promise<{
36
+ lockfileHash: string | null;
37
+ manifest: ManifestEntry[];
38
+ }>;
39
+ /** Most-recent `limit` manifests from lockfile history (newest first). */
40
+ recentHistory(limit: number): Promise<ManifestEntry[][]>;
41
+ /** Prune history rows beyond the most-recent `keep` hashes. */
42
+ pruneHistory(keep: number): Promise<void>;
43
+ /** Every indexed blob with size + created_at. */
44
+ listBlobs(): Promise<GcBlob[]>;
45
+ /** Remove a blob's index row (after its bytes are deleted). */
46
+ removeBlobRow(integrity: string): Promise<void>;
47
+ /** True while an install OR storage migration is in flight. */
48
+ isBusy(): Promise<boolean>;
49
+ /** Persist the run summary for the admin UI. */
50
+ recordRun(input: { deleted: number; bytesReclaimed: number }): Promise<void>;
51
+ /** Keep current + this many previous hashes (default 1). */
52
+ retainPrevious?: number;
53
+ /** Grace window in ms (default 24h). */
54
+ graceMs?: number;
55
+ logger?: { debug(msg: string): void; error(msg: string): void };
56
+ }
57
+
58
+ export function createBlobGcTrigger(
59
+ deps: BlobGcRunnerDeps,
60
+ ): () => Promise<BlobGcSummary> {
61
+ const retainPrevious = deps.retainPrevious ?? DEFAULT_BLOB_GC_RETAIN_PREVIOUS;
62
+ const graceMs = deps.graceMs ?? DEFAULT_BLOB_GC_GRACE_MS;
63
+
64
+ return async function triggerBlobGc(): Promise<BlobGcSummary> {
65
+ // Election: hold the installer lock for the whole pass so an install /
66
+ // migration cannot start concurrently (they contend for the same lock).
67
+ const lock = await deps.installerLock.tryInstallerLock();
68
+ if (!lock) {
69
+ const reason =
70
+ "An install or storage migration holds the installer lock; skipping blob GC.";
71
+ deps.logger?.debug(reason);
72
+ return {
73
+ ran: false,
74
+ reason,
75
+ candidates: 0,
76
+ deleted: 0,
77
+ keptWithinGrace: 0,
78
+ bytesReclaimed: 0,
79
+ };
80
+ }
81
+
82
+ try {
83
+ // Retain current + previous N. We pull `retainPrevious + 1` history
84
+ // rows (the current hash is itself recorded in history on install) and
85
+ // additionally union the live install-state manifest in case history
86
+ // hasn't caught up yet — when in doubt, retain.
87
+ return await runBlobGc({
88
+ deps: {
89
+ retainedManifests: async () => {
90
+ const manifests = await deps.recentHistory(retainPrevious + 1);
91
+ const current = await deps.loadCurrent();
92
+ if (current.lockfileHash && current.manifest.length > 0) {
93
+ manifests.push(current.manifest);
94
+ }
95
+ return manifests;
96
+ },
97
+ listBlobs: deps.listBlobs,
98
+ deleteBlob: async ({ integrity, backend }) => {
99
+ // Route the byte-delete to the blob's recorded backend, then drop
100
+ // the index row. If the backend isn't registered (shouldn't
101
+ // happen), skip the byte-delete but still drop the row so a
102
+ // dangling reference doesn't linger.
103
+ if (deps.blobStores.has(backend)) {
104
+ await deps.blobStores.get(backend).delete({ integrity });
105
+ } else {
106
+ deps.logger?.error(
107
+ `Blob GC: backend "${backend}" for ${integrity} not registered; dropping index row only.`,
108
+ );
109
+ }
110
+ await deps.removeBlobRow(integrity);
111
+ },
112
+ isBusy: deps.isBusy,
113
+ recordRun: deps.recordRun,
114
+ graceMs,
115
+ logger: deps.logger,
116
+ },
117
+ });
118
+ } catch (error) {
119
+ const message = extractErrorMessage(error);
120
+ deps.logger?.error(`Blob GC pass failed: ${message}`);
121
+ return {
122
+ ran: false,
123
+ reason: message,
124
+ candidates: 0,
125
+ deleted: 0,
126
+ keptWithinGrace: 0,
127
+ bytesReclaimed: 0,
128
+ };
129
+ } finally {
130
+ // Prune history beyond the retained window (best-effort), then release.
131
+ await deps.pruneHistory(retainPrevious + 1).catch((error) => {
132
+ deps.logger?.error(
133
+ `Blob GC: pruning lockfile history failed: ${extractErrorMessage(error)}`,
134
+ );
135
+ });
136
+ await lock.release();
137
+ }
138
+ };
139
+ }
@@ -0,0 +1,182 @@
1
+ import { describe, expect, test } from "bun:test";
2
+ import type { ManifestEntry } from "@checkstack/script-packages-common";
3
+ import { runBlobGc, type BlobGcDeps, type GcBlob } from "./blob-gc";
4
+
5
+ const NOW = 1_700_000_000_000;
6
+ const HOUR = 60 * 60 * 1000;
7
+ const DAY = 24 * HOUR;
8
+
9
+ function entry(name: string, integrity: string): ManifestEntry {
10
+ return { name, version: "1.0.0", integrity };
11
+ }
12
+
13
+ function blob(over: Partial<GcBlob> & { integrity: string }): GcBlob {
14
+ return {
15
+ backend: "postgres",
16
+ sizeBytes: 100,
17
+ createdAt: new Date(NOW - 2 * DAY), // past-grace by default
18
+ ...over,
19
+ };
20
+ }
21
+
22
+ function makeDeps(
23
+ over: Partial<BlobGcDeps> = {},
24
+ ): {
25
+ deps: BlobGcDeps;
26
+ deletes: { integrity: string; backend: string }[];
27
+ recorded: { deleted: number; bytesReclaimed: number }[];
28
+ } {
29
+ const deletes: { integrity: string; backend: string }[] = [];
30
+ const recorded: { deleted: number; bytesReclaimed: number }[] = [];
31
+ const deps: BlobGcDeps = {
32
+ retainedManifests: async () => [[entry("a", "sha-a")]],
33
+ listBlobs: async () => [blob({ integrity: "sha-a" })],
34
+ deleteBlob: async (input) => void deletes.push(input),
35
+ isBusy: async () => false,
36
+ recordRun: async (r) => void recorded.push(r),
37
+ now: () => NOW,
38
+ ...over,
39
+ };
40
+ return { deps, deletes, recorded };
41
+ }
42
+
43
+ describe("runBlobGc", () => {
44
+ test("never deletes a blob referenced by a retained manifest", async () => {
45
+ const { deps, deletes } = makeDeps({
46
+ retainedManifests: async () => [[entry("a", "sha-a")]],
47
+ listBlobs: async () => [
48
+ blob({ integrity: "sha-a" }),
49
+ blob({ integrity: "sha-orphan" }),
50
+ ],
51
+ });
52
+ const out = await runBlobGc({ deps });
53
+ expect(out.ran).toBe(true);
54
+ // Only the orphan is a candidate; the referenced blob is untouched.
55
+ expect(deletes.map((d) => d.integrity)).toEqual(["sha-orphan"]);
56
+ expect(out.candidates).toBe(1);
57
+ expect(out.deleted).toBe(1);
58
+ });
59
+
60
+ test("retains blobs referenced by ANY retained manifest (current + previous)", async () => {
61
+ const { deps, deletes } = makeDeps({
62
+ // current references sha-a, previous references sha-b.
63
+ retainedManifests: async () => [
64
+ [entry("a", "sha-a")],
65
+ [entry("b", "sha-b")],
66
+ ],
67
+ listBlobs: async () => [
68
+ blob({ integrity: "sha-a" }),
69
+ blob({ integrity: "sha-b" }),
70
+ blob({ integrity: "sha-old" }),
71
+ ],
72
+ });
73
+ const out = await runBlobGc({ deps });
74
+ expect(deletes.map((d) => d.integrity)).toEqual(["sha-old"]);
75
+ expect(out.deleted).toBe(1);
76
+ });
77
+
78
+ test("deletes unreferenced blobs past the grace window", async () => {
79
+ const { deps, deletes } = makeDeps({
80
+ retainedManifests: async () => [[entry("a", "sha-a")]],
81
+ listBlobs: async () => [
82
+ blob({ integrity: "sha-old", createdAt: new Date(NOW - 2 * DAY) }),
83
+ ],
84
+ graceMs: DAY,
85
+ });
86
+ const out = await runBlobGc({ deps });
87
+ expect(deletes.map((d) => d.integrity)).toEqual(["sha-old"]);
88
+ expect(out.deleted).toBe(1);
89
+ expect(out.keptWithinGrace).toBe(0);
90
+ });
91
+
92
+ test("keeps unreferenced blobs still within the grace window", async () => {
93
+ const { deps, deletes } = makeDeps({
94
+ retainedManifests: async () => [[entry("a", "sha-a")]],
95
+ listBlobs: async () => [
96
+ blob({ integrity: "sha-fresh", createdAt: new Date(NOW - HOUR) }),
97
+ ],
98
+ graceMs: DAY,
99
+ });
100
+ const out = await runBlobGc({ deps });
101
+ expect(deletes).toHaveLength(0);
102
+ expect(out.candidates).toBe(1);
103
+ expect(out.deleted).toBe(0);
104
+ expect(out.keptWithinGrace).toBe(1);
105
+ });
106
+
107
+ test("refuses to run while an install / migration is in flight", async () => {
108
+ const { deps, deletes } = makeDeps({
109
+ isBusy: async () => true,
110
+ listBlobs: async () => [blob({ integrity: "sha-old" })],
111
+ });
112
+ const out = await runBlobGc({ deps });
113
+ expect(out.ran).toBe(false);
114
+ expect(out.reason).toMatch(/install|migration/i);
115
+ expect(deletes).toHaveLength(0);
116
+ });
117
+
118
+ test("routes each delete to the blob's recorded backend", async () => {
119
+ const { deps, deletes } = makeDeps({
120
+ retainedManifests: async () => [[]],
121
+ listBlobs: async () => [
122
+ blob({ integrity: "sha-pg", backend: "postgres" }),
123
+ blob({ integrity: "sha-s3", backend: "s3" }),
124
+ ],
125
+ });
126
+ await runBlobGc({ deps });
127
+ expect(deletes).toEqual([
128
+ { integrity: "sha-pg", backend: "postgres" },
129
+ { integrity: "sha-s3", backend: "s3" },
130
+ ]);
131
+ });
132
+
133
+ test("accounts for reclaimed bytes", async () => {
134
+ const { deps, recorded } = makeDeps({
135
+ retainedManifests: async () => [[]],
136
+ listBlobs: async () => [
137
+ blob({ integrity: "x", sizeBytes: 500 }),
138
+ blob({ integrity: "y", sizeBytes: 250 }),
139
+ ],
140
+ });
141
+ const out = await runBlobGc({ deps });
142
+ expect(out.bytesReclaimed).toBe(750);
143
+ expect(recorded).toEqual([{ deleted: 2, bytesReclaimed: 750 }]);
144
+ });
145
+
146
+ test("is idempotent: a re-run after deletion reclaims nothing", async () => {
147
+ let store = [blob({ integrity: "x" }), blob({ integrity: "orphan" })];
148
+ const deps: BlobGcDeps = {
149
+ retainedManifests: async () => [[entry("x", "x")]],
150
+ listBlobs: async () => store,
151
+ deleteBlob: async ({ integrity }) => {
152
+ store = store.filter((b) => b.integrity !== integrity);
153
+ },
154
+ isBusy: async () => false,
155
+ now: () => NOW,
156
+ };
157
+ const first = await runBlobGc({ deps });
158
+ expect(first.deleted).toBe(1);
159
+ const second = await runBlobGc({ deps });
160
+ expect(second.deleted).toBe(0);
161
+ expect(second.candidates).toBe(0);
162
+ });
163
+
164
+ test("a failed single delete is logged and skipped, not fatal", async () => {
165
+ const errors: string[] = [];
166
+ const { deps } = makeDeps({
167
+ retainedManifests: async () => [[]],
168
+ listBlobs: async () => [
169
+ blob({ integrity: "bad" }),
170
+ blob({ integrity: "good" }),
171
+ ],
172
+ deleteBlob: async ({ integrity }) => {
173
+ if (integrity === "bad") throw new Error("backend offline");
174
+ },
175
+ logger: { debug: () => {}, error: (m) => void errors.push(m) },
176
+ });
177
+ const out = await runBlobGc({ deps });
178
+ // "good" still deleted; "bad" retained for next pass.
179
+ expect(out.deleted).toBe(1);
180
+ expect(errors.some((e) => /bad/.test(e))).toBe(true);
181
+ });
182
+ });
package/src/blob-gc.ts ADDED
@@ -0,0 +1,161 @@
1
+ import type {
2
+ BlobGcSummary,
3
+ ManifestEntry,
4
+ } from "@checkstack/script-packages-common";
5
+ import { DEFAULT_BLOB_GC_GRACE_MS } from "@checkstack/script-packages-common";
6
+ import { extractErrorMessage } from "@checkstack/common";
7
+
8
+ /**
9
+ * Blob garbage collection: prune content-addressed blobs no longer referenced
10
+ * by any RETAINED lockfile manifest (the current desired hash + the previous
11
+ * N), older than a grace window.
12
+ *
13
+ * GC is destructive, so every deletion is provably safe:
14
+ * - **Retained set:** the union of blob integrities across the retained
15
+ * manifests (current + previous N). Any blob NOT in that union is a
16
+ * candidate. When in doubt we retain.
17
+ * - **Grace window:** a candidate is only deleted when it is older than the
18
+ * grace window (keyed on `script_package_blob.created_at`), so a pod /
19
+ * satellite mid-reconcile toward a just-superseded hash can still pull a
20
+ * blob that was just dropped from the retained set.
21
+ * - **Mutual exclusion:** the caller holds the installer-election advisory
22
+ * lock while GC runs AND we re-check `isBusy()` (install OR migration in
23
+ * flight) before deleting, so GC can never race a concurrent install that
24
+ * is publishing blobs for a new hash.
25
+ * - **Per-backend routing:** each blob is deleted from the backend recorded
26
+ * in its index row, then the index row is removed.
27
+ *
28
+ * All collaborators are injected so the orchestration is unit-testable
29
+ * without a DB or a real blob store.
30
+ */
31
+
32
+ /** One indexed blob, as the GC needs to see it. */
33
+ export interface GcBlob {
34
+ integrity: string;
35
+ backend: string;
36
+ sizeBytes: number;
37
+ createdAt: Date;
38
+ }
39
+
40
+ export interface BlobGcDeps {
41
+ /**
42
+ * The retained manifests: the current desired manifest plus the previous N
43
+ * (most-recent-first or any order; only the union of integrities matters).
44
+ */
45
+ retainedManifests(): Promise<ManifestEntry[][]>;
46
+ /** Every indexed blob (integrity + backend + size + created_at). */
47
+ listBlobs(): Promise<GcBlob[]>;
48
+ /** Delete a blob's bytes from `backend`, then its index row. Idempotent. */
49
+ deleteBlob(input: { integrity: string; backend: string }): Promise<void>;
50
+ /**
51
+ * True while an install OR a storage migration is in flight. Re-checked
52
+ * AFTER the lock is held (an install on this very pod could be mid-publish
53
+ * without holding the installer lock for the whole window).
54
+ */
55
+ isBusy(): Promise<boolean>;
56
+ /** Record the run summary (for the admin UI). Best-effort. */
57
+ recordRun?(input: {
58
+ deleted: number;
59
+ bytesReclaimed: number;
60
+ }): Promise<void>;
61
+ /** Now, injectable for deterministic grace tests. */
62
+ now?(): number;
63
+ /** Grace window in ms (default 24h). */
64
+ graceMs?: number;
65
+ logger?: { debug(msg: string): void; error(msg: string): void };
66
+ }
67
+
68
+ /**
69
+ * Run one blob-GC pass. The caller MUST already hold the installer-election
70
+ * advisory lock (so an install/migration cannot start mid-pass); this
71
+ * function additionally refuses if `isBusy()` reports an active install /
72
+ * migration, as a belt-and-braces guard.
73
+ *
74
+ * Idempotent: a second run with the same inputs finds the just-deleted blobs
75
+ * gone and reports zero further deletions.
76
+ */
77
+ export async function runBlobGc({
78
+ deps,
79
+ }: {
80
+ deps: BlobGcDeps;
81
+ }): Promise<BlobGcSummary> {
82
+ const graceMs = deps.graceMs ?? DEFAULT_BLOB_GC_GRACE_MS;
83
+ const now = deps.now?.() ?? Date.now();
84
+
85
+ if (await deps.isBusy()) {
86
+ const reason =
87
+ "An install or storage migration is in flight; skipping blob GC.";
88
+ deps.logger?.debug(reason);
89
+ return {
90
+ ran: false,
91
+ reason,
92
+ candidates: 0,
93
+ deleted: 0,
94
+ keptWithinGrace: 0,
95
+ bytesReclaimed: 0,
96
+ };
97
+ }
98
+
99
+ // Retained set = union of integrities across every retained manifest.
100
+ const retained = new Set<string>();
101
+ for (const manifest of await deps.retainedManifests()) {
102
+ for (const entry of manifest) retained.add(entry.integrity);
103
+ }
104
+
105
+ const blobs = await deps.listBlobs();
106
+ const candidates = blobs.filter((b) => !retained.has(b.integrity));
107
+
108
+ let deleted = 0;
109
+ let keptWithinGrace = 0;
110
+ let bytesReclaimed = 0;
111
+
112
+ for (const blob of candidates) {
113
+ const ageMs = now - blob.createdAt.getTime();
114
+ if (ageMs < graceMs) {
115
+ keptWithinGrace++;
116
+ deps.logger?.debug(
117
+ `Blob GC: keeping unreferenced ${blob.integrity} (age ${Math.round(
118
+ ageMs / 1000,
119
+ )}s < grace ${Math.round(graceMs / 1000)}s).`,
120
+ );
121
+ continue;
122
+ }
123
+ try {
124
+ await deps.deleteBlob({
125
+ integrity: blob.integrity,
126
+ backend: blob.backend,
127
+ });
128
+ deleted++;
129
+ bytesReclaimed += blob.sizeBytes;
130
+ deps.logger?.debug(
131
+ `Blob GC: deleted ${blob.integrity} from "${blob.backend}" ` +
132
+ `(${blob.sizeBytes} bytes).`,
133
+ );
134
+ } catch (error) {
135
+ // A single failed delete must not abort the whole pass; the blob is
136
+ // simply retained until the next run. Never silently swallow: log it.
137
+ deps.logger?.error(
138
+ `Blob GC: failed to delete ${blob.integrity} from "${blob.backend}": ${extractErrorMessage(
139
+ error,
140
+ )}. Retaining for the next pass.`,
141
+ );
142
+ }
143
+ }
144
+
145
+ await deps.recordRun?.({ deleted, bytesReclaimed }).catch(() => {
146
+ // Recording the summary is best-effort UI state; never fail the pass.
147
+ });
148
+
149
+ deps.logger?.debug(
150
+ `Blob GC complete: ${candidates.length} candidate(s), ${deleted} deleted ` +
151
+ `(${bytesReclaimed} bytes), ${keptWithinGrace} kept within grace.`,
152
+ );
153
+
154
+ return {
155
+ ran: true,
156
+ candidates: candidates.length,
157
+ deleted,
158
+ keptWithinGrace,
159
+ bytesReclaimed,
160
+ };
161
+ }
@@ -0,0 +1,70 @@
1
+ import { describe, expect, test } from "bun:test";
2
+ import { blobSha256, toUint8Array, verifyBlobSha256 } from "./blob-hash";
3
+
4
+ const bytes = new TextEncoder().encode("the real blob");
5
+ const hash = blobSha256(bytes);
6
+
7
+ describe("blobSha256 byte-type boundary", () => {
8
+ // Regression: a real-package install reads blob bytes from a store backend
9
+ // whose `get` can yield a raw ArrayBuffer (e.g. an S3/HTTP transport).
10
+ // `crypto.Hash.update()` rejects a bare ArrayBuffer
11
+ // ("Received an instance of ArrayBuffer"), failing the install with
12
+ // status=error. The hash boundary must normalize ArrayBuffer -> Uint8Array.
13
+ test("hashes a raw ArrayBuffer without throwing", () => {
14
+ const u8 = new TextEncoder().encode("blob-from-an-arraybuffer-source");
15
+ // A standalone ArrayBuffer holding exactly these bytes.
16
+ const ab = u8.buffer.slice(u8.byteOffset, u8.byteOffset + u8.byteLength);
17
+ expect(ab).toBeInstanceOf(ArrayBuffer);
18
+ // Must not throw, and must equal the Uint8Array digest over the same bytes.
19
+ expect(blobSha256(ab)).toBe(blobSha256(u8));
20
+ });
21
+
22
+ test("a Uint8Array view hashes identically to its ArrayBuffer", () => {
23
+ const u8 = new Uint8Array([0, 1, 2, 250, 255, 128, 64]);
24
+ const ab = u8.buffer.slice(0);
25
+ expect(blobSha256(ab)).toBe(blobSha256(u8));
26
+ });
27
+
28
+ test("verifyBlobSha256 accepts an ArrayBuffer", () => {
29
+ const u8 = new TextEncoder().encode("verify me");
30
+ const ab = u8.buffer.slice(u8.byteOffset, u8.byteOffset + u8.byteLength);
31
+ const verdict = verifyBlobSha256({
32
+ entry: { blobSha256: blobSha256(u8) },
33
+ bytes: ab,
34
+ });
35
+ expect(verdict.ok).toBe(true);
36
+ });
37
+
38
+ test("toUint8Array returns the same instance for a Uint8Array", () => {
39
+ const u8 = new Uint8Array([1, 2, 3]);
40
+ expect(toUint8Array(u8)).toBe(u8);
41
+ });
42
+ });
43
+
44
+ describe("verifyBlobSha256", () => {
45
+ test("ok when the hash matches", () => {
46
+ const verdict = verifyBlobSha256({ entry: { blobSha256: hash }, bytes });
47
+ expect(verdict.ok).toBe(true);
48
+ });
49
+
50
+ test("rejects tampered bytes with expected + actual hashes", () => {
51
+ const tampered = new TextEncoder().encode("the EVIL blob");
52
+ const verdict = verifyBlobSha256({
53
+ entry: { blobSha256: hash },
54
+ bytes: tampered,
55
+ });
56
+ expect(verdict.ok).toBe(false);
57
+ if (!verdict.ok) {
58
+ expect(verdict.expected).toBe(hash);
59
+ expect(verdict.actual).toBe(blobSha256(tampered));
60
+ expect(verdict.actual).not.toBe(verdict.expected);
61
+ }
62
+ });
63
+
64
+ test("backward-safe: no recorded hash skips verification", () => {
65
+ // Entries published before blobSha256 must still seed (until re-install
66
+ // regenerates the manifest). Any bytes pass.
67
+ const verdict = verifyBlobSha256({ entry: {}, bytes });
68
+ expect(verdict.ok).toBe(true);
69
+ });
70
+ });