dpdp-erasure-cli 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/.env.example +55 -0
  2. package/Dockerfile +33 -0
  3. package/compliance.worker.yaml +64 -0
  4. package/package.json +41 -0
  5. package/src/constants/index.ts +1 -0
  6. package/src/errors/fail.ts +110 -0
  7. package/src/errors/index.ts +4 -0
  8. package/src/errors/inferer.ts +166 -0
  9. package/src/errors/registry.ts +122 -0
  10. package/src/errors/types.ts +65 -0
  11. package/src/errors/worker.ts +161 -0
  12. package/src/index.ts +328 -0
  13. package/src/lib/crypto/digest.ts +22 -0
  14. package/src/lib/crypto/encoding.ts +78 -0
  15. package/src/lib/crypto/index.ts +2 -0
  16. package/src/lib/index.ts +1 -0
  17. package/src/modules/bootstrap/index.ts +2 -0
  18. package/src/modules/bootstrap/integrity.ts +38 -0
  19. package/src/modules/bootstrap/preflight.ts +296 -0
  20. package/src/modules/cli/check-integrity.ts +48 -0
  21. package/src/modules/cli/dry-run.ts +90 -0
  22. package/src/modules/cli/graph.ts +87 -0
  23. package/src/modules/cli/index.ts +184 -0
  24. package/src/modules/cli/init.ts +115 -0
  25. package/src/modules/cli/inspect.ts +86 -0
  26. package/src/modules/cli/introspector.ts +117 -0
  27. package/src/modules/cli/keygen.ts +38 -0
  28. package/src/modules/cli/scan.ts +126 -0
  29. package/src/modules/cli/sign.ts +50 -0
  30. package/src/modules/cli/ui.ts +61 -0
  31. package/src/modules/cli/verify-schema.ts +31 -0
  32. package/src/modules/cli/verify.ts +85 -0
  33. package/src/modules/config/compatibility.ts +271 -0
  34. package/src/modules/config/index.ts +4 -0
  35. package/src/modules/config/reader.ts +149 -0
  36. package/src/modules/config/signature.ts +69 -0
  37. package/src/modules/config/validation.ts +658 -0
  38. package/src/modules/crypto/aes.ts +158 -0
  39. package/src/modules/crypto/envelope.ts +48 -0
  40. package/src/modules/crypto/hmac.ts +60 -0
  41. package/src/modules/crypto/index.ts +3 -0
  42. package/src/modules/db/drift.ts +36 -0
  43. package/src/modules/db/graph.ts +203 -0
  44. package/src/modules/db/index.ts +4 -0
  45. package/src/modules/db/migrations.ts +254 -0
  46. package/src/modules/db/sql-debug.ts +61 -0
  47. package/src/modules/engine/blob/index.ts +3 -0
  48. package/src/modules/engine/blob/s3.ts +455 -0
  49. package/src/modules/engine/blob/store.ts +236 -0
  50. package/src/modules/engine/blob/types.ts +44 -0
  51. package/src/modules/engine/helpers/identity.ts +47 -0
  52. package/src/modules/engine/helpers/index.ts +4 -0
  53. package/src/modules/engine/helpers/outbox.ts +118 -0
  54. package/src/modules/engine/helpers/runtime.ts +115 -0
  55. package/src/modules/engine/helpers/types.ts +61 -0
  56. package/src/modules/engine/index.ts +6 -0
  57. package/src/modules/engine/notifier/config.ts +147 -0
  58. package/src/modules/engine/notifier/dispatcher.ts +300 -0
  59. package/src/modules/engine/notifier/index.ts +3 -0
  60. package/src/modules/engine/notifier/payload.ts +51 -0
  61. package/src/modules/engine/notifier/reservation.ts +153 -0
  62. package/src/modules/engine/notifier/types.ts +38 -0
  63. package/src/modules/engine/shredder.ts +254 -0
  64. package/src/modules/engine/types.ts +146 -0
  65. package/src/modules/engine/vault/compiled-targets.ts +562 -0
  66. package/src/modules/engine/vault/context.ts +254 -0
  67. package/src/modules/engine/vault/dry-run.ts +94 -0
  68. package/src/modules/engine/vault/execution.ts +485 -0
  69. package/src/modules/engine/vault/index.ts +3 -0
  70. package/src/modules/engine/vault/purge.ts +82 -0
  71. package/src/modules/engine/vault/retention.ts +124 -0
  72. package/src/modules/engine/vault/satellite-mutation.ts +193 -0
  73. package/src/modules/engine/vault/satellite.ts +103 -0
  74. package/src/modules/engine/vault/shadow.ts +36 -0
  75. package/src/modules/engine/vault/static-plan.ts +116 -0
  76. package/src/modules/engine/vault/store.ts +34 -0
  77. package/src/modules/engine/vault/vault.ts +84 -0
  78. package/src/modules/introspector/classifier.ts +502 -0
  79. package/src/modules/introspector/dag.ts +276 -0
  80. package/src/modules/introspector/index.ts +7 -0
  81. package/src/modules/introspector/naming.ts +75 -0
  82. package/src/modules/introspector/report.ts +153 -0
  83. package/src/modules/introspector/run.ts +123 -0
  84. package/src/modules/introspector/s3-sampler.ts +227 -0
  85. package/src/modules/introspector/types.ts +131 -0
  86. package/src/modules/introspector/yaml.ts +101 -0
  87. package/src/modules/network/api/control-plane.ts +275 -0
  88. package/src/modules/network/api/index.ts +1 -0
  89. package/src/modules/network/api/validation.ts +71 -0
  90. package/src/modules/network/index.ts +4 -0
  91. package/src/modules/network/object-store/aws/client.ts +444 -0
  92. package/src/modules/network/object-store/aws/credentials.ts +271 -0
  93. package/src/modules/network/object-store/aws/index.ts +2 -0
  94. package/src/modules/network/object-store/aws/sigv4.ts +190 -0
  95. package/src/modules/network/object-store/aws/type.ts +6 -0
  96. package/src/modules/network/object-store/index.ts +1 -0
  97. package/src/modules/network/outbox/dispatcher.ts +183 -0
  98. package/src/modules/network/outbox/index.ts +3 -0
  99. package/src/modules/network/outbox/process.ts +133 -0
  100. package/src/modules/network/outbox/shared.ts +56 -0
  101. package/src/modules/network/outbox/store.ts +346 -0
  102. package/src/modules/network/outbox/types.ts +54 -0
  103. package/src/modules/network/request-signing.ts +61 -0
  104. package/src/modules/worker/index.ts +2 -0
  105. package/src/modules/worker/tasks.ts +58 -0
  106. package/src/modules/worker/types.ts +89 -0
  107. package/src/modules/worker/worker.ts +243 -0
  108. package/src/secrets/index.ts +4 -0
  109. package/src/secrets/kms/index.ts +2 -0
  110. package/src/secrets/kms/signature.ts +82 -0
  111. package/src/secrets/kms/validation.ts +64 -0
  112. package/src/secrets/reader.ts +42 -0
  113. package/src/secrets/repository/crypto.ts +89 -0
  114. package/src/secrets/repository/index.ts +2 -0
  115. package/src/secrets/repository/methods.ts +37 -0
  116. package/src/secrets/resolvers.ts +247 -0
  117. package/src/secrets/signature.ts +78 -0
  118. package/src/types/index.ts +1 -0
  119. package/src/types/types.ts +23 -0
  120. package/src/utils/identifiers.ts +48 -0
  121. package/src/utils/index.ts +3 -0
  122. package/src/utils/json.ts +35 -0
  123. package/src/utils/logger.ts +161 -0
  124. package/src/validation/zod.ts +70 -0
  125. package/tests/adversarial.test.ts +464 -0
  126. package/tests/blob-s3.test.ts +216 -0
  127. package/tests/config.test.ts +395 -0
  128. package/tests/control-plane-client.test.ts +108 -0
  129. package/tests/crypto.test.ts +106 -0
  130. package/tests/errors.test.ts +69 -0
  131. package/tests/fetch-dispatcher.test.ts +213 -0
  132. package/tests/graph.test.ts +84 -0
  133. package/tests/helpers/index.ts +101 -0
  134. package/tests/index-preflight.test.ts +168 -0
  135. package/tests/introspector-classifier.test.ts +62 -0
  136. package/tests/introspector-report.test.ts +85 -0
  137. package/tests/introspector.test.ts +394 -0
  138. package/tests/kms.test.ts +124 -0
  139. package/tests/logger.test.ts +61 -0
  140. package/tests/notifier.test.ts +303 -0
  141. package/tests/outbox.test.ts +478 -0
  142. package/tests/purge-policy.test.ts +124 -0
  143. package/tests/retention.test.ts +103 -0
  144. package/tests/s3-client.test.ts +110 -0
  145. package/tests/satellite.test.ts +119 -0
  146. package/tests/schema-compatibility.test.ts +237 -0
  147. package/tests/schema-integrity.test.ts +64 -0
  148. package/tests/shredder.test.ts +163 -0
  149. package/tests/vault.compiled-targets.test.ts +243 -0
  150. package/tests/vault.replica.test.ts +59 -0
  151. package/tests/vault.test.ts +279 -0
  152. package/tests/worker.retry.test.ts +291 -0
  153. package/tests/worker.test.ts +200 -0
  154. package/tsconfig.json +19 -0
  155. package/vitest.config.ts +13 -0
@@ -0,0 +1,275 @@
1
+ import { asWorkerError, workerError } from "@/errors";
2
+ import { getLogger } from "@/utils";
3
+ import type { ApiClient, SyncTaskResponse, TaskAckPayload } from "@modules/worker";
4
+ import { computeRequestSignature } from "../request-signing";
5
+ import { syncResponseSchema } from "./validation";
6
+
7
+ const logger = getLogger({ component: "control-plane" });
8
+
9
+ /**
10
+ * HTTP endpoint and credentials required to communicate with the Control Plane.
11
+ */
12
+ interface ControlPlaneApiClientOptions {
13
+ syncUrl: string;
14
+ ackBaseUrl: string;
15
+ workerAuthHeaders: {
16
+ "x-client-id": string;
17
+ authorization: string;
18
+ };
19
+ workerConfigHash: string;
20
+ workerConfigVersion?: string;
21
+ workerDpoIdentifier?: string;
22
+ pushOutboxEvent: ApiClient["pushOutboxEvent"];
23
+ requestSigningSecret?: string;
24
+ timeoutMs?: number;
25
+ }
26
+
27
+ function buildControlPlaneHttpError(
28
+ operation: "sync" | "ack" | "heartbeat",
29
+ status: number,
30
+ context: Record<string, unknown> = {},
31
+ ) {
32
+ if (status === 429 || status >= 500) {
33
+ return workerError({
34
+ code: "CONTROL_PLANE_UNAVAILABLE",
35
+ title: "Control Plane unavailable",
36
+ detail: `Control Plane ${operation} request failed with HTTP ${status}.`,
37
+ category: "network",
38
+ retryable: true,
39
+ context: {
40
+ operation,
41
+ status,
42
+ ...context,
43
+ },
44
+ });
45
+ }
46
+
47
+ if (status === 401 || status === 403) {
48
+ return workerError({
49
+ code: "CONTROL_PLANE_AUTH_REJECTED",
50
+ title: "Control Plane authentication rejected",
51
+ detail: `Control Plane ${operation} request was rejected with HTTP ${status}.`,
52
+ category: "configuration",
53
+ retryable: false,
54
+ fatal: true,
55
+ context: {
56
+ operation,
57
+ status,
58
+ ...context,
59
+ },
60
+ });
61
+ }
62
+
63
+ return workerError({
64
+ code: "CONTROL_PLANE_PROTOCOL_REJECTED",
65
+ title: "Control Plane protocol rejected",
66
+ detail: `Control Plane ${operation} request failed with HTTP ${status}.`,
67
+ category: "external",
68
+ retryable: false,
69
+ fatal: true,
70
+ context: {
71
+ operation,
72
+ status,
73
+ ...context,
74
+ },
75
+ });
76
+ };
77
+
78
+ async function fetchWithTimeout(
79
+ url: string,
80
+ init: RequestInit,
81
+ timeoutMs: number,
82
+ ): Promise<Response> {
83
+ const controller = new AbortController();
84
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
85
+
86
+ try {
87
+ return await fetch(url, {
88
+ ...init,
89
+ signal: controller.signal,
90
+ redirect: "error",
91
+ })
92
+ } catch (error) {
93
+ throw asWorkerError({
94
+ code: "CONTROL_PLANE_REQUEST_FAILED",
95
+ title: "Control Plane request failed",
96
+ detail: `Failed to reach ${url}.`,
97
+ category: "network",
98
+ retryable: true,
99
+ context: {
100
+ url,
101
+ },
102
+ });
103
+ } finally {
104
+ clearTimeout(timer);
105
+ }
106
+ }
107
+
108
+ async function signWorkerRequest(
109
+ secret: string | undefined,
110
+ clientId: string,
111
+ method: string,
112
+ url: string,
113
+ bodyText: string
114
+ ): Promise<Record<string, string>> {
115
+ if (!secret) {
116
+ return {};
117
+ }
118
+
119
+ const timestamp = String(Date.now());
120
+ const nonce = crypto.randomUUID();
121
+ const pathname = new URL(url).pathname;
122
+ const signature = await computeRequestSignature(
123
+ secret,
124
+ method,
125
+ pathname,
126
+ clientId,
127
+ timestamp,
128
+ bodyText,
129
+ nonce
130
+ )
131
+
132
+ return {
133
+ "x-dpdp-timestamp": timestamp,
134
+ "x-dpdp-nonce": nonce,
135
+ "x-dpdp-signature": signature,
136
+ };
137
+ }
138
+
139
+
140
+ /**
141
+ * Builds a strict Control Plane client that validates response payloads before task execution.
142
+ *
143
+ * @param options - Control Plane endpoints, worker auth headers, outbox push transport, and timeout.
144
+ * @returns API client implementation consumed by `ComplianceWorker`.
145
+ * @throws {WorkerError} For transport failures, auth failures, or protocol/schema violations.
146
+ */
147
+ export function createControlPlaneApiClient(options: ControlPlaneApiClientOptions): ApiClient {
148
+ const timeoutMs = options.timeoutMs ?? 10_000;
149
+
150
+ return {
151
+ async syncTask(): Promise<SyncTaskResponse> {
152
+ const response = await fetchWithTimeout(
153
+ options.syncUrl,
154
+ {
155
+ headers: {
156
+ ...options.workerAuthHeaders,
157
+ "x-worker-config-hash": options.workerConfigHash,
158
+ ...(options.workerConfigVersion
159
+ ? { "x-worker-config-version": options.workerConfigVersion }
160
+ : {}),
161
+ ...(options.workerDpoIdentifier
162
+ ? { "x-worker-dpo-identifier": options.workerDpoIdentifier }
163
+ : {}),
164
+ ...(await signWorkerRequest(
165
+ options.requestSigningSecret,
166
+ options.workerAuthHeaders["x-client-id"],
167
+ "GET",
168
+ options.syncUrl,
169
+ ""
170
+ )),
171
+ },
172
+ },
173
+ timeoutMs
174
+ );
175
+
176
+ if (response.status === 204) {
177
+ return { pending: false };
178
+ }
179
+
180
+ if (!response.ok) {
181
+ throw buildControlPlaneHttpError("sync", response.status, { url: options.syncUrl });
182
+ }
183
+
184
+ let parsedBody: unknown;
185
+ try {
186
+ parsedBody = await response.json();
187
+ return syncResponseSchema.parse(parsedBody);
188
+ } catch (error) {
189
+ throw asWorkerError({
190
+ code: "CONTROL_PLANE_RESPONSE_INVALID",
191
+ title: "Invalid Control Plane response",
192
+ detail: "Control Plane sync response failed schema validation.",
193
+ category: "external",
194
+ retryable: false,
195
+ fatal: true,
196
+ context: {
197
+ url: options.syncUrl,
198
+ },
199
+ });
200
+ }
201
+ },
202
+
203
+ async ackTask(
204
+ taskId: string,
205
+ status: "completed" | "failed",
206
+ result: TaskAckPayload
207
+ ): Promise<boolean> {
208
+ const url = `${options.ackBaseUrl}/${taskId}/ack`;
209
+ const bodyText = JSON.stringify({ status, result });
210
+ const response = await fetchWithTimeout(
211
+ url,
212
+ {
213
+ method: "POST",
214
+ headers: {
215
+ "content-type": "application/json",
216
+ ...options.workerAuthHeaders,
217
+ ...(await signWorkerRequest(
218
+ options.requestSigningSecret,
219
+ options.workerAuthHeaders["x-client-id"],
220
+ "POST",
221
+ url,
222
+ bodyText
223
+ )),
224
+ },
225
+ body: bodyText,
226
+ },
227
+ timeoutMs
228
+ );
229
+
230
+ if (!response.ok) {
231
+ throw buildControlPlaneHttpError("ack", response.status, {
232
+ url,
233
+ taskId,
234
+ status,
235
+ });
236
+ }
237
+
238
+ logger.debug({ taskId, status }, "Control Plane acknowledged task");
239
+ return true;
240
+ },
241
+
242
+ async heartbeatTask(taskId: string): Promise<boolean> {
243
+ const url = `${options.ackBaseUrl}/${taskId}/heartbeat`;
244
+ const bodyText = "";
245
+ const response = await fetchWithTimeout(url,
246
+ {
247
+ method: "POST",
248
+ headers: {
249
+ ...options.workerAuthHeaders,
250
+ ...(await signWorkerRequest(
251
+ options.requestSigningSecret,
252
+ options.workerAuthHeaders["x-client-id"],
253
+ "POST",
254
+ url,
255
+ bodyText
256
+ )),
257
+ },
258
+ },
259
+ timeoutMs
260
+ );
261
+
262
+ if (!response.ok) {
263
+ throw buildControlPlaneHttpError("heartbeat", response.status, {
264
+ url,
265
+ taskId,
266
+ });
267
+ }
268
+
269
+ logger.debug({ taskId }, "Control Plane extended task lease");
270
+ return true;
271
+ },
272
+
273
+ pushOutboxEvent: options.pushOutboxEvent,
274
+ };
275
+ }
@@ -0,0 +1 @@
1
+ export * from "./control-plane";
@@ -0,0 +1,71 @@
1
+ import { z } from "zod";
2
+
3
+ export const isoDateStringSchema = z.iso.datetime({ offset: true });
4
+
5
+ export const taskPayloadBaseSchema = z.object({
6
+ request_id: z.uuid().optional(),
7
+ subject_opaque_id: z.string().min(1).optional(),
8
+ idempotency_key: z.uuid().optional(),
9
+ trigger_source: z.string().min(1).optional(),
10
+ actor_opaque_id: z.string().min(1).optional(),
11
+ legal_framework: z.string().min(1).optional(),
12
+ request_timestamp: isoDateStringSchema.optional(),
13
+ tenant_id: z.string().min(1).optional(),
14
+ cooldown_days: z.number().int().min(0).optional(),
15
+ shadow_mode: z.boolean().optional(),
16
+ webhook_url: z.url().optional(),
17
+ userId: z.number().int().positive().optional(),
18
+ now: isoDateStringSchema.optional(),
19
+ })
20
+ .strict();
21
+
22
+ export const syncTaskSchema = z.discriminatedUnion("task_type", [
23
+ z.object({
24
+ id: z.string().min(1),
25
+ task_type: z.literal("COMPILE_DAG"),
26
+ payload: taskPayloadBaseSchema.extend({
27
+ erasure_job_id: z.uuid().optional(),
28
+ }),
29
+ })
30
+ .strict(),
31
+ z.object({
32
+ id: z.string().min(1),
33
+ task_type: z.literal("VAULT_USER"),
34
+ payload: taskPayloadBaseSchema.extend({
35
+ shadowMode: z.boolean().optional(),
36
+ }).superRefine((value, ctx) => {
37
+ if (!value.subject_opaque_id && value.userId === undefined) {
38
+ ctx.addIssue({
39
+ code: "custom",
40
+ message: "VAULT_USER payload must include subject_opaque_id or userId.",
41
+ path: ["subject_opaque_id"],
42
+ });
43
+ }
44
+ }),
45
+ })
46
+ .strict(),
47
+ z.object({
48
+ id: z.string().min(1),
49
+ task_type: z.literal("NOTIFY_USER"),
50
+ payload: taskPayloadBaseSchema,
51
+ })
52
+ .strict(),
53
+ z.object({
54
+ id: z.string().min(1),
55
+ task_type: z.literal("SHRED_USER"),
56
+ payload: taskPayloadBaseSchema,
57
+ })
58
+ .strict(),
59
+ ]);
60
+
61
+ export const syncResponseSchema = z.union([
62
+ z.object({
63
+ pending: z.literal(false),
64
+ })
65
+ .strict(),
66
+ z.object({
67
+ pending: z.literal(true),
68
+ task: syncTaskSchema,
69
+ })
70
+ .strict(),
71
+ ]);
@@ -0,0 +1,4 @@
1
+ export * from "./request-signing";
2
+ export * from "./api";
3
+ export * from "./object-store";
4
+ export * from "./outbox";