dpdp-erasure-cli 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/.env.example +55 -0
  2. package/Dockerfile +33 -0
  3. package/compliance.worker.yaml +64 -0
  4. package/package.json +41 -0
  5. package/src/constants/index.ts +1 -0
  6. package/src/errors/fail.ts +110 -0
  7. package/src/errors/index.ts +4 -0
  8. package/src/errors/inferer.ts +166 -0
  9. package/src/errors/registry.ts +122 -0
  10. package/src/errors/types.ts +65 -0
  11. package/src/errors/worker.ts +161 -0
  12. package/src/index.ts +328 -0
  13. package/src/lib/crypto/digest.ts +22 -0
  14. package/src/lib/crypto/encoding.ts +78 -0
  15. package/src/lib/crypto/index.ts +2 -0
  16. package/src/lib/index.ts +1 -0
  17. package/src/modules/bootstrap/index.ts +2 -0
  18. package/src/modules/bootstrap/integrity.ts +38 -0
  19. package/src/modules/bootstrap/preflight.ts +296 -0
  20. package/src/modules/cli/check-integrity.ts +48 -0
  21. package/src/modules/cli/dry-run.ts +90 -0
  22. package/src/modules/cli/graph.ts +87 -0
  23. package/src/modules/cli/index.ts +184 -0
  24. package/src/modules/cli/init.ts +115 -0
  25. package/src/modules/cli/inspect.ts +86 -0
  26. package/src/modules/cli/introspector.ts +117 -0
  27. package/src/modules/cli/keygen.ts +38 -0
  28. package/src/modules/cli/scan.ts +126 -0
  29. package/src/modules/cli/sign.ts +50 -0
  30. package/src/modules/cli/ui.ts +61 -0
  31. package/src/modules/cli/verify-schema.ts +31 -0
  32. package/src/modules/cli/verify.ts +85 -0
  33. package/src/modules/config/compatibility.ts +271 -0
  34. package/src/modules/config/index.ts +4 -0
  35. package/src/modules/config/reader.ts +149 -0
  36. package/src/modules/config/signature.ts +69 -0
  37. package/src/modules/config/validation.ts +658 -0
  38. package/src/modules/crypto/aes.ts +158 -0
  39. package/src/modules/crypto/envelope.ts +48 -0
  40. package/src/modules/crypto/hmac.ts +60 -0
  41. package/src/modules/crypto/index.ts +3 -0
  42. package/src/modules/db/drift.ts +36 -0
  43. package/src/modules/db/graph.ts +203 -0
  44. package/src/modules/db/index.ts +4 -0
  45. package/src/modules/db/migrations.ts +254 -0
  46. package/src/modules/db/sql-debug.ts +61 -0
  47. package/src/modules/engine/blob/index.ts +3 -0
  48. package/src/modules/engine/blob/s3.ts +455 -0
  49. package/src/modules/engine/blob/store.ts +236 -0
  50. package/src/modules/engine/blob/types.ts +44 -0
  51. package/src/modules/engine/helpers/identity.ts +47 -0
  52. package/src/modules/engine/helpers/index.ts +4 -0
  53. package/src/modules/engine/helpers/outbox.ts +118 -0
  54. package/src/modules/engine/helpers/runtime.ts +115 -0
  55. package/src/modules/engine/helpers/types.ts +61 -0
  56. package/src/modules/engine/index.ts +6 -0
  57. package/src/modules/engine/notifier/config.ts +147 -0
  58. package/src/modules/engine/notifier/dispatcher.ts +300 -0
  59. package/src/modules/engine/notifier/index.ts +3 -0
  60. package/src/modules/engine/notifier/payload.ts +51 -0
  61. package/src/modules/engine/notifier/reservation.ts +153 -0
  62. package/src/modules/engine/notifier/types.ts +38 -0
  63. package/src/modules/engine/shredder.ts +254 -0
  64. package/src/modules/engine/types.ts +146 -0
  65. package/src/modules/engine/vault/compiled-targets.ts +562 -0
  66. package/src/modules/engine/vault/context.ts +254 -0
  67. package/src/modules/engine/vault/dry-run.ts +94 -0
  68. package/src/modules/engine/vault/execution.ts +485 -0
  69. package/src/modules/engine/vault/index.ts +3 -0
  70. package/src/modules/engine/vault/purge.ts +82 -0
  71. package/src/modules/engine/vault/retention.ts +124 -0
  72. package/src/modules/engine/vault/satellite-mutation.ts +193 -0
  73. package/src/modules/engine/vault/satellite.ts +103 -0
  74. package/src/modules/engine/vault/shadow.ts +36 -0
  75. package/src/modules/engine/vault/static-plan.ts +116 -0
  76. package/src/modules/engine/vault/store.ts +34 -0
  77. package/src/modules/engine/vault/vault.ts +84 -0
  78. package/src/modules/introspector/classifier.ts +502 -0
  79. package/src/modules/introspector/dag.ts +276 -0
  80. package/src/modules/introspector/index.ts +7 -0
  81. package/src/modules/introspector/naming.ts +75 -0
  82. package/src/modules/introspector/report.ts +153 -0
  83. package/src/modules/introspector/run.ts +123 -0
  84. package/src/modules/introspector/s3-sampler.ts +227 -0
  85. package/src/modules/introspector/types.ts +131 -0
  86. package/src/modules/introspector/yaml.ts +101 -0
  87. package/src/modules/network/api/control-plane.ts +275 -0
  88. package/src/modules/network/api/index.ts +1 -0
  89. package/src/modules/network/api/validation.ts +71 -0
  90. package/src/modules/network/index.ts +4 -0
  91. package/src/modules/network/object-store/aws/client.ts +444 -0
  92. package/src/modules/network/object-store/aws/credentials.ts +271 -0
  93. package/src/modules/network/object-store/aws/index.ts +2 -0
  94. package/src/modules/network/object-store/aws/sigv4.ts +190 -0
  95. package/src/modules/network/object-store/aws/type.ts +6 -0
  96. package/src/modules/network/object-store/index.ts +1 -0
  97. package/src/modules/network/outbox/dispatcher.ts +183 -0
  98. package/src/modules/network/outbox/index.ts +3 -0
  99. package/src/modules/network/outbox/process.ts +133 -0
  100. package/src/modules/network/outbox/shared.ts +56 -0
  101. package/src/modules/network/outbox/store.ts +346 -0
  102. package/src/modules/network/outbox/types.ts +54 -0
  103. package/src/modules/network/request-signing.ts +61 -0
  104. package/src/modules/worker/index.ts +2 -0
  105. package/src/modules/worker/tasks.ts +58 -0
  106. package/src/modules/worker/types.ts +89 -0
  107. package/src/modules/worker/worker.ts +243 -0
  108. package/src/secrets/index.ts +4 -0
  109. package/src/secrets/kms/index.ts +2 -0
  110. package/src/secrets/kms/signature.ts +82 -0
  111. package/src/secrets/kms/validation.ts +64 -0
  112. package/src/secrets/reader.ts +42 -0
  113. package/src/secrets/repository/crypto.ts +89 -0
  114. package/src/secrets/repository/index.ts +2 -0
  115. package/src/secrets/repository/methods.ts +37 -0
  116. package/src/secrets/resolvers.ts +247 -0
  117. package/src/secrets/signature.ts +78 -0
  118. package/src/types/index.ts +1 -0
  119. package/src/types/types.ts +23 -0
  120. package/src/utils/identifiers.ts +48 -0
  121. package/src/utils/index.ts +3 -0
  122. package/src/utils/json.ts +35 -0
  123. package/src/utils/logger.ts +161 -0
  124. package/src/validation/zod.ts +70 -0
  125. package/tests/adversarial.test.ts +464 -0
  126. package/tests/blob-s3.test.ts +216 -0
  127. package/tests/config.test.ts +395 -0
  128. package/tests/control-plane-client.test.ts +108 -0
  129. package/tests/crypto.test.ts +106 -0
  130. package/tests/errors.test.ts +69 -0
  131. package/tests/fetch-dispatcher.test.ts +213 -0
  132. package/tests/graph.test.ts +84 -0
  133. package/tests/helpers/index.ts +101 -0
  134. package/tests/index-preflight.test.ts +168 -0
  135. package/tests/introspector-classifier.test.ts +62 -0
  136. package/tests/introspector-report.test.ts +85 -0
  137. package/tests/introspector.test.ts +394 -0
  138. package/tests/kms.test.ts +124 -0
  139. package/tests/logger.test.ts +61 -0
  140. package/tests/notifier.test.ts +303 -0
  141. package/tests/outbox.test.ts +478 -0
  142. package/tests/purge-policy.test.ts +124 -0
  143. package/tests/retention.test.ts +103 -0
  144. package/tests/s3-client.test.ts +110 -0
  145. package/tests/satellite.test.ts +119 -0
  146. package/tests/schema-compatibility.test.ts +237 -0
  147. package/tests/schema-integrity.test.ts +64 -0
  148. package/tests/shredder.test.ts +163 -0
  149. package/tests/vault.compiled-targets.test.ts +243 -0
  150. package/tests/vault.replica.test.ts +59 -0
  151. package/tests/vault.test.ts +279 -0
  152. package/tests/worker.retry.test.ts +291 -0
  153. package/tests/worker.test.ts +200 -0
  154. package/tsconfig.json +19 -0
  155. package/vitest.config.ts +13 -0
@@ -0,0 +1,85 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import {
3
+ type IntrospectorDraft,
4
+ buildIntrospectorReport,
5
+ renderIntrospectorJson,
6
+ renderIntrospectorMarkdown,
7
+ } from "@modules/introspector";
8
+
9
+ const draft: IntrospectorDraft = {
10
+ root: { schema: "public", table: "users" },
11
+ maxDepth: 32,
12
+ generatedAt: "2026-05-11T00:00:00.000Z",
13
+ schemaHash: "a".repeat(64),
14
+ potentialLogicalLinks: [
15
+ {
16
+ sourceTable: { schema: "public", table: "orders" },
17
+ targetTable: { schema: "public", table: "support_events" },
18
+ column: "user_id",
19
+ reason: "Both tables expose user_id but no physical foreign key was found.",
20
+ },
21
+ ],
22
+ targets: [
23
+ {
24
+ table: { schema: "public", table: "users" },
25
+ parentTable: null,
26
+ fkCondition: "ROOT",
27
+ childColumns: [],
28
+ parentColumns: [],
29
+ depth: 0,
30
+ piiColumns: [
31
+ {
32
+ table: { schema: "public", table: "users" },
33
+ column: "email",
34
+ dataType: "text",
35
+ metadataScore: 0.92,
36
+ contentMatchRatio: 1,
37
+ confidence: 0.95,
38
+ sampleSize: 100,
39
+ matchedSignatures: ["email"],
40
+ },
41
+ {
42
+ table: { schema: "public", table: "users" },
43
+ column: "bank_account_number",
44
+ dataType: "text",
45
+ metadataScore: 0.82,
46
+ contentMatchRatio: 0,
47
+ confidence: 0.82,
48
+ sampleSize: 0,
49
+ matchedSignatures: [],
50
+ },
51
+ ],
52
+ },
53
+ ],
54
+ };
55
+
56
+ describe("Introspector report rendering", () => {
57
+ it("builds actionable report summaries without leaking sampled values", () => {
58
+ const report = buildIntrospectorReport(draft);
59
+
60
+ expect(report.summary.rootTable).toBe("public.users");
61
+ expect(report.summary.piiColumnCount).toBe(2);
62
+ expect(report.summary.highConfidenceCount).toBe(1);
63
+ expect(report.summary.reviewRequiredCount).toBe(1);
64
+ expect(report.summary.potentialLogicalLinkCount).toBe(1);
65
+ expect(report.findings.map((finding) => finding.column)).toEqual(["email", "bank_account_number"]);
66
+ expect(JSON.stringify(report)).not.toContain("alpha@example.com");
67
+ });
68
+
69
+ it("renders deterministic Markdown and JSON artifacts for developer review", () => {
70
+ const report = buildIntrospectorReport(draft);
71
+ const markdown = renderIntrospectorMarkdown(report);
72
+ const json = renderIntrospectorJson(report);
73
+
74
+ expect(markdown).toContain("# Compliance Introspector Report");
75
+ expect(markdown).toContain("`public.users`");
76
+ expect(markdown).toContain("Potential Logical Links");
77
+ expect(markdown).toContain("compliance-worker check-integrity");
78
+ expect(JSON.parse(json)).toMatchObject({
79
+ summary: {
80
+ rootTable: "public.users",
81
+ piiColumnCount: 2,
82
+ },
83
+ });
84
+ });
85
+ });
@@ -0,0 +1,394 @@
1
+ import { afterAll, beforeAll, describe, expect, it } from "vitest";
2
+ import { writeFile, mkdir } from "node:fs/promises";
3
+ import { dirname } from "node:path";
4
+ import {
5
+ classifyLeaf,
6
+ compileStaticDag,
7
+ extractLeafValues,
8
+ runIntrospector,
9
+ sampleS3ObjectChunk,
10
+ sampleS3ObjectForClassification,
11
+ validateAadhaar,
12
+ validateGstin,
13
+ validateLuhn,
14
+ validatePan,
15
+ verifySchemaIntegrity,
16
+ } from "@modules/introspector";
17
+ import { detectSchemaDrift } from "@modules/db";
18
+ import { createTestSql, dropSchemas, uniqueSchema } from "./helpers";
19
+ import type { Sql } from "@/types";
20
+
21
+ async function gzipBytes(bytes: Uint8Array): Promise<Uint8Array> {
22
+ const copy = new Uint8Array(new ArrayBuffer(bytes.byteLength));
23
+ copy.set(bytes);
24
+
25
+ if (typeof Bun !== "undefined") {
26
+ return Bun.gzipSync(copy);
27
+ }
28
+
29
+ const stream = new Blob([copy.buffer]).stream().pipeThrough(new CompressionStream("gzip"));
30
+ return new Uint8Array(await new Response(stream).arrayBuffer());
31
+ }
32
+
33
+ async function writeText(path: string, value: string): Promise<void> {
34
+ await mkdir(dirname(path), { recursive: true });
35
+
36
+ if (typeof Bun !== "undefined") {
37
+ await Bun.write(path, value);
38
+ } else {
39
+ await writeFile(path, value, "utf8");
40
+ }
41
+ }
42
+
43
+ async function readText(path: string): Promise<string> {
44
+ let content: string;
45
+
46
+ if (typeof Bun !== "undefined") {
47
+ content = await Bun.file(path).text();
48
+ } else {
49
+ const { readFile } = await import("node:fs/promises");
50
+ content = await readFile(path, "utf8");
51
+ }
52
+
53
+ return content.trim();
54
+ }
55
+
56
+ describe("Offline Introspector", () => {
57
+ let sql: Sql;
58
+ const schemasToDrop: string[] = [];
59
+
60
+ beforeAll(() => {
61
+ sql = createTestSql();
62
+ });
63
+
64
+ afterAll(async () => {
65
+ await dropSchemas(sql, ...schemasToDrop);
66
+ await sql.end();
67
+ }, 60_000);
68
+
69
+ async function prepareSchema() {
70
+ const schema = uniqueSchema("introspector");
71
+ schemasToDrop.push(schema);
72
+ await dropSchemas(sql, schema);
73
+ await sql`CREATE SCHEMA ${sql(schema)}`;
74
+ await sql`
75
+ CREATE TABLE ${sql(schema)}.users (
76
+ id SERIAL PRIMARY KEY,
77
+ email TEXT NOT NULL,
78
+ phone TEXT NOT NULL,
79
+ full_name TEXT NOT NULL,
80
+ upi_id TEXT NOT NULL,
81
+ card_number TEXT NOT NULL,
82
+ gstin TEXT NOT NULL,
83
+ random_digits TEXT NOT NULL
84
+ )
85
+ `;
86
+ await sql`
87
+ CREATE TABLE ${sql(schema)}.profiles (
88
+ id SERIAL PRIMARY KEY,
89
+ user_id INTEGER NOT NULL REFERENCES ${sql(schema)}.users(id),
90
+ pan TEXT NOT NULL,
91
+ aadhaar_payload JSONB NOT NULL,
92
+ nested_payload JSONB NOT NULL
93
+ )
94
+ `;
95
+ await sql`
96
+ CREATE TABLE ${sql(schema)}.orders (
97
+ id SERIAL PRIMARY KEY,
98
+ user_id INTEGER NOT NULL REFERENCES ${sql(schema)}.users(id),
99
+ receipt_code TEXT NOT NULL
100
+ )
101
+ `;
102
+ await sql`
103
+ CREATE TABLE ${sql(schema)}.kyc_reviews (
104
+ id SERIAL PRIMARY KEY,
105
+ profile_id INTEGER NOT NULL REFERENCES ${sql(schema)}.profiles(id),
106
+ reviewer_note TEXT NOT NULL
107
+ )
108
+ `;
109
+ await sql`
110
+ CREATE TABLE ${sql(schema)}.support_events (
111
+ id SERIAL PRIMARY KEY,
112
+ user_id INTEGER NOT NULL,
113
+ event_name TEXT NOT NULL
114
+ )
115
+ `;
116
+ await sql`
117
+ INSERT INTO ${sql(schema)}.users (email, phone, full_name, upi_id, card_number, gstin, random_digits)
118
+ VALUES
119
+ ('alpha@example.com', '+91-9876543210', 'Alpha User', 'alpha.user@upi', '4111 1111 1111 1111', '27ABCPE1234F1Z5', '123456789012'),
120
+ ('beta@example.com', '9876543211', 'Beta User', 'beta.user@upi', '5555 5555 5555 4444', '27PQRPT9876Z1Z5', '987654321234')
121
+ `;
122
+ await sql`
123
+ INSERT INTO ${sql(schema)}.profiles (user_id, pan, aadhaar_payload, nested_payload)
124
+ VALUES
125
+ (1, 'ABCPE1234F', ${sql.json({ government_id: "2345 6789 1238" })}, ${sql.json({ profile: { contact_email: "redacted" } })}),
126
+ (2, 'PQRPT9876Z', ${sql.json({ nested: { aadhaar: "3456 7891 2342" } })}, ${sql.json({ pii: { phone_number: "not-collected" } })})
127
+ `;
128
+ await sql`
129
+ INSERT INTO ${sql(schema)}.orders (user_id, receipt_code)
130
+ VALUES (1, 'R-100'), (2, 'R-200')
131
+ `;
132
+ await sql`
133
+ INSERT INTO ${sql(schema)}.kyc_reviews (profile_id, reviewer_note)
134
+ VALUES (1, 'verified'), (2, 'verified')
135
+ `;
136
+ await sql`
137
+ INSERT INTO ${sql(schema)}.support_events (user_id, event_name)
138
+ VALUES (1, 'ticket_opened'), (2, 'ticket_closed')
139
+ `;
140
+ return schema;
141
+ }
142
+
143
+ it("uses checksum-backed signatures and iterative JSON flattening to reduce false positives", () => {
144
+ expect(validateAadhaar("2345 6789 1238")).toBe(true);
145
+ expect(validateAadhaar("2345 6789 1234")).toBe(false);
146
+ expect(validateAadhaar("9999 9999 9999")).toBe(false);
147
+ expect(validateLuhn("4111 1111 1111 1111")).toBe(true);
148
+ expect(validateLuhn("4111 1111 1111 1112")).toBe(false);
149
+ expect(validatePan("ABCPE1234F")).toBe(true);
150
+ expect(validatePan("ABCDE1234F")).toBe(false);
151
+ expect(validateGstin("27ABCPE1234F1Z5")).toBe(true);
152
+ expect(validateGstin("27ABCDE1234F1Z5")).toBe(false);
153
+
154
+ expect(classifyLeaf("2345 6789 1234")).not.toContain("aadhaar");
155
+ expect(classifyLeaf("2345 6789 1238")).toContain("aadhaar");
156
+ expect(classifyLeaf("4111 1111 1111 1112")).not.toContain("credit_card");
157
+ expect(classifyLeaf("123456789012")).not.toContain("bank_account");
158
+ expect(classifyLeaf("123456789012", "bank_account_number")).toContain("bank_account");
159
+ expect(classifyLeaf("Alpha User")).toEqual([]);
160
+
161
+ const deepPayload = { a: { b: { c: { d: { e: { f: { g: { h: { i: { j: { k: "too-deep@example.com" } } } } } } } } } } };
162
+ expect(extractLeafValues(deepPayload, "jsonb")).toEqual([]);
163
+ });
164
+
165
+ it("compiles a static FK DAG from the root table with bounded depth", async () => {
166
+ const schema = await prepareSchema();
167
+
168
+ const dag = await compileStaticDag({
169
+ sql,
170
+ rootTable: `${schema}.users`,
171
+ maxDepth: 32,
172
+ });
173
+
174
+ expect(dag.map((target) => `${target.depth}:${target.table.schema}.${target.table.table}`)).toEqual([
175
+ `0:${schema}.users`,
176
+ `1:${schema}.orders`,
177
+ `1:${schema}.profiles`,
178
+ `2:${schema}.kyc_reviews`,
179
+ ]);
180
+ expect(dag.find((target) => target.table.table === "profiles")?.fkCondition).toBe(
181
+ `${schema}.users.id = ${schema}.profiles.user_id`
182
+ );
183
+ }, 60_000);
184
+
185
+ it("classifies PII with metadata plus bounded content sampling and renders a draft YAML", async () => {
186
+ const schema = await prepareSchema();
187
+
188
+ const { draft, yaml } = await runIntrospector({
189
+ sql,
190
+ rootTable: `${schema}.users`,
191
+ samplePercent: 100,
192
+ sampleLimit: 100,
193
+ threshold: 0.75,
194
+ generatedAt: new Date("2026-04-27T00:00:00.000Z"),
195
+ });
196
+
197
+ const users = draft.targets.find((target) => target.table.table === "users");
198
+ const profiles = draft.targets.find((target) => target.table.table === "profiles");
199
+ const orders = draft.targets.find((target) => target.table.table === "orders");
200
+
201
+ expect(users?.piiColumns.map((column) => column.column).sort()).toEqual([
202
+ "card_number",
203
+ "email",
204
+ "gstin",
205
+ "phone",
206
+ "upi_id",
207
+ ]);
208
+ expect(profiles?.piiColumns.map((column) => column.column).sort()).toEqual([
209
+ "aadhaar_payload",
210
+ "nested_payload",
211
+ "pan",
212
+ ]);
213
+ expect(orders?.piiColumns).toEqual([]);
214
+ expect(yaml).toContain("rules:");
215
+ expect(yaml).toContain(`root_table: ${schema}.users`);
216
+ expect(yaml).toContain(`table: ${schema}.profiles`);
217
+ expect(yaml).toContain("pii_columns: [pan, aadhaar_payload, nested_payload]");
218
+ expect(yaml).not.toContain("full_name");
219
+ expect(yaml).toContain("schema_hash:");
220
+ expect(yaml).toContain("generated_by: compliance-introspector-v1");
221
+ expect(yaml).toContain("legal_disclaimer:");
222
+ expect(yaml).toContain("[Potential Logical Link]");
223
+ expect(yaml).toContain(`${schema}.orders.user_id <-> ${schema}.support_events.user_id`);
224
+ expect(yaml).toContain("# REVIEW REQUIRED");
225
+ }, 60_000);
226
+
227
+ it("uses a bounded S3 Range request for object-store sampling", async () => {
228
+ const body = new Uint8Array([65, 66, 67]);
229
+ const calls: Array<{ url: string; range: string | null; authorization: string | null }> = [];
230
+ const fetchFn = (async (url, init) => {
231
+ const headers = new Headers(init?.headers);
232
+ calls.push({
233
+ url: String(url),
234
+ range: headers.get("range"),
235
+ authorization: headers.get("authorization"),
236
+ });
237
+ return new Response(body.slice(), { status: 206 });
238
+ }) as typeof fetch;
239
+
240
+ const chunk = await sampleS3ObjectChunk({
241
+ bucket: "client-vault",
242
+ key: "kyc/user-1.pdf",
243
+ region: "ap-south-1",
244
+ maxBytes: 1024,
245
+ fetchFn,
246
+ credentials: {
247
+ accessKeyId: "AKIA_TEST",
248
+ secretAccessKey: "secret",
249
+ },
250
+ });
251
+
252
+ try {
253
+ expect(new TextDecoder().decode(chunk)).toBe("ABC");
254
+ expect(calls).toHaveLength(1);
255
+ expect(calls[0]?.range).toBe("bytes=0-1023");
256
+ expect(calls[0]?.authorization).toContain("AWS4-HMAC-SHA256");
257
+ expect(calls[0]?.url).toBe("https://client-vault.s3.ap-south-1.amazonaws.com/kyc/user-1.pdf");
258
+ } finally {
259
+ chunk.fill(0);
260
+ }
261
+ });
262
+
263
+ it("decompresses gzip prefixes and flags binary structured formats before regex scanning", async () => {
264
+ const gzipped = await gzipBytes(new TextEncoder().encode("alpha@example.com"));
265
+ const gzipFetch = (async () =>
266
+ new Response(gzipped.slice(), {
267
+ status: 206,
268
+ headers: { "content-type": "application/gzip" },
269
+ })) as unknown as typeof fetch;
270
+
271
+ const gzipSample = await sampleS3ObjectForClassification({
272
+ bucket: "client-vault",
273
+ key: "logs/users.json.gz",
274
+ region: "ap-south-1",
275
+ fetchFn: gzipFetch,
276
+ credentials: {
277
+ accessKeyId: "AKIA_TEST",
278
+ secretAccessKey: "secret",
279
+ },
280
+ });
281
+
282
+ try {
283
+ expect(gzipSample.decompressed).toBe(true);
284
+ expect(new TextDecoder().decode(gzipSample.bytes)).toBe("alpha@example.com");
285
+ } finally {
286
+ gzipSample.bytes.fill(0);
287
+ }
288
+
289
+ const parquetFetch = (async () =>
290
+ new Response(new Uint8Array([0x50, 0x41, 0x52, 0x31, 0x00]).slice(), { status: 206 })) as unknown as typeof fetch;
291
+ const parquetSample = await sampleS3ObjectForClassification({
292
+ bucket: "client-vault",
293
+ key: "warehouse/users.parquet",
294
+ region: "ap-south-1",
295
+ fetchFn: parquetFetch,
296
+ credentials: {
297
+ accessKeyId: "AKIA_TEST",
298
+ secretAccessKey: "secret",
299
+ },
300
+ });
301
+
302
+ expect(parquetSample.binaryFormat).toBe("parquet");
303
+ expect(parquetSample.bytes).toHaveLength(0);
304
+ expect(parquetSample.warnings).toContain("BINARY_FORMAT_DETECTED: Structural Metadata Scan Required.");
305
+ });
306
+
307
+ it("verifies legal-attested schema hashes for CI/CD gates", async () => {
308
+ const schema = await prepareSchema();
309
+ const liveHash = await detectSchemaDrift(sql, schema);
310
+ const path = `/tmp/compliance-introspector-${crypto.randomUUID()}.yml`;
311
+ await writeText(
312
+ path,
313
+ `
314
+ version: "1.0"
315
+ database:
316
+ app_schema: ${schema}
317
+ engine_schema: dpdp_engine
318
+ compliance_policy:
319
+ default_retention_years: 0
320
+ notice_window_hours: 48
321
+ retention_rules: []
322
+ graph:
323
+ root_table: users
324
+ root_id_column: id
325
+ max_depth: 32
326
+ root_pii_columns:
327
+ email: HMAC
328
+ satellite_targets:
329
+ - table: support_events
330
+ lookup_column: user_id
331
+ action: hard_delete
332
+ blob_targets: []
333
+ rules:
334
+ - id: dpdp_standard
335
+ root_table: ${schema}.users
336
+ targets:
337
+ - table: ${schema}.users
338
+ pii_columns: [email]
339
+ - table: ${schema}.orders
340
+ parent: ${schema}.users
341
+ join: "${schema}.users.id = ${schema}.orders.user_id"
342
+ pii_columns: []
343
+ - table: ${schema}.profiles
344
+ parent: ${schema}.users
345
+ join: "${schema}.users.id = ${schema}.profiles.user_id"
346
+ pii_columns: [pan, aadhaar_payload, nested_payload]
347
+ - table: ${schema}.kyc_reviews
348
+ parent: ${schema}.profiles
349
+ join: "${schema}.profiles.id = ${schema}.kyc_reviews.profile_id"
350
+ pii_columns: []
351
+ - table: ${schema}.support_events
352
+ pii_columns: []
353
+ outbox:
354
+ batch_size: 10
355
+ lease_seconds: 60
356
+ max_attempts: 3
357
+ security:
358
+ master_key_env: DPDP_MASTER_KEY
359
+ hmac_key_env: DPDP_HMAC_KEY
360
+ integrity:
361
+ expected_schema_hash: "${"0".repeat(64)}"
362
+ legal_attestation:
363
+ dpo_identifier: dpo@example.com
364
+ configuration_version: v-test
365
+ legal_review_date: "2026-04-20"
366
+ schema_hash: "${liveHash}"
367
+ generated_by: compliance-introspector-v1
368
+ acknowledgment: reviewed
369
+ `
370
+ );
371
+
372
+ const result = await verifySchemaIntegrity({
373
+ sql,
374
+ configPath: path,
375
+ env: {
376
+ DPDP_MASTER_KEY: "0".repeat(64),
377
+ DPDP_HMAC_KEY: "0".repeat(64),
378
+ },
379
+ });
380
+ expect(result).toBe(liveHash);
381
+
382
+ await writeText(path, (await readText(path)).replace(liveHash, "1".repeat(64)));
383
+ await expect(
384
+ verifySchemaIntegrity({
385
+ sql,
386
+ configPath: path,
387
+ env: {
388
+ DPDP_MASTER_KEY: "0".repeat(64),
389
+ DPDP_HMAC_KEY: "0".repeat(64),
390
+ },
391
+ })
392
+ ).rejects.toThrow(/does not match legal attestation hash/i);
393
+ });
394
+ });
@@ -0,0 +1,124 @@
1
+ import { describe, expect, it, vi } from "vitest";
2
+ import { keySourceSchema, resolveConfiguredKey } from "@/secrets";
3
+ import { bytesToBase64 } from "@/lib";
4
+
5
+ const keyBytes = new Uint8Array(32).fill(0x7a);
6
+ const keyBase64 = bytesToBase64(keyBytes);
7
+
8
+ function jsonResponse(body: unknown, status = 200): Response {
9
+ return new Response(JSON.stringify(body), {
10
+ status,
11
+ headers: {
12
+ "content-type": "application/json",
13
+ },
14
+ });
15
+ }
16
+
17
+ describe("runtime KMS key sources", () => {
18
+ it("resolves AWS KMS plaintext through a signed native fetch request", async () => {
19
+ const fetchFn = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
20
+ const headers = new Headers(init?.headers);
21
+ expect(headers.get("authorization")).toMatch(/^AWS4-HMAC-SHA256 /);
22
+ expect(headers.get("x-amz-target")).toBe("TrentService.Decrypt");
23
+ expect(headers.get("x-amz-content-sha256")).toMatch(/^[0-9a-f]{64}$/);
24
+ return jsonResponse({ Plaintext: keyBase64 });
25
+ }) as unknown as typeof fetch;
26
+
27
+ const resolved = await resolveConfiguredKey({
28
+ env: {
29
+ AWS_ACCESS_KEY_ID: "AKIATEST",
30
+ AWS_SECRET_ACCESS_KEY: "secret",
31
+ },
32
+ keyName: "DPDP_MASTER_KEY",
33
+ legacyEnvName: "DPDP_MASTER_KEY",
34
+ fetchFn,
35
+ source: keySourceSchema.parse({
36
+ provider: "aws_kms",
37
+ region: "ap-south-1",
38
+ endpoint: "https://kms.ap-south-1.amazonaws.com/",
39
+ ciphertext_blob_base64: "ciphertext",
40
+ }),
41
+ });
42
+
43
+ expect(resolved).toEqual(keyBytes);
44
+ });
45
+
46
+ it("resolves GCP Secret Manager payload.data as URL-safe base64", async () => {
47
+ const fetchFn = vi.fn(async () =>
48
+ jsonResponse({
49
+ payload: {
50
+ data: keyBase64.replace(/\+/g, "-").replace(/\//g, "_"),
51
+ },
52
+ })
53
+ ) as unknown as typeof fetch;
54
+
55
+ const resolved = await resolveConfiguredKey({
56
+ env: {
57
+ GCP_ACCESS_TOKEN: "token",
58
+ },
59
+ keyName: "DPDP_MASTER_KEY",
60
+ legacyEnvName: "DPDP_MASTER_KEY",
61
+ fetchFn,
62
+ source: keySourceSchema.parse({
63
+ provider: "gcp_secret_manager",
64
+ secret_version: "projects/p/secrets/dpdp-master-key/versions/latest",
65
+ }),
66
+ });
67
+
68
+ expect(resolved).toEqual(keyBytes);
69
+ });
70
+
71
+ it("resolves HashiCorp Vault KV v2 data.data field without exposing the token", async () => {
72
+ const fetchFn = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => {
73
+ const headers = new Headers(init?.headers);
74
+ expect(headers.get("x-vault-token")).toBe("vault-token");
75
+ return jsonResponse({
76
+ data: {
77
+ data: {
78
+ key: `base64:${keyBase64}`,
79
+ },
80
+ },
81
+ });
82
+ }) as unknown as typeof fetch;
83
+
84
+ const resolved = await resolveConfiguredKey({
85
+ env: {
86
+ VAULT_ADDR: "https://vault.example.com",
87
+ VAULT_TOKEN: "vault-token",
88
+ },
89
+ keyName: "DPDP_MASTER_KEY",
90
+ legacyEnvName: "DPDP_MASTER_KEY",
91
+ fetchFn,
92
+ source: keySourceSchema.parse({
93
+ provider: "hashicorp_vault",
94
+ mount: "secret",
95
+ path: "dpdp/master-key",
96
+ field: "key",
97
+ }),
98
+ });
99
+
100
+ expect(resolved).toEqual(keyBytes);
101
+ });
102
+
103
+ it("fails closed when a remote provider returns malformed key material", async () => {
104
+ const fetchFn = vi.fn(async () => jsonResponse({ Plaintext: bytesToBase64(new Uint8Array(8)) })) as unknown as typeof fetch;
105
+
106
+ await expect(
107
+ resolveConfiguredKey({
108
+ env: {
109
+ AWS_ACCESS_KEY_ID: "AKIATEST",
110
+ AWS_SECRET_ACCESS_KEY: "secret",
111
+ },
112
+ keyName: "DPDP_MASTER_KEY",
113
+ legacyEnvName: "DPDP_MASTER_KEY",
114
+ fetchFn,
115
+ source: keySourceSchema.parse({
116
+ provider: "aws_kms",
117
+ region: "ap-south-1",
118
+ endpoint: "https://kms.ap-south-1.amazonaws.com/",
119
+ ciphertext_blob_base64: "ciphertext",
120
+ }),
121
+ })
122
+ ).rejects.toThrow(/exactly 32 bytes/i);
123
+ });
124
+ });
@@ -0,0 +1,61 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import type { DestinationStream } from "pino";
3
+ import { createWorkerLogger } from "@/utils";
4
+ import { redactSqlDebugParameters } from "@modules/db";
5
+
6
+ describe("Pino worker logger", () => {
7
+ it("redacts sensitive fields before they leave the process", async () => {
8
+ const chunks: string[] = [];
9
+ const destination = {
10
+ write(chunk: string) {
11
+ chunks.push(chunk);
12
+ return true;
13
+ },
14
+ } as DestinationStream;
15
+ const logger = createWorkerLogger({}, destination);
16
+
17
+ logger.info({
18
+ authorization: "Bearer secret-token",
19
+ email: "pii@example.com",
20
+ full_name: "Sensitive User",
21
+ payload: {
22
+ data: "ciphertext",
23
+ email: "payload@example.com",
24
+ full_name: "Payload User",
25
+ },
26
+ encrypted_pii: {
27
+ data: "top-secret",
28
+ },
29
+ }, "redaction-check");
30
+
31
+ const logRecord = JSON.parse(chunks.join("").trim()) as Record<string, unknown>;
32
+
33
+ expect(logRecord.authorization).toBe("[REDACTED]");
34
+ expect(logRecord.email).toBe("[REDACTED]");
35
+ expect(logRecord.full_name).toBe("[REDACTED]");
36
+ expect(logRecord.payload).toEqual({
37
+ data: "[REDACTED]",
38
+ email: "[REDACTED]",
39
+ full_name: "[REDACTED]",
40
+ });
41
+ expect(logRecord.encrypted_pii).toBe("[REDACTED]");
42
+ });
43
+
44
+ it("redacts postgres.js debug parameters when SQL references configured PII columns", () => {
45
+ expect(
46
+ redactSqlDebugParameters(
47
+ "UPDATE tenant.users SET email = $1 WHERE id = $2",
48
+ ["alice@example.com", "usr_123"],
49
+ ["email", "full_name"]
50
+ )
51
+ ).toEqual(["[REDACTED]", "[REDACTED]"]);
52
+
53
+ expect(
54
+ redactSqlDebugParameters(
55
+ "SELECT id FROM tenant.orders WHERE user_id = $1",
56
+ ["usr_123"],
57
+ ["email", "full_name"]
58
+ )
59
+ ).toEqual(["usr_123"]);
60
+ });
61
+ });