@nodii/dsar 0.28.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/bind-pattern-a.d.ts.map +1 -1
- package/dist/bind-pattern-a.js +38 -9
- package/dist/bind-pattern-a.js.map +1 -1
- package/dist/holds.d.ts +12 -0
- package/dist/holds.d.ts.map +1 -1
- package/dist/holds.js +41 -2
- package/dist/holds.js.map +1 -1
- package/dist/index.d.ts +9 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -5
- package/dist/index.js.map +1 -1
- package/dist/migrations/index.d.ts.map +1 -1
- package/dist/migrations/index.js +22 -0
- package/dist/migrations/index.js.map +1 -1
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +40 -10
- package/dist/orchestrator.js.map +1 -1
- package/dist/retention-sweeper.d.ts +117 -16
- package/dist/retention-sweeper.d.ts.map +1 -1
- package/dist/retention-sweeper.js +482 -187
- package/dist/retention-sweeper.js.map +1 -1
- package/dist/s3-artifact.d.ts +10 -0
- package/dist/s3-artifact.d.ts.map +1 -1
- package/dist/s3-artifact.js +15 -0
- package/dist/s3-artifact.js.map +1 -1
- package/dist/tenant-erase.d.ts +16 -0
- package/dist/tenant-erase.d.ts.map +1 -0
- package/dist/tenant-erase.js +31 -0
- package/dist/tenant-erase.js.map +1 -0
- package/dist/types.d.ts +17 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +25 -0
- package/dist/types.js.map +1 -1
- package/package.json +3 -3
- package/src/migrations/001-lifecycle-requests.sql +22 -0
|
@@ -1,5 +1,13 @@
|
|
|
1
|
-
// D720 — the PURPOSE-SPENT SWEEPER:
|
|
2
|
-
// window
|
|
1
|
+
// D720 — the PURPOSE-SPENT SWEEPER: marks a record `retention_period_expired`
|
|
2
|
+
// once its retention window ends, and hard-deletes it only when the Ops control
|
|
3
|
+
// "Auto-delete after retention ends" is on for its data class.
|
|
4
|
+
//
|
|
5
|
+
// EXPIRY IS NOT DELETION (RETN-56 to RETN-58). Whether a record must, may or
|
|
6
|
+
// must not be deleted when its legal period ends is a lawyer's question, and
|
|
7
|
+
// the control stays off for a class until a lawyer has ruled on it. So the
|
|
8
|
+
// default outcome of a sweep is a status change, and `autoDelete` is the only
|
|
9
|
+
// way to reach `onPurge`. A sweeper with no `autoDelete`, or whose `autoDelete`
|
|
10
|
+
// cannot be read, never deletes anything.
|
|
3
11
|
//
|
|
4
12
|
// WHY A SWEEPER AND NOT A DSAR HANDLER. DPDP s.8(7) owes erasure WITHOUT a
|
|
5
13
|
// request once the purpose is spent, and GDPR storage limitation says the same
|
|
@@ -12,14 +20,15 @@
|
|
|
12
20
|
// THE STATUTORY BUCKET'S SECOND DEATH. A statutory record survives every
|
|
13
21
|
// per-subject erase by construction (it is under the tenant retention DEK, not
|
|
14
22
|
// the person's). This job is how it eventually dies: its `retention_schedules`
|
|
15
|
-
// window expires, no hold covers it, and `onPurge` deletes
|
|
23
|
+
// window expires, no hold covers it, it is marked expired, and `onPurge` deletes
|
|
24
|
+
// the row once the class's control is on.
|
|
16
25
|
//
|
|
17
26
|
// SHAPE mirrors the lifecycle reaper (`state-store/reaper-job.ts`): a
|
|
18
27
|
// framework-agnostic `{ name, intervalMs, pool, handler }` the consumer hands to
|
|
19
28
|
// whatever scheduler it runs. The consumer owns the class→table mapping (only
|
|
20
29
|
// the service knows which of its tables holds which data class) and owns the
|
|
21
|
-
// DELETE (`onPurge`); the library owns schedule resolution, the hold gate,
|
|
22
|
-
// the per-record purge event.
|
|
30
|
+
// DELETE (`onPurge`); the library owns schedule resolution, the hold gate, the
|
|
31
|
+
// expired status, the auto-delete decision and the per-record purge event.
|
|
23
32
|
//
|
|
24
33
|
// AMENDS D593. D593 observed that the platform has "not a per-record purge log".
|
|
25
34
|
// That was a description of an absence, not a prohibition — D720 makes the
|
|
@@ -36,6 +45,22 @@ export const RETENTION_SWEEPER_JOB_NAME = "dsar.retention_sweeper";
|
|
|
36
45
|
export const RETENTION_SWEEPER_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
37
46
|
/** Default per-target row budget for one sweep. */
|
|
38
47
|
export const RETENTION_SWEEPER_BATCH_SIZE = 500;
|
|
48
|
+
/**
|
|
49
|
+
* The status a record enters when its effective retention period ends (RETN-56).
|
|
50
|
+
* Every page that lists or shows the record labels it "Retention period expired".
|
|
51
|
+
*/
|
|
52
|
+
export const RETENTION_PERIOD_EXPIRED = "retention_period_expired";
|
|
53
|
+
/** One row per purge run, per tenant and class. Shipped by `applyDsarMigrations`. */
|
|
54
|
+
const PURGE_RUNS_TABLE = "retention_purge_runs";
|
|
55
|
+
/** Default name of the column on a swept table that holds the retention status. */
|
|
56
|
+
export const RETENTION_STATUS_COLUMN = "retention_status";
|
|
57
|
+
/**
|
|
58
|
+
* The audit action of one purge run (RETN-58, AUDT-50):
|
|
59
|
+
* `<serviceName>.retention.purge_run`.
|
|
60
|
+
*/
|
|
61
|
+
export function retentionPurgeRunAuditAction(serviceName) {
|
|
62
|
+
return `${serviceName}.retention.purge_run`;
|
|
63
|
+
}
|
|
39
64
|
/**
|
|
40
65
|
* D720 amendment to D593 — the per-record purge topic:
|
|
41
66
|
* `<serviceName>.retention.record_purged.v1`.
|
|
@@ -61,6 +86,17 @@ export class RetentionSweepHoldConsultFailed extends Error {
|
|
|
61
86
|
this.name = "RetentionSweepHoldConsultFailed";
|
|
62
87
|
}
|
|
63
88
|
}
|
|
89
|
+
/**
|
|
90
|
+
* A column compared with a text parameter. The parameter is bound as text and cast
|
|
91
|
+
* on the server, so any driver can bind it and the column's index stays usable. A
|
|
92
|
+
* character column is compared as text instead, because `::character` means
|
|
93
|
+
* char(1) and would truncate the value. The type name comes from Postgres.
|
|
94
|
+
*/
|
|
95
|
+
function columnEquals(column, type, param) {
|
|
96
|
+
return /^(text|character|character varying)$/.test(type)
|
|
97
|
+
? `${column} = ${param}::text`
|
|
98
|
+
: `${column} = (${param}::text)::${type}`;
|
|
99
|
+
}
|
|
64
100
|
function ensureIdent(value, what) {
|
|
65
101
|
if (!IDENT_RE.test(value)) {
|
|
66
102
|
throw new Error(`makeRetentionSweeperJob: ${what} ${JSON.stringify(value)} is not a bare SQL identifier (/^[a-z_][a-z0-9_]*$/). Table + column names are interpolated into the scan statement, so they are validated rather than parameterised.`);
|
|
@@ -80,6 +116,7 @@ function validateTarget(t) {
|
|
|
80
116
|
anchorColumn: ensureIdent(t.anchorColumn, "anchorColumn"),
|
|
81
117
|
anchor: t.anchor,
|
|
82
118
|
idColumn: ensureIdent(t.idColumn ?? "id", "idColumn"),
|
|
119
|
+
statusColumn: ensureIdent(t.statusColumn ?? RETENTION_STATUS_COLUMN, "statusColumn"),
|
|
83
120
|
tenantColumn: ensureIdent(t.tenantColumn ?? "tenant_id", "tenantColumn"),
|
|
84
121
|
subjectColumn: t.subjectColumn
|
|
85
122
|
? ensureIdent(t.subjectColumn, "subjectColumn")
|
|
@@ -96,17 +133,22 @@ function validateTarget(t) {
|
|
|
96
133
|
* D720 — build the schedulable purpose-spent sweeper.
|
|
97
134
|
*
|
|
98
135
|
* Per sweep, per target:
|
|
99
|
-
* 1. SELECT the batch (id, tenant, subject?, jurisdiction?, anchor);
|
|
136
|
+
* 1. SELECT the batch (id, tenant, subject?, jurisdiction?, anchor, status);
|
|
100
137
|
* 2. resolve the governing schedule via {@link applyRetentionSchedules} —
|
|
101
138
|
* no schedule ⇒ SKIP (absence of a rule is never a licence to delete);
|
|
102
|
-
* 3. still inside its window ⇒ SKIP;
|
|
139
|
+
* 3. still inside its window ⇒ SKIP, clearing a stale expired status;
|
|
103
140
|
* 4. `extended_by_proceeding` ⇒ SKIP unless a hold consult says otherwise
|
|
104
141
|
* (an open proceeding rides the hold mechanism per D720);
|
|
105
142
|
* 5. hold consult (FAIL-CLOSED — a consult error aborts the sweep);
|
|
106
|
-
* 6. `
|
|
107
|
-
*
|
|
143
|
+
* 6. set `retention_period_expired` and commit it, so a later failure still
|
|
144
|
+
* leaves the record visibly expired;
|
|
145
|
+
* 7. control off (or unreadable) ⇒ STOP; the record stays expired;
|
|
146
|
+
* 8. `onPurge(record, tx)` — the consumer's hard delete;
|
|
147
|
+
* 9. emit `<svc>.retention.record_purged.v1`.
|
|
148
|
+
*
|
|
149
|
+
* After every target: one `auditPurgeRun` per class that attempted a purge.
|
|
108
150
|
*
|
|
109
|
-
* Steps
|
|
151
|
+
* Steps 8+9 run in ONE transaction when `db` exposes `begin`, so the delete and
|
|
110
152
|
* its event commit together. Without a transaction runner the event is emitted
|
|
111
153
|
* after the delete resolves — a dropped event is recoverable, an
|
|
112
154
|
* announced-but-undeleted row is not.
|
|
@@ -115,8 +157,16 @@ export function makeRetentionSweeperJob(opts) {
|
|
|
115
157
|
if (!opts.db || typeof opts.db.unsafe !== "function") {
|
|
116
158
|
throw new Error("makeRetentionSweeperJob requires `db` — an OWNER-pool-backed SQL client (the scan SELECT + the purge DELETE are owner-granted per the RLS doctrine).");
|
|
117
159
|
}
|
|
118
|
-
if (
|
|
119
|
-
|
|
160
|
+
if (opts.autoDelete) {
|
|
161
|
+
if (typeof opts.onPurge !== "function") {
|
|
162
|
+
throw new Error("makeRetentionSweeperJob: `autoDelete` requires `onPurge` — the consumer's HARD DELETE. The library never deletes a service's rows itself, so a control that is on would have nothing to run.");
|
|
163
|
+
}
|
|
164
|
+
if (typeof opts.outboxEmitter !== "function") {
|
|
165
|
+
throw new Error("makeRetentionSweeperJob: `autoDelete` requires `outboxEmitter` — each delete's record_purged event is its durable record (RETN-23).");
|
|
166
|
+
}
|
|
167
|
+
if (typeof opts.auditPurgeRun !== "function") {
|
|
168
|
+
throw new Error("makeRetentionSweeperJob: `autoDelete` requires `auditPurgeRun` — every purge run is audited against the staff member who switched the control on (RETN-58, AUDT-50).");
|
|
169
|
+
}
|
|
120
170
|
}
|
|
121
171
|
if (!Array.isArray(opts.targets) || opts.targets.length === 0) {
|
|
122
172
|
throw new Error("makeRetentionSweeperJob requires a non-empty `targets` — the service's data_class → table mapping. Only the service knows which of its tables holds which class.");
|
|
@@ -134,10 +184,43 @@ export function makeRetentionSweeperJob(opts) {
|
|
|
134
184
|
const loadSchedules = async () => typeof opts.schedules === "function"
|
|
135
185
|
? await opts.schedules()
|
|
136
186
|
: opts.schedules;
|
|
187
|
+
/** One class's control, read at most once per sweep. */
|
|
188
|
+
const readAutoDelete = async (dataClass) => {
|
|
189
|
+
if (!opts.autoDelete)
|
|
190
|
+
return { control: { enabled: false } };
|
|
191
|
+
try {
|
|
192
|
+
const control = await opts.autoDelete(dataClass);
|
|
193
|
+
if (control?.enabled === false)
|
|
194
|
+
return { control: { enabled: false } };
|
|
195
|
+
if (control?.enabled !== true) {
|
|
196
|
+
return {
|
|
197
|
+
control: { enabled: false },
|
|
198
|
+
error: `auto-delete control for ${dataClass} returned a malformed reading. Treated as off.`,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
if (typeof control.switchedOnBy !== "string" ||
|
|
202
|
+
control.switchedOnBy.trim() === "") {
|
|
203
|
+
return {
|
|
204
|
+
control: { enabled: false },
|
|
205
|
+
error: `auto-delete control for ${dataClass} is on but names no staff member, so a purge run could not be audited (RETN-58). Treated as off.`,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
return { control };
|
|
209
|
+
}
|
|
210
|
+
catch (err) {
|
|
211
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
212
|
+
return {
|
|
213
|
+
control: { enabled: false },
|
|
214
|
+
error: `could not read the auto-delete control for ${dataClass}. Treated as off. Underlying error: ${detail}`,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
};
|
|
137
218
|
const handler = async () => {
|
|
138
219
|
const at = now();
|
|
139
220
|
const schedules = await loadSchedules();
|
|
140
221
|
const result = {
|
|
222
|
+
expired: 0,
|
|
223
|
+
alreadyExpired: 0,
|
|
141
224
|
purged: 0,
|
|
142
225
|
inWindow: 0,
|
|
143
226
|
skippedHeld: 0,
|
|
@@ -147,30 +230,97 @@ export function makeRetentionSweeperJob(opts) {
|
|
|
147
230
|
errors: [],
|
|
148
231
|
ranAt: at.toISOString(),
|
|
149
232
|
};
|
|
233
|
+
const controls = new Map();
|
|
234
|
+
const runId = crypto.randomUUID();
|
|
235
|
+
const startedRuns = new Set();
|
|
236
|
+
const blockedClasses = new Set();
|
|
237
|
+
const deliverPurgeRunAudits = async () => {
|
|
238
|
+
const auditPurgeRun = opts.auditPurgeRun;
|
|
239
|
+
if (!opts.autoDelete || !auditPurgeRun)
|
|
240
|
+
return;
|
|
241
|
+
const rows = (await opts.db.unsafe(`SELECT run_id::text AS run_id, tenant_id::text AS tenant_id, data_class,
|
|
242
|
+
switched_on_by, switched_on_at, purged, failed,
|
|
243
|
+
to_char(ran_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS ran_at
|
|
244
|
+
FROM ${PURGE_RUNS_TABLE} WHERE audited_at IS NULL ORDER BY ran_at, run_id`));
|
|
245
|
+
for (const row of rows) {
|
|
246
|
+
try {
|
|
247
|
+
await auditPurgeRun({
|
|
248
|
+
runId: row.run_id,
|
|
249
|
+
action: retentionPurgeRunAuditAction(opts.serviceName),
|
|
250
|
+
serviceName: opts.serviceName,
|
|
251
|
+
dataClass: row.data_class,
|
|
252
|
+
tenantId: row.tenant_id,
|
|
253
|
+
switchedOnBy: row.switched_on_by,
|
|
254
|
+
...(row.switched_on_at ? { switchedOnAt: row.switched_on_at } : {}),
|
|
255
|
+
purged: row.purged,
|
|
256
|
+
failed: row.failed,
|
|
257
|
+
ranAt: row.ran_at,
|
|
258
|
+
});
|
|
259
|
+
await opts.db.unsafe(`UPDATE ${PURGE_RUNS_TABLE} SET audited_at = now()
|
|
260
|
+
WHERE run_id = $1::uuid AND tenant_id = $2::uuid AND data_class = $3`, [row.run_id, row.tenant_id, row.data_class]);
|
|
261
|
+
}
|
|
262
|
+
catch (err) {
|
|
263
|
+
blockedClasses.add(row.data_class);
|
|
264
|
+
result.errors.push({
|
|
265
|
+
recordId: "",
|
|
266
|
+
tableName: PURGE_RUNS_TABLE,
|
|
267
|
+
error: `could not audit the purge run for ${row.data_class} in tenant ${row.tenant_id}. The class purges nothing until this succeeds: ${err instanceof Error ? err.message : String(err)}`,
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
};
|
|
272
|
+
// Runs a dead process left unaudited come first. If this table cannot be read,
|
|
273
|
+
// the sweep stops here: it could not know what is owed, or record new purges.
|
|
274
|
+
await deliverPurgeRunAudits();
|
|
275
|
+
// A missing status column stops the sweep before any target is touched, so a
|
|
276
|
+
// half-migrated service never expires or purges one table and then fails on
|
|
277
|
+
// the next.
|
|
150
278
|
for (const t of targets) {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
279
|
+
try {
|
|
280
|
+
await opts.db.unsafe(`SELECT 1 FROM ${t.tableName} WHERE ${t.statusColumn} = '${RETENTION_PERIOD_EXPIRED}' LIMIT 0`);
|
|
281
|
+
}
|
|
282
|
+
catch (err) {
|
|
283
|
+
throw new Error(`makeRetentionSweeperJob: could not check status column ${t.statusColumn} on ${t.tableName}, so the sweep stops before changing anything. If the column is missing or is not text, add a nullable text column. Underlying error: ${err instanceof Error ? err.message : String(err)}`);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
try {
|
|
287
|
+
for (const t of targets) {
|
|
288
|
+
const cols = [
|
|
289
|
+
`${t.idColumn}::text AS record_id`,
|
|
290
|
+
`${t.tenantColumn}::text AS tenant_id`,
|
|
291
|
+
t.subjectColumn
|
|
292
|
+
? `${t.subjectColumn}::text AS subject_id`
|
|
293
|
+
: "NULL::text AS subject_id",
|
|
294
|
+
t.jurisdictionColumn
|
|
295
|
+
? `${t.jurisdictionColumn}::text AS jurisdiction`
|
|
296
|
+
: "NULL::text AS jurisdiction",
|
|
297
|
+
`${t.statusColumn}::text AS retention_status`,
|
|
298
|
+
`pg_typeof(${t.idColumn})::text AS id_type`,
|
|
299
|
+
`pg_typeof(${t.tenantColumn})::text AS tenant_type`,
|
|
300
|
+
`to_char(${t.anchorColumn} AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS anchored_at`,
|
|
301
|
+
].join(",\n ");
|
|
302
|
+
// Two passes, each with its own cursor and budget. With the control off,
|
|
303
|
+
// expired rows stay forever at the oldest end of the table. In one shared
|
|
304
|
+
// scan they would fill `maxScan` on every tick, and newer rows would never
|
|
305
|
+
// expire. Already-expired rows go first, so a row this sweep has just
|
|
306
|
+
// marked is not read a second time.
|
|
307
|
+
for (const expiredPass of [true, false]) {
|
|
308
|
+
const statusClause = expiredPass
|
|
309
|
+
? `${t.statusColumn} = '${RETENTION_PERIOD_EXPIRED}'`
|
|
310
|
+
: `${t.statusColumn} IS DISTINCT FROM '${RETENTION_PERIOD_EXPIRED}'`;
|
|
311
|
+
// KEYSET CURSOR over `(anchor, id::text)`.
|
|
312
|
+
//
|
|
313
|
+
// The tiebreak is `id::text`, not the raw id, because the id column may be
|
|
314
|
+
// uuid, text or int and the cursor needs only a TOTAL order, not a
|
|
315
|
+
// semantically meaningful one. `ORDER BY` uses the identical expression so
|
|
316
|
+
// the comparison and the ordering can never disagree — an int column
|
|
317
|
+
// orders '10' before '9' under text collation, which is harmless while
|
|
318
|
+
// both sides agree and silently skips rows the moment they don't.
|
|
319
|
+
const stmt = `SELECT
|
|
171
320
|
${cols}
|
|
172
321
|
FROM ${t.tableName}
|
|
173
322
|
WHERE ${t.anchorColumn} IS NOT NULL
|
|
323
|
+
AND ${statusClause}
|
|
174
324
|
AND (
|
|
175
325
|
$1::text IS NULL
|
|
176
326
|
OR ${t.anchorColumn} > $1::timestamptz
|
|
@@ -178,170 +328,315 @@ export function makeRetentionSweeperJob(opts) {
|
|
|
178
328
|
)
|
|
179
329
|
ORDER BY ${t.anchorColumn} ASC, ${t.idColumn}::text ASC
|
|
180
330
|
LIMIT $3`;
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
const record = {
|
|
269
|
-
dataClass: t.dataClass,
|
|
270
|
-
tableName: t.tableName,
|
|
271
|
-
recordId: row.record_id,
|
|
272
|
-
tenantId: row.tenant_id,
|
|
273
|
-
subjectId: row.subject_id,
|
|
274
|
-
jurisdiction,
|
|
275
|
-
documentClass: t.documentClass,
|
|
276
|
-
entityKind: t.entityKind,
|
|
277
|
-
anchoredAt: row.anchored_at,
|
|
278
|
-
retainedUntil: match.retainUntil,
|
|
279
|
-
schedule: match.schedule,
|
|
280
|
-
};
|
|
281
|
-
const buildEvent = () => ({
|
|
282
|
-
topic,
|
|
283
|
-
serviceName: opts.serviceName,
|
|
284
|
-
dataClass: t.dataClass,
|
|
285
|
-
tableName: t.tableName,
|
|
286
|
-
recordId: row.record_id,
|
|
287
|
-
tenantId: row.tenant_id,
|
|
288
|
-
subjectId: row.subject_id,
|
|
289
|
-
jurisdiction,
|
|
290
|
-
documentClass: t.documentClass,
|
|
291
|
-
entityKind: t.entityKind,
|
|
292
|
-
retentionPeriod: match.schedule.retention_period,
|
|
293
|
-
anchor: match.schedule.anchor,
|
|
294
|
-
anchoredAt: row.anchored_at,
|
|
295
|
-
retainedUntil: match.retainUntil,
|
|
296
|
-
...(match.schedule.statute_citation
|
|
297
|
-
? { statuteCitation: match.schedule.statute_citation }
|
|
298
|
-
: {}),
|
|
299
|
-
purgedAt: new Date().toISOString(),
|
|
300
|
-
});
|
|
301
|
-
try {
|
|
302
|
-
if (opts.db.begin) {
|
|
303
|
-
// Atomic leg: the DELETE and its purge event commit together, so
|
|
304
|
-
// the feed can never diverge from what is actually gone.
|
|
305
|
-
await opts.db.begin(async (tx) => {
|
|
306
|
-
await opts.onPurge(record, tx);
|
|
307
|
-
if (opts.outboxEmitter) {
|
|
308
|
-
await opts.outboxEmitter(tx, buildEvent());
|
|
331
|
+
let cursorAnchor = null;
|
|
332
|
+
let cursorId = null;
|
|
333
|
+
let actionable = 0;
|
|
334
|
+
let scannedForTarget = 0;
|
|
335
|
+
let targetExhausted = false;
|
|
336
|
+
// Page until the ACTIONABLE budget fills, the ceiling is hit, or the
|
|
337
|
+
// table runs out. Skipped rows advance the cursor WITHOUT spending
|
|
338
|
+
// budget, so the next page starts beyond them inside this same tick —
|
|
339
|
+
// which is the whole fix: under the old fixed `LIMIT` with no cursor, a
|
|
340
|
+
// block of permanently-skipped rows at the head (oldest anchors) refilled
|
|
341
|
+
// every batch on every tick and nothing behind them was examined again,
|
|
342
|
+
// while the sweep still reported success.
|
|
343
|
+
while (actionable < batchSize && scannedForTarget < maxScan) {
|
|
344
|
+
const pageSize = Math.min(batchSize, maxScan - scannedForTarget);
|
|
345
|
+
const rows = (await opts.db.unsafe(stmt, [
|
|
346
|
+
cursorAnchor,
|
|
347
|
+
cursorId,
|
|
348
|
+
pageSize,
|
|
349
|
+
]));
|
|
350
|
+
if (rows.length === 0) {
|
|
351
|
+
targetExhausted = true;
|
|
352
|
+
break;
|
|
353
|
+
}
|
|
354
|
+
scannedForTarget += rows.length;
|
|
355
|
+
result.scanned += rows.length;
|
|
356
|
+
for (const row of rows) {
|
|
357
|
+
// Advance the cursor for EVERY row, whatever its outcome. This line
|
|
358
|
+
// is the anti-starvation guarantee.
|
|
359
|
+
cursorAnchor = row.anchored_at;
|
|
360
|
+
cursorId = row.record_id;
|
|
361
|
+
const jurisdiction = t.jurisdiction ?? row.jurisdiction;
|
|
362
|
+
if (!jurisdiction) {
|
|
363
|
+
// No jurisdiction ⇒ no resolvable window ⇒ no purge.
|
|
364
|
+
result.skippedNoSchedule++;
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
const match = applyRetentionSchedules(schedules, {
|
|
368
|
+
jurisdiction,
|
|
369
|
+
documentClass: t.documentClass,
|
|
370
|
+
entityKind: t.entityKind,
|
|
371
|
+
anchorAt: row.anchored_at,
|
|
372
|
+
at,
|
|
373
|
+
});
|
|
374
|
+
// A schedule anchored on a DIFFERENT timestamp than this target's
|
|
375
|
+
// column is not this target's schedule — counting an 8-year window
|
|
376
|
+
// from the wrong instant is worse than not purging at all.
|
|
377
|
+
if (!match || match.schedule.anchor !== t.anchor) {
|
|
378
|
+
result.skippedNoSchedule++;
|
|
379
|
+
continue;
|
|
380
|
+
}
|
|
381
|
+
const wasExpired = row.retention_status === RETENTION_PERIOD_EXPIRED;
|
|
382
|
+
const statusFilter = `${columnEquals(t.idColumn, row.id_type, "$1")} AND ${columnEquals(t.tenantColumn, row.tenant_type, "$2")}`;
|
|
383
|
+
if (new Date(match.retainUntil).getTime() > at.getTime()) {
|
|
384
|
+
if (wasExpired) {
|
|
385
|
+
// A new schedule version or a tenant extension moved this record
|
|
386
|
+
// back inside its window, so "expired" is no longer true.
|
|
387
|
+
try {
|
|
388
|
+
await opts.db.unsafe(`UPDATE ${t.tableName} SET ${t.statusColumn} = NULL WHERE ${statusFilter}`, [row.record_id, row.tenant_id]);
|
|
389
|
+
}
|
|
390
|
+
catch (err) {
|
|
391
|
+
result.errors.push({
|
|
392
|
+
recordId: row.record_id,
|
|
393
|
+
tableName: t.tableName,
|
|
394
|
+
error: err instanceof Error ? err.message : String(err),
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
// The second pass reads it again and counts it as in-window.
|
|
398
|
+
actionable++;
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
result.inWindow++;
|
|
402
|
+
actionable++;
|
|
403
|
+
continue;
|
|
404
|
+
}
|
|
405
|
+
let control = controls.get(t.dataClass);
|
|
406
|
+
if (!control) {
|
|
407
|
+
const read = blockedClasses.has(t.dataClass)
|
|
408
|
+
? { control: { enabled: false }, error: undefined }
|
|
409
|
+
: await readAutoDelete(t.dataClass);
|
|
410
|
+
control = read.control;
|
|
411
|
+
controls.set(t.dataClass, control);
|
|
412
|
+
if (read.error) {
|
|
413
|
+
result.errors.push({
|
|
414
|
+
recordId: row.record_id,
|
|
415
|
+
tableName: t.tableName,
|
|
416
|
+
error: read.error,
|
|
417
|
+
});
|
|
309
418
|
}
|
|
419
|
+
}
|
|
420
|
+
const onPurge = opts.onPurge;
|
|
421
|
+
const willPurge = control.enabled && Boolean(onPurge);
|
|
422
|
+
if (wasExpired && !willPurge) {
|
|
423
|
+
// RETN-58: the record stays in `retention_period_expired`. Nothing
|
|
424
|
+
// changes this tick, so it needs no hold read and spends no budget
|
|
425
|
+
// (RETN-24).
|
|
426
|
+
result.alreadyExpired++;
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
// Hold gate — FAIL-CLOSED. `isHeld` throws on any SQL/connection
|
|
430
|
+
// error; we do NOT catch it into a skip, we abort the sweep (see
|
|
431
|
+
// RetentionSweepHoldConsultFailed).
|
|
432
|
+
const consultHold = async () => {
|
|
433
|
+
try {
|
|
434
|
+
const consult = await isHeld({
|
|
435
|
+
sql: opts.db,
|
|
436
|
+
tenantId: row.tenant_id,
|
|
437
|
+
...(row.subject_id ? { subjectId: row.subject_id } : {}),
|
|
438
|
+
holdKind: blockingKinds,
|
|
439
|
+
dataClass: t.dataClass,
|
|
440
|
+
tableName: holdsTableName,
|
|
441
|
+
});
|
|
442
|
+
return consult.held;
|
|
443
|
+
}
|
|
444
|
+
catch (err) {
|
|
445
|
+
throw new RetentionSweepHoldConsultFailed(t.tableName, row.record_id, err);
|
|
446
|
+
}
|
|
447
|
+
};
|
|
448
|
+
const heldNow = await consultHold();
|
|
449
|
+
if (heldNow) {
|
|
450
|
+
// Held rows do NOT spend the actionable budget — they are exactly
|
|
451
|
+
// the rows that used to starve everything behind them.
|
|
452
|
+
result.skippedHeld++;
|
|
453
|
+
continue;
|
|
454
|
+
}
|
|
455
|
+
if (match.extendedByProceeding) {
|
|
456
|
+
// The window has expired but the schedule says an open proceeding
|
|
457
|
+
// extends it. Proceedings ride the HOLD mechanism (D720:
|
|
458
|
+
// preservation holds are the exceptional-purpose surface), and the
|
|
459
|
+
// consult above found none — so there is no proceeding and the
|
|
460
|
+
// record expires. The branch exists to make that reasoning explicit
|
|
461
|
+
// rather than implicit in the absence of a check.
|
|
462
|
+
}
|
|
463
|
+
// The status commits on its own, before any purge, so a purge that
|
|
464
|
+
// fails and rolls back leaves the record visibly expired.
|
|
465
|
+
if (!wasExpired) {
|
|
466
|
+
try {
|
|
467
|
+
const updated = (await opts.db.unsafe(`UPDATE ${t.tableName} SET ${t.statusColumn} = $3 WHERE ${statusFilter}`, [row.record_id, row.tenant_id, RETENTION_PERIOD_EXPIRED]));
|
|
468
|
+
// The row went away since the scan, so there is nothing to
|
|
469
|
+
// mark and nothing to purge.
|
|
470
|
+
if (updated?.count === 0) {
|
|
471
|
+
actionable++;
|
|
472
|
+
continue;
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
catch (err) {
|
|
476
|
+
result.errors.push({
|
|
477
|
+
recordId: row.record_id,
|
|
478
|
+
tableName: t.tableName,
|
|
479
|
+
error: err instanceof Error ? err.message : String(err),
|
|
480
|
+
});
|
|
481
|
+
actionable++;
|
|
482
|
+
continue;
|
|
483
|
+
}
|
|
484
|
+
result.expired++;
|
|
485
|
+
}
|
|
486
|
+
if (!control.enabled || !onPurge) {
|
|
487
|
+
actionable++;
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
// RETN-20: check again immediately before the delete. This narrows the
|
|
491
|
+
// window for a hold placed during the sweep. It is not a lock, so a
|
|
492
|
+
// hold committed after this read and before the delete is missed.
|
|
493
|
+
if (await consultHold()) {
|
|
494
|
+
result.skippedHeld++;
|
|
495
|
+
continue;
|
|
496
|
+
}
|
|
497
|
+
const record = {
|
|
498
|
+
dataClass: t.dataClass,
|
|
499
|
+
tableName: t.tableName,
|
|
500
|
+
recordId: row.record_id,
|
|
501
|
+
tenantId: row.tenant_id,
|
|
502
|
+
subjectId: row.subject_id,
|
|
503
|
+
jurisdiction,
|
|
504
|
+
documentClass: t.documentClass,
|
|
505
|
+
entityKind: t.entityKind,
|
|
506
|
+
anchoredAt: row.anchored_at,
|
|
507
|
+
retainedUntil: match.retainUntil,
|
|
508
|
+
schedule: match.schedule,
|
|
509
|
+
};
|
|
510
|
+
const buildEvent = () => ({
|
|
511
|
+
topic,
|
|
512
|
+
serviceName: opts.serviceName,
|
|
513
|
+
dataClass: t.dataClass,
|
|
514
|
+
tableName: t.tableName,
|
|
515
|
+
recordId: row.record_id,
|
|
516
|
+
tenantId: row.tenant_id,
|
|
517
|
+
subjectId: row.subject_id,
|
|
518
|
+
jurisdiction,
|
|
519
|
+
documentClass: t.documentClass,
|
|
520
|
+
entityKind: t.entityKind,
|
|
521
|
+
retentionPeriod: match.schedule.retention_period,
|
|
522
|
+
anchor: match.schedule.anchor,
|
|
523
|
+
anchoredAt: row.anchored_at,
|
|
524
|
+
retainedUntil: match.retainUntil,
|
|
525
|
+
...(match.schedule.statute_citation
|
|
526
|
+
? { statuteCitation: match.schedule.statute_citation }
|
|
527
|
+
: {}),
|
|
528
|
+
purgedAt: new Date().toISOString(),
|
|
529
|
+
switchedOnBy: control.switchedOnBy,
|
|
310
530
|
});
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
531
|
+
const countRun = (column) => `UPDATE ${PURGE_RUNS_TABLE} SET ${column} = ${column} + 1
|
|
532
|
+
WHERE run_id = $1::uuid AND tenant_id = $2::uuid AND data_class = $3`;
|
|
533
|
+
const runArgs = [runId, row.tenant_id, t.dataClass];
|
|
534
|
+
const runKey = JSON.stringify([t.dataClass, row.tenant_id]);
|
|
535
|
+
if (!startedRuns.has(runKey)) {
|
|
536
|
+
// Committed before the first delete, so the audit is owed even if
|
|
537
|
+
// this process dies straight after it.
|
|
538
|
+
try {
|
|
539
|
+
await opts.db.unsafe(`INSERT INTO ${PURGE_RUNS_TABLE}
|
|
540
|
+
(run_id, tenant_id, data_class, switched_on_by, switched_on_at, ran_at)
|
|
541
|
+
VALUES ($1::uuid, $2::uuid, $3, $4, $5, $6::timestamptz)`, [
|
|
542
|
+
runId,
|
|
543
|
+
row.tenant_id,
|
|
544
|
+
t.dataClass,
|
|
545
|
+
control.switchedOnBy,
|
|
546
|
+
control.switchedOnAt ?? null,
|
|
547
|
+
result.ranAt,
|
|
548
|
+
]);
|
|
549
|
+
}
|
|
550
|
+
catch (err) {
|
|
551
|
+
result.errors.push({
|
|
552
|
+
recordId: row.record_id,
|
|
553
|
+
tableName: t.tableName,
|
|
554
|
+
error: `could not record the purge run, so nothing was deleted: ${err instanceof Error ? err.message : String(err)}`,
|
|
555
|
+
});
|
|
556
|
+
actionable++;
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
startedRuns.add(runKey);
|
|
560
|
+
}
|
|
561
|
+
try {
|
|
562
|
+
if (opts.db.begin) {
|
|
563
|
+
// Atomic leg: the DELETE and its purge event commit together, so
|
|
564
|
+
// the feed can never diverge from what is actually gone.
|
|
565
|
+
await opts.db.begin(async (tx) => {
|
|
566
|
+
await onPurge(record, tx);
|
|
567
|
+
if (opts.outboxEmitter) {
|
|
568
|
+
await opts.outboxEmitter(tx, buildEvent());
|
|
569
|
+
}
|
|
570
|
+
await tx.unsafe(countRun("purged"), runArgs);
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
else {
|
|
574
|
+
// No transaction runner. Delete first, then announce — a dropped
|
|
575
|
+
// event is recoverable, an announced-but-undeleted row is not.
|
|
576
|
+
await onPurge(record);
|
|
577
|
+
if (opts.outboxEmitter) {
|
|
578
|
+
await opts.outboxEmitter(opts.db, buildEvent());
|
|
579
|
+
}
|
|
580
|
+
await opts.db.unsafe(countRun("purged"), runArgs);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
catch (err) {
|
|
584
|
+
result.errors.push({
|
|
585
|
+
recordId: row.record_id,
|
|
586
|
+
tableName: t.tableName,
|
|
587
|
+
error: err instanceof Error ? err.message : String(err),
|
|
588
|
+
});
|
|
589
|
+
// A failed purge SPENDS budget: it was actionable, it was attempted,
|
|
590
|
+
// and it must not be retried forever inside one tick.
|
|
591
|
+
try {
|
|
592
|
+
await opts.db.unsafe(countRun("failed"), runArgs);
|
|
593
|
+
}
|
|
594
|
+
catch {
|
|
595
|
+
// The failure is already in `errors`; only the run's count is short.
|
|
596
|
+
}
|
|
597
|
+
actionable++;
|
|
598
|
+
continue;
|
|
318
599
|
}
|
|
600
|
+
result.purged++;
|
|
601
|
+
actionable++;
|
|
602
|
+
}
|
|
603
|
+
if (rows.length < pageSize) {
|
|
604
|
+
targetExhausted = true;
|
|
605
|
+
break;
|
|
319
606
|
}
|
|
320
607
|
}
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
608
|
+
// Not reading a target to its end means rows beyond the cursor were never
|
|
609
|
+
// examined this tick. Say so rather than letting the tick look complete.
|
|
610
|
+
if (!targetExhausted)
|
|
611
|
+
result.exhausted = false;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
catch (err) {
|
|
616
|
+
// The purges made before the abort still happened and still owe their audit
|
|
617
|
+
// event, and a failure to write it must not vanish with the result. What
|
|
618
|
+
// cannot be delivered now stays in the table for the next sweep.
|
|
619
|
+
try {
|
|
620
|
+
await deliverPurgeRunAudits();
|
|
621
|
+
}
|
|
622
|
+
catch (auditErr) {
|
|
623
|
+
result.errors.push({
|
|
624
|
+
recordId: "",
|
|
625
|
+
tableName: PURGE_RUNS_TABLE,
|
|
626
|
+
error: `could not read the purge runs: ${auditErr instanceof Error ? auditErr.message : String(auditErr)}`,
|
|
627
|
+
});
|
|
628
|
+
}
|
|
629
|
+
if (opts.onRun) {
|
|
630
|
+
try {
|
|
631
|
+
await opts.onRun(result);
|
|
334
632
|
}
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
break;
|
|
633
|
+
catch {
|
|
634
|
+
// The abort's own error is the one to report.
|
|
338
635
|
}
|
|
339
636
|
}
|
|
340
|
-
|
|
341
|
-
// examined this tick. Say so rather than letting the tick look complete.
|
|
342
|
-
if (!targetExhausted)
|
|
343
|
-
result.exhausted = false;
|
|
637
|
+
throw err;
|
|
344
638
|
}
|
|
639
|
+
await deliverPurgeRunAudits();
|
|
345
640
|
if (opts.onRun)
|
|
346
641
|
await opts.onRun(result);
|
|
347
642
|
return result;
|