@cosmicdrift/kumiko-bundled-features 0.221.0 → 0.223.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -9
- package/src/audit/__tests__/audit-log-screen.test.tsx +5 -5
- package/src/audit/__tests__/audit-screens.boot.test.ts +2 -1
- package/src/audit/__tests__/audit.integration.test.ts +3 -1
- package/src/audit/__tests__/constants.test.ts +6 -1
- package/src/audit/constants.ts +10 -1
- package/src/audit/feature.ts +2 -0
- package/src/audit/i18n.ts +2 -0
- package/src/audit/web/audit-log-detail-screen.tsx +7 -3
- package/src/audit/web/audit-log-screen.tsx +27 -29
- package/src/auth-email-password/handlers/invite-create.write.ts +7 -1
- package/src/auth-email-password/seeding.ts +3 -1
- package/src/auth-mfa/hmac-token-codec.ts +75 -0
- package/src/auth-mfa/mfa-challenge-token.ts +10 -65
- package/src/auth-mfa/mfa-preauth-setup-token.ts +10 -61
- package/src/cap-counter/stock-cap-guard.ts +1 -1
- package/src/channel-email/__tests__/email-channel.test.ts +19 -0
- package/src/channel-email/__tests__/pii-guard.test.ts +11 -0
- package/src/channel-email/email-channel.ts +14 -7
- package/src/channel-email/pii-guard.ts +8 -0
- package/src/compliance-profiles/handlers/set-profile.write.ts +1 -4
- package/src/crypto-shredding/constants.ts +1 -0
- package/src/crypto-shredding/feature.ts +6 -1
- package/src/crypto-shredding/handlers/forget-subject.write.ts +25 -1
- package/src/crypto-shredding/index.ts +2 -0
- package/src/custom-fields/__tests__/audit-integration.integration.test.ts +9 -1
- package/src/file-derivatives/feature.ts +2 -2
- package/src/file-derivatives/handlers/public-variant.query.ts +4 -5
- package/src/files-provider-s3/__tests__/s3-provider.test.ts +58 -1
- package/src/files-provider-s3/s3-provider.ts +45 -0
- package/src/files-tenant-data/__tests__/hooks.integration.test.ts +271 -0
- package/src/files-tenant-data/__tests__/sweep-orphaned-derivatives.job.integration.test.ts +275 -0
- package/src/files-tenant-data/changes.json +1 -0
- package/src/files-tenant-data/handlers/sweep-orphaned-derivatives.job.ts +173 -0
- package/src/files-tenant-data/hooks.ts +93 -0
- package/src/files-tenant-data/index.ts +58 -0
- package/src/form-draft/db/queries/cleanup.ts +17 -0
- package/src/form-draft/handlers/cleanup.job.ts +20 -10
- package/src/form-draft/handlers/discard.write.ts +25 -7
- package/src/form-draft/handlers/get.query.ts +4 -2
- package/src/form-draft/handlers/list.query.ts +4 -1
- package/src/form-draft/lookup.ts +12 -5
- package/src/inbound-mail-foundation/watch-supervisor.ts +28 -4
- package/src/jobs/__tests__/jobs-pii-kms.integration.test.ts +128 -3
- package/src/jobs/constants.ts +1 -0
- package/src/jobs/handlers/retry.write.ts +26 -6
- package/src/jobs/i18n.ts +3 -0
- package/src/sessions/session-callbacks.ts +1 -0
- package/src/tags/__tests__/tag-screens.boot.test.ts +13 -0
- package/src/tenant/__tests__/tenant-timezone-boot.integration.test.ts +165 -0
- package/src/tenant/changes.json +1 -1
- package/src/tenant/constants.ts +5 -1
- package/src/tenant/handlers/remove-member.write.ts +12 -0
- package/src/tenant/handlers/update-member-roles.write.ts +10 -37
- package/src/tenant/last-tenant-admin.ts +39 -0
- package/src/tenant/screens.ts +2 -0
- package/src/tenant/seeding.ts +3 -1
- package/src/tenant-lifecycle/feature.ts +1 -0
- package/src/tenant-lifecycle/handlers/cancel-destruction.write.ts +1 -5
- package/src/tenant-lifecycle/run-tenant-destroy.ts +5 -0
- package/src/tenant-lifecycle/stages.ts +5 -0
- package/src/tier-engine/web/tier-admin-screen.tsx +1 -2
- package/src/user/__tests__/email-unique-blind-index.integration.test.ts +25 -13
- package/src/user/changes.json +12 -5
- package/src/user-data-rights/handlers/cancel-deletion.write.ts +3 -10
- package/src/user-data-rights/handlers/confirm-deletion-by-token.write.ts +2 -8
- package/src/user-data-rights/handlers/deletion-grace-period.ts +2 -2
- package/src/user-data-rights/handlers/lift-restriction.write.ts +2 -2
- package/src/user-data-rights/handlers/restrict-account.write.ts +3 -3
- package/src/user-data-rights/web/__tests__/privacy-center-screen.test.tsx +10 -6
- package/src/user-data-rights-defaults/__tests__/user-data-rights-defaults.integration.test.ts +114 -1
- package/src/user-data-rights-defaults/hooks/file-ref.userdata-hook.ts +63 -12
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
// Backfill/GC job for derivatives orphaned by a forget/tenant-destroy that ran
|
|
2
|
+
// before #2461 wired binary cleanup into those flows (#2474). Those older runs
|
|
3
|
+
// left the FileRef row gone with no audit trail of what was ever forgotten —
|
|
4
|
+
// so this sweep works forward from the deterministic derivative-key grammar
|
|
5
|
+
// alone: for every key under a tenant's storage prefix that parses as a
|
|
6
|
+
// derivative (parseDerivativeKey), delete it only if NO fileRef row (live or
|
|
7
|
+
// trashed) backs its reconstructed original. A row backing the original — even
|
|
8
|
+
// a soft-deleted one sitting in trash — means the derivative is still
|
|
9
|
+
// legitimate; only genuinely unreferenced derivatives are swept.
|
|
10
|
+
//
|
|
11
|
+
// Manual trigger only (no cron) — an operator runs this once after upgrading
|
|
12
|
+
// past #2461, or on demand for spot-checks.
|
|
13
|
+
//
|
|
14
|
+
// Row unit for runChunkedMigration is a discovered derivative CANDIDATE KEY,
|
|
15
|
+
// not a tenant: one tenant's storage listing produces a variable number of
|
|
16
|
+
// candidates, which doesn't fit "one row -> one migrate outcome" if the row
|
|
17
|
+
// were a tenant. nextBatch pages tenants internally and buffers their
|
|
18
|
+
// candidates into a flat queue so the shared deadline/circuit-breaker/signal
|
|
19
|
+
// handling still applies per-key.
|
|
20
|
+
|
|
21
|
+
import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
22
|
+
import { parseDerivativeKey } from "@cosmicdrift/kumiko-framework/derivatives";
|
|
23
|
+
import type { JobHandlerFn, TenantId } from "@cosmicdrift/kumiko-framework/engine";
|
|
24
|
+
import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
|
|
25
|
+
import {
|
|
26
|
+
assertSafeStorageKey,
|
|
27
|
+
type FileProviderResolver,
|
|
28
|
+
type FileStorageProvider,
|
|
29
|
+
fileRefsTable,
|
|
30
|
+
} from "@cosmicdrift/kumiko-framework/files";
|
|
31
|
+
import { z } from "zod";
|
|
32
|
+
import { runChunkedMigration } from "../../shared";
|
|
33
|
+
import { tenantTable } from "../../tenant";
|
|
34
|
+
|
|
35
|
+
const DEFAULT_TENANT_PAGE_SIZE = 50;
|
|
36
|
+
const DEFAULT_CANDIDATE_BATCH_SIZE = 100;
|
|
37
|
+
const DEFAULT_MAX_FAILURES = 10;
|
|
38
|
+
|
|
39
|
+
export const sweepOrphanedDerivativesPayloadSchema = z.object({
|
|
40
|
+
dryRun: z.boolean().optional(),
|
|
41
|
+
batchSize: z.number().int().positive().optional(),
|
|
42
|
+
maxDurationMs: z.number().int().positive().optional(),
|
|
43
|
+
maxFailures: z.number().int().positive().optional(),
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
type DerivativeCandidate = {
|
|
47
|
+
readonly tenantId: TenantId;
|
|
48
|
+
readonly key: string;
|
|
49
|
+
readonly originalKey: string;
|
|
50
|
+
readonly provider: FileStorageProvider;
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
export const sweepOrphanedDerivativesJob: JobHandlerFn = async (rawPayload, ctx): Promise<void> => {
|
|
54
|
+
const payload = sweepOrphanedDerivativesPayloadSchema.parse(rawPayload ?? {});
|
|
55
|
+
if (!ctx.db) {
|
|
56
|
+
throw new InternalError({
|
|
57
|
+
message:
|
|
58
|
+
"[files-tenant-data:sweep] ctx.db missing — job context requires a database connection.",
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
const db = ctx.db;
|
|
62
|
+
// Re-bound to a definitely-assigned const: TS control-flow narrowing from
|
|
63
|
+
// the guard above doesn't cross into fillQueue's nested closure below.
|
|
64
|
+
const maybeResolver = ctx._fileProviderResolver;
|
|
65
|
+
if (!maybeResolver) {
|
|
66
|
+
ctx.log?.warn(
|
|
67
|
+
"[files-tenant-data:sweep] no _fileProviderResolver wired — nothing to sweep, skipping run",
|
|
68
|
+
);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const resolver: FileProviderResolver = maybeResolver;
|
|
72
|
+
|
|
73
|
+
const batchSize = payload.batchSize ?? DEFAULT_CANDIDATE_BATCH_SIZE;
|
|
74
|
+
const maxFailures = payload.maxFailures ?? DEFAULT_MAX_FAILURES;
|
|
75
|
+
const deadline = payload.maxDurationMs
|
|
76
|
+
? Date.now() + payload.maxDurationMs
|
|
77
|
+
: Number.POSITIVE_INFINITY;
|
|
78
|
+
const dryRun = payload.dryRun ?? false;
|
|
79
|
+
|
|
80
|
+
let queue: DerivativeCandidate[] = [];
|
|
81
|
+
let tenantCursor: string | undefined;
|
|
82
|
+
let tenantsExhausted = false;
|
|
83
|
+
let dryRunWouldDelete = 0;
|
|
84
|
+
|
|
85
|
+
async function fillQueue(): Promise<void> {
|
|
86
|
+
while (queue.length === 0 && !tenantsExhausted) {
|
|
87
|
+
const tenants = await selectMany<{ id: string }>(
|
|
88
|
+
db,
|
|
89
|
+
tenantTable,
|
|
90
|
+
tenantCursor ? { id: { gt: tenantCursor } } : {},
|
|
91
|
+
{ orderBy: [{ col: "id", direction: "asc" }], limit: DEFAULT_TENANT_PAGE_SIZE },
|
|
92
|
+
);
|
|
93
|
+
if (tenants.length === 0) {
|
|
94
|
+
tenantsExhausted = true;
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
tenantCursor = tenants.at(-1)?.id;
|
|
98
|
+
for (const tenant of tenants) {
|
|
99
|
+
const tenantId = tenant.id as TenantId;
|
|
100
|
+
let provider: FileStorageProvider;
|
|
101
|
+
try {
|
|
102
|
+
provider = await resolver(tenantId);
|
|
103
|
+
} catch (err) {
|
|
104
|
+
ctx.log?.warn?.(
|
|
105
|
+
`[files-tenant-data:sweep] no file provider resolvable for tenant=${tenantId}, skipping: ${err instanceof Error ? err.message : String(err)}`,
|
|
106
|
+
);
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
let keys: readonly string[];
|
|
110
|
+
try {
|
|
111
|
+
keys = await provider.list(`${tenantId}/`);
|
|
112
|
+
} catch (err) {
|
|
113
|
+
ctx.log?.warn?.(
|
|
114
|
+
`[files-tenant-data:sweep] provider.list() failed for tenant=${tenantId}, skipping: ${err instanceof Error ? err.message : String(err)}`,
|
|
115
|
+
);
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
for (const key of keys) {
|
|
119
|
+
const parsed = parseDerivativeKey(key);
|
|
120
|
+
if (!parsed) continue;
|
|
121
|
+
queue.push({ tenantId, key, originalKey: parsed.originalKey, provider });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async function nextBatch(): Promise<readonly DerivativeCandidate[]> {
|
|
128
|
+
await fillQueue();
|
|
129
|
+
const batch = queue.slice(0, batchSize);
|
|
130
|
+
queue = queue.slice(batch.length);
|
|
131
|
+
return batch;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async function migrateRow(
|
|
135
|
+
candidate: DerivativeCandidate,
|
|
136
|
+
): Promise<"migrated" | "skipped" | "failed"> {
|
|
137
|
+
// No isDeleted filter: a soft-deleted (trashed, not yet forgotten) fileRef
|
|
138
|
+
// still legitimately backs its derivatives — only an ABSENT row (forgotten,
|
|
139
|
+
// or a tenant destroyed before #2461 with no row left at all) makes a
|
|
140
|
+
// derivative orphaned.
|
|
141
|
+
const owners = await selectMany(
|
|
142
|
+
db,
|
|
143
|
+
fileRefsTable,
|
|
144
|
+
{ tenantId: candidate.tenantId, storageKey: candidate.originalKey },
|
|
145
|
+
{ limit: 1 },
|
|
146
|
+
);
|
|
147
|
+
if (owners.length > 0) return "skipped";
|
|
148
|
+
if (dryRun) {
|
|
149
|
+
dryRunWouldDelete++;
|
|
150
|
+
return "skipped";
|
|
151
|
+
}
|
|
152
|
+
assertSafeStorageKey(candidate.key);
|
|
153
|
+
await candidate.provider.delete(candidate.key);
|
|
154
|
+
return "migrated";
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const outcome = await runChunkedMigration<DerivativeCandidate>({
|
|
158
|
+
nextBatch,
|
|
159
|
+
migrateRow,
|
|
160
|
+
maxFailures,
|
|
161
|
+
deadlineAt: deadline,
|
|
162
|
+
signal: ctx.signal,
|
|
163
|
+
onRowError: (candidate, err) => {
|
|
164
|
+
ctx.log?.warn?.(
|
|
165
|
+
`[files-tenant-data:sweep] failed to delete key=${candidate.key} tenant=${candidate.tenantId}: ${err instanceof Error ? err.message : String(err)}`,
|
|
166
|
+
);
|
|
167
|
+
},
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
ctx.log?.info?.(
|
|
171
|
+
`[files-tenant-data:sweep] complete dryRun=${dryRun} deleted=${outcome.migrated} wouldDelete=${dryRunWouldDelete} skipped=${outcome.skipped} failed=${outcome.failed} batches=${outcome.batchesProcessed} stoppedReason=${outcome.stoppedReason}`,
|
|
172
|
+
);
|
|
173
|
+
};
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// EXT_TENANT_DATA + EXT_STORAGE_PROVIDER destroy hooks for the `fileRef`
|
|
2
|
+
// entity (#2474). Kept apart from the `files` feature (like folders-user-data
|
|
3
|
+
// does for EXT_USER_DATA) so file consumers without the tenant-lifecycle
|
|
4
|
+
// pipeline don't pull a hard dependency — `files` stays usable standalone.
|
|
5
|
+
//
|
|
6
|
+
// Two hooks cover two different destroy stages:
|
|
7
|
+
// - fileRefTenantDestroyHook ("app-data" stage, EXT_TENANT_DATA): purges
|
|
8
|
+
// every fileRef ROW for the tenant via forget() — rebuild-safe, mirrors
|
|
9
|
+
// document-ingest-foundation's documentExtractTenantDestroyHook. No
|
|
10
|
+
// isDeleted filter: tenant-destroy purges trashed rows too, a destroyed
|
|
11
|
+
// tenant has no "restore from trash" future.
|
|
12
|
+
// - fileRefStorageDestroyHook ("files" stage, EXT_STORAGE_PROVIDER): wipes
|
|
13
|
+
// every BINARY (original + derivatives + anything else) under the
|
|
14
|
+
// tenant's storage prefix. A full `${tenantId}/` prefix sweep, not a
|
|
15
|
+
// per-row derivative lookup — buildStorageKey always puts tenantId first,
|
|
16
|
+
// so this needs only the tenantId, stays correct even though the "files"
|
|
17
|
+
// stage runs after "app-data" already forgot the rows, and can never
|
|
18
|
+
// cross into another tenant's keys (the provider's own list() prefix
|
|
19
|
+
// already scopes it).
|
|
20
|
+
|
|
21
|
+
import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
22
|
+
import { createEventStoreExecutor, createTenantDb } from "@cosmicdrift/kumiko-framework/db";
|
|
23
|
+
import {
|
|
24
|
+
createSystemUser,
|
|
25
|
+
type StorageProviderDestroyTenantHook,
|
|
26
|
+
type TenantDataDestroyHook,
|
|
27
|
+
} from "@cosmicdrift/kumiko-framework/engine";
|
|
28
|
+
import {
|
|
29
|
+
assertSafeStorageKey,
|
|
30
|
+
fileRefEntity,
|
|
31
|
+
fileRefsTable,
|
|
32
|
+
} from "@cosmicdrift/kumiko-framework/files";
|
|
33
|
+
|
|
34
|
+
const crud = createEventStoreExecutor(fileRefsTable, fileRefEntity, { entityName: "fileRef" });
|
|
35
|
+
|
|
36
|
+
export const fileRefTenantDestroyHook: TenantDataDestroyHook = async (ctx) => {
|
|
37
|
+
const rows = await selectMany<{ id: string }>(ctx.db, fileRefsTable, {
|
|
38
|
+
tenantId: ctx.tenantId,
|
|
39
|
+
});
|
|
40
|
+
const user = createSystemUser(ctx.tenantId);
|
|
41
|
+
const db = createTenantDb(ctx.db, ctx.tenantId, "system");
|
|
42
|
+
for (const row of rows) {
|
|
43
|
+
const result = await crud.forget({ id: row.id }, user, db);
|
|
44
|
+
// Executor writes return {isSuccess:false} instead of throwing — a
|
|
45
|
+
// discarded result would report this destroy stage "succeeded" while the
|
|
46
|
+
// row (and its PII fileName) survives. Throw so the pipeline's
|
|
47
|
+
// retry/abandon handling sees it.
|
|
48
|
+
if (!result.isSuccess) {
|
|
49
|
+
throw new Error(
|
|
50
|
+
`files-tenant-data: failed to forget fileRef ${row.id} for tenant ${ctx.tenantId}: ${result.error.message}`,
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
export const fileRefStorageDestroyHook: StorageProviderDestroyTenantHook = async (
|
|
57
|
+
tenantId,
|
|
58
|
+
ctx,
|
|
59
|
+
) => {
|
|
60
|
+
if (!ctx.fileProviderResolver) {
|
|
61
|
+
// Resolution unavailable (no file-provider-* feature mounted, or the job
|
|
62
|
+
// ctx didn't wire one) degrades to a no-op — same "not fail-closed"
|
|
63
|
+
// stance as the per-user forget hook's resolveProvider: a misconfigured
|
|
64
|
+
// store must not block the rest of the destroy pipeline forever. The row
|
|
65
|
+
// purge from the "app-data" stage already ran regardless.
|
|
66
|
+
ctx.log?.(
|
|
67
|
+
`[files-tenant-data] no fileProviderResolver wired — tenant ${tenantId}'s file binaries are NOT deleted on destroy`,
|
|
68
|
+
);
|
|
69
|
+
// skip: no resolver wired for this destroy run — already logged above, and
|
|
70
|
+
// the row purge from the "app-data" stage ran regardless.
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
let provider: Awaited<ReturnType<typeof ctx.fileProviderResolver>>;
|
|
74
|
+
try {
|
|
75
|
+
provider = await ctx.fileProviderResolver(tenantId);
|
|
76
|
+
} catch (err) {
|
|
77
|
+
ctx.log?.(
|
|
78
|
+
`[files-tenant-data] no file provider resolvable for tenant ${tenantId}: ${err instanceof Error ? err.message : String(err)} — binaries NOT deleted`,
|
|
79
|
+
);
|
|
80
|
+
// skip: provider resolution failed for this tenant — already logged
|
|
81
|
+
// above, binaries are left in place rather than blocking the pipeline.
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
// Provider resolved — a list()/delete() failure from here IS fail-closed:
|
|
85
|
+
// the "files" stage throws, tenant-lifecycle's retry/abandon handling sees
|
|
86
|
+
// it, and the next sweep tick retries (list+delete are idempotent, so this
|
|
87
|
+
// converges rather than double-deleting or erroring on a missing key).
|
|
88
|
+
const keys = await provider.list(`${tenantId}/`);
|
|
89
|
+
for (const key of keys) {
|
|
90
|
+
assertSafeStorageKey(key);
|
|
91
|
+
await provider.delete(key);
|
|
92
|
+
}
|
|
93
|
+
};
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// GDPR tenant-destroy + orphaned-derivative backfill coverage for the `files`
|
|
2
|
+
// feature's `fileRef` entity (#2474). Kept apart from `files` — like
|
|
3
|
+
// folders-user-data does for EXT_USER_DATA — so file consumers without the
|
|
4
|
+
// tenant-lifecycle pipeline don't pull a hard dependency; `files` stays usable
|
|
5
|
+
// standalone (e.g. user-data-rights-demo mounts `files` + user-data-rights
|
|
6
|
+
// without tenant-lifecycle).
|
|
7
|
+
//
|
|
8
|
+
// tenant-lifecycle is the hard dependency (EXT_TENANT_DATA + EXT_STORAGE_PROVIDER
|
|
9
|
+
// host). `files` is OPTIONAL: if it's mounted toggleable (default=false), a hard
|
|
10
|
+
// r.requires would throw an "effectively disabled" boot warning even though the
|
|
11
|
+
// fileRef entity exists and the hooks work fine — same reasoning as
|
|
12
|
+
// folders-user-data's optionalRequires("folders").
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
defineFeature,
|
|
16
|
+
EXT_STORAGE_PROVIDER,
|
|
17
|
+
EXT_TENANT_DATA,
|
|
18
|
+
type FeatureDefinition,
|
|
19
|
+
} from "@cosmicdrift/kumiko-framework/engine";
|
|
20
|
+
import {
|
|
21
|
+
sweepOrphanedDerivativesJob,
|
|
22
|
+
sweepOrphanedDerivativesPayloadSchema,
|
|
23
|
+
} from "./handlers/sweep-orphaned-derivatives.job";
|
|
24
|
+
import { fileRefStorageDestroyHook, fileRefTenantDestroyHook } from "./hooks";
|
|
25
|
+
|
|
26
|
+
export function createFilesTenantDataFeature(): FeatureDefinition {
|
|
27
|
+
return defineFeature("files-tenant-data", (r) => {
|
|
28
|
+
r.describe(
|
|
29
|
+
"GDPR coverage for the `files` feature's `fileRef` entity: tenant-destroy row purge (EXT_TENANT_DATA) and full storage-prefix binary wipe (EXT_STORAGE_PROVIDER), plus a manual backfill/GC job that sweeps derivatives orphaned by a forget/tenant-destroy that ran before #2461 wired binary cleanup into those flows. Requires `tenant-lifecycle`, optionalRequires `files`.",
|
|
30
|
+
);
|
|
31
|
+
r.uiHints({
|
|
32
|
+
displayLabel: "Files · Tenant-Destroy & Derivative GC",
|
|
33
|
+
category: "compliance",
|
|
34
|
+
recommended: false,
|
|
35
|
+
});
|
|
36
|
+
r.requires("tenant-lifecycle");
|
|
37
|
+
r.optionalRequires("files");
|
|
38
|
+
|
|
39
|
+
r.useExtension(EXT_TENANT_DATA, "fileRef", { destroy: fileRefTenantDestroyHook });
|
|
40
|
+
r.useExtension(EXT_STORAGE_PROVIDER, "fileRef", { destroyTenant: fileRefStorageDestroyHook });
|
|
41
|
+
|
|
42
|
+
r.job(
|
|
43
|
+
"sweep-orphaned-derivatives",
|
|
44
|
+
{
|
|
45
|
+
trigger: { manual: true },
|
|
46
|
+
concurrency: "skip",
|
|
47
|
+
schema: sweepOrphanedDerivativesPayloadSchema,
|
|
48
|
+
},
|
|
49
|
+
sweepOrphanedDerivativesJob,
|
|
50
|
+
);
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export {
|
|
55
|
+
sweepOrphanedDerivativesJob,
|
|
56
|
+
sweepOrphanedDerivativesPayloadSchema,
|
|
57
|
+
} from "./handlers/sweep-orphaned-derivatives.job";
|
|
58
|
+
export { fileRefStorageDestroyHook, fileRefTenantDestroyHook } from "./hooks";
|
|
@@ -53,3 +53,20 @@ export async function selectStaleDraftsBatch(
|
|
|
53
53
|
insertedAt: Temporal.Instant.fromEpochMilliseconds(+row.inserted_at), // @cast-boundary db-row
|
|
54
54
|
}));
|
|
55
55
|
}
|
|
56
|
+
|
|
57
|
+
/** Re-check staleness after the select→release window so a freshly saved draft is not deleted. */
|
|
58
|
+
export async function isDraftStillStale(
|
|
59
|
+
db: DbConnection,
|
|
60
|
+
id: string,
|
|
61
|
+
olderThanDays: number,
|
|
62
|
+
): Promise<boolean> {
|
|
63
|
+
const rows = (await unsafeReadRetrying(
|
|
64
|
+
db,
|
|
65
|
+
`SELECT 1 FROM "read_form_drafts"
|
|
66
|
+
WHERE "id" = $1
|
|
67
|
+
AND COALESCE("modified_at", "inserted_at") < now() - ($2::int * interval '1 day')
|
|
68
|
+
LIMIT 1`,
|
|
69
|
+
[id, olderThanDays],
|
|
70
|
+
)) as readonly unknown[];
|
|
71
|
+
return rows.length > 0;
|
|
72
|
+
}
|
|
@@ -24,7 +24,11 @@ import {
|
|
|
24
24
|
} from "@cosmicdrift/kumiko-framework/engine";
|
|
25
25
|
import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
|
|
26
26
|
import { fileRefEntity, fileRefsTable } from "@cosmicdrift/kumiko-framework/files";
|
|
27
|
-
import {
|
|
27
|
+
import {
|
|
28
|
+
isDraftStillStale,
|
|
29
|
+
type StaleDraftRow,
|
|
30
|
+
selectStaleDraftsBatch,
|
|
31
|
+
} from "../db/queries/cleanup";
|
|
28
32
|
import { filterOwnedFileRefs } from "../db/queries/owned-file-refs";
|
|
29
33
|
import { formDraftExecutor } from "../executor";
|
|
30
34
|
import { collectDraftFileRefKeys, releaseDraftFileRefs } from "../release-file-refs";
|
|
@@ -121,16 +125,21 @@ export function groupStaleDraftIdsByTenant(
|
|
|
121
125
|
async function deleteStaleDraftsBatch(
|
|
122
126
|
batch: readonly StaleDraftRow[],
|
|
123
127
|
db: DbConnection,
|
|
128
|
+
retentionDays: number,
|
|
124
129
|
log: AppContext["log"],
|
|
125
|
-
): Promise<
|
|
126
|
-
|
|
130
|
+
): Promise<readonly StaleDraftRow[]> {
|
|
131
|
+
const deletedRows: StaleDraftRow[] = [];
|
|
132
|
+
const byId = new Map(batch.map((row) => [row.id, row]));
|
|
127
133
|
for (const [tenantId, ids] of groupStaleDraftIdsByTenant(batch)) {
|
|
128
134
|
const systemUser = createSystemUser(tenantId);
|
|
129
135
|
const tenantDb = createTenantDb(db, tenantId, "system");
|
|
130
136
|
for (const id of ids) {
|
|
137
|
+
// Race: user may have saved between select and here — skip if no longer stale.
|
|
138
|
+
if (!(await isDraftStillStale(db, id, retentionDays))) continue;
|
|
131
139
|
const result = await formDraftExecutor.delete({ id }, systemUser, tenantDb);
|
|
132
140
|
if (result.isSuccess) {
|
|
133
|
-
|
|
141
|
+
const row = byId.get(id);
|
|
142
|
+
if (row) deletedRows.push(row);
|
|
134
143
|
} else {
|
|
135
144
|
log?.warn?.(
|
|
136
145
|
`[form-draft:cleanup] failed to delete stale draft id=${id} tenant=${tenantId}: ${result.error.message}`,
|
|
@@ -138,7 +147,7 @@ async function deleteStaleDraftsBatch(
|
|
|
138
147
|
}
|
|
139
148
|
}
|
|
140
149
|
}
|
|
141
|
-
return
|
|
150
|
+
return deletedRows;
|
|
142
151
|
}
|
|
143
152
|
|
|
144
153
|
export const cleanupDraftsJob: JobHandlerFn = async (_payload, ctx) => {
|
|
@@ -182,13 +191,14 @@ export const cleanupDraftsJob: JobHandlerFn = async (_payload, ctx) => {
|
|
|
182
191
|
// tenant per job run and would be wrong here, so each row resolves its
|
|
183
192
|
// own tenant's provider via _fileProviderResolver instead. Skipped
|
|
184
193
|
// entirely for rows with no FileRefs in the blob, the common case.
|
|
185
|
-
|
|
194
|
+
// Delete first (with staleness re-check), then release binaries from rows
|
|
195
|
+
// that actually disappeared — never release a draft the user just saved.
|
|
196
|
+
const deletedRows = await deleteStaleDraftsBatch(batch, db, retentionDays, ctx.log);
|
|
197
|
+
for (const row of deletedRows) {
|
|
186
198
|
await releaseRowFileRefs(row, db, fileProviderResolver, ctx.log);
|
|
187
199
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
deleted += batchDeleted;
|
|
191
|
-
if (batchDeleted === 0 || batch.length < DEFAULT_BATCH_SIZE) break;
|
|
200
|
+
deleted += deletedRows.length;
|
|
201
|
+
if (deletedRows.length === 0 || batch.length < DEFAULT_BATCH_SIZE) break;
|
|
192
202
|
}
|
|
193
203
|
ctx.log?.info?.(`[form-draft:cleanup] deleted=${deleted} retentionDays=${retentionDays}`);
|
|
194
204
|
};
|
|
@@ -74,13 +74,31 @@ export const discardDraftWrite = defineWriteHandler({
|
|
|
74
74
|
const forgetResult = await fileRefExecutor.forget({ id: ref.id }, event.user, ctx.db);
|
|
75
75
|
if (!forgetResult.isSuccess) return forgetResult;
|
|
76
76
|
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
77
|
+
// Binary delete only after commit — in-tx delete would orphan pointers
|
|
78
|
+
// if the surrounding write transaction rolls back after this handler.
|
|
79
|
+
const keys = ownedRefs.map((ref) => ref.storageKey);
|
|
80
|
+
const log = ctx.log;
|
|
81
|
+
const schedule = ctx.scheduleAfterCommit;
|
|
82
|
+
if (schedule) {
|
|
83
|
+
schedule(async () => {
|
|
84
|
+
await releaseDraftFileRefs(
|
|
85
|
+
keys,
|
|
86
|
+
async (key) => {
|
|
87
|
+
await files.ref(key).delete();
|
|
88
|
+
},
|
|
89
|
+
log,
|
|
90
|
+
);
|
|
91
|
+
});
|
|
92
|
+
} else {
|
|
93
|
+
// Unit/harness paths without a commit sink — best-effort inline.
|
|
94
|
+
await releaseDraftFileRefs(
|
|
95
|
+
keys,
|
|
96
|
+
async (key) => {
|
|
97
|
+
await files.ref(key).delete();
|
|
98
|
+
},
|
|
99
|
+
log,
|
|
100
|
+
);
|
|
101
|
+
}
|
|
84
102
|
}
|
|
85
103
|
return { isSuccess: true as const, data: { discarded: true } };
|
|
86
104
|
},
|
|
@@ -2,7 +2,7 @@ import { defineQueryHandler } from "@cosmicdrift/kumiko-framework/engine";
|
|
|
2
2
|
import { FORM_DRAFT_ACCESS } from "../constants";
|
|
3
3
|
import { lookupDraft } from "../lookup";
|
|
4
4
|
import type { FormDraftBlob } from "../schemas";
|
|
5
|
-
import { getDraftPayloadSchema } from "../schemas";
|
|
5
|
+
import { formDraftBlobSchema, getDraftPayloadSchema } from "../schemas";
|
|
6
6
|
|
|
7
7
|
export type GetDraftResult = { readonly draft: FormDraftBlob | null };
|
|
8
8
|
|
|
@@ -21,6 +21,8 @@ export const getDraftQuery = defineQueryHandler({
|
|
|
21
21
|
query.user.id,
|
|
22
22
|
query.payload.draftKey,
|
|
23
23
|
);
|
|
24
|
-
return { draft:
|
|
24
|
+
if (!row) return { draft: null };
|
|
25
|
+
const parsed = formDraftBlobSchema.safeParse(row.draft);
|
|
26
|
+
return { draft: parsed.success ? parsed.data : null };
|
|
25
27
|
},
|
|
26
28
|
});
|
|
@@ -43,7 +43,10 @@ export const listDraftsQuery = defineQueryHandler({
|
|
|
43
43
|
stepIndex: row.draft.stepIndex,
|
|
44
44
|
savedAt: row.draft.savedAt,
|
|
45
45
|
}))
|
|
46
|
-
.sort(byNewestFirst)
|
|
46
|
+
.sort(byNewestFirst)
|
|
47
|
+
// Matches listDraftsByScreen selectMany limit — JS sort cannot push the
|
|
48
|
+
// ceiling into SQL while savedAt lives inside jsonb.
|
|
49
|
+
.slice(0, 200);
|
|
47
50
|
return { drafts };
|
|
48
51
|
},
|
|
49
52
|
});
|
package/src/form-draft/lookup.ts
CHANGED
|
@@ -47,9 +47,16 @@ export async function listDraftsByScreen(
|
|
|
47
47
|
ownerId: string,
|
|
48
48
|
screenId: string,
|
|
49
49
|
): Promise<readonly FormDraftSummaryRow[]> {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
50
|
+
// ponytail: hard cap — savedAt lives in jsonb so SQL ORDER BY+LIMIT is wrong;
|
|
51
|
+
// promote savedAt to a column when callers need correct newest-N under the ceiling.
|
|
52
|
+
return selectMany<FormDraftSummaryRow>(
|
|
53
|
+
db,
|
|
54
|
+
formDraftTable,
|
|
55
|
+
{
|
|
56
|
+
tenantId,
|
|
57
|
+
ownerId,
|
|
58
|
+
draftKey: { like: `${escapeLikePattern(screenId)}:%` },
|
|
59
|
+
},
|
|
60
|
+
{ limit: 200 },
|
|
61
|
+
);
|
|
55
62
|
}
|
|
@@ -127,6 +127,10 @@ type WatcherState = {
|
|
|
127
127
|
* watch lease; null when unclaimed (no `deps.lock`, or claim lost). */
|
|
128
128
|
lockToken: string | null;
|
|
129
129
|
renewTimer: ReturnType<typeof setTimeout> | null;
|
|
130
|
+
/** Consecutive renew() throws — tear down once failures span a full TTL. */
|
|
131
|
+
renewFailures: number;
|
|
132
|
+
/** True while plugin.watch() is in flight (re-entrancy guard). */
|
|
133
|
+
starting: boolean;
|
|
130
134
|
};
|
|
131
135
|
|
|
132
136
|
type WatchLeaseResult = "acquired" | "no-token" | "stale";
|
|
@@ -386,12 +390,14 @@ export function createInboundMailSupervisor(
|
|
|
386
390
|
// lifecycle — it keeps renewing across reconnect backoff, not just while
|
|
387
391
|
// `plugin.watch()` is actually connected, so a flaky IMAP link doesn't
|
|
388
392
|
// make this worker lose the account to a peer mid-backoff.
|
|
393
|
+
// kumiko-lint-ignore complexity-budget watch lease renew backoff/re-acquire loop
|
|
389
394
|
function scheduleRenew(
|
|
390
395
|
account: MailAccountRecord,
|
|
391
396
|
state: WatcherState,
|
|
392
397
|
generation: number,
|
|
393
398
|
): void {
|
|
394
399
|
state.renewTimer = setTimeout(() => {
|
|
400
|
+
// kumiko-lint-ignore complexity-budget renew timer callback (renew/backoff/re-acquire)
|
|
395
401
|
void (async () => {
|
|
396
402
|
state.renewTimer = null;
|
|
397
403
|
// skip: supervisor stopped, generation superseded, or lease already lost — stale timer fire, nothing to renew.
|
|
@@ -403,14 +409,25 @@ export function createInboundMailSupervisor(
|
|
|
403
409
|
log(
|
|
404
410
|
`inbound-mail: watch lease renew for account ${account.id} failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
405
411
|
);
|
|
406
|
-
|
|
407
|
-
//
|
|
412
|
+
state.renewFailures += 1;
|
|
413
|
+
// After failures spanning a full lease TTL, tear down — otherwise a
|
|
414
|
+
// peer can acquire the expired key while we still hold an IDLE conn.
|
|
415
|
+
if (state.renewFailures * renewIntervalMs >= leaseTtlSeconds * 1000) {
|
|
416
|
+
log(
|
|
417
|
+
`inbound-mail: watch lease renew for account ${account.id} failed across TTL — tearing down local watcher`,
|
|
418
|
+
);
|
|
419
|
+
state.lockToken = null;
|
|
420
|
+
await stopWatcher(account.id);
|
|
421
|
+
// skip: local watcher torn down after renew failures spanning a full lease TTL.
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
408
424
|
if (running && state.generation === generation && state.lockToken) {
|
|
409
425
|
scheduleRenew(account, state, generation);
|
|
410
426
|
}
|
|
411
427
|
// skip: renew already rescheduled above (or conditions no longer hold) — nothing left to do this tick.
|
|
412
428
|
return;
|
|
413
429
|
}
|
|
430
|
+
state.renewFailures = 0;
|
|
414
431
|
if (!renewed) {
|
|
415
432
|
log(
|
|
416
433
|
`inbound-mail: watch lease for account ${account.id} lost — tearing down local watcher`,
|
|
@@ -427,6 +444,7 @@ export function createInboundMailSupervisor(
|
|
|
427
444
|
}, renewIntervalMs);
|
|
428
445
|
}
|
|
429
446
|
|
|
447
|
+
// kumiko-lint-ignore complexity-budget yield-point stale-map check after acquire is intentional (#2460)
|
|
430
448
|
async function acquireWatchLease(
|
|
431
449
|
account: MailAccountRecord,
|
|
432
450
|
state: WatcherState,
|
|
@@ -448,6 +466,7 @@ export function createInboundMailSupervisor(
|
|
|
448
466
|
return "acquired";
|
|
449
467
|
}
|
|
450
468
|
|
|
469
|
+
// kumiko-lint-ignore complexity-budget watch lifecycle (start/stop/restart) stays cohesive
|
|
451
470
|
async function ensureWatcher(
|
|
452
471
|
account: MailAccountRecord,
|
|
453
472
|
plugin: InboundMailProviderPlugin,
|
|
@@ -455,8 +474,8 @@ export function createInboundMailSupervisor(
|
|
|
455
474
|
// skip: Provider ohne Live-Push — der Reconciliation-Poll deckt den Account ab.
|
|
456
475
|
if (!plugin.watch) return;
|
|
457
476
|
const existing = watchers.get(account.id);
|
|
458
|
-
// skip: Watcher läuft bereits
|
|
459
|
-
if (existing?.stop || existing?.restartTimer) return;
|
|
477
|
+
// skip: Watcher läuft bereits, startet gerade, oder Restart ist geplant.
|
|
478
|
+
if (existing?.stop || existing?.restartTimer || existing?.starting) return;
|
|
460
479
|
|
|
461
480
|
const state: WatcherState = existing ?? {
|
|
462
481
|
stop: null,
|
|
@@ -465,6 +484,8 @@ export function createInboundMailSupervisor(
|
|
|
465
484
|
generation: 0,
|
|
466
485
|
lockToken: null,
|
|
467
486
|
renewTimer: null,
|
|
487
|
+
renewFailures: 0,
|
|
488
|
+
starting: false,
|
|
468
489
|
};
|
|
469
490
|
watchers.set(account.id, state);
|
|
470
491
|
|
|
@@ -501,6 +522,7 @@ export function createInboundMailSupervisor(
|
|
|
501
522
|
}, delay);
|
|
502
523
|
};
|
|
503
524
|
|
|
525
|
+
state.starting = true;
|
|
504
526
|
try {
|
|
505
527
|
const stop = await plugin.watch(deps.providerCtx, account, {
|
|
506
528
|
onMessages: async (msgs) => {
|
|
@@ -545,6 +567,8 @@ export function createInboundMailSupervisor(
|
|
|
545
567
|
} catch (err) {
|
|
546
568
|
const keepRunning = await handleSyncError(account, err);
|
|
547
569
|
if (keepRunning) scheduleRestart(err);
|
|
570
|
+
} finally {
|
|
571
|
+
state.starting = false;
|
|
548
572
|
}
|
|
549
573
|
}
|
|
550
574
|
|