@cosmicdrift/kumiko-bundled-features 0.110.0 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/auth-email-password/__tests__/account-lockout-no-redis.integration.test.ts +3 -3
- package/src/auth-email-password/__tests__/account-lockout.integration.test.ts +3 -3
- package/src/auth-email-password/__tests__/auth-claims.integration.test.ts +9 -10
- package/src/auth-email-password/__tests__/auth.integration.test.ts +5 -5
- package/src/auth-email-password/__tests__/email-verification.integration.test.ts +7 -4
- package/src/auth-email-password/__tests__/identity-v3-login.integration.test.ts +3 -3
- package/src/auth-email-password/__tests__/password-reset.integration.test.ts +3 -4
- package/src/auth-email-password/__tests__/public-routes-rate-limit.integration.test.ts +3 -3
- package/src/auth-email-password/__tests__/session-callbacks.integration.test.ts +3 -3
- package/src/config/__tests__/config.integration.test.ts +7 -8
- package/src/config/__tests__/encrypted-legacy-migration.integration.test.ts +179 -0
- package/src/config/__tests__/inherited-redaction.integration.test.ts +4 -3
- package/src/config/feature.ts +14 -8
- package/src/config/handlers/reencrypt.job.ts +197 -0
- package/src/config/handlers/set.write.ts +2 -2
- package/src/config/index.ts +4 -3
- package/src/config/resolver.ts +34 -10
- package/src/file-foundation/__tests__/file-foundation.integration.test.ts +4 -3
- package/src/folders/__tests__/folders.integration.test.ts +3 -0
- package/src/folders-user-data/__tests__/hooks.integration.test.ts +1 -0
- package/src/jobs/__tests__/job-system-user.integration.test.ts +4 -8
- package/src/mail-foundation/__tests__/mail-foundation.integration.test.ts +4 -3
- package/src/personal-access-tokens/__tests__/pat.integration.test.ts +3 -4
- package/src/readiness/__tests__/readiness.integration.test.ts +4 -3
- package/src/secrets/handlers/rotate.job.ts +77 -98
- package/src/secrets/secrets-context.ts +7 -38
- package/src/secrets/table.ts +6 -7
- package/src/sessions/__tests__/password-auto-revoke.integration.test.ts +6 -4
- package/src/sessions/__tests__/sessions.integration.test.ts +3 -3
- package/src/shared/chunked-entity-migration.ts +89 -0
- package/src/shared/index.ts +7 -0
- package/src/subscription-stripe/__tests__/stripe-foundation.integration.test.ts +6 -5
- package/src/tenant/__tests__/tenant.integration.test.ts +8 -4
- package/src/tier-engine/__tests__/tier-engine.integration.test.ts +7 -4
- package/src/user-data-rights/__tests__/download.integration.test.ts +7 -4
- package/src/user-data-rights/__tests__/export-encrypted-fields.integration.test.ts +152 -0
- package/src/user-data-rights/__tests__/forget-hook-registry.integration.test.ts +108 -0
- package/src/user-data-rights/__tests__/read-users-rebuild-survives-lifecycle.integration.test.ts +3 -3
- package/src/user-data-rights/__tests__/restriction-flow.integration.test.ts +3 -3
- package/src/user-data-rights/run-forget-cleanup.ts +1 -0
- package/src/user-data-rights/run-user-export.ts +43 -4
- package/src/user-data-rights-defaults/__tests__/user-data-rights-defaults.integration.test.ts +31 -9
|
@@ -26,7 +26,12 @@ import {
|
|
|
26
26
|
} from "@cosmicdrift/kumiko-framework/db";
|
|
27
27
|
import type { JobHandlerFn, SessionUser, TenantId } from "@cosmicdrift/kumiko-framework/engine";
|
|
28
28
|
import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
|
|
29
|
-
import {
|
|
29
|
+
import {
|
|
30
|
+
decodeStoredEnvelope,
|
|
31
|
+
encodeStoredEnvelope,
|
|
32
|
+
rewrapDek,
|
|
33
|
+
} from "@cosmicdrift/kumiko-framework/secrets";
|
|
34
|
+
import { type ChunkedMigrationStopReason, runChunkedMigration } from "../../shared";
|
|
30
35
|
import { type StoredEnvelope, tenantSecretEntity, tenantSecretsTable } from "../table";
|
|
31
36
|
|
|
32
37
|
const DEFAULT_BATCH_SIZE = 100;
|
|
@@ -47,7 +52,7 @@ export type RotateJobResult = {
|
|
|
47
52
|
readonly migrated: number;
|
|
48
53
|
readonly failed: number;
|
|
49
54
|
readonly batchesProcessed: number;
|
|
50
|
-
readonly stoppedReason:
|
|
55
|
+
readonly stoppedReason: ChunkedMigrationStopReason;
|
|
51
56
|
};
|
|
52
57
|
|
|
53
58
|
export const rotateJob: JobHandlerFn = async (rawPayload, ctx): Promise<void> => {
|
|
@@ -71,11 +76,6 @@ export const rotateJob: JobHandlerFn = async (rawPayload, ctx): Promise<void> =>
|
|
|
71
76
|
? Date.now() + payload.maxDurationMs
|
|
72
77
|
: Number.POSITIVE_INFINITY;
|
|
73
78
|
|
|
74
|
-
let migrated = 0;
|
|
75
|
-
let failed = 0;
|
|
76
|
-
let batchesProcessed = 0;
|
|
77
|
-
let stoppedReason: RotateJobResult["stoppedReason"] = "empty";
|
|
78
|
-
|
|
79
79
|
// Reuse a TenantDb-per-tenant map so we don't rebuild the wrapper for
|
|
80
80
|
// each row in the same tenant. Rotation typically hits one tenant in a
|
|
81
81
|
// batch; the map trims an allocation without adding complexity.
|
|
@@ -89,101 +89,80 @@ export const rotateJob: JobHandlerFn = async (rawPayload, ctx): Promise<void> =>
|
|
|
89
89
|
return existing;
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
kekVersion:
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
if (batch.length
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
if (failed >= maxFailures) {
|
|
116
|
-
stoppedReason = "too_many_failures";
|
|
117
|
-
break;
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
for (const row of batch) {
|
|
121
|
-
if (failed >= maxFailures) {
|
|
122
|
-
stoppedReason = "too_many_failures";
|
|
123
|
-
break;
|
|
124
|
-
}
|
|
125
|
-
try {
|
|
126
|
-
const oldEnvelope = {
|
|
127
|
-
ciphertext: Buffer.from(row.envelope.ciphertext, "base64"),
|
|
128
|
-
iv: Buffer.from(row.envelope.iv, "base64"),
|
|
129
|
-
authTag: Buffer.from(row.envelope.authTag, "base64"),
|
|
130
|
-
encryptedDek: Buffer.from(row.envelope.encryptedDek, "base64"),
|
|
131
|
-
kekVersion: row.envelope.kekVersion,
|
|
132
|
-
};
|
|
133
|
-
const rotated = await rewrapDek(oldEnvelope, provider);
|
|
134
|
-
|
|
135
|
-
if (rotated.kekVersion === row.kekVersion) continue;
|
|
92
|
+
type SecretRow = {
|
|
93
|
+
id: string;
|
|
94
|
+
tenantId: string;
|
|
95
|
+
version: number;
|
|
96
|
+
envelope: StoredEnvelope;
|
|
97
|
+
kekVersion: number;
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
// A partial batch means the ne-filter is exhausted — the next re-query
|
|
101
|
+
// would only re-serve rows that failed above; end the run instead.
|
|
102
|
+
let sawPartialBatch = false;
|
|
103
|
+
async function nextBatch(): Promise<readonly SecretRow[]> {
|
|
104
|
+
if (sawPartialBatch) return [];
|
|
105
|
+
const batch = await selectMany<SecretRow>(
|
|
106
|
+
db,
|
|
107
|
+
tenantSecretsTable,
|
|
108
|
+
{ kekVersion: { ne: provider.currentVersion() } },
|
|
109
|
+
{ limit: batchSize },
|
|
110
|
+
);
|
|
111
|
+
if (batch.length < batchSize) sawPartialBatch = true;
|
|
112
|
+
return batch;
|
|
113
|
+
}
|
|
136
114
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
115
|
+
async function migrateRow(row: SecretRow): Promise<"migrated" | "skipped" | "failed"> {
|
|
116
|
+
const rotated = await rewrapDek(decodeStoredEnvelope(row.envelope), provider);
|
|
117
|
+
if (rotated.kekVersion === row.kekVersion) return "skipped";
|
|
118
|
+
|
|
119
|
+
const actor: SessionUser = {
|
|
120
|
+
id: "system",
|
|
121
|
+
tenantId: row.tenantId as TenantId,
|
|
122
|
+
roles: SYSTEM_ROLES,
|
|
123
|
+
};
|
|
124
|
+
const result = await executor.update(
|
|
125
|
+
{
|
|
126
|
+
id: row.id,
|
|
127
|
+
version: row.version,
|
|
128
|
+
changes: {
|
|
129
|
+
envelope: encodeStoredEnvelope(rotated),
|
|
142
130
|
kekVersion: rotated.kekVersion,
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
},
|
|
159
|
-
},
|
|
160
|
-
actor,
|
|
161
|
-
tdbFor(row.tenantId as TenantId),
|
|
162
|
-
);
|
|
163
|
-
|
|
164
|
-
// version_conflict == another writer (secrets.set or a parallel
|
|
165
|
-
// rotation worker) beat us. Count as "skipped" and move on — the
|
|
166
|
-
// row is already in a valid state, potentially even past target.
|
|
167
|
-
if (!result.isSuccess) {
|
|
168
|
-
if (result.error.code === "version_conflict") continue;
|
|
169
|
-
failed++;
|
|
170
|
-
ctx.log?.warn?.(`[secrets:rotate] executor rejected row ${row.id}`, {
|
|
171
|
-
code: result.error.code,
|
|
172
|
-
});
|
|
173
|
-
continue;
|
|
174
|
-
}
|
|
175
|
-
} catch (err) {
|
|
176
|
-
failed++;
|
|
177
|
-
ctx.log?.warn?.(`[secrets:rotate] failed to rotate row ${row.id}`, { err });
|
|
178
|
-
continue;
|
|
179
|
-
}
|
|
180
|
-
migrated++;
|
|
131
|
+
},
|
|
132
|
+
},
|
|
133
|
+
actor,
|
|
134
|
+
tdbFor(row.tenantId as TenantId),
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
// version_conflict == another writer (secrets.set or a parallel
|
|
138
|
+
// rotation worker) beat us — the row is already in a valid state,
|
|
139
|
+
// potentially even past target.
|
|
140
|
+
if (!result.isSuccess) {
|
|
141
|
+
if (result.error.code === "version_conflict") return "skipped";
|
|
142
|
+
ctx.log?.warn?.(`[secrets:rotate] executor rejected row ${row.id}`, {
|
|
143
|
+
code: result.error.code,
|
|
144
|
+
});
|
|
145
|
+
return "failed";
|
|
181
146
|
}
|
|
182
|
-
|
|
183
|
-
if (stoppedReason === "too_many_failures") break;
|
|
184
|
-
if (batch.length < batchSize) break;
|
|
147
|
+
return "migrated";
|
|
185
148
|
}
|
|
186
149
|
|
|
187
|
-
const
|
|
150
|
+
const outcome = await runChunkedMigration<SecretRow>({
|
|
151
|
+
nextBatch,
|
|
152
|
+
migrateRow,
|
|
153
|
+
maxFailures,
|
|
154
|
+
deadlineAt: deadline,
|
|
155
|
+
signal: ctx.signal,
|
|
156
|
+
onRowError: (row, err) => {
|
|
157
|
+
ctx.log?.warn?.(`[secrets:rotate] failed to rotate row ${row.id}`, { err });
|
|
158
|
+
},
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
const result: RotateJobResult = {
|
|
162
|
+
migrated: outcome.migrated,
|
|
163
|
+
failed: outcome.failed,
|
|
164
|
+
batchesProcessed: outcome.batchesProcessed,
|
|
165
|
+
stoppedReason: outcome.stoppedReason,
|
|
166
|
+
};
|
|
188
167
|
ctx.log?.info?.(`[secrets:rotate] complete: ${JSON.stringify(result)}`);
|
|
189
168
|
};
|
|
@@ -25,10 +25,13 @@ import {
|
|
|
25
25
|
createDekCache,
|
|
26
26
|
createSecret,
|
|
27
27
|
type DekCache,
|
|
28
|
+
decodeStoredEnvelope,
|
|
28
29
|
decryptValue,
|
|
30
|
+
encodeStoredEnvelope,
|
|
29
31
|
encryptValue,
|
|
30
32
|
type MasterKeyProvider,
|
|
31
33
|
type SecretsContext,
|
|
34
|
+
withDekCache,
|
|
32
35
|
} from "@cosmicdrift/kumiko-framework/secrets";
|
|
33
36
|
import { generateId } from "@cosmicdrift/kumiko-framework/utils";
|
|
34
37
|
import { z } from "zod";
|
|
@@ -79,37 +82,9 @@ function resolveKey(keyOrHandle: string | { readonly name: string }): string {
|
|
|
79
82
|
return typeof keyOrHandle === "string" ? keyOrHandle : keyOrHandle.name;
|
|
80
83
|
}
|
|
81
84
|
|
|
82
|
-
// Wrap a provider so its unwrapDek goes through the cache. Lets decryptValue
|
|
83
|
-
// use the full provider contract without knowing about caching — separation
|
|
84
|
-
// of concerns: decryptValue handles crypto, cache handles cost.
|
|
85
|
-
function cachedProvider(provider: MasterKeyProvider, cache: DekCache): MasterKeyProvider {
|
|
86
|
-
return {
|
|
87
|
-
wrapDek: provider.wrapDek.bind(provider),
|
|
88
|
-
unwrapDek: (encryptedDek, version) => cache.unwrapDek(encryptedDek, version, provider),
|
|
89
|
-
currentVersion: provider.currentVersion.bind(provider),
|
|
90
|
-
isAvailable: provider.isAvailable.bind(provider),
|
|
91
|
-
};
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
function decodeEnvelope(stored: StoredEnvelope): {
|
|
95
|
-
ciphertext: Buffer;
|
|
96
|
-
iv: Buffer;
|
|
97
|
-
authTag: Buffer;
|
|
98
|
-
encryptedDek: Buffer;
|
|
99
|
-
kekVersion: number;
|
|
100
|
-
} {
|
|
101
|
-
return {
|
|
102
|
-
ciphertext: Buffer.from(stored.ciphertext, "base64"),
|
|
103
|
-
iv: Buffer.from(stored.iv, "base64"),
|
|
104
|
-
authTag: Buffer.from(stored.authTag, "base64"),
|
|
105
|
-
encryptedDek: Buffer.from(stored.encryptedDek, "base64"),
|
|
106
|
-
kekVersion: stored.kekVersion,
|
|
107
|
-
};
|
|
108
|
-
}
|
|
109
|
-
|
|
110
85
|
export function createSecretsContext(opts: SecretsContextOptions): SecretsContext {
|
|
111
86
|
const { db, masterKeyProvider } = opts;
|
|
112
|
-
const provider =
|
|
87
|
+
const provider = withDekCache(masterKeyProvider, opts.dekCache ?? createDekCache());
|
|
113
88
|
|
|
114
89
|
type SecretLookupRow = {
|
|
115
90
|
readonly id: string;
|
|
@@ -133,7 +108,7 @@ export function createSecretsContext(opts: SecretsContextOptions): SecretsContex
|
|
|
133
108
|
if (!auditCtx) {
|
|
134
109
|
const existing = await lookup(tenantId, key);
|
|
135
110
|
if (!existing) return undefined;
|
|
136
|
-
const plaintext = await decryptValue(
|
|
111
|
+
const plaintext = await decryptValue(decodeStoredEnvelope(existing.envelope), provider);
|
|
137
112
|
return createSecret(plaintext);
|
|
138
113
|
}
|
|
139
114
|
|
|
@@ -142,7 +117,7 @@ export function createSecretsContext(opts: SecretsContextOptions): SecretsContex
|
|
|
142
117
|
// type doesn't widen to the transaction object cleanly.
|
|
143
118
|
const envelope = await selectTenantSecretEnvelope(tx, tenantId, key);
|
|
144
119
|
if (!envelope) return undefined;
|
|
145
|
-
const pt = await decryptValue(
|
|
120
|
+
const pt = await decryptValue(decodeStoredEnvelope(envelope), provider);
|
|
146
121
|
|
|
147
122
|
// One event per read on its own aggregate-stream (fresh UUID as
|
|
148
123
|
// aggregateId). Avoids version-conflicts between parallel reads —
|
|
@@ -190,13 +165,7 @@ export function createSecretsContext(opts: SecretsContextOptions): SecretsContex
|
|
|
190
165
|
async set(tenantId, keyOrHandle, value, setOpts = {}) {
|
|
191
166
|
const key = resolveKey(keyOrHandle);
|
|
192
167
|
const envelope = await encryptValue(value, masterKeyProvider);
|
|
193
|
-
const stored: StoredEnvelope =
|
|
194
|
-
ciphertext: envelope.ciphertext.toString("base64"),
|
|
195
|
-
iv: envelope.iv.toString("base64"),
|
|
196
|
-
authTag: envelope.authTag.toString("base64"),
|
|
197
|
-
encryptedDek: envelope.encryptedDek.toString("base64"),
|
|
198
|
-
kekVersion: envelope.kekVersion,
|
|
199
|
-
};
|
|
168
|
+
const stored: StoredEnvelope = encodeStoredEnvelope(envelope);
|
|
200
169
|
const metadata: StoredMetadata = {
|
|
201
170
|
...(setOpts.redact ? { redactedPreview: setOpts.redact(value) } : {}),
|
|
202
171
|
...(setOpts.hint ? { hint: setOpts.hint } : {}),
|
package/src/secrets/table.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
createNumberField,
|
|
14
14
|
createTextField,
|
|
15
15
|
} from "@cosmicdrift/kumiko-framework/engine";
|
|
16
|
+
import type { StoredEnvelope } from "@cosmicdrift/kumiko-framework/secrets";
|
|
16
17
|
|
|
17
18
|
// Envelope stored as a single jsonb blob. All ops are upsert-by-(tenantId, key)
|
|
18
19
|
// so there's no value in decomposing the envelope into separate columns —
|
|
@@ -22,13 +23,11 @@ import {
|
|
|
22
23
|
// `WHERE kek_version != currentVersion()` with an index on just that column
|
|
23
24
|
// without deserializing the jsonb. Duplicated inside envelope too — the two
|
|
24
25
|
// always stay in sync via the write path.
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
readonly kekVersion: number;
|
|
31
|
-
};
|
|
26
|
+
//
|
|
27
|
+
// StoredEnvelope itself is canonical in @cosmicdrift/kumiko-framework/secrets
|
|
28
|
+
// (shared with EnvelopeCipher for TEXT-column stores) — re-exported here so
|
|
29
|
+
// existing consumers keep their import path.
|
|
30
|
+
export type { StoredEnvelope } from "@cosmicdrift/kumiko-framework/secrets";
|
|
32
31
|
|
|
33
32
|
export type StoredMetadata = {
|
|
34
33
|
readonly redactedPreview?: string;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
3
|
import { asRawClient, selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
4
|
-
import { createEncryptionProvider } from "@cosmicdrift/kumiko-framework/db";
|
|
5
4
|
import type { TenantId } from "@cosmicdrift/kumiko-framework/engine";
|
|
6
5
|
import {
|
|
7
6
|
setupTestStack,
|
|
@@ -10,7 +9,10 @@ import {
|
|
|
10
9
|
unsafeCreateEntityTable,
|
|
11
10
|
unsafePushTables,
|
|
12
11
|
} from "@cosmicdrift/kumiko-framework/stack";
|
|
13
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
createLateBoundHolder,
|
|
14
|
+
createTestEnvelopeCipher,
|
|
15
|
+
} from "@cosmicdrift/kumiko-framework/testing";
|
|
14
16
|
import { AuthHandlers } from "../../auth-email-password/constants";
|
|
15
17
|
import { createAuthEmailPasswordFeature } from "../../auth-email-password/feature";
|
|
16
18
|
import { createConfigFeature } from "../../config";
|
|
@@ -49,8 +51,8 @@ const encryptionKey = randomBytes(32).toString("base64");
|
|
|
49
51
|
const TENANT: TenantId = testTenantId(1);
|
|
50
52
|
|
|
51
53
|
beforeAll(async () => {
|
|
52
|
-
const encryption =
|
|
53
|
-
const resolver = createConfigResolver({ encryption });
|
|
54
|
+
const encryption = createTestEnvelopeCipher(encryptionKey);
|
|
55
|
+
const resolver = createConfigResolver({ cipher: encryption });
|
|
54
56
|
const bound = sessionCallbacksFromLateBound(callbacks);
|
|
55
57
|
const baseRevoker = bound.asMassRevoker();
|
|
56
58
|
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
3
|
import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
4
|
-
import { createEncryptionProvider } from "@cosmicdrift/kumiko-framework/db";
|
|
5
4
|
import type { TenantId } from "@cosmicdrift/kumiko-framework/engine";
|
|
6
5
|
import {
|
|
7
6
|
setupTestStack,
|
|
@@ -12,6 +11,7 @@ import {
|
|
|
12
11
|
} from "@cosmicdrift/kumiko-framework/stack";
|
|
13
12
|
import {
|
|
14
13
|
createLateBoundHolder,
|
|
14
|
+
createTestEnvelopeCipher,
|
|
15
15
|
deleteRows,
|
|
16
16
|
resetTestTables,
|
|
17
17
|
updateRows,
|
|
@@ -47,8 +47,8 @@ const encryptionKey = randomBytes(32).toString("base64");
|
|
|
47
47
|
const TENANT: TenantId = testTenantId(1);
|
|
48
48
|
|
|
49
49
|
beforeAll(async () => {
|
|
50
|
-
const encryption =
|
|
51
|
-
const resolver = createConfigResolver({ encryption });
|
|
50
|
+
const encryption = createTestEnvelopeCipher(encryptionKey);
|
|
51
|
+
const resolver = createConfigResolver({ cipher: encryption });
|
|
52
52
|
const bound = sessionCallbacksFromLateBound(callbacks);
|
|
53
53
|
|
|
54
54
|
stack = await setupTestStack({
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// Shared loop for chunked, idempotent row migrations (secrets KEK rotation,
|
|
2
|
+
// config re-encrypt, future entity-field re-encrypts). Owns the operational
|
|
3
|
+
// mechanics — abort signal, deadline, failure circuit-breaker, batch
|
|
4
|
+
// accounting — while the caller owns the domain: what a batch is and how a
|
|
5
|
+
// single row migrates.
|
|
6
|
+
//
|
|
7
|
+
// Contract notes:
|
|
8
|
+
// - nextBatch() returning an empty array ends the run ("done"). Callers
|
|
9
|
+
// whose batch query re-evaluates (WHERE kek_version != current) converge
|
|
10
|
+
// naturally; full-scan callers return their one batch, then [].
|
|
11
|
+
// - migrateRow returns "migrated" | "skipped" (already current, lost a
|
|
12
|
+
// version_conflict race) | "failed" (counted against maxFailures);
|
|
13
|
+
// throws count as "failed" too and go through onRowError.
|
|
14
|
+
// - Rows that keep failing MAY be re-served by a re-evaluating nextBatch —
|
|
15
|
+
// the circuit-breaker (maxFailures) is what terminates that loop, same
|
|
16
|
+
// semantics the secrets rotate job always had.
|
|
17
|
+
|
|
18
|
+
export type MigrationRowOutcome = "migrated" | "skipped" | "failed";
|
|
19
|
+
|
|
20
|
+
export type ChunkedMigrationStopReason = "done" | "timeout" | "signal" | "too_many_failures";
|
|
21
|
+
|
|
22
|
+
export type ChunkedMigrationResult = {
|
|
23
|
+
readonly migrated: number;
|
|
24
|
+
readonly skipped: number;
|
|
25
|
+
readonly failed: number;
|
|
26
|
+
readonly batchesProcessed: number;
|
|
27
|
+
readonly stoppedReason: ChunkedMigrationStopReason;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export type ChunkedMigrationOptions<Row> = {
|
|
31
|
+
readonly nextBatch: () => Promise<readonly Row[]>;
|
|
32
|
+
readonly migrateRow: (row: Row) => Promise<MigrationRowOutcome>;
|
|
33
|
+
readonly maxFailures: number;
|
|
34
|
+
// Epoch ms; Number.POSITIVE_INFINITY for no time bound.
|
|
35
|
+
readonly deadlineAt: number;
|
|
36
|
+
readonly signal?: AbortSignal | undefined;
|
|
37
|
+
readonly onRowError?: (row: Row, err: unknown) => void;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export async function runChunkedMigration<Row>(
|
|
41
|
+
opts: ChunkedMigrationOptions<Row>,
|
|
42
|
+
): Promise<ChunkedMigrationResult> {
|
|
43
|
+
let migrated = 0;
|
|
44
|
+
let skipped = 0;
|
|
45
|
+
let failed = 0;
|
|
46
|
+
let batchesProcessed = 0;
|
|
47
|
+
let stoppedReason: ChunkedMigrationStopReason = "done";
|
|
48
|
+
|
|
49
|
+
outer: while (true) {
|
|
50
|
+
if (opts.signal?.aborted) {
|
|
51
|
+
stoppedReason = "signal";
|
|
52
|
+
break;
|
|
53
|
+
}
|
|
54
|
+
if (Date.now() >= opts.deadlineAt) {
|
|
55
|
+
stoppedReason = "timeout";
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const batch = await opts.nextBatch();
|
|
60
|
+
if (batch.length === 0) break;
|
|
61
|
+
|
|
62
|
+
batchesProcessed++;
|
|
63
|
+
|
|
64
|
+
if (failed >= opts.maxFailures) {
|
|
65
|
+
stoppedReason = "too_many_failures";
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
for (const row of batch) {
|
|
70
|
+
if (failed >= opts.maxFailures) {
|
|
71
|
+
stoppedReason = "too_many_failures";
|
|
72
|
+
break outer;
|
|
73
|
+
}
|
|
74
|
+
let outcome: MigrationRowOutcome;
|
|
75
|
+
try {
|
|
76
|
+
outcome = await opts.migrateRow(row);
|
|
77
|
+
} catch (err) {
|
|
78
|
+
opts.onRowError?.(row, err);
|
|
79
|
+
failed++;
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
if (outcome === "migrated") migrated++;
|
|
83
|
+
else if (outcome === "skipped") skipped++;
|
|
84
|
+
else failed++;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return { migrated, skipped, failed, batchesProcessed, stoppedReason };
|
|
89
|
+
}
|
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
type SubscriptionProviderPlugin,
|
|
21
21
|
subscriptionAggregateId,
|
|
22
22
|
} from "@cosmicdrift/kumiko-bundled-features/billing-foundation";
|
|
23
|
-
import {
|
|
23
|
+
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
24
24
|
import { SYSTEM_TENANT_ID, type TenantId } from "@cosmicdrift/kumiko-framework/engine";
|
|
25
25
|
import { createEventsTable, loadAggregate } from "@cosmicdrift/kumiko-framework/event-store";
|
|
26
26
|
import { createEnvMasterKeyProvider } from "@cosmicdrift/kumiko-framework/secrets";
|
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
testTenantId,
|
|
32
32
|
unsafePushTables,
|
|
33
33
|
} from "@cosmicdrift/kumiko-framework/stack";
|
|
34
|
+
import { createTestEnvelopeCipher } from "@cosmicdrift/kumiko-framework/testing";
|
|
34
35
|
import { Hono } from "hono";
|
|
35
36
|
import Stripe from "stripe";
|
|
36
37
|
import { configValuesTable, createConfigFeature } from "../../config";
|
|
@@ -79,8 +80,8 @@ beforeAll(async () => {
|
|
|
79
80
|
priceToTier: PRICE_TO_TIER,
|
|
80
81
|
});
|
|
81
82
|
|
|
82
|
-
const encryption =
|
|
83
|
-
const resolver = createConfigResolver({ encryption });
|
|
83
|
+
const encryption = createTestEnvelopeCipher(randomBytes(32).toString("base64"));
|
|
84
|
+
const resolver = createConfigResolver({ cipher: encryption });
|
|
84
85
|
const masterKeyProvider = createEnvMasterKeyProvider({
|
|
85
86
|
env: {
|
|
86
87
|
KUMIKO_SECRETS_MASTER_KEY_V1: randomBytes(32).toString("base64"),
|
|
@@ -470,8 +471,8 @@ describe("scenario 6: billing-live gate end-to-end (#104)", () => {
|
|
|
470
471
|
|
|
471
472
|
beforeAll(async () => {
|
|
472
473
|
const stripeFeature = createSubscriptionStripeFeature({ priceToTier: PRICE_TO_TIER });
|
|
473
|
-
const encryption =
|
|
474
|
-
const resolver = createConfigResolver({ encryption });
|
|
474
|
+
const encryption = createTestEnvelopeCipher(randomBytes(32).toString("base64"));
|
|
475
|
+
const resolver = createConfigResolver({ cipher: encryption });
|
|
475
476
|
const masterKeyProvider = createEnvMasterKeyProvider({
|
|
476
477
|
env: {
|
|
477
478
|
KUMIKO_SECRETS_MASTER_KEY_V1: randomBytes(32).toString("base64"),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
|
-
import {
|
|
3
|
+
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
4
4
|
import { createEventsTable } from "@cosmicdrift/kumiko-framework/event-store";
|
|
5
5
|
import {
|
|
6
6
|
createTestUser,
|
|
@@ -10,7 +10,11 @@ import {
|
|
|
10
10
|
unsafeCreateEntityTable,
|
|
11
11
|
unsafePushTables,
|
|
12
12
|
} from "@cosmicdrift/kumiko-framework/stack";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
createTestEnvelopeCipher,
|
|
15
|
+
expectErrorIncludes,
|
|
16
|
+
rolesOf,
|
|
17
|
+
} from "@cosmicdrift/kumiko-framework/testing";
|
|
14
18
|
import { createConfigAccessor, createConfigFeature } from "../../config";
|
|
15
19
|
import { ConfigHandlers, ConfigQueries } from "../../config/constants";
|
|
16
20
|
import { type ConfigResolver, createConfigResolver } from "../../config/resolver";
|
|
@@ -33,8 +37,8 @@ const tenantFeature = createTenantFeature();
|
|
|
33
37
|
const testEncryptionKey = randomBytes(32).toString("base64");
|
|
34
38
|
|
|
35
39
|
beforeAll(async () => {
|
|
36
|
-
const encryption =
|
|
37
|
-
resolver = createConfigResolver({ encryption });
|
|
40
|
+
const encryption = createTestEnvelopeCipher(testEncryptionKey);
|
|
41
|
+
resolver = createConfigResolver({ cipher: encryption });
|
|
38
42
|
|
|
39
43
|
stack = await setupTestStack({
|
|
40
44
|
features: [configFeature, tenantFeature],
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
|
-
import {
|
|
3
|
+
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
4
4
|
import { createEventsTable } from "@cosmicdrift/kumiko-framework/event-store";
|
|
5
5
|
import {
|
|
6
6
|
createTestUser,
|
|
@@ -11,7 +11,10 @@ import {
|
|
|
11
11
|
unsafeCreateEntityTable,
|
|
12
12
|
unsafePushTables,
|
|
13
13
|
} from "@cosmicdrift/kumiko-framework/stack";
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
createTestEnvelopeCipher,
|
|
16
|
+
expectErrorIncludes,
|
|
17
|
+
} from "@cosmicdrift/kumiko-framework/testing";
|
|
15
18
|
import { createConfigFeature } from "../../config";
|
|
16
19
|
import { type ConfigResolver, createConfigResolver } from "../../config/resolver";
|
|
17
20
|
import { configValuesTable } from "../../config/table";
|
|
@@ -38,8 +41,8 @@ const tenantFeature = createTenantFeature();
|
|
|
38
41
|
const testEncryptionKey = randomBytes(32).toString("base64");
|
|
39
42
|
|
|
40
43
|
beforeAll(async () => {
|
|
41
|
-
const encryption =
|
|
42
|
-
resolver = createConfigResolver({ encryption });
|
|
44
|
+
const encryption = createTestEnvelopeCipher(testEncryptionKey);
|
|
45
|
+
resolver = createConfigResolver({ cipher: encryption });
|
|
43
46
|
|
|
44
47
|
stack = await setupTestStack({
|
|
45
48
|
features: [configFeature, tenantFeature, tierEngineFeature],
|
|
@@ -10,7 +10,6 @@
|
|
|
10
10
|
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
11
11
|
import { randomBytes } from "node:crypto";
|
|
12
12
|
import { asRawClient, selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
13
|
-
import { createEncryptionProvider } from "@cosmicdrift/kumiko-framework/db";
|
|
14
13
|
import { defineFeature } from "@cosmicdrift/kumiko-framework/engine";
|
|
15
14
|
import { createEventsTable, eventsTable } from "@cosmicdrift/kumiko-framework/event-store";
|
|
16
15
|
import {
|
|
@@ -25,7 +24,11 @@ import {
|
|
|
25
24
|
unsafeCreateEntityTable,
|
|
26
25
|
unsafePushTables,
|
|
27
26
|
} from "@cosmicdrift/kumiko-framework/stack";
|
|
28
|
-
import {
|
|
27
|
+
import {
|
|
28
|
+
createTestEnvelopeCipher,
|
|
29
|
+
resetTestTables,
|
|
30
|
+
updateRows,
|
|
31
|
+
} from "@cosmicdrift/kumiko-framework/testing";
|
|
29
32
|
import { getTemporal } from "@cosmicdrift/kumiko-framework/time";
|
|
30
33
|
import {
|
|
31
34
|
createComplianceProfilesFeature,
|
|
@@ -100,8 +103,8 @@ const noSignedUrlProviderFeature = defineFeature("test-no-signed-url-provider",
|
|
|
100
103
|
});
|
|
101
104
|
|
|
102
105
|
beforeAll(async () => {
|
|
103
|
-
const encryption =
|
|
104
|
-
const resolver = createConfigResolver({ encryption });
|
|
106
|
+
const encryption = createTestEnvelopeCipher(testEncryptionKey);
|
|
107
|
+
const resolver = createConfigResolver({ cipher: encryption });
|
|
105
108
|
|
|
106
109
|
stack = await setupTestStack({
|
|
107
110
|
features: [
|