@cosmicdrift/kumiko-testing 0.413.1 → 0.414.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,190 @@
1
+ // @runtime dev
2
+ import { randomBytes } from "node:crypto";
3
+ import {
4
+ createDbConnection,
5
+ dropDatabaseIfExists,
6
+ listDatabaseNamesWithPrefix,
7
+ } from "@cosmicdrift/kumiko-framework/internal/stack-support";
8
+ import { generateId } from "@cosmicdrift/kumiko-framework/utils";
9
+
10
+ // Set by the integration runner; every process of the run shares it, and the runner keeps it alive.
11
+ export const TEST_RUN_ID_ENV = "KUMIKO_TEST_RUN_ID";
12
+
13
+ const TEST_RUN_ALIVE_TTL_SECONDS = 15 * 60;
14
+ const TEST_RUN_ALIVE_REFRESH_MS = 60_000;
15
+ const SWEEP_SCAN_COUNT = 1000;
16
+ const TEST_DATABASE_PREFIX = "kumiko_test_";
17
+
18
+ // Anchored on the exact run-id width, so legacy `kt:<8 hex>:` keys and hand-named test DBs never match.
19
+ const RUN_SCOPED_REDIS_KEY = /^(?:kt:|bull:kt-)([0-9a-f]{12})[:-]/;
20
+ const RUN_SCOPED_DATABASE = /^kumiko_test_(?:tpl_)?([0-9a-f]{12})_/;
21
+
22
+ export function createTestRunId(): string {
23
+ return randomBytes(6).toString("hex");
24
+ }
25
+
26
+ export function runIdOfTestRedisKey(key: string): string | undefined {
27
+ return RUN_SCOPED_REDIS_KEY.exec(key)?.[1];
28
+ }
29
+
30
+ export function runIdOfTestDatabase(dbName: string): string | undefined {
31
+ return RUN_SCOPED_DATABASE.exec(dbName)?.[1];
32
+ }
33
+
34
+ export function testRunAliveKey(runId: string): string {
35
+ return `kt:${runId}:alive`;
36
+ }
37
+
38
+ async function markTestRunAlive(redisUrl: string, runId: string): Promise<void> {
39
+ const Redis = (await import("ioredis")).Redis;
40
+ const redis = new Redis(redisUrl, { lazyConnect: true });
41
+ try {
42
+ await redis.connect();
43
+ await redis.set(testRunAliveKey(runId), "1", "EX", TEST_RUN_ALIVE_TTL_SECONDS);
44
+ } finally {
45
+ redis.disconnect();
46
+ }
47
+ }
48
+
49
+ // A short connection per refresh: a held client would keep the test process from exiting.
50
+ export async function keepTestRunAlive(
51
+ redisUrl: string,
52
+ runId: string,
53
+ ): Promise<{ readonly stop: () => void }> {
54
+ await markTestRunAlive(redisUrl, runId);
55
+ const timer = setInterval(() => {
56
+ markTestRunAlive(redisUrl, runId).catch(() => {
57
+ // skip: the next tick retries, and the TTL outlasts several missed refreshes
58
+ });
59
+ }, TEST_RUN_ALIVE_REFRESH_MS);
60
+ timer.unref();
61
+ return { stop: () => clearInterval(timer) };
62
+ }
63
+
64
+ let processTestRunId: string | undefined;
65
+ let processTestRunAlive: Promise<unknown> | undefined;
66
+
67
+ // Outside the runner (a single `bun test`, dev and e2e servers) the process owns a run of its own.
68
+ export function currentTestRunId(): string {
69
+ const runnerRunId = process.env[TEST_RUN_ID_ENV];
70
+ if (runnerRunId !== undefined) return runnerRunId;
71
+ processTestRunId ??= createTestRunId();
72
+ return processTestRunId;
73
+ }
74
+
75
+ // A process-owned run keeps itself alive, so a runner's sweep can tell its keys from a dead run's.
76
+ export async function ensureCurrentTestRunAlive(redisUrl: string): Promise<string> {
77
+ const runId = currentTestRunId();
78
+ if (process.env[TEST_RUN_ID_ENV] !== undefined) return runId;
79
+ processTestRunAlive ??= keepTestRunAlive(redisUrl, runId);
80
+ try {
81
+ await processTestRunAlive;
82
+ } catch (error) {
83
+ processTestRunAlive = undefined;
84
+ throw error;
85
+ }
86
+ return runId;
87
+ }
88
+
89
+ // Run-scoped only inside the runner: outside it no Redis key could prove the database is in use.
90
+ export function testDatabaseName(): string {
91
+ const runnerRunId = process.env[TEST_RUN_ID_ENV];
92
+ const suffix = generateId().slice(-8);
93
+ return runnerRunId === undefined
94
+ ? `${TEST_DATABASE_PREFIX}${suffix}`
95
+ : `${TEST_DATABASE_PREFIX}${runnerRunId}_${suffix}`;
96
+ }
97
+
98
+ /** Queue-name prefix for a JobRunner a test builds itself (`jobs.queueNamePrefix`,
99
+ * `createJobRunner`): it sits in the test run's namespace, so the run sweep removes its BullMQ
100
+ * keys even when the test never cleans them up. BullMQ rejects `:` in `label`. */
101
+ export function testRunQueueNamePrefix(label: string): string {
102
+ return `kt-${currentTestRunId()}-${label}-${generateId().slice(-8)}`;
103
+ }
104
+
105
+ export type FinishedTestRunSweep = {
106
+ readonly runId: string;
107
+ readonly redisUrl: string;
108
+ readonly adminDatabaseUrl: string;
109
+ };
110
+
111
+ export type FinishedTestRunSweepResult = {
112
+ readonly redisKeys: number;
113
+ readonly databases: number;
114
+ };
115
+
116
+ type IoRedis = import("ioredis").Redis;
117
+
118
+ function createRemovableRunCheck(
119
+ redis: IoRedis,
120
+ ownRunId: string,
121
+ ): (runId: string) => Promise<boolean> {
122
+ const verdicts = new Map<string, boolean>();
123
+ return async (runId) => {
124
+ if (runId === ownRunId) return true;
125
+ const known = verdicts.get(runId);
126
+ if (known !== undefined) return known;
127
+ const removable = (await redis.exists(testRunAliveKey(runId))) === 0;
128
+ verdicts.set(runId, removable);
129
+ return removable;
130
+ };
131
+ }
132
+
133
+ async function sweepTestRunRedisKeys(
134
+ redis: IoRedis,
135
+ isRemovableRun: (runId: string) => Promise<boolean>,
136
+ ): Promise<number> {
137
+ let removed = 0;
138
+ for (const match of ["kt:*", "bull:kt-*"]) {
139
+ for await (const batch of redis.scanStream({ match, count: SWEEP_SCAN_COUNT })) {
140
+ const keys: string[] = [];
141
+ for (const key of batch) {
142
+ const runId = runIdOfTestRedisKey(key);
143
+ if (runId !== undefined && (await isRemovableRun(runId))) keys.push(key);
144
+ }
145
+ if (keys.length > 0) removed += await redis.unlink(...keys);
146
+ }
147
+ }
148
+ return removed;
149
+ }
150
+
151
+ async function sweepTestRunDatabases(
152
+ adminUrl: string,
153
+ isRemovableRun: (runId: string) => Promise<boolean>,
154
+ ): Promise<number> {
155
+ const admin = createDbConnection({ url: adminUrl, maxConnections: 1, rowLevelSecurity: "off" });
156
+ let dropped = 0;
157
+ try {
158
+ for (const dbName of await listDatabaseNamesWithPrefix(admin.db, TEST_DATABASE_PREFIX)) {
159
+ const runId = runIdOfTestDatabase(dbName);
160
+ if (runId === undefined || !(await isRemovableRun(runId))) continue;
161
+ try {
162
+ await dropDatabaseIfExists(admin.db, dbName);
163
+ dropped += 1;
164
+ } catch {
165
+ // skip: a database with open connections is still in use; a later sweep retries it
166
+ }
167
+ }
168
+ } finally {
169
+ await admin.close();
170
+ }
171
+ return dropped;
172
+ }
173
+
174
+ // Once per run: removes what the run itself left behind, plus whatever runs without a live
175
+ // alive key (killed or crashed) left in Redis and Postgres.
176
+ export async function sweepFinishedTestRuns(
177
+ sweep: FinishedTestRunSweep,
178
+ ): Promise<FinishedTestRunSweepResult> {
179
+ const Redis = (await import("ioredis")).Redis;
180
+ const redis = new Redis(sweep.redisUrl, { lazyConnect: true });
181
+ try {
182
+ await redis.connect();
183
+ const isRemovableRun = createRemovableRunCheck(redis, sweep.runId);
184
+ const redisKeys = await sweepTestRunRedisKeys(redis, isRemovableRun);
185
+ const databases = await sweepTestRunDatabases(sweep.adminDatabaseUrl, isRemovableRun);
186
+ return { redisKeys, databases };
187
+ } finally {
188
+ redis.disconnect();
189
+ }
190
+ }
@@ -0,0 +1,289 @@
1
+ // @runtime dev
2
+ import { buildServer } from "@cosmicdrift/kumiko-framework/api";
3
+ import {
4
+ appHookScopedRunner,
5
+ DEFAULT_ROW_LEVEL_SECURITY_MODE,
6
+ systemScopedRunner,
7
+ } from "@cosmicdrift/kumiko-framework/internal/row-level-security";
8
+ import { createRateLimitResolver } from "@cosmicdrift/kumiko-framework/internal/stack-support";
9
+ import type { JobRunner } from "@cosmicdrift/kumiko-framework/jobs";
10
+ import type { EventDispatcher } from "@cosmicdrift/kumiko-framework/pipeline";
11
+ import {
12
+ createEventDedup,
13
+ createIdempotencyGuard,
14
+ dispatcherToWriteRef,
15
+ } from "@cosmicdrift/kumiko-framework/pipeline";
16
+ import type { ClosableCacheSyncBus } from "@cosmicdrift/kumiko-framework/redis";
17
+ import type { SearchAdapter } from "@cosmicdrift/kumiko-framework/search";
18
+ import { propertyOf } from "@cosmicdrift/kumiko-types/type-guards";
19
+ import { drainJobs } from "./drain-jobs.ts";
20
+ import type { TestRedis } from "./redis.ts";
21
+ import { createRequestHelper } from "./request-helper.ts";
22
+ import type { TestStack, TestStackOptions } from "./test-stack.ts";
23
+ import type { JobFailureTracker, StackServices } from "./test-stack-services.ts";
24
+ import { runCleanupsIgnoringErrors, type StackStorage } from "./test-stack-storage.ts";
25
+
26
+ function collectSearchableFields(features: TestStackOptions["features"]): string[] {
27
+ const searchableFields: string[] = [];
28
+ for (const feature of features) {
29
+ for (const [, entity] of Object.entries(feature.entities ?? {})) {
30
+ for (const [fieldName, field] of Object.entries(entity.fields)) {
31
+ if ((field.type === "text" || field.type === "longText") && field.searchable) {
32
+ searchableFields.push(fieldName);
33
+ }
34
+ if (field.type === "embedded") {
35
+ for (const [subName, subField] of Object.entries(field.schema)) {
36
+ if (subField.searchable) {
37
+ searchableFields.push(`${fieldName}_${subName}`);
38
+ }
39
+ }
40
+ }
41
+ }
42
+ }
43
+ }
44
+ return searchableFields;
45
+ }
46
+
47
+ // Auto-configure search for tenant 1 based on registry
48
+ async function configureStackSearch(
49
+ options: TestStackOptions,
50
+ enabledHooks: ReadonlyArray<string>,
51
+ searchAdapter: SearchAdapter,
52
+ ): Promise<void> {
53
+ if (enabledHooks.includes("search")) {
54
+ const searchableFields = collectSearchableFields(options.features);
55
+ if (options.searchConfig) {
56
+ await searchAdapter.configure(options.searchConfig.tenantId, {
57
+ searchableFields: options.searchConfig.searchableFields,
58
+ rankingFields: options.searchConfig.rankingFields,
59
+ });
60
+ } else if (searchableFields.length > 0) {
61
+ await searchAdapter.configure("00000000-0000-4000-8000-000000000001", {
62
+ searchableFields,
63
+ rankingFields: searchableFields,
64
+ });
65
+ }
66
+ }
67
+ }
68
+
69
+ // buildServer only auto-wires an L3 RateLimitResolver from
70
+ // `context.redis` — a degradation test that nulls `redis` via
71
+ // extraContext (to exercise a handler's own ctx.services.redis-down guard) also
72
+ // silently strips rate-limiting, even though every handler with a
73
+ // `rateLimit` declaration still requires a resolver at dispatch time.
74
+ // Since e7bf543ad, an IP-keyed bucket key is built unconditionally
75
+ // (falls back to a shared "unknown" IP rather than skipping), so the
76
+ // missing resolver now throws instead of merely skipping enforcement.
77
+ // Mirrors idempotency/eventDedup: built from the test-stack's own
78
+ // always-real `testRedis.redis`, independent of whatever extraContext
79
+ // did to `appContext.redis` — an explicit `context.rateLimit` from
80
+ // extraContext still wins (same precedence buildServer documents).
81
+ function resolveStackRateLimitFallback(
82
+ options: TestStackOptions,
83
+ appContext: StackServices["appContext"],
84
+ testRedis: TestRedis,
85
+ ): ReturnType<typeof createRateLimitResolver> | undefined {
86
+ const explicitRateLimit = propertyOf(appContext, "rateLimit");
87
+ const needsOwnRateLimitResolver =
88
+ explicitRateLimit === undefined &&
89
+ (appContext.redis === undefined || options.rateLimitNowMs !== undefined);
90
+ if (!needsOwnRateLimitResolver) return undefined;
91
+ return createRateLimitResolver({
92
+ // An app-supplied redis keeps buckets and app redis on the same instance.
93
+ redis: appContext.redis ?? testRedis.redis,
94
+ ...(options.rateLimitNowMs ? { nowMs: options.rateLimitNowMs } : {}),
95
+ });
96
+ }
97
+
98
+ function testCodeDb<T extends object>(options: TestStackOptions, db: T): T {
99
+ return (options.rowLevelSecurity ?? DEFAULT_ROW_LEVEL_SECURITY_MODE) === "on"
100
+ ? systemScopedRunner(db, "test stack seeds and reads rows across tenants")
101
+ : db;
102
+ }
103
+
104
+ async function resolveStackAnonymousAccess(
105
+ options: TestStackOptions,
106
+ testDb: StackStorage["testDb"],
107
+ testRedis: TestRedis,
108
+ services: StackServices,
109
+ ): Promise<Pick<ServerBuildOptions, "anonymousAccess">> {
110
+ const { registry, sseBroker } = services;
111
+ const hookDb = appHookScopedRunner(
112
+ testDb.db,
113
+ options.rowLevelSecurity ?? DEFAULT_ROW_LEVEL_SECURITY_MODE,
114
+ );
115
+ const baseAnon =
116
+ typeof options.anonymousAccess === "function"
117
+ ? options.anonymousAccess({ registry, db: hookDb, sseBroker, redis: testRedis.redis })
118
+ : options.anonymousAccess;
119
+ const resolvedAnon = options.enrichAnonymousAccess
120
+ ? await options.enrichAnonymousAccess(baseAnon, { registry, db: hookDb })
121
+ : baseAnon;
122
+ return resolvedAnon ? { anonymousAccess: resolvedAnon } : {};
123
+ }
124
+
125
+ type ServerBuildOptions = Parameters<typeof buildServer>[0];
126
+
127
+ // Default tests to no login rate-limiter so existing suites that loop
128
+ // over logins don't hit a 429 after 10 attempts. Suites specifically
129
+ // testing the limiter can override via authConfig.loginRateLimit.
130
+ function buildStackAuthConfig(
131
+ authConfig: TestStackOptions["authConfig"],
132
+ ): Pick<ServerBuildOptions, "auth"> {
133
+ if (!authConfig) return {};
134
+ return {
135
+ auth: {
136
+ ...authConfig,
137
+ ...(authConfig.loginRateLimit === undefined ? { loginRateLimit: null } : {}),
138
+ },
139
+ };
140
+ }
141
+
142
+ // Tests drive the dispatcher via stack.eventDispatcher.runOnce() for
143
+ // deterministic drains — no timer-induced flakiness. pollIntervalMs
144
+ // stays short anyway in case a test opts into `.start()`. pgClient
145
+ // plumbs through the LISTEN wake-up for tests that want to measure
146
+ // post-commit latency (Sprint E.4).
147
+ function stackEventDispatcherOptions(
148
+ options: TestStackOptions,
149
+ testDb: StackStorage["testDb"],
150
+ enabledHooks: ReadonlyArray<string>,
151
+ ): NonNullable<ServerBuildOptions["eventDispatcher"]> {
152
+ return {
153
+ pollIntervalMs: options.eventDispatcherPollIntervalMs ?? 50,
154
+ pgClient: testDb.client,
155
+ systemConsumers: {
156
+ sse: enabledHooks.includes("sse"),
157
+ search: enabledHooks.includes("search"),
158
+ accessInvalidation: enabledHooks.includes("sse"),
159
+ },
160
+ };
161
+ }
162
+
163
+ export type StackServerInput = {
164
+ readonly options: TestStackOptions;
165
+ readonly jwtSecret: string;
166
+ readonly enabledHooks: ReadonlyArray<string>;
167
+ readonly testDb: StackStorage["testDb"];
168
+ readonly testRedis: TestRedis;
169
+ readonly cacheSync: ClosableCacheSyncBus;
170
+ readonly services: StackServices;
171
+ readonly jobRunner: JobRunner | undefined;
172
+ readonly jobFailureTracker: JobFailureTracker;
173
+ };
174
+
175
+ type StackServer = ReturnType<typeof buildServer>;
176
+
177
+ function buildStackServerOptions(
178
+ input: StackServerInput,
179
+ anonymousAccess: Pick<ServerBuildOptions, "anonymousAccess">,
180
+ ): ServerBuildOptions {
181
+ const { options, jwtSecret, enabledHooks, testDb, testRedis, cacheSync, services, jobRunner } =
182
+ input;
183
+ const { registry, sseBroker, appContext } = services;
184
+ const idempotency = createIdempotencyGuard(testRedis.redis, { ttlSeconds: 60 });
185
+ const eventDedup = createEventDedup(testRedis.redis, { ttlSeconds: 60 });
186
+ const rateLimitFallback = resolveStackRateLimitFallback(options, appContext, testRedis);
187
+ return {
188
+ registry,
189
+ context: rateLimitFallback ? { ...appContext, rateLimit: rateLimitFallback } : appContext,
190
+ jwtSecret,
191
+ ...(options.observability ? { observability: options.observability } : {}),
192
+ ...(options.lifecycleSystemHooks && { systemHooks: options.lifecycleSystemHooks }),
193
+ dispatcherOptions: {
194
+ idempotency,
195
+ cacheSync,
196
+ outsideTransactionDb: testDb.outsideTransactionDb,
197
+ ...(options.effectiveFeatures && { effectiveFeatures: options.effectiveFeatures }),
198
+ ...(jobRunner && { jobRunner }),
199
+ },
200
+ eventDedup,
201
+ sseBroker,
202
+ ...(options.trustedProxyHops !== undefined && { trustedProxyHops: options.trustedProxyHops }),
203
+ ...(options.errorDocs && { errorDocs: options.errorDocs }),
204
+ ...(options.extraRoutes && { extraRoutes: options.extraRoutes }),
205
+ ...(options.metrics && { metrics: options.metrics }),
206
+ eventDispatcher: stackEventDispatcherOptions(options, testDb, enabledHooks),
207
+ ...buildStackAuthConfig(options.authConfig),
208
+ ...(options.lifecycle ? { lifecycle: options.lifecycle } : {}),
209
+ ...(options.rateLimit ? { rateLimit: options.rateLimit } : {}),
210
+ ...anonymousAccess,
211
+ };
212
+ }
213
+
214
+ function assembleTestStack(
215
+ input: StackServerInput,
216
+ server: StackServer,
217
+ eventDispatcher: EventDispatcher | undefined,
218
+ ): TestStack {
219
+ const { options, testDb, testRedis, cacheSync, services, jobRunner, jobFailureTracker } = input;
220
+ const http = createRequestHelper(server.app, server.jwt, {
221
+ ...(options.authConfig?.sessionCreator !== undefined && {
222
+ sessionCreator: options.authConfig.sessionCreator,
223
+ }),
224
+ });
225
+ // Test code and the drain see every tenant's rows; the services keep the enforced pool.
226
+ const crossTenantDb = testCodeDb(options, testDb.db);
227
+ return {
228
+ app: server.app,
229
+ jwt: server.jwt,
230
+ registry: services.registry,
231
+ db: crossTenantDb,
232
+ unscopedDb: testDb.db,
233
+ dbPoolMax: testDb.poolMax,
234
+ redis: testRedis,
235
+ cacheSync,
236
+ search: services.searchAdapter,
237
+ events: services.events,
238
+ http,
239
+ observability: server.observability,
240
+ sseBroker: services.sseBroker,
241
+ dispatcher: server.dispatcher,
242
+ context: server.context,
243
+ ...(services.clock !== undefined && { clock: services.clock }),
244
+ ...(eventDispatcher ? { eventDispatcher } : {}),
245
+ ...(server.lifecycle ? { lifecycle: server.lifecycle } : {}),
246
+ ...(jobRunner ? { jobRunner } : {}),
247
+ drainJobs: () =>
248
+ drainJobs({ db: crossTenantDb, eventDispatcher, jobRunner }, jobFailureTracker),
249
+ cleanup: async () => {
250
+ if (jobRunner) await jobRunner.stop();
251
+ if (eventDispatcher) await eventDispatcher.stop();
252
+ await server.observability.shutdown();
253
+ await cacheSync.close();
254
+ await Promise.all([testDb.cleanup(), testRedis.cleanup()]);
255
+ },
256
+ };
257
+ }
258
+
259
+ export async function startStackServer(input: StackServerInput): Promise<TestStack> {
260
+ const { options, enabledHooks, testDb, testRedis, services, jobRunner } = input;
261
+ await configureStackSearch(options, enabledHooks, services.searchAdapter);
262
+
263
+ const anonymousAccess = await resolveStackAnonymousAccess(options, testDb, testRedis, services);
264
+ const server = buildServer(buildStackServerOptions(input, anonymousAccess));
265
+
266
+ // Attach before start, as the prod entrypoints do: start() runs the worker and enqueues the
267
+ // runOnBoot jobs, and one that reaches JobContext.write before the attach fails its run.
268
+ if (jobRunner) {
269
+ jobRunner.attachDispatcher(dispatcherToWriteRef(server.dispatcher));
270
+ try {
271
+ await jobRunner.start();
272
+ } catch (error) {
273
+ // setupTestStack's failure cleanup predates the server; a failed boot gate must not leak its observability.
274
+ await runCleanupsIgnoringErrors([() => server.observability.shutdown()]);
275
+ throw error;
276
+ }
277
+ }
278
+
279
+ const eventDispatcher: EventDispatcher | undefined = server.eventDispatcher;
280
+
281
+ // Pre-register consumer state rows so tests can call runOnce() directly
282
+ // without a preceding explicit start(). Timer fires at pollIntervalMs=50
283
+ // but each consumer runs at most one turn at a time — tests that drain via
284
+ // runOnce() remain deterministic. Tests that specifically exercise the
285
+ // timer loop call start() again (idempotent) after setup.
286
+ if (eventDispatcher) await eventDispatcher.ensureRegistered();
287
+
288
+ return assembleTestStack(input, server, eventDispatcher);
289
+ }