@cosmicdrift/kumiko-framework 0.227.0 → 0.229.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-framework",
-  "version": "0.227.0",
+  "version": "0.229.0",
   "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -198,7 +198,7 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-types": "0.227.0",
+    "@cosmicdrift/kumiko-types": "0.229.0",
     "bullmq": "^5.76.7",
     "bun-types": "^1.3.13",
     "hono": "^4.13.1",
@@ -214,7 +214,7 @@
     "zod": "^4.4.3"
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-dispatcher-live": "0.227.0",
+    "@cosmicdrift/kumiko-dispatcher-live": "0.229.0",
     "bun-types": "^1.3.13",
     "pino-pretty": "^13.1.3"
   },

package/src/jobs/__tests__/jobs.integration.test.ts

@@ -865,6 +865,287 @@ describe("error handling", () => {
   });
 });
 
+// --- Multiple runner instances against the same Redis (prod runs N replicas
+// of the worker process) — proves the scheduler dedups across instances and
+// that per-tenant children distribute across whichever worker picks them up,
+// not just the instance that ran the fan-out. Own registries/log per test,
+// not the shared testFeature/jobLog — a per-second cron must not tick
+// during every other test in this file, and the two scenarios below must
+// not observe each other's fan-outs. Job handlers tag their entry with
+// `ctx.db`, which each runner instance sets to a literal string instead of
+// a real DbConnection — a cheap already-typed, already-threaded channel for
+// "which runner instance ran this" (mirrors the `ctx["systemUser"] as
+// SessionUser` cast the multi-tenant.integration.test.ts billing job uses).
+describe("perTenant across multiple runner instances", () => {
+  test("a cron tick fires the perTenant wrapper exactly once, not once per runner instance", async () => {
+    const log: Array<{ tenantId: string; runnerTag: string }> = [];
+    const cronFeature = defineFeature("multicron", (r) => {
+      r.job(
+        "fanout",
+        { trigger: { cron: "* * * * * *" }, perTenant: true },
+        async (_payload, ctx) => {
+          log.push({
+            tenantId: String(ctx.systemUser.tenantId),
+            runnerTag: ctx.db as unknown as string, // @cast-boundary test fixture
+          });
+        },
+      );
+    });
+
+    const tenants = ["multi-cron-a", "multi-cron-b"] as TenantId[];
+    let wrapperCalls = 0;
+    const getActiveTenantIds = async () => {
+      wrapperCalls++;
+      return tenants;
+    };
+    const registry = createRegistry([cronFeature]);
+    const queueNamePrefix = `kumiko-test-multi-cron-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    const runnerA = createJobRunner({
+      registry,
+      context: { db: "runner-a" as unknown as AppContext["db"] },
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      getActiveTenantIds,
+    });
+    const runnerB = createJobRunner({
+      registry,
+      context: { db: "runner-b" as unknown as AppContext["db"] },
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      getActiveTenantIds,
+    });
+    // Query BullMQ directly rather than inferring dedup from log ratios —
+    // wrapperCalls is incremented by every wrapper run, so a ratio against
+    // it stays constant whether the wrapper fired once or twice per tick.
+    const rawQueue = new Queue(`${queueNamePrefix}-worker`, {
+      connection: { host: testRedis.redis.options.host, port: testRedis.redis.options.port },
+    });
+    // A post-close 'error' here is otherwise unhandled and bun:test
+    // attributes it to whichever test runs next (fw#1805).
+    rawQueue.on("error", () => {});
+    try {
+      // Both instances call start() → both call upsertJobScheduler() with
+      // the SAME schedulerId against the SAME Redis. If that didn't dedup,
+      // there would be two scheduler entries and two independent repeat
+      // timers driving the tick below.
+      await runnerA.start();
+      await runnerB.start();
+      // The actual once-per-tick proof: exactly one scheduler entry exists
+      // for this job in Redis, regardless of how many runner instances
+      // called upsertJobScheduler() against it. Two entries here means two
+      // independent repeat timers, which would double-fire every tick.
+      expect(await rawQueue.getJobSchedulersCount()).toBe(1);
+      await waitFor(() => expect(wrapperCalls).toBeGreaterThanOrEqual(1), {
+        delays: [2000, 3000, 5000],
+      });
+    } finally {
+      // Stop immediately after catching the first tick so no further ticks
+      // can land mid-assertion — the invariant below still holds however
+      // many ticks slipped in before stop() took effect.
+      await runnerA.stop();
+      await runnerB.stop();
+      await rawQueue.close();
+    }
+    await sleep(300); // let any in-flight children land
+
+    // Secondary sanity check only — this ratio holds whether the wrapper
+    // fired once or twice per tick (wrapperCalls scales with it either
+    // way), so it does not by itself prove the once-per-tick invariant.
+    // It only confirms each wrapper run that did happen fanned out to
+    // every tenant exactly once.
+    expect(log.length).toBe(wrapperCalls * tenants.length);
+    const seenTenants = log.map((e) => e.tenantId).sort();
+    const expectedTenants = Array.from({ length: wrapperCalls }, () => tenants)
+      .flat()
+      .sort();
+    expect(seenTenants).toEqual(expectedTenants);
+  });
+
+  test("manual per-tenant fan-out distributes children across both worker instances", async () => {
+    const log: Array<{ tenantId: string; runnerTag: string }> = [];
+    const manualFeature = defineFeature("multiworker", (r) => {
+      r.job("fanout", { trigger: { manual: true }, perTenant: true }, async (_payload, ctx) => {
+        // Widen the window both worker instances are actively racing for
+        // children in — without it, a fast single worker could drain a
+        // small batch before the other instance's blocking queue-pop even
+        // wakes up, which would prove nothing either way.
+        await sleep(80);
+        log.push({
+          tenantId: String(ctx.systemUser.tenantId),
+          runnerTag: ctx.db as unknown as string, // @cast-boundary test fixture
+        });
+      });
+    });
+
+    // 12 > a single worker's hardcoded concurrency (5) — with both
+    // instances already polling before dispatch, the overflow past one
+    // worker's cap can only be served by the other.
+    const tenants = Array.from({ length: 12 }, (_, i) => `multi-worker-${i}`) as TenantId[];
+    const getActiveTenantIds = async () => tenants;
+    const registry = createRegistry([manualFeature]);
+    const queueNamePrefix = `kumiko-test-multi-worker-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    const runnerA = createJobRunner({
+      registry,
+      context: { db: "runner-a" as unknown as AppContext["db"] },
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      getActiveTenantIds,
+    });
+    const runnerB = createJobRunner({
+      registry,
+      context: { db: "runner-b" as unknown as AppContext["db"] },
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      getActiveTenantIds,
+    });
+    try {
+      await runnerA.start();
+      await runnerB.start();
+      await runnerA.dispatch("multiworker:job:fanout");
+      await waitFor(
+        () => {
+          expect(log.length).toBe(tenants.length);
+        },
+        { delays: [200, 300, 500, 1000] },
+      );
+    } finally {
+      await runnerA.stop();
+      await runnerB.stop();
+    }
+
+    // Every tenant ran exactly once — no duplicate fan-out.
+    expect(log.map((e) => e.tenantId).sort()).toEqual([...tenants].sort());
+    // Both worker instances actually processed children, not just the one
+    // that happened to run the fan-out.
+    const tagsUsed = new Set(log.map((e) => e.runnerTag));
+    expect(tagsUsed.has("runner-a")).toBe(true);
+    expect(tagsUsed.has("runner-b")).toBe(true);
+  });
+});
+
+// A perTenant wrapper's own BullMQ job id is reused to derive each child's
+// job id (see perTenantChildJobId in job-runner.ts) — a wrapper that runs
+// twice for the SAME trigger (retry after failure, a Redis drop, an
+// instance restarting mid-run) must not double the fan-out, while two
+// DIFFERENT triggers (e.g. two cron ticks) must each still fan out fully.
+// These are the two failure modes on either side of that invariant.
+describe("perTenant wrapper re-run for the same trigger", () => {
+  test("a wrapper that fires twice for the same trigger still enqueues exactly one child per tenant", async () => {
+    const log: Array<{ tenantId: string }> = [];
+    const dedupFeature = defineFeature("multidedup", (r) => {
+      r.job("fanout", { trigger: { manual: true }, perTenant: true }, async (_payload, ctx) => {
+        log.push({ tenantId: String(ctx.systemUser.tenantId) });
+      });
+    });
+
+    const tenants = ["dedup-a", "dedup-b", "dedup-c"] as TenantId[];
+    const getActiveTenantIds = async () => tenants;
+    const registry = createRegistry([dedupFeature]);
+    const queueNamePrefix = `kumiko-test-dedup-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    const runner = createJobRunner({
+      registry,
+      context: {},
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      getActiveTenantIds,
+    });
+    const rawQueue = new Queue(`${queueNamePrefix}-worker`, {
+      connection: { host: testRedis.redis.options.host, port: testRedis.redis.options.port },
+    });
+    // A post-close 'error' here is otherwise unhandled and bun:test
+    // attributes it to whichever test runs next (fw#1805).
+    rawQueue.on("error", () => {});
+
+    const wrapperJobId = "fixed-wrapper-run";
+    try {
+      await runner.start();
+
+      // First run of the wrapper for this trigger — the real Worker picks
+      // it up and fans out one child per tenant.
+      await rawQueue.add("_perTenant:multidedup:job:fanout", {}, { jobId: wrapperJobId });
+      await waitFor(() => expect(log.length).toBe(tenants.length), {
+        delays: [200, 300, 500, 1000],
+      });
+
+      // Simulate the SAME trigger firing the wrapper a second time. Remove
+      // the now-completed wrapper's bookkeeping and re-add a genuinely
+      // separate Job with the SAME id, so handleJob() runs again with the
+      // identical bullJob.id — the actual "wrapper ran twice" scenario the
+      // deterministic child id is meant to cover.
+      const completedWrapper = await rawQueue.getJob(wrapperJobId);
+      await completedWrapper?.remove();
+      await rawQueue.add("_perTenant:multidedup:job:fanout", {}, { jobId: wrapperJobId });
+
+      // Give the second run's fan-out a chance to land. It re-derives the
+      // SAME child ids as the first run, so BullMQ's existing-jobId add()
+      // should return the already-completed children rather than re-run
+      // the handler.
+      await sleep(500);
+    } finally {
+      await runner.stop();
+      await rawQueue.close();
+    }
+
+    expect(log.length).toBe(tenants.length);
+    expect(log.map((e) => e.tenantId).sort()).toEqual([...tenants].sort());
+  });
+
+  test("two consecutive cron ticks each still enqueue one child per tenant", async () => {
+    const log: Array<{ tenantId: string }> = [];
+    const cronFeature = defineFeature("multidedupcron", (r) => {
+      r.job(
+        "fanout",
+        { trigger: { cron: "* * * * * *" }, perTenant: true },
+        async (_payload, ctx) => {
+          log.push({ tenantId: String(ctx.systemUser.tenantId) });
+        },
+      );
+    });
+
+    const tenants = ["dedup-cron-a", "dedup-cron-b"] as TenantId[];
+    let wrapperCalls = 0;
+    const getActiveTenantIds = async () => {
+      wrapperCalls++;
+      return tenants;
+    };
+    const registry = createRegistry([cronFeature]);
+    const queueNamePrefix = `kumiko-test-dedup-cron-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    const runner = createJobRunner({
+      registry,
+      context: {},
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+      getActiveTenantIds,
+    });
+    try {
+      await runner.start();
+      // Two distinct ticks get two distinct repeat-job ids (different
+      // millis) — each must still produce its own full batch of children;
+      // a wrongly cross-tick dedup would collapse the second tick's log
+      // entries into the first.
+      await waitFor(() => expect(wrapperCalls).toBeGreaterThanOrEqual(2), {
+        delays: [1500, 2000, 3000],
+      });
+    } finally {
+      await runner.stop();
+    }
+    await sleep(300); // let any in-flight children land
+
+    expect(log.length).toBe(wrapperCalls * tenants.length);
+    const seenTenants = log.map((e) => e.tenantId).sort();
+    const expectedTenants = Array.from({ length: wrapperCalls }, () => tenants)
+      .flat()
+      .sort();
+    expect(seenTenants).toEqual(expectedTenants);
+  });
+});
+
 describe("handleEvent maxPerTenant", () => {
   test("skips enqueue when tenant is already at the cap", async () => {
     clearLog();

package/src/jobs/job-runner.ts

@@ -27,6 +27,7 @@ import {
 } from "../observability";
 import { createDistributedLock, type DistributedLock } from "../pipeline/distributed-lock";
 import { RedisKeys } from "../pipeline/redis-keys";
+import { bridgeStub } from "../testing/handler-context";
 
 // Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
 // ("kumiko-jobs") — it must match between enqueuers and consumers, and an
@@ -38,6 +39,15 @@ function queueNameFor(prefix: string, lane: JobRunIn): string {
   return `${prefix}-${lane}`;
 }
 
+// perTenant jobs need a source of truth for "which tenants are active" —
+// see createJobRunner's defaultGetActiveTenantIds below. The tenant feature
+// (bundled-features package) registers a query under this exact name when
+// mounted. framework must not depend on bundled-features (wrong direction —
+// bundled-features already depends on framework), so this is a name-only
+// coupling: createJobRunner looks the handler up in the registry by string
+// instead of importing it.
+const ACTIVE_TENANT_IDS_QUERY_NAME = "tenant:query:active-tenant-ids";
+
 /**
  * BullMQ job ids are `repeat:<schedulerId>:<millis>`. Colons inside the
  * scheduler id push the segment count to ≥5, which BullMQ's legacy heuristic
@@ -60,6 +70,24 @@ function legacySchedulerIdForJobName(jobName: string): string {
   return `scheduler-${jobName.replace(/\./g, "-")}`;
 }
 
+// Deterministic per-tenant child job id, derived from the perTenant
+// wrapper's own BullMQ job id. A cron tick's wrapper id is
+// `repeat:<schedulerId>:<millis>` (stable for that tick, including across
+// BullMQ's own retry/stalled-job redelivery of the *same* job); a manual
+// dispatch's wrapper id is BullMQ's auto-generated id, equally stable
+// across retries of that job. Reusing it means a wrapper that runs twice
+// for the same trigger (retry after failure, a Redis drop, an instance
+// restarting mid-run) re-derives the *same* child ids, and BullMQ's
+// existing-jobId add() no-ops the second batch instead of creating
+// duplicates — the same dedup-on-id mechanism bootJobIdForJobName already
+// relies on above. Same colon-in-BullMQ-id hazard as schedulerIdForJobName
+// (fw#1603/#1604) — the wrapper id already contains ":", so strip
+// separators from both halves before joining instead of interpolating raw.
+function perTenantChildJobId(wrapperJobId: string, tenantId: string): string {
+  const sanitize = (value: string) => value.replace(/[.:]/g, "-");
+  return `child-${sanitize(wrapperJobId)}-${sanitize(tenantId)}`;
+}
+
 export type JobLogEntry = {
   level: "info" | "warn" | "error";
   message: string;
@@ -237,6 +265,38 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
   // is true keeps those out of error-rate alerts without losing them.
   let stopping = false;
 
+  // Default for perTenant fan-out when the caller didn't supply
+  // options.getActiveTenantIds: if the tenant feature is mounted (its
+  // active-tenant-ids query is in the registry), resolve tenants through
+  // that instead of forcing every app to wire the option by hand. Same
+  // hand-built-context pattern delivery-service.ts uses to invoke a
+  // registered handler without a live dispatcher — no dispatcher exists yet
+  // at createJobRunner() time, and this call runs with system-level access
+  // by construction, so going through the dispatcher's per-request identity
+  // checks would add nothing.
+  function defaultGetActiveTenantIds(): (() => Promise<TenantId[]>) | undefined {
+    const handler = registry.getQueryHandler(ACTIVE_TENANT_IDS_QUERY_NAME);
+    const db = context.db as DbConnection | undefined; // @cast-boundary db-operator
+    if (!handler || !db) return undefined;
+    return async () => {
+      const systemUser = createSystemUser(SYSTEM_TENANT_ID);
+      const systemModeDb = createTenantDb(db, SYSTEM_TENANT_ID, "system");
+      const result = await handler.handler(
+        { type: ACTIVE_TENANT_IDS_QUERY_NAME, payload: {}, user: systemUser },
+        {
+          db: systemModeDb,
+          dbOutsideTransaction: systemModeDb,
+          systemDb: createUncheckedSystemDb(systemModeDb),
+          registry,
+          ...bridgeStub(),
+        },
+      );
+      // @cast-boundary engine-payload — generic query-handler return for typed convention
+      return result as TenantId[];
+    };
+  }
+  const getActiveTenantIds = options.getActiveTenantIds ?? defaultGetActiveTenantIds();
+
   const allJobs = registry.getAllJobs();
 
   // Resolve the lane for a job — "worker" is the default because that's the
@@ -344,17 +404,34 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
     // (it's picked up by this runner's own worker).
     if (rawName.startsWith("_perTenant:")) {
       const actualName = rawName.slice("_perTenant:".length);
-      if (!options.getActiveTenantIds) {
-        throw new Error(`perTenant job "${actualName}" requires getActiveTenantIds option`);
+      if (!getActiveTenantIds) {
+        throw new Error(
+          `perTenant job "${actualName}" requires either options.getActiveTenantIds or the ` +
+            `tenant feature mounted (it registers "${ACTIVE_TENANT_IDS_QUERY_NAME}", which the ` +
+            "framework uses to resolve active tenants on its own)",
+        );
       }
       const actualDef = allJobs.get(actualName);
       if (!actualDef) {
         throw new Error(`Unknown job: ${actualName}`);
       }
-      const tenantIds = await options.getActiveTenantIds();
+      const tenantIds = await getActiveTenantIds();
       const targetQueue = queues[laneForJob(actualDef)];
+      // wrapperJobId is only absent for a Job BullMQ somehow never assigned
+      // an id to (not observed in practice) — falling back to a constant
+      // there would collide every such run's children onto the same ids
+      // and silently swallow later runs as "duplicates", which is worse
+      // than the duplicate this is meant to prevent. Skip id-based dedup
+      // for that edge case instead: enqueue plainly, same as before.
+      const wrapperJobId = bullJob.id;
       for (const tenantId of tenantIds) {
-        await targetQueue.add(actualName, { ...bullJob.data, _tenantId: tenantId });
+        await targetQueue.add(
+          actualName,
+          { ...bullJob.data, _tenantId: tenantId },
+          wrapperJobId !== undefined
+            ? { jobId: perTenantChildJobId(wrapperJobId, tenantId) }
+            : undefined,
+        );
       }
       // skip: fan-out dispatcher job, per-tenant children enqueued
       return;

package/src/stack/test-stack.ts

@@ -8,7 +8,7 @@ import { extractTableInfo } from "../db/query";
 import { createRegistry } from "../engine/registry";
 import type { AppContext, FeatureDefinition, JobRunIn, Registry, TenantId } from "../engine/types";
 import { createArchivedStreamsTable, createEventsTable } from "../event-store";
-import { createJobRunner, type JobRunner } from "../jobs";
+import { createJobRunner, type JobRunner, type JobRunnerOptions } from "../jobs";
 import type { Lifecycle } from "../lifecycle";
 import { createNoopProvider, type ObservabilityProvider } from "../observability";
 import type { Dispatcher, EventDispatcher } from "../pipeline";
@@ -154,6 +154,12 @@ export type TestStackOptions = {
   jobs?: {
     consumerLane?: JobRunIn;
     queueNamePrefix?: string;
+    /** Source of active tenant ids for `perTenant: true` jobs. Omit when the
+     *  tenant feature is among `options.features` — the framework then
+     *  resolves tenants on its own via that feature's active-tenant-ids
+     *  query, same as prod. Only needed for suites that fan a job over
+     *  tenants without mounting the tenant feature. */
+    getActiveTenantIds?: JobRunnerOptions["getActiveTenantIds"];
   };
   /** Override the event dispatcher's polling-timer interval. Default 50ms.
    *  Tests that assert LISTEN/NOTIFY wake-up latency need this pushed far
@@ -322,6 +328,9 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
       ...(options.jobs.queueNamePrefix !== undefined && {
         queueNamePrefix: options.jobs.queueNamePrefix,
       }),
+      ...(options.jobs.getActiveTenantIds !== undefined && {
+        getActiveTenantIds: options.jobs.getActiveTenantIds,
+      }),
     });
   }