@cosmicdrift/kumiko-samples 0.363.0 → 0.364.0

package/packages/bundled-features/src/inbound-mail-foundation/oauth-access-token.ts

@@ -140,14 +140,17 @@ export function createOAuthAccessTokenManager(
     const secretKey = inboundCredentialSecretKey(account.id);
     return transaction(deps.db, async (tx) => {
       await acquireNamespacedAdvisoryLock(tx, OAUTH_REFRESH_LOCK_NAMESPACE, lockKey(account));
-      const stored = await deps.secrets.get(account.tenantId, secretKey);
+      // Secret read/write join the lock transaction: a second pool connection per refresh
+      // would exhaust a small pool when many accounts refresh at once.
+      const secrets = deps.secrets.withRunner(tx);
+      const stored = await secrets.get(account.tenantId, secretKey);
       if (!stored) {
         throw new InboundAuthError("no refresh token stored for account");
       }
       const currentRefreshToken = stored.reveal();
       const tokens = await refreshWithTimeout(plugin, account, currentRefreshToken);
       if (tokens.refreshToken && tokens.refreshToken !== currentRefreshToken) {
-        await deps.secrets.set(
+        await secrets.set(
           account.tenantId,
           secretKey,
           tokens.refreshToken,
@@ -180,12 +183,14 @@ export function createOAuthAccessTokenManager(
     async storeRefreshToken(account, refreshToken) {
       await transaction(deps.db, async (tx) => {
         await acquireNamespacedAdvisoryLock(tx, OAUTH_REFRESH_LOCK_NAMESPACE, lockKey(account));
-        await deps.secrets.set(
-          account.tenantId,
-          inboundCredentialSecretKey(account.id),
-          refreshToken,
-          inboundRefreshTokenSecretOptions(account.id),
-        );
+        await deps.secrets
+          .withRunner(tx)
+          .set(
+            account.tenantId,
+            inboundCredentialSecretKey(account.id),
+            refreshToken,
+            inboundRefreshTokenSecretOptions(account.id),
+          );
       });
       cache.delete(account.id);
     },

package/packages/bundled-features/src/inbound-mail-foundation/watch-supervisor.ts

@@ -231,8 +231,10 @@ export function createInboundMailSupervisor(
         : row;
       const decryptedAddress = decrypted["address"];
       // Falling back to row.address would hand the ciphertext to a provider.
+      // One undecryptable account must not stop polling of every other tenant's accounts.
       if (typeof decryptedAddress !== "string") {
-        throw new Error(`inbound-mail: address of mail account ${row.id} did not decrypt to text`);
+        log(`inbound-mail: account ${row.id} skipped — address did not decrypt to text`);
+        continue;
       }
       records.push({
         id: row.id,

package/packages/bundled-features/src/jobs/changes.json

@@ -1,4 +1,23 @@
 [
+  {
+    "version": "0.364.0",
+    "type": "fix",
+    "title": "jobs list query caps the page size at 200",
+    "detail": "limit above 200 is rejected with a validation error."
+  },
+  {
+    "version": "0.364.0",
+    "type": "fix",
+    "title": "Stale-run sweep uses the job-run retention window for queued runs",
+    "detail": "The sweep now only fails queued runs that are older than the configured retention days instead of every queued run past the stale cutoff, so freshly queued runs of a long backlog are no longer failed."
+  },
+  {
+    "version": "0.364.0",
+    "type": "fix",
+    "title": "Tenant-visible job runs no longer collide across tenants, jobs and lanes",
+    "detail": "store_tenant_job_runs was unique on bull_job_id alone, but BullMQ ids count per queue, so a job on the api lane and one on the worker lane (or two tenants after a Redis reset) shared an id and overwrote each other's row. The unique index now covers (tenant_id, job_name, bull_job_id), every write filters by all three, and an enqueue replaces a finished row left over under the same key. onJobDropped now receives { tenantId } as a third argument.",
+    "migration": "Run `kumiko-schema generate` in your app to add the migration that creates store_tenant_job_runs_tenant_job_bull_job_id_unique and drops store_tenant_job_runs_bull_job_id_unique (generated as a commented DROP INDEX; uncomment it). A custom onJobDropped callback passed to the job runner now gets a third argument { tenantId }."
+  },
   {
     "version": "0.360.0",
     "type": "fix",

package/packages/bundled-features/src/jobs/db/queries/stale-run-sweep.ts

@@ -15,6 +15,7 @@
 import { deleteManyBatched, selectMany, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
 import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
 import { mapWithConcurrency } from "../../../shared/index.js";
+import { DEFAULT_JOB_RUN_RETENTION_DAYS } from "../../handlers/retention-cleanup.job.js";
 import { encryptFailureError } from "../../job-run-logger.js";
 import { jobRunsTable } from "../../job-run-table.js";
 import { tenantJobRunsTable } from "../../tenant-job-run-table.js";
@@ -31,18 +32,24 @@ export type StaleRunSweepResult = {
   readonly tenantQueuedDropped: number;
 };
 
+// A queued tenant row can legitimately wait much longer than a run takes (backlog, delay/backoff,
+// sequential requeue), so it is only treated as lost once it is older than the retention window,
+// not after the running-timeout.
 export async function markStaleJobRunsFailed(
   db: DbConnection,
   timeoutHours: number,
+  queuedRetentionDays: number = DEFAULT_JOB_RUN_RETENTION_DAYS,
 ): Promise<StaleRunSweepResult> {
   const cutoff = Temporal.Now.instant().subtract({ hours: timeoutHours });
+  const queuedCutoff = Temporal.Now.instant().subtract({ hours: queuedRetentionDays * 24 });
   const now = Temporal.Now.instant();
 
   // Fetch the matched rows first: `error` carries the same per-triggering-
   // user subject annotation as job-run-logger.ts's onJobFailed path
   // (personal: { of: "triggeredById" } on job-run-table.ts), so it is encrypted
-  // under that subject's DEK. Runs are grouped by subject: one encrypt and one
-  // updateMany per subject instead of per run.
+  // under the triggering user's DEK, or the system tenant key when
+  // triggeredById is null (whenAbsent: "tenant", fw#3654). Runs are grouped by
+  // that subject: one encrypt and one updateMany per subject instead of per run.
   const stale = await selectMany<{ id: string; triggeredById: string | null }>(db, jobRunsTable, {
     status: "running",
     startedAt: { lt: cutoff },
@@ -86,7 +93,8 @@ export async function markStaleJobRunsFailed(
 
   // Tenant-visible run state follows the same clock: a running row whose
   // worker died is a failure, a queued row that never started (job lost from
-  // Redis, enqueue hook without a run) would otherwise show "waiting" forever.
+  // Redis, enqueue hook without a run) would otherwise show "waiting" forever;
+  // it is dropped once it has outlived the retention window.
   const tenantRunsFailed = await updateMany(
     db,
     tenantJobRunsTable,
@@ -96,7 +104,7 @@ export async function markStaleJobRunsFailed(
   const tenantQueuedStale = await deleteManyBatched(
     db,
     tenantJobRunsTable,
-    { status: "queued", updatedAt: { lt: cutoff } },
+    { status: "queued", updatedAt: { lt: queuedCutoff } },
     { limit: STALE_DELETE_BATCH_SIZE },
   );
 

package/packages/bundled-features/src/jobs/feature.ts

@@ -36,7 +36,8 @@ import { tenantJobRunsTableMeta } from "./tenant-job-run-table.js";
 
 export type JobsFeatureOptions = {
   // How long a job run (and its logs) stays in store_job_runs/
-  // store_job_run_logs before the daily retention-cleanup job deletes it.
+  // store_job_run_logs before the daily retention-cleanup job deletes it. Also the age after
+  // which a tenant-visible "queued" row counts as lost and is dropped by the stale-run sweep.
   readonly retentionDays?: number;
   // How long a run can sit at status "running" before the hourly
   // stale-run-sweep job marks it "failed" (#2246 — a process kill mid-run
@@ -51,7 +52,7 @@ export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefi
   const staleRunTimeoutHours = options.staleRunTimeoutHours ?? DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS;
   return defineFeature("jobs", (r) => {
     r.describe(
-      "Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI. A job that declares `tenantVisibleFailure` also records its last failed attempt per tenant and subject in `store_tenant_job_failures`, which the tenant itself reads through `jobs:query:failures` — a translation key only, never the provider's message (fw#3079). A job that declares `tenantVisibleRun` also keeps its run state per tenant and subject in `store_tenant_job_runs` (queued, running, plus the latest completed or failed run, no payload, error text or logs), which the tenant reads through `jobs:query:tenant-runs` (fw#3616); the stale-run sweep fails its stuck `running` rows and drops its stuck `queued` rows.",
+      "Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI. A job that declares `tenantVisibleFailure` also records its last failed attempt per tenant and subject in `store_tenant_job_failures`, which the tenant itself reads through `jobs:query:failures` — a translation key only, never the provider's message (fw#3079). A job that declares `tenantVisibleRun` also keeps its run state per tenant and subject in `store_tenant_job_runs` (queued, running, plus the latest completed or failed run, no payload, error text or logs), which the tenant reads through `jobs:query:tenant-runs` (fw#3616); the stale-run sweep fails its stuck `running` rows and drops `queued` rows older than `retentionDays`.",
     );
     r.uiHints({
       displayLabel: "Jobs · Audit & Operator UI",
@@ -61,7 +62,9 @@ export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefi
     r.systemScope();
     r.storeTable(jobRunsTableMeta, {
       reason: "direct_write.job_runs",
-      // payload/error carry personal:{of:"triggeredById"} (job-run-table.ts).
+      // payload/error carry personal:{of:"triggeredById"} (job-run-table.ts): encrypted under the
+      // triggering user's DEK, or the system tenant key when triggeredById is null
+      // (whenAbsent: "tenant", fw#3654).
       // job-run-logger.ts manually encrypts both before every insert/update
       // (encryptStartedPayload/encryptFailureError) since this store bypasses
       // the executor's automatic PII pipeline (#2243).
@@ -115,7 +118,7 @@ export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefi
     r.job(
       "stale-run-sweep",
       { trigger: { cron: "0 * * * *" }, concurrency: "skip" },
-      createStaleRunSweepJob(staleRunTimeoutHours),
+      createStaleRunSweepJob(staleRunTimeoutHours, retentionDays),
     );
 
     const handlers = {

package/packages/bundled-features/src/jobs/handlers/list.query.ts

@@ -8,6 +8,8 @@ import { jobRunsTable } from "../job-run-table.js";
 
 const KMS_POOL_CONCURRENCY = 4;
 const DEFAULT_PAGE_SIZE = 50;
+// The offset cursor re-reads everything up to the page end, so the page size bounds that read.
+const MAX_LIST_LIMIT = 200;
 
 const jobRunStatusSchema = z.enum(["queued", "running", "completed", "failed"]);
 
@@ -68,7 +70,7 @@ export const listQuery = defineQueryHandler({
       .optional(),
     sort: z.enum(["jobName", "status", "startedAt", "duration"]).optional(),
     sortDirection: z.enum(["asc", "desc"]).optional(),
-    limit: z.number().int().min(1).optional(),
+    limit: z.number().int().min(1).max(MAX_LIST_LIMIT).optional(),
     cursor: z.string().optional(),
     totalCount: z.boolean().optional(),
   }),

package/packages/bundled-features/src/jobs/handlers/stale-run-sweep.job.ts

@@ -13,7 +13,10 @@ import { markStaleJobRunsFailed } from "../db/queries/stale-run-sweep.js";
 // Configurable per-app via JobsFeatureOptions.staleRunTimeoutHours.
 export const DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS = 24;
 
-export function createStaleRunSweepJob(timeoutHours: number): JobHandlerFn {
+export function createStaleRunSweepJob(
+  timeoutHours: number,
+  queuedRetentionDays: number,
+): JobHandlerFn {
   return async (_payload, ctx) => {
     if (!ctx.systemDb) {
       throw new InternalError({
@@ -24,7 +27,7 @@ export function createStaleRunSweepJob(timeoutHours: number): JobHandlerFn {
     const db = ctx.systemDb.unsafeRaw(
       "stale-run sweep marks stuck runs of every tenant as failed",
     ) as DbConnection; // @cast-boundary db-operator — DbRunner narrows to DbConnection, jobs never run inside a DbTx
-    const result = await markStaleJobRunsFailed(db, timeoutHours);
+    const result = await markStaleJobRunsFailed(db, timeoutHours, queuedRetentionDays);
     ctx.log?.info?.(`[jobs:stale-run-sweep] complete: ${JSON.stringify(result)}`);
   };
 }

package/packages/bundled-features/src/jobs/job-run-logger.ts

@@ -19,6 +19,7 @@ import {
 import { acquireNamespacedAdvisoryLock, type DbConnection } from "@cosmicdrift/kumiko-framework/db";
 import { type Registry, SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
 import type {
+  JobDroppedMeta,
   JobLogEntry,
   JobMeta,
   JobOutcomeMeta,
@@ -169,6 +170,7 @@ async function encryptStartedPayload(
 const TENANT_JOB_FAILURE_LOCK_NAMESPACE = 0x746a666c;
 // 'tjrl': serialises the finished-row write + prune per tenant run key.
 const TENANT_JOB_RUN_LOCK_NAMESPACE = 0x746a726c;
+const TENANT_RUN_CONFLICT_KEYS = ["tenantId", "jobName", "bullJobId"] as const;
 
 // fw#3079 — which row the tenant-visible failure record lives in: one per
 // (tenant, job, subject), so the next outcome of the same work replaces or
@@ -227,8 +229,11 @@ async function clearTenantJobFailure(
 }
 
 // fw#3616 — tenant-visible run state. One row per BullMQ job in
-// store_tenant_job_runs; the writes below only ever touch rows of the same
-// bullJobId, except pruning older finished rows of the same key.
+// store_tenant_job_runs, addressed by (tenant, job name, bullJobId): BullMQ
+// ids are only unique per queue (api and worker lane both count from 1) and
+// restart after a Redis reset, so the id alone must never pick a row. The
+// writes below only touch rows of that triple, except pruning older finished
+// rows of the same key.
 type TenantRunKey = {
   readonly tenantId: string;
   readonly jobName: string;
@@ -255,14 +260,27 @@ async function recordTenantRunQueued(
   // skip: system-tenant run, no tenant-scoped query reads it
   if (!key) return;
   const now = Temporal.Now.instant();
-  // DO NOTHING: a fast worker can report the start before this enqueue hook
-  // returns, and a repeated jobId (debounce, fan-out dedup) is the same job.
-  await insertOnConflictDoNothing(
-    db,
-    tenantJobRunsTable,
-    { ...key, bullJobId, status: "queued", queuedAt: now, updatedAt: now },
-    { conflictKeys: ["bullJobId"] },
-  );
+  await transaction(db, async (tx) => {
+    // A finished row under the same id is left over from before a Redis reset
+    // (ids restart at 1), not this job. Only rows that went through this hook
+    // (queuedAt set) qualify: a fast worker can finish before the hook runs,
+    // and that row is this job's own and must stay.
+    await deleteMany(tx, tenantJobRunsTable, {
+      tenantId: key.tenantId,
+      jobName,
+      bullJobId,
+      status: { in: ["completed", "failed"] },
+      queuedAt: { gt: Temporal.Instant.fromEpochMilliseconds(0) },
+    });
+    // DO NOTHING: a fast worker can report the start before this enqueue hook
+    // returns, and a repeated jobId (debounce, fan-out dedup) is the same job.
+    await insertOnConflictDoNothing(
+      tx,
+      tenantJobRunsTable,
+      { ...key, bullJobId, status: "queued", queuedAt: now, updatedAt: now },
+      { conflictKeys: [...TENANT_RUN_CONFLICT_KEYS] },
+    );
+  });
 }
 
 async function recordTenantRunStarted(
@@ -282,7 +300,7 @@ async function recordTenantRunStarted(
     tenantJobRunsTable,
     { ...key, bullJobId, status: "running", startedAt: now, updatedAt: now },
     {
-      conflictKeys: ["bullJobId"],
+      conflictKeys: [...TENANT_RUN_CONFLICT_KEYS],
       update: { status: "running", startedAt: now, finishedAt: null, updatedAt: now },
     },
   );
@@ -308,7 +326,7 @@ async function recordTenantRunOutcome(
       db,
       tenantJobRunsTable,
       { status: "queued", startedAt: null, finishedAt: null, updatedAt: now },
-      { bullJobId },
+      { tenantId: key.tenantId, jobName, bullJobId },
     );
     // skip: retry pending, the row stays active until the final attempt
     return;
@@ -327,7 +345,10 @@ async function recordTenantRunOutcome(
       tx,
       tenantJobRunsTable,
       { ...key, bullJobId, status: result, finishedAt: now, updatedAt: now },
-      { conflictKeys: ["bullJobId"], update: { status: result, finishedAt: now, updatedAt: now } },
+      {
+        conflictKeys: [...TENANT_RUN_CONFLICT_KEYS],
+        update: { status: result, finishedAt: now, updatedAt: now },
+      },
     );
     await deleteMany(tx, tenantJobRunsTable, {
       ...key,
@@ -337,8 +358,18 @@ async function recordTenantRunOutcome(
   });
 }
 
-async function dropTenantRunQueued(db: DbConnection, bullJobId: string): Promise<void> {
-  await deleteMany(db, tenantJobRunsTable, { bullJobId, status: "queued" });
+async function dropTenantRunQueued(
+  db: DbConnection,
+  jobName: string,
+  bullJobId: string,
+  meta: JobDroppedMeta,
+): Promise<void> {
+  await deleteMany(db, tenantJobRunsTable, {
+    tenantId: meta.tenantId,
+    jobName,
+    bullJobId,
+    status: "queued",
+  });
 }
 
 const log = createFallbackLogger("job-run-logger");
@@ -458,8 +489,12 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
       );
     },
 
-    onJobDropped: async (jobName: string, bullJobId: string) => {
-      await isolateTenantRunWrite(() => dropTenantRunQueued(db, bullJobId), jobName, bullJobId);
+    onJobDropped: async (jobName: string, bullJobId: string, meta: JobDroppedMeta) => {
+      await isolateTenantRunWrite(
+        () => dropTenantRunQueued(db, jobName, bullJobId, meta),
+        jobName,
+        bullJobId,
+      );
     },
 
     onJobStart: async (jobName: string, bullJobId: string, meta: JobMeta) => {

package/packages/bundled-features/src/jobs/tenant-job-run-table.ts

@@ -21,7 +21,7 @@ export type TenantJobRunStatus = "queued" | "running" | "completed" | "failed";
 // job is invisible there. This table holds no payload, error text or logs by
 // design — the tenant learns only that work is waiting, running or finished.
 //
-// One row per BullMQ job. Active rows (queued/running) are kept as they are;
+// One row per BullMQ job and (tenant, job name). Active rows (queued/running) are kept as they are;
 // a finished row replaces earlier finished rows of the same (tenant, job,
 // subject), so only the latest outcome stays. `subject` is the canonical JSON
 // of the job's declared subjectFields, or NULL for none — stored in clear, so
@@ -41,7 +41,12 @@ export const tenantJobRunsTable = pgTable(
     updatedAt: instant("updated_at").default(sql`now()`).notNull(),
   },
   (t) => [
-    uniqueIndex("store_tenant_job_runs_bull_job_id_unique").on(t.bullJobId),
+    // BullMQ ids are unique per queue only; the id alone would let two tenants collide.
+    uniqueIndex("store_tenant_job_runs_tenant_job_bull_job_id_unique").on(
+      t.tenantId,
+      t.jobName,
+      t.bullJobId,
+    ),
     // Finishing a run prunes older outcomes by this key; the tenant query reads by it.
     index("store_tenant_job_runs_tenant_job_subject_idx").on(t.tenantId, t.jobName, t.subject),
     // Stale sweep and retention filter on age alone.

package/packages/bundled-features/src/ledger/changes.json

@@ -1,4 +1,11 @@
 [
+  {
+    "version": "0.364.0",
+    "type": "fix",
+    "title": "Document the duplicate check required before the (tenant_id, source_ref) unique index",
+    "detail": "The unique index on read_ledger_schedules (tenant_id, source_ref) is created in place and fails on existing duplicate pairs, and a projection rebuild over duplicate events fails the same way. Replay handling for duplicate events is not part of this change.",
+    "migration": "Before upgrading, check each database for duplicates and clean them up: SELECT tenant_id, source_ref, count(*) FROM read_ledger_schedules WHERE source_ref IS NOT NULL GROUP BY 1, 2 HAVING count(*) > 1; Any row returned makes the unique index migration abort."
+  },
   {
     "version": "0.354.0",
     "type": "fix",

package/packages/bundled-features/src/ledger/entity.ts

@@ -176,7 +176,9 @@ export const scheduleEntity = createEntity({
   },
   // Find-then-create by sourceRef races across tabs/devices; only the DB closes
   // that gap. Partial because sourceRef is optional. A new name (not the old
-  // non-unique default) keeps the migration an in-place CREATE UNIQUE INDEX.
+  // non-unique default) keeps the migration an in-place CREATE UNIQUE INDEX;
+  // that statement, like a projection rebuild, fails on existing duplicate
+  // (tenant_id, source_ref) pairs, so duplicates must be cleaned up beforehand.
   indexes: [
     { columns: ["tenantId", "subjectType", "subjectId"] },
     {

package/packages/bundled-features/src/renderer-foundation/types.ts

@@ -62,6 +62,8 @@ export type RenderResponse = (
       html: string;
       text: string;
       previewText?: string;
+      fromName?: string;
+      replyTo?: string;
       attachments?: readonly MailAttachmentRef[];
     }
   | {

package/packages/bundled-features/src/renderer-pdf/changes.json

@@ -1,4 +1,16 @@
 [
+  {
+    "version": "0.364.0",
+    "type": "fix",
+    "title": "renderTablePdf rejects tables above 10000 rows, 30 columns or 5000 characters per cell",
+    "detail": "The table document schema now caps rows at 10000, columns at 30 and each cell string at 5000 characters. Larger input fails with invalid_payload instead of exhausting memory during layout."
+  },
+  {
+    "version": "0.364.0",
+    "type": "fix",
+    "title": "render-table-pdf validates input and paginates its intro",
+    "detail": "Table documents reject negative margins and unusable column widths, normalize CRLF and paginate the intro text and the first header row."
+  },
   {
     "version": "0.360.0",
     "type": "improvement",