@cosmicdrift/kumiko-samples 0.338.0

package/packages/bundled-features/src/jobs/job-run-logger.ts

@@ -0,0 +1,457 @@
+import {
+  deleteMany,
+  fetchOne,
+  insertMany,
+  insertOne,
+  updateMany,
+} from "@cosmicdrift/kumiko-framework/bun-db";
+import {
+  configuredPiiSubjectKms,
+  encryptPiiValueForSubject,
+  KeyErasedError,
+  type LocalKeyKmsAdapter,
+  PII_ERASED_SENTINEL,
+} from "@cosmicdrift/kumiko-framework/crypto";
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import { type Registry, SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
+import type {
+  JobLogEntry,
+  JobMeta,
+  JobOutcomeMeta,
+  JobRunnerOptions,
+} from "@cosmicdrift/kumiko-framework/jobs";
+import { createFallbackLogger } from "@cosmicdrift/kumiko-framework/logging";
+import { generateId } from "@cosmicdrift/kumiko-framework/utils";
+import { mapWithConcurrency } from "../shared/index.js";
+import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events.js";
+import { parseJobInstant } from "./job-instant.js";
+import { jobRunLogsTable, jobRunsTable } from "./job-run-table.js";
+import { tenantJobFailuresTable } from "./tenant-job-failure-table.js";
+
+// Matches PgKmsAdapter's default pool size (see tenant/handlers/*.query.ts) —
+// bounds concurrent getOrCreateDek calls so a large log batch doesn't claim
+// every connection in the pool.
+const KMS_POOL_CONCURRENCY = 4;
+
+// Direct-write job-run log (#2243): onJobStart/-Complete/-Failed write
+// straight into jobRunsTable / jobRunLogsTable instead of appending to the
+// event store and replaying through inline projections. Pre-#2243 every run
+// left two permanent `kumiko_events` rows that nothing else ever replayed
+// or MSP-subscribed to — in two production apps that was ~99% of all
+// events. Same tables, same shape, no event stream in between.
+//
+// BullMQ callbacks don't carry a tenantId (jobs are cross-tenant). We
+// anchor every run on SYSTEM_TENANT_ID — mirrors how config system-scope
+// rows use the sentinel.
+
+export type JobRunLoggerOptions = {
+  readonly db: DbConnection;
+  readonly registry: Registry;
+};
+
+export type JobRunLoggerCallbacks = Pick<
+  JobRunnerOptions,
+  "onJobStart" | "onJobComplete" | "onJobFailed"
+>;
+
+// Default cap on the bullJobId → runId cache. A worker that starts jobs
+// without ever seeing complete/failed callbacks (e.g. crashes mid-run)
+// would otherwise leak entries indefinitely. 10k fits ~1 hour of
+// high-throughput jobs; past that we evict oldest. DB-lookup recovers
+// evicted entries, so correctness isn't at stake — only memory bounds.
+const DEFAULT_CACHE_MAX_ENTRIES = 10_000;
+// Entry TTL. A run that hangs longer than this is either a real stuck
+// worker (ops should alert) or a test-environment run that never fired
+// complete/failed; either way the cache entry has no value. Falls back
+// to DB-lookup if actually needed.
+const DEFAULT_CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
+
+// Same per-subject encryption as encryptFailureError/encryptStartedPayload,
+// applied to a single value — shared so payload/message/error all go
+// through the identical erased-key fallback. A key erased right before (or
+// between) any of these callbacks must not blow up the write (status is
+// already committed by the time logs/error land): fall back to the
+// sentinel like decryptPiiValueForSubject does on the read side, instead of
+// throwing and dropping the rest of the batch — or, for onJobStart, the
+// whole run row.
+async function encryptOrSentinel(
+  kms: LocalKeyKmsAdapter,
+  subjectUserId: string,
+  value: string,
+  field: string,
+): Promise<string> {
+  try {
+    return await encryptPiiValueForSubject(
+      kms,
+      { kind: "user", userId: subjectUserId },
+      value,
+      { requestId: "jobs:job-run-logger" },
+      field,
+    );
+  } catch (e) {
+    if (!(e instanceof KeyErasedError)) throw e;
+    return PII_ERASED_SENTINEL;
+  }
+}
+
+// Same per-subject encryption as encryptStartedPayload, applied to each
+// batched log line's `message` (#2247) before insertMany into
+// jobRunLogsTable — that table is unmanaged/direct-write, so there is no
+// event-piiFields catalog to lean on here either. "message" is the AAD
+// field name and must match the field passed to decryptStoredPii on read
+// (detail.query.ts) or decrypt fails loud. Same skip rules as the payload:
+// null subject (system/cron runs) and absent KMS (rollout mode) both stay
+// plaintext. Bounded concurrency (#2307) — unbounded Promise.all would fire
+// one getOrCreateDek per log line at once against the KMS adapter's pool.
+async function encryptLogMessages<T extends { readonly message: string }>(
+  logs: readonly T[],
+  triggeredById: string | null,
+): Promise<T[]> {
+  if (triggeredById === null) return [...logs];
+  const kms = configuredPiiSubjectKms();
+  if (!kms) return [...logs];
+  return mapWithConcurrency(logs, KMS_POOL_CONCURRENCY, async (log) => ({
+    ...log,
+    message: await encryptOrSentinel(kms, triggeredById, log.message, "message"),
+  }));
+}
+
+// Mirrors encryptFailureError for the failure-path `error` column (#2307):
+// stored alongside logs[].message under the same triggering user's DEK, so
+// a log/error pair from the same failed run decrypt together or erase
+// together. Exported so stale-run-sweep.ts's crash-recovery write (a
+// separate, per-row batch path outside onJobFailed) applies the identical
+// per-subject encryption instead of writing `error` in the clear.
+export async function encryptFailureError(
+  error: string,
+  triggeredById: string | null,
+): Promise<string> {
+  if (triggeredById === null) return error;
+  const kms = configuredPiiSubjectKms();
+  if (!kms) return error;
+  return encryptOrSentinel(kms, triggeredById, error, "error");
+}
+
+// The run-started payload can carry arbitrary user data; triggeredById
+// names its owning user. No event-PII catalog involved (#2243 removed the
+// jobRun r.defineEvent registrations, so there is nothing to catalog) —
+// the subject is known statically, so we encrypt directly. A null subject
+// (system cron runs, recipient-less triggers) stays plaintext: there is no
+// user key to shred, mirroring the previous event-pii catalog's own skip
+// rule. Absent KMS adapter stays plaintext too (rollout mode, unchanged).
+// A key already erased by the time the trigger lands (#2307) must not fail
+// the whole run-start write — sentinel like the log/error paths.
+async function encryptStartedPayload(
+  payload: string | null,
+  triggeredById: string | null,
+): Promise<string | null> {
+  if (payload === null || triggeredById === null) return payload;
+  const kms = configuredPiiSubjectKms();
+  if (!kms) return payload;
+  return encryptOrSentinel(kms, triggeredById, payload, "payload");
+}
+
+// fw#3079 — which row the tenant-visible failure record lives in: one per
+// (tenant, job, subject), so the next outcome of the same work replaces or
+// clears it. `subject` is null for a job that declares no subjectFields.
+// Null target = nothing to write or clear: the job did not opt in, the run
+// was tenant-less (cron resolves to SYSTEM_TENANT_ID, where no tenant-scoped
+// query could ever read the row), or the caller predates fw#3079 and passes
+// no outcome at all.
+function tenantJobFailureTarget(
+  jobName: string,
+  outcome: JobOutcomeMeta | undefined,
+): Record<string, unknown> | null {
+  if (!outcome?.tenantVisible || outcome.tenantId === SYSTEM_TENANT_ID) return null;
+  return { tenantId: outcome.tenantId, jobName, subject: outcome.tenantVisible.subject };
+}
+
+async function recordTenantJobFailure(
+  db: DbConnection,
+  jobName: string,
+  outcome: JobOutcomeMeta | undefined,
+): Promise<void> {
+  const where = tenantJobFailureTarget(jobName, outcome);
+  const messageKey = outcome?.tenantVisible?.messageKey;
+  // skip: no tenant-visible target, or an attempt BullMQ may still retry — a
+  // non-final failure must not show the tenant a failure the next attempt
+  // may still resolve.
+  if (!where || !messageKey || outcome?.finalAttempt !== true) return;
+  // ponytail: delete-then-insert instead of an upsert — two runs of the same
+  // key finishing at once can leave two rows, and the query returns the
+  // newest. Add a unique index + ON CONFLICT if that ever matters.
+  await deleteMany(db, tenantJobFailuresTable, where);
+  await insertOne(db, tenantJobFailuresTable, {
+    ...where,
+    messageKey,
+    failedAt: Temporal.Now.instant(),
+  });
+}
+
+async function clearTenantJobFailure(
+  db: DbConnection,
+  jobName: string,
+  outcome: JobOutcomeMeta | undefined,
+): Promise<void> {
+  const where = tenantJobFailureTarget(jobName, outcome);
+  // skip: no tenant-visible target — nothing was ever recorded
+  if (!where) return;
+  await deleteMany(db, tenantJobFailuresTable, where);
+}
+
+const log = createFallbackLogger("job-run-logger");
+
+// The tenant failure record is an optional side write: a throw here must not
+// skip the run-row update or turn a completed run into a BullMQ retry.
+async function isolateTenantFailureWrite(
+  write: () => Promise<void>,
+  jobName: string,
+  bullJobId: string,
+): Promise<void> {
+  try {
+    await write();
+  } catch (e) {
+    log.error("tenant job failure record write failed", {
+      jobName,
+      bullJobId,
+      error: e instanceof Error ? e.message : String(e),
+    });
+  }
+}
+
+export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallbacks {
+  const { db } = opts;
+
+  // bullJobId → run uuid. BullMQ hands us the bullJobId on every callback,
+  // but the run row is keyed by a fresh UUID we mint on start. The cache
+  // threads that UUID from onJobStart through to onJobComplete/onJobFailed
+  // so the completion-write lands on the same row as the start-write.
+  //
+  // Bounded cache (LRU-ish with TTL) — worker-crash between start and
+  // complete would otherwise leak entries. DB-lookup recovers evicted
+  // entries via bull_job_id on jobRunsTable.
+  // triggeredById rides along with runId (#2247) — resolved once at start
+  // (or on cache-miss DB fallback) so onJobComplete/-Failed know the log
+  // subject without an extra always-on DB round trip.
+  type CacheEntry = {
+    readonly runId: string;
+    readonly triggeredById: string | null;
+    readonly expiresAt: number;
+  };
+  const runIdByBullJobId = new Map<string, CacheEntry>();
+
+  function cachePut(bullJobId: string, runId: string, triggeredById: string | null): void {
+    // Enforce max-size BEFORE insert. Map iteration returns insertion
+    // order, so dropping the first entry is the oldest.
+    if (runIdByBullJobId.size >= DEFAULT_CACHE_MAX_ENTRIES) {
+      const oldest = runIdByBullJobId.keys().next().value;
+      if (oldest !== undefined) runIdByBullJobId.delete(oldest);
+    }
+    runIdByBullJobId.set(bullJobId, {
+      runId,
+      triggeredById,
+      expiresAt: Date.now() + DEFAULT_CACHE_TTL_MS,
+    });
+  }
+
+  function cacheGet(bullJobId: string): CacheEntry | undefined {
+    const entry = runIdByBullJobId.get(bullJobId);
+    if (!entry) return undefined;
+    if (Date.now() >= entry.expiresAt) {
+      runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
+      return undefined;
+    }
+    return entry;
+  }
+
+  async function resolveRun(
+    bullJobId: string,
+  ): Promise<{ readonly runId: string; readonly triggeredById: string | null } | undefined> {
+    const cached = cacheGet(bullJobId);
+    if (cached) return { runId: cached.runId, triggeredById: cached.triggeredById };
+    const row = await fetchOne<{ id: string | number; triggeredById: string | null }>(
+      db,
+      jobRunsTable,
+      { bullJobId },
+    );
+    if (!row) return undefined;
+    // buildBaseColumns's signature types `id` as `string | number` because
+    // it returns both branches of the idType union. We know this table
+    // was built with idType: "uuid" (see job-run-table.ts), so narrowing
+    // via String() is safe runtime-wise. A proper framework-level fix
+    // would overload buildBaseColumns per idType — scoped out of this
+    // follow-up as its return type has four branches (with/without
+    // softDelete × serial/uuid).
+    const runId = String(row.id);
+    const triggeredById = row.triggeredById ?? null;
+    cachePut(bullJobId, runId, triggeredById);
+    return { runId, triggeredById };
+  }
+
+  return {
+    onJobStart: async (jobName: string, bullJobId: string, meta: JobMeta) => {
+      const runId = generateId();
+      const triggeredById = meta.triggeredById ?? null;
+      cachePut(bullJobId, runId, triggeredById);
+      // Parse against the registered schema so out-of-dispatcher writes
+      // get the same validation guarantee as ctx.appendEvent. A shape
+      // drift between feature + logger fails loudly at the source
+      // instead of silently landing on the table.
+      const payload = runStartedSchema.parse({
+        jobName,
+        bullJobId,
+        status: "running",
+        payload: meta.payload ?? null,
+        triggeredById,
+        startedAt: Temporal.Now.instant().toString(),
+        attempt: meta.attempt ?? 1,
+      });
+      const encryptedPayload = await encryptStartedPayload(payload.payload, payload.triggeredById);
+      await insertOne(db, jobRunsTable, {
+        id: runId,
+        tenantId: SYSTEM_TENANT_ID,
+        insertedById: "system",
+        jobName: payload.jobName,
+        bullJobId: payload.bullJobId,
+        status: payload.status,
+        payload: encryptedPayload,
+        attempt: payload.attempt,
+        startedAt: parseJobInstant(payload.startedAt),
+        triggeredById: payload.triggeredById,
+      });
+    },
+
+    onJobComplete: async (
+      jobName: string,
+      bullJobId: string,
+      duration: number,
+      logs: JobLogEntry[],
+      outcome?: JobOutcomeMeta,
+    ) => {
+      // Before the run-row write and independent of it: a successful run
+      // clears the tenant's failure record even when the run row itself is
+      // unreachable (the state-loss return below).
+      await isolateTenantFailureWrite(
+        () => clearTenantJobFailure(db, jobName, outcome),
+        jobName,
+        bullJobId,
+      );
+      const resolved = await resolveRun(bullJobId);
+      // skip: state loss between start + complete (worker restart, cache
+      // evicted AND DB has no matching bull_job_id). Rare edge case; we
+      // drop the completion write rather than forging a run row from
+      // scratch — forensics still has the original BullMQ lifecycle.
+      if (!resolved) return;
+      const { runId, triggeredById } = resolved;
+      const payload = runCompletedSchema.parse({
+        duration,
+        finishedAt: Temporal.Now.instant().toString(),
+        logs: logs.map((l) => ({
+          level: l.level,
+          message: l.message,
+          timestamp: l.timestamp.toString(),
+        })),
+      });
+      await updateMany(
+        db,
+        jobRunsTable,
+        {
+          status: "completed",
+          // Clear stale-sweep error if the run finished after being marked failed.
+          error: null,
+          // integer column — match onJobFailed's Math.round so fractional
+          // BullMQ timings don't trip Postgres integer syntax.
+          duration: Math.round(payload.duration),
+          finishedAt: parseJobInstant(payload.finishedAt),
+          modifiedAt: Temporal.Now.instant(),
+          modifiedById: "system",
+        },
+        { id: runId },
+      );
+      // skip: empty log batch — the worker ran silent. No child rows to
+      // insert; the status update above already recorded completion.
+      if (payload.logs.length > 0) {
+        const encryptedLogs = await encryptLogMessages(payload.logs, triggeredById);
+        await insertMany(
+          db,
+          jobRunLogsTable,
+          encryptedLogs.map((log) => ({
+            runId,
+            level: log.level,
+            message: log.message,
+            timestamp: parseJobInstant(log.timestamp),
+          })),
+        );
+      }
+      runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
+    },
+
+    onJobFailed: async (
+      jobName: string,
+      bullJobId: string,
+      error: string,
+      logs: JobLogEntry[],
+      outcome?: JobOutcomeMeta,
+    ) => {
+      // Mirror of onJobComplete: recorded independently of the run row, so a
+      // tenant still learns their job failed if the row is unreachable.
+      await isolateTenantFailureWrite(
+        () => recordTenantJobFailure(db, jobName, outcome),
+        jobName,
+        bullJobId,
+      );
+      const resolved = await resolveRun(bullJobId);
+      // skip: same rare state-loss case as in onJobComplete — drop the
+      // failure write rather than forge a run row from scratch.
+      if (!resolved) return;
+      const { runId, triggeredById } = resolved;
+      // Read started_at off the row so we can compute duration
+      // symmetrically to onJobComplete (which gets duration from the
+      // worker). The row already has started_at from onJobStart.
+      const row = await fetchOne<{ startedAt: Temporal.Instant }>(db, jobRunsTable, { id: runId });
+      const now = Temporal.Now.instant();
+      const duration = row
+        ? Math.round(Number(now.since(row.startedAt).total({ unit: "millisecond" })))
+        : 0;
+      const payload = runFailedSchema.parse({
+        duration,
+        finishedAt: now.toString(),
+        error,
+        logs: logs.map((l) => ({
+          level: l.level,
+          message: l.message,
+          timestamp: l.timestamp.toString(),
+        })),
+      });
+      const encryptedError = await encryptFailureError(payload.error, triggeredById);
+      await updateMany(
+        db,
+        jobRunsTable,
+        {
+          status: "failed",
+          error: encryptedError,
+          duration: payload.duration,
+          finishedAt: parseJobInstant(payload.finishedAt),
+          modifiedAt: now,
+          modifiedById: "system",
+        },
+        { id: runId },
+      );
+      // skip: empty log batch — mirror of onJobComplete
+      if (payload.logs.length > 0) {
+        const encryptedLogs = await encryptLogMessages(payload.logs, triggeredById);
+        await insertMany(
+          db,
+          jobRunLogsTable,
+          encryptedLogs.map((log) => ({
+            runId,
+            level: log.level,
+            message: log.message,
+            timestamp: parseJobInstant(log.timestamp),
+          })),
+        );
+      }
+      runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
+    },
+  };
+}

package/packages/bundled-features/src/jobs/job-run-table.ts

@@ -0,0 +1,101 @@
+import {
+  defineUnmanagedTable,
+  deriveEntityTableMeta,
+  type EntityTableMeta,
+  instant,
+  table as pgTable,
+  serial,
+  text,
+} from "@cosmicdrift/kumiko-framework/db";
+import {
+  createEntity,
+  createNumberField,
+  createTextField,
+  createTimestampField,
+} from "@cosmicdrift/kumiko-framework/engine";
+
+export type JobRunStatus = "queued" | "running" | "completed" | "failed";
+export type JobLogLevel = "info" | "warn" | "error";
+
+// jobRun is a system-scoped direct-write store (#2243): every job execution
+// writes straight into jobRunsTable / jobRunLogsTable from the BullMQ
+// callbacks (onJobStart / -Complete / -Failed, see job-run-logger.ts) —
+// no event-store detour. Pre-#2243 this was an events-only aggregate
+// replayed through two inline projections; that generated two permanent
+// `kumiko_events` rows per run for data that is itself already the
+// system of record (no other consumer replays or MSP-subscribes to it).
+//
+// Logs are batched onto the completed/failed callback as an array —
+// "Option B" from the original design discussion: one write per run
+// instead of one write per log line, no log duplication across status
+// transitions. job-run-logger.ts expands the batch into N rows in
+// jobRunLogsTable.
+//
+// Entity-derived table (query API + migration meta share one field
+// definition). status/$type<JobRunStatus> is not modeled in the entity
+// schema — the column type is text, with the status union enforced at the
+// app boundary (same pattern as template-resolver kind/scope).
+// `table: "store_job_runs"` (not `read_*`) because this is no longer a
+// rebuildable projection — `defineUnmanagedTable`/`deriveEntityTableMeta`
+// reject the `read_` prefix for `source: "unmanaged"` (#1208/#1220).
+export const jobRunEntity = createEntity({
+  table: "store_job_runs",
+  fields: {
+    jobName: createTextField({ required: true, personal: false, reason: "system_metadata" }),
+    bullJobId: createTextField({ required: true, personal: false, reason: "system_metadata" }),
+    status: createTextField({ required: true, personal: false, reason: "system_metadata" }),
+    // The run-started payload can carry arbitrary caller-supplied data,
+    // encrypted by job-run-logger.ts under the triggering user's DEK (see
+    // jobs-pii-kms.integration.test.ts) — same subject as `triggeredById`.
+    payload: createTextField({ personal: { of: "triggeredById" }, find: "none" }),
+    // Failure messages can echo payload content back — same subject/DEK as
+    // `payload` (job-run-logger.ts encryptFailureError).
+    error: createTextField({ personal: { of: "triggeredById" }, find: "none" }),
+    attempt: createNumberField({ required: true, default: 1, integer: true }),
+    startedAt: createTimestampField({ required: true }),
+    finishedAt: createTimestampField(),
+    duration: createNumberField({ integer: true }),
+    triggeredById: createTextField({ personal: false, reason: "pseudonymous_fk" }),
+  },
+  indexes: [{ columns: ["startedAt"] }],
+});
+
+// Plain EntityTableMeta, NOT a branded EntityTable (buildEntityTable would
+// mark it executor-only): job-run is an unmanaged direct-write store, so
+// onJobStart/-Complete/-Failed need to write via ctx.db/insertOne directly
+// (same pattern as sessions/schema/user-session.ts).
+export const jobRunsTable: EntityTableMeta = deriveEntityTableMeta("job-run", jobRunEntity, {
+  source: "unmanaged",
+});
+export const jobRunsTableMeta = jobRunsTable;
+
+// Child projection keyed by the jobRun aggregate id. Pre-ES used a serial
+// PK + integer runId; post-ES runId is still exposed but now holds the
+// uuid of the parent jobRun. Existing detail-query callers treat it as an
+// opaque identifier, so the type-switch is backward-compatible at the
+// query surface.
+export const jobRunLogsTable = pgTable("store_job_run_logs", {
+  id: serial("id").primaryKey(),
+  runId: text("run_id").notNull(),
+  level: text("level").notNull().$type<JobLogLevel>(),
+  message: text("message").notNull(),
+  timestamp: instant("timestamp").notNull(),
+});
+
+// **Unmanaged table** — deliberately no createEntity. Reasoning:
+//   - serial PK (not uuid) — pre-ES legacy, compatible with existing rows
+//   - no tenant_id — child table of jobRun, tenant context lives on the parent
+//   - no base columns (no version/inserted_at/inserted_by_id) — append-only
+//     log, no in-place update, no audit columns needed
+// pgTable stays the source of truth for the query API; this meta mirrors it
+// for migration generation.
+export const jobRunLogsTableMeta: EntityTableMeta = defineUnmanagedTable({
+  tableName: "store_job_run_logs",
+  columns: [
+    { name: "id", pgType: "serial", notNull: true, primaryKey: true },
+    { name: "run_id", pgType: "text", notNull: true },
+    { name: "level", pgType: "text", notNull: true },
+    { name: "message", pgType: "text", notNull: true },
+    { name: "timestamp", pgType: "timestamptz", notNull: true },
+  ],
+});

package/packages/bundled-features/src/jobs/tenant-job-failure-table.ts

@@ -0,0 +1,38 @@
+import { requireEntityTableMeta } from "@cosmicdrift/kumiko-framework/bun-db";
+import {
+  type EntityTableMeta,
+  instant,
+  table as pgTable,
+  serial,
+  sql,
+  text,
+  uuid,
+} from "@cosmicdrift/kumiko-framework/db";
+
+// The tenant's own view of a failed job (fw#3079). Direct-write store like
+// store_job_runs: job-run-logger.ts writes it from the BullMQ callbacks,
+// outside any dispatcher transaction.
+//
+// Deliberately holds no error text. `message_key` is a translation key
+// (the thrown KumikoError's i18nKey or the one declared at the job), so a
+// provider message can never reach the tenant through this table — it stays
+// on store_job_runs.error and in store_job_run_logs, both SystemAdmin-only.
+//
+// One row per (tenant, job, subject): a later final-attempt failure replaces
+// it, a later successful run of the same key deletes it. `subject` is the
+// canonical JSON of the job's declared subjectFields, or NULL for a job that
+// declares none — stored in clear, so a job must not declare a PII field
+// as its subject.
+export const tenantJobFailuresTable = pgTable("store_tenant_job_failures", {
+  id: serial("id").primaryKey(),
+  tenantId: uuid("tenant_id").notNull(),
+  jobName: text("job_name").notNull(),
+  subject: text("subject"),
+  messageKey: text("message_key").notNull(),
+  failedAt: instant("failed_at").default(sql`now()`).notNull(),
+});
+
+export const tenantJobFailuresTableMeta: EntityTableMeta = requireEntityTableMeta(
+  tenantJobFailuresTable,
+  "tenantJobFailuresTable",
+);

package/packages/bundled-features/src/jobs/web/client-plugin.tsx

@@ -0,0 +1,16 @@
+// @runtime client
+import { mergeTranslations, type TranslationsByLocale } from "@cosmicdrift/kumiko-renderer";
+import type { ClientFeatureDefinition } from "@cosmicdrift/kumiko-renderer-web";
+import { JOBS_FEATURE } from "../constants.js";
+import { defaultTranslations } from "./i18n.js";
+
+export type JobsClientOptions = {
+  readonly translations?: TranslationsByLocale;
+};
+
+export function jobsClient(options?: JobsClientOptions): ClientFeatureDefinition {
+  return {
+    name: JOBS_FEATURE,
+    translations: mergeTranslations(defaultTranslations, options?.translations ?? {}),
+  };
+}

package/packages/bundled-features/src/jobs/web/i18n.ts

@@ -0,0 +1,8 @@
+// @runtime client
+import {
+  type TranslationsByLocale,
+  translationsByLocaleFromKeys,
+} from "@cosmicdrift/kumiko-renderer";
+import { JOBS_I18N } from "../i18n.js";
+
+export const defaultTranslations: TranslationsByLocale = translationsByLocaleFromKeys(JOBS_I18N);

package/packages/bundled-features/src/jobs/web/index.ts

@@ -0,0 +1,2 @@
+// @runtime client
+export { type JobsClientOptions, jobsClient } from "./client-plugin.js";

package/packages/bundled-features/src/ledger/changes.json

@@ -0,0 +1,16 @@
+[
+  {
+    "version": "0.331.0",
+    "type": "improvement",
+    "title": "Ledger schedules get an optional filterable sourceRef field",
+    "detail": "`read_ledger_schedules` gets a nullable text column `source_ref` (max 128) plus the non-unique index `(tenant_id, source_ref)`. Callers set it on `createSchedule` / `updateSchedule` to find a schedule again without parsing `description`. Existing schedules keep `sourceRef = null`.",
+    "migration": "Run `kumiko schema generate`. For a new nullable column and a new non-unique index on a managed projection it emits an in-place `ALTER TABLE \"read_ledger_schedules\" ADD COLUMN \"source_ref\" text` plus a `CREATE INDEX`, with no DROP/CREATE and no `.rebuild.json`. No backfill: do not write the column with SQL. Old schedules stay at `sourceRef = null`, so callers should fall back to their previous lookup key for them."
+  },
+  {
+    "version": "0.320.0",
+    "type": "breaking",
+    "title": "Account code is unique per tenant (partial unique index read_ledger_accounts_tenant_id_code_uidx)",
+    "detail": "`accountEntity` declares `read_ledger_accounts_tenant_id_code_uidx`, a unique index on `(tenant_id, code)` that only covers accounts with a code. Two parallel find-then-create calls for the same code (two tabs, two devices) could each create an account and leave a tenant with duplicate codes. The second write now fails with `unique_violation` (HTTP 409). This applies to `createAccount` and to an `updateAccount` that changes the code. Accounts without a code are unaffected, and the same code in different tenants stays allowed.",
+    "migration": "Before you apply the migration, check for existing duplicates:\n\n  SELECT tenant_id, code, count(*) FROM read_ledger_accounts\n  WHERE code IS NOT NULL GROUP BY tenant_id, code HAVING count(*) > 1;\n\n`kumiko schema generate` treats a new unique index on a managed projection as destructive. It emits DROP TABLE + CREATE TABLE plus a `.rebuild.json`, which means a full event replay of `read_ledger_accounts`. A rebuild creates the table's indexes before it replays. So if a tenant's event history ever contained two accounts with the same code at the same time, the replay hits the unique index and fails, even after one of them was renamed. For an app with existing ledger data, hand-edit the generated migration before committing it. Replace the DROP/CREATE with an in-place\n\n  CREATE UNIQUE INDEX IF NOT EXISTS \"read_ledger_accounts_tenant_id_code_uidx\"\n    ON \"read_ledger_accounts\" (\"tenant_id\", \"code\") WHERE \"code\" IS NOT NULL;\n\nand delete the `.rebuild.json`. The migrations snapshot stays as generated. If the duplicate check found rows, resolve them first through real writes, for example an `updateAccount` that gives the extra account another code. Never edit the table directly. For such a tenant, a later full rebuild of `read_ledger_accounts` still replays the historical duplicate and fails. An app that already created this index by hand under the same name (money-horse migration 0024) gets a no-op from the in-place statement.\n\nCode that does find-then-create by code should expect `unique_violation` on create and re-read the account that won the race."
+  }
+]