@cosmicdrift/kumiko-samples 0.365.0 → 0.365.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-samples",
-  "version": "0.365.0",
+  "version": "0.365.1",
   "description": "Source trees of the Kumiko sample recipes, sample apps and bundled features in repo layout, for tooling such as few-shot corpus builds.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",

package/packages/bundled-features/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-bundled-features",
-  "version": "0.365.0",
+  "version": "0.365.1",
   "description": "Built-in features — tenant, user, auth, delivery. The stuff you'd rewrite anyway, already typed.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",

package/packages/bundled-features/src/crypto-shredding/changes.json

@@ -1,4 +1,11 @@
 [
+  {
+    "version": "0.365.1",
+    "type": "fix",
+    "title": "eraseKey leaves a tombstone for subjects that never had a key",
+    "detail": "eraseKey now records an erased entry even when the subject has no key (pre-KMS plaintext). A later createKey or write for that subject fails with subject_erased instead of minting a fresh key and re-encrypting the plaintext. Erase flows must write with writePurpose erasure after eraseKey. kumiko_subject_keys now gets one tombstone row per forgotten record subject and per step dispatch. The tenant destruction stage tenant-row now fails loudly when marking the tenant destroyed does not succeed.",
+    "migration": "Run backfill-pii before erasing Pre-KMS data, otherwise untouched plaintext stays readable after eraseKey."
+  },
   {
     "version": "0.364.0",
     "type": "fix",

package/packages/bundled-features/src/step-dispatcher/changes.json

@@ -1,4 +1,11 @@
 [
+  {
+    "version": "0.365.1",
+    "type": "fix",
+    "title": "step-dispatcher delivers each dispatch as a retried job, so a slow receiver no longer delays other tenants",
+    "detail": "The step-dispatcher projection now only enqueues one `step-dispatcher:job:deliver` job per dispatch request, and the job performs the webhook or mail delivery. Redis holds only the dispatch stream id and the tenant, never the request payload. Transient failures (network error, timeout, 5xx, 408, 429, any mail adapter failure) are retried 3 times with exponential backoff; the outcome event and the payload key erase follow after the final attempt. Permanent failures (invalid url, blocked host, other 4xx, unavailable auth secret) are recorded at once. At most 2 deliveries per tenant run at the same time, so one tenant's slow receivers cannot hold up the others. This corrects the 0.346.0 note that a hanging receiver no longer stalls the dispatcher for every tenant: the shared consumer still delivered one request at a time.",
+    "migration": "step-dispatcher now needs a job runner (Redis) and a worker consuming the default worker lane. Without a job runner the projection throws instead of delivering. Test stacks must pass `jobs` to setupTestStack and call `drainJobs()` after the event dispatcher ran. `r.job` accepts `maxPerTenant: { limit, onLimit: \"defer\", deferMs? }`; a plain number still drops excess enqueues. A `webhook.send` request that gets a retryable status (5xx, 408, 429) is now sent up to 4 times under the same Idempotency-Key. Worker pods must know the deliver job before the projection enqueues it, so deploy worker and API together or the worker first. A dev server only consumes jobs when the app mounts `createJobsFeature()`."
+  },
   {
     "version": "0.347.0",
     "type": "fix",
@@ -21,7 +28,7 @@
     "version": "0.346.0",
     "type": "fix",
     "title": "Webhook requests time out after 10 seconds instead of blocking the dispatcher",
-    "detail": "Webhook requests time out after 10 seconds\nA hanging receiver now ends as `step.dispatch-failed` and no longer stalls the step-dispatcher for every tenant."
+    "detail": "Webhook requests time out after 10 seconds\nA hanging receiver now ends as `step.dispatch-failed` after the timeout instead of waiting indefinitely. Until the step-dispatcher delivered through jobs, the shared consumer still processed requests one at a time, so a slow receiver of one tenant delayed the others."
   },
   {
     "version": "0.346.0",

package/packages/bundled-features/src/step-dispatcher/constants.ts

@@ -0,0 +1,22 @@
+import { QnTypes, qn } from "@cosmicdrift/kumiko-framework/engine";
+
+export const STEP_DISPATCHER_FEATURE = "step-dispatcher";
+
+export const StepDispatcherJobNames = {
+  deliver: "deliver",
+} as const;
+
+export const StepDispatcherJobs = {
+  deliver: qn(STEP_DISPATCHER_FEATURE, QnTypes.job, StepDispatcherJobNames.deliver),
+} as const;
+
+// Stable per dispatch stream: BullMQ ignores a second add() with the same id,
+// so an MSP redelivery cannot enqueue a second delivery.
+export function deliverJobId(aggregateId: string): string {
+  return `step-dispatch-${aggregateId}`;
+}
+
+// Per-tenant cap on simultaneous deliveries, so one tenant's slow receivers
+// cannot occupy every worker slot.
+export const DELIVER_MAX_PER_TENANT = 2;
+export const DELIVER_RETRIES = 3;

package/packages/bundled-features/src/step-dispatcher/deliver-dispatch-request.ts

@@ -0,0 +1,236 @@
+import { requestContext } from "@cosmicdrift/kumiko-framework/api";
+import { configuredPiiSubjectKms } from "@cosmicdrift/kumiko-framework/crypto";
+import {
+  STEP_DISPATCH_AGGREGATE_TYPE,
+  STEP_DISPATCH_FAILED_TYPE,
+  STEP_DISPATCH_REQUESTED_TYPE,
+  STEP_DISPATCHED_TYPE,
+} from "@cosmicdrift/kumiko-framework/engine";
+import { createFallbackLogger } from "@cosmicdrift/kumiko-framework/logging";
+import type { MultiStreamApplyContext } from "@cosmicdrift/kumiko-framework/pipeline";
+import { SYSTEM_USER_ID, type TenantId } from "@cosmicdrift/kumiko-types/identifiers";
+import { redactEmailAddresses } from "../shared/index.js";
+import {
+  buildDispatchSpec,
+  dispatchRequestedPayloadSchema,
+  rawStepKindOf,
+  readPayloadFields,
+} from "./dispatch-payload.js";
+import { type MailDispatchResult, performMailDispatch } from "./mail-runner.js";
+import { performWebhookDispatch } from "./webhook-runner.js";
+
+const log = createFallbackLogger("step-dispatcher");
+
+// zod issue messages can echo the invalid value (e.g. a rejected url) back
+// into the tenant-visible dispatch-failed event — keep this generic.
+const INVALID_DISPATCH_PAYLOAD_ERROR = "invalid dispatch payload";
+const PAYLOAD_UNREADABLE_ERROR = "dispatch payload is not readable";
+// Adapter errors (e.g. SMTP 550) usually echo the recipient address; the
+// dispatch-failed text outlives the payload erase, so only this is persisted.
+const MAIL_DELIVERY_FAILED_ERROR = "mail delivery failed";
+const PAYLOAD_ERASED_ERROR = "dispatch payload erased before an outcome was recorded";
+// Thrown messages land in BullMQ's failedReason and the run log, so this never
+// carries the receiver's error text or the target url.
+const DELIVERY_RETRY_ERROR = "step delivery failed, retrying";
+const DISPATCH_THREW_ERROR = "delivery attempt threw unexpectedly";
+const FINAL_ATTEMPT_ABORTED_ERROR = "delivery aborted on the final attempt";
+const FINAL_ATTEMPT_UNRECORDED_ERROR = "step delivery failed and its outcome could not be recorded";
+
+export function isDispatchOutcome(e: { readonly type: string }): boolean {
+  return e.type === STEP_DISPATCHED_TYPE || e.type === STEP_DISPATCH_FAILED_TYPE;
+}
+
+export async function eraseDispatchKey(aggregateId: string): Promise<void> {
+  await configuredPiiSubjectKms()?.eraseKey(
+    { kind: "record", entity: STEP_DISPATCH_AGGREGATE_TYPE, id: aggregateId },
+    {
+      requestId: requestContext.get()?.requestId ?? "step-dispatcher",
+      eraseReason: "step-dispatch-outcome-recorded",
+    },
+  );
+}
+
+type DeliveryResult =
+  | { readonly ok: true; readonly status: number }
+  | { readonly ok: false; readonly error: string; readonly retryable: boolean };
+
+// Mail adapter failures carry no status to classify, so every one is retried.
+function mailResultAsDelivery(result: MailDispatchResult): DeliveryResult {
+  return result.ok ? result : { ...result, retryable: true };
+}
+
+export type DeliverDispatchRequestArgs = {
+  readonly aggregateId: string;
+  readonly tenantId: TenantId;
+  // 1-based run number of the job; finalAttempt = no retry follows a throw.
+  readonly attempt: number;
+  readonly finalAttempt: boolean;
+  // The outcome events are stamped with the requesting user, which is only
+  // known after the stream was loaded — hence a factory instead of a ready ctx.
+  readonly applyContextFor: (userId: string) => MultiStreamApplyContext;
+};
+
+// Outcome events are plaintext and generic; the request payload's per-dispatch
+// key is erased right after, so the PII dies with the dispatch instead of
+// living in the event log. The outcome is appended before the erase on
+// purpose: if the erase throws, the outcome survives and a redelivery only
+// repeats the erase. A crash between the send and the outcome append still
+// re-sends under the same Idempotency-Key — the one remaining double-send case.
+// A transient failure before the final attempt throws without an outcome or
+// erase: the key must survive until the outcome is final.
+export async function deliverDispatchRequest(args: DeliverDispatchRequestArgs): Promise<void> {
+  const { aggregateId, finalAttempt } = args;
+  const stream = await args.applyContextFor(SYSTEM_USER_ID).loadAggregate(aggregateId);
+  if (stream.some(isDispatchOutcome)) {
+    await eraseDispatchKey(aggregateId);
+    // skip: redelivery after an outcome was recorded — only the erase is repeated
+    return;
+  }
+
+  const requested = stream.find((e) => e.type === STEP_DISPATCH_REQUESTED_TYPE);
+  if (!requested) {
+    // The key ref carries no tenant, so a job triggered with a foreign or
+    // unknown aggregateId must not erase anything or append to that stream.
+    log.warn("step dispatch request not found, nothing delivered", { aggregateId });
+    return;
+  }
+  const ctx = args.applyContextFor(requested.metadata.userId || SYSTEM_USER_ID);
+  const recorder = createOutcomeRecorder(ctx, aggregateId, args.attempt);
+
+  try {
+    await deliverRequested(args, requested, ctx, recorder);
+  } catch (err) {
+    if (!finalAttempt) throw err;
+    await abortOnFinalAttempt(err, ctx, recorder, aggregateId, requested.payload);
+  }
+}
+
+type OutcomeRecorder = {
+  readonly record: (
+    type: typeof STEP_DISPATCHED_TYPE | typeof STEP_DISPATCH_FAILED_TYPE,
+    payload: Record<string, unknown>,
+  ) => Promise<void>;
+  readonly recordFailure: (stepKind: string, error: string) => Promise<void>;
+};
+
+function createOutcomeRecorder(
+  ctx: MultiStreamApplyContext,
+  aggregateId: string,
+  attempt: number,
+): OutcomeRecorder {
+  const record: OutcomeRecorder["record"] = async (type, payload) => {
+    await ctx.unsafeAppendEvent({
+      aggregateId,
+      aggregateType: STEP_DISPATCH_AGGREGATE_TYPE,
+      type,
+      payload,
+    });
+    await eraseDispatchKey(aggregateId);
+  };
+  return {
+    record,
+    recordFailure: (stepKind, error) =>
+      record(STEP_DISPATCH_FAILED_TYPE, { stepKind, error, attempt }),
+  };
+}
+
+type RequestedEvent = Awaited<ReturnType<MultiStreamApplyContext["loadAggregate"]>>[number];
+
+async function deliverRequested(
+  args: DeliverDispatchRequestArgs,
+  requested: RequestedEvent,
+  ctx: MultiStreamApplyContext,
+  recorder: OutcomeRecorder,
+): Promise<void> {
+  const { aggregateId, tenantId, attempt, finalAttempt } = args;
+  const { record, recordFailure } = recorder;
+
+  const parsed = dispatchRequestedPayloadSchema.safeParse(requested.payload);
+  if (!parsed.success) {
+    return recordFailure(rawStepKindOf(requested.payload), INVALID_DISPATCH_PAYLOAD_ERROR);
+  }
+  const payload = parsed.data;
+
+  const read = await readPayloadFields(payload);
+  if (read.kind === "unreadable") return recordFailure(payload.stepKind, PAYLOAD_UNREADABLE_ERROR);
+  if (read.kind === "erased") return recordFailure(payload.stepKind, PAYLOAD_ERASED_ERROR);
+
+  const dispatchSpec = buildDispatchSpec(payload, read.fields);
+  if (!dispatchSpec) return recordFailure(payload.stepKind, INVALID_DISPATCH_PAYLOAD_ERROR);
+
+  const result = await sendDispatch(dispatchSpec, {
+    aggregateId,
+    tenantId,
+    userId: requested.metadata.userId || SYSTEM_USER_ID,
+    secrets: ctx.secrets,
+  });
+  if (result.ok) {
+    return record(STEP_DISPATCHED_TYPE, { stepKind: payload.stepKind, status: result.status });
+  }
+
+  const loggedReason =
+    payload.stepKind === "mail.send" ? redactEmailAddresses(result.error) : result.error;
+  if (result.retryable && !finalAttempt) {
+    log.warn("step delivery failed, will retry", { aggregateId, attempt, reason: loggedReason });
+    throw new Error(DELIVERY_RETRY_ERROR);
+  }
+  if (payload.stepKind === "mail.send") {
+    log.warn("mail dispatch failed", { aggregateId, reason: loggedReason });
+    return recordFailure(payload.stepKind, MAIL_DELIVERY_FAILED_ERROR);
+  }
+  return recordFailure(payload.stepKind, result.error);
+}
+
+async function sendDispatch(
+  dispatchSpec: NonNullable<ReturnType<typeof buildDispatchSpec>>,
+  target: {
+    readonly aggregateId: string;
+    readonly tenantId: TenantId;
+    readonly userId: string;
+    readonly secrets: MultiStreamApplyContext["secrets"];
+  },
+): Promise<DeliveryResult> {
+  try {
+    return dispatchSpec.stepKind === "webhook.send"
+      ? await performWebhookDispatch(dispatchSpec.spec, {
+          tenantId: target.tenantId,
+          userId: target.userId,
+          secrets: target.secrets,
+          idempotencyKey: target.aggregateId,
+        })
+      : await performMailDispatch(dispatchSpec.spec).then(mailResultAsDelivery);
+  } catch (err) {
+    // err.message can echo the url or an address; only the class name is logged.
+    log.warn("step delivery threw", {
+      aggregateId: target.aggregateId,
+      errorName: err instanceof Error ? err.name : "unknown",
+    });
+    return { ok: false, error: DISPATCH_THREW_ERROR, retryable: true };
+  }
+}
+
+async function abortOnFinalAttempt(
+  err: unknown,
+  ctx: MultiStreamApplyContext,
+  recorder: OutcomeRecorder,
+  aggregateId: string,
+  requestedPayload: unknown,
+): Promise<void> {
+  // err.message can echo the url or an address; only the class name is logged.
+  log.warn("step delivery aborted on the final attempt", {
+    aggregateId,
+    errorName: err instanceof Error ? err.name : "unknown",
+  });
+  try {
+    // An outcome may already be appended when only the erase threw: repeat the
+    // erase, never a second outcome.
+    const current = await ctx.loadAggregate(aggregateId);
+    if (current.some(isDispatchOutcome)) {
+      await eraseDispatchKey(aggregateId);
+    } else {
+      await recorder.recordFailure(rawStepKindOf(requestedPayload), FINAL_ATTEMPT_ABORTED_ERROR);
+    }
+  } catch {
+    throw new Error(FINAL_ATTEMPT_UNRECORDED_ERROR);
+  }
+}

package/packages/bundled-features/src/step-dispatcher/feature.ts

@@ -1,41 +1,40 @@
 // step-dispatcher — bundled-feature that drains deferred Tier-2 step
 // requests (webhook.send, mail.send, ...) after their TX commits.
 //
-// Listens on the `kumiko:system:step.dispatch-requested` system event
-// (registry-bypassed, see SYSTEM_EVENT_PREFIX).
-// Performs the side-effect and emits `kumiko:system:step.dispatched`
-// or `kumiko:system:step.dispatch-failed` back onto the same stream so
-// the audit trail lives in the event log only — no separate status table.
+// The `kumiko:system:step.dispatch-requested` system event (registry-bypassed,
+// see SYSTEM_EVENT_PREFIX) is picked up by an MSP that only enqueues a
+// per-dispatch job. The job performs the side-effect and emits
+// `kumiko:system:step.dispatched` or `kumiko:system:step.dispatch-failed` back
+// onto the same stream so the audit trail lives in the event log only — no
+// separate status table. The job's Redis data carries only the stream id and
+// tenant, never the (encrypted) request payload.
 
-import { requestContext } from "@cosmicdrift/kumiko-framework/api";
-import { configuredPiiSubjectKms } from "@cosmicdrift/kumiko-framework/crypto";
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
 import {
   defineFeature,
   type FeatureDefinition,
   STEP_DISPATCH_AGGREGATE_TYPE,
-  STEP_DISPATCH_FAILED_TYPE,
   STEP_DISPATCH_REQUESTED_TYPE,
-  STEP_DISPATCHED_TYPE,
 } from "@cosmicdrift/kumiko-framework/engine";
-import { createFallbackLogger } from "@cosmicdrift/kumiko-framework/logging";
-import { SYSTEM_USER_ID } from "@cosmicdrift/kumiko-types/identifiers";
+import { createMultiStreamApplyContext } from "@cosmicdrift/kumiko-framework/pipeline";
 import * as z from "zod";
-import { redactEmailAddresses } from "../shared/index.js";
 import {
-  buildDispatchSpec,
-  dispatchRequestedPayloadSchema,
-  rawStepKindOf,
-  readPayloadFields,
-} from "./dispatch-payload.js";
-import { performMailDispatch } from "./mail-runner.js";
+  DELIVER_MAX_PER_TENANT,
+  DELIVER_RETRIES,
+  deliverJobId,
+  STEP_DISPATCHER_FEATURE,
+  StepDispatcherJobs,
+} from "./constants.js";
+import {
+  deliverDispatchRequest,
+  eraseDispatchKey,
+  isDispatchOutcome,
+} from "./deliver-dispatch-request.js";
 import {
-  performWebhookDispatch,
   WEBHOOK_ALLOWED_PRIVATE_HOSTS_ENV_VAR,
   WEBHOOK_AUTH_SECRET_NAMESPACE_OPTIONS,
 } from "./webhook-runner.js";
 
-const log = createFallbackLogger("step-dispatcher");
-
 export const stepDispatcherEnvSchema = z.object({
   [WEBHOOK_ALLOWED_PRIVATE_HOSTS_ENV_VAR]: z
     .string()
@@ -47,23 +46,16 @@ export const stepDispatcherEnvSchema = z.object({
 
 export { STEP_DISPATCH_AGGREGATE_TYPE };
 
-// zod issue messages can echo the invalid value (e.g. a rejected url) back
-// into the tenant-visible dispatch-failed event — keep this generic.
-const INVALID_DISPATCH_PAYLOAD_ERROR = "invalid dispatch payload";
-const PAYLOAD_UNREADABLE_ERROR = "dispatch payload is not readable";
-// Adapter errors (e.g. SMTP 550) usually echo the recipient address; the
-// dispatch-failed text outlives the payload erase, so only this is persisted.
-const MAIL_DELIVERY_FAILED_ERROR = "mail delivery failed";
-const PAYLOAD_ERASED_ERROR = "dispatch payload erased before an outcome was recorded";
+const NEEDS_JOB_RUNNER_ERROR = "step-dispatcher needs a job runner (Redis) to deliver";
+const DELIVER_ESCAPE_HATCH_REASON =
+  "reads the dispatch stream and appends its outcome event, which needs a raw connection";
 
-function isDispatchOutcome(e: { readonly type: string }): boolean {
-  return e.type === STEP_DISPATCHED_TYPE || e.type === STEP_DISPATCH_FAILED_TYPE;
-}
+const deliverJobPayloadSchema = z.object({ aggregateId: z.string() });
 
 export function createStepDispatcherFeature(): FeatureDefinition {
   return defineFeature("step-dispatcher", (r) => {
     r.describe(
-      "Internal system feature that drains deferred Tier-2 side-effects (currently `webhook.send` and `mail.send`) after their originating transaction commits. Listens via `r.multiStreamProjection` on the `kumiko:system:step.dispatch-requested` system event, performs the actual HTTP or mail delivery, then appends `kumiko:system:step.dispatched` or `kumiko:system:step.dispatch-failed` back onto the same stream so the outcome is recorded in the event log without a separate status table. Mount this feature explicitly via `createStepDispatcherFeature()` in your app's feature list alongside any features that use `r.step.webhook.send` or `r.step.mail.send`. Requires the `secrets` feature (`createSecretsFeature()`) to be mounted — `webhook.send` auth resolves per-tenant through it, under `step-dispatcher:webhook-auth.<name>`. Every `webhook.send` request carries `Idempotency-Key: <dispatch stream id>`, stable across redeliveries of the same dispatch request; an explicit `Idempotency-Key` in `headers` takes precedence.",
+      "Internal system feature that drains deferred Tier-2 side-effects (currently `webhook.send` and `mail.send`) after their originating transaction commits. An `r.multiStreamProjection` on the `kumiko:system:step.dispatch-requested` system event enqueues one `step-dispatcher:job:deliver` BullMQ job per dispatch (Redis carries only the stream id and tenant, never the request payload). The job performs the actual HTTP or mail delivery and appends `kumiko:system:step.dispatched` or `kumiko:system:step.dispatch-failed` back onto the same stream, so the outcome is recorded in the event log without a separate status table. Transient failures (network error, timeout, 5xx, 408, 429, any mail adapter failure) are retried 3 times with exponential backoff; the outcome is recorded after the final attempt and the payload key is erased only then. Permanent failures (invalid url, blocked host, other 4xx, unavailable auth secret) are recorded immediately. At most 2 deliveries per tenant run at once (`maxPerTenant` with `onLimit: \"defer\"`), so a slow receiver of one tenant does not delay other tenants. Requires a job runner (Redis) and a worker consuming the job's default `worker` lane; without a job runner the projection fails loudly. Mount this feature explicitly via `createStepDispatcherFeature()` in your app's feature list alongside any features that use `r.step.webhook.send` or `r.step.mail.send`. Requires the `secrets` feature (`createSecretsFeature()`) to be mounted — `webhook.send` auth resolves per-tenant through it, under `step-dispatcher:webhook-auth.<name>`. Every `webhook.send` request carries `Idempotency-Key: <dispatch stream id>`, stable across retries and redeliveries of the same dispatch request; an explicit `Idempotency-Key` in `headers` takes precedence.",
     );
     r.secretNamespace("webhook-auth", WEBHOOK_AUTH_SECRET_NAMESPACE_OPTIONS);
     r.uiHints({
@@ -74,100 +66,55 @@ export function createStepDispatcherFeature(): FeatureDefinition {
     r.envSchema(stepDispatcherEnvSchema);
     r.requires("secrets");
 
+    // Literal name: the few-shot corpus parser only reads string-literal job names.
+    // StepDispatcherJobNames.deliver mirrors it; feature.boot.test.ts pins the two together.
+    r.job(
+      "deliver",
+      {
+        trigger: { manual: true },
+        retries: DELIVER_RETRIES,
+        backoff: "exponential",
+        maxPerTenant: { limit: DELIVER_MAX_PER_TENANT, onLimit: "defer" },
+        escapeHatch: { reason: DELIVER_ESCAPE_HATCH_REASON },
+      },
+      async (payload, ctx) => {
+        const { aggregateId } = deliverJobPayloadSchema.parse(payload);
+        const db = ctx.db.unsafeRaw() as DbConnection; // @cast-boundary db-operator — jobs never run inside a DbTx
+        const tenantId = ctx.systemUser.tenantId;
+        await deliverDispatchRequest({
+          aggregateId,
+          tenantId,
+          attempt: ctx.attempt,
+          finalAttempt: ctx.finalAttempt,
+          applyContextFor: (userId) =>
+            createMultiStreamApplyContext({
+              registry: ctx.registry,
+              db,
+              tenantId,
+              userId,
+              callerFeature: STEP_DISPATCHER_FEATURE,
+              ...(ctx.secrets && { secrets: ctx.secrets }),
+            }),
+        });
+      },
+    );
+
     r.multiStreamProjection({
       name: "step-dispatcher",
       apply: {
         [STEP_DISPATCH_REQUESTED_TYPE]: async (event, _tx, ctx) => {
-          const kms = configuredPiiSubjectKms();
-          const requestId = requestContext.get()?.requestId ?? "step-dispatcher";
-
-          const eraseDispatchKey = async (): Promise<void> => {
-            await kms?.eraseKey(
-              { kind: "record", entity: STEP_DISPATCH_AGGREGATE_TYPE, id: event.aggregateId },
-              { requestId, eraseReason: "step-dispatch-outcome-recorded" },
-            );
-          };
-
-          // Outcome events are plaintext and generic; the request payload's
-          // per-dispatch key is erased right after, so the PII dies with the
-          // dispatch instead of living in the event log. The outcome is
-          // appended before the erase on purpose: if the erase throws, the
-          // outcome survives and a redelivery only repeats the erase. A crash
-          // between the send and the outcome append still re-sends under the
-          // same Idempotency-Key — the one remaining double-send case.
-          const recordOutcome = async (
-            type: typeof STEP_DISPATCHED_TYPE | typeof STEP_DISPATCH_FAILED_TYPE,
-            payload: Record<string, unknown>,
-          ): Promise<void> => {
-            await ctx.unsafeAppendEvent({
-              aggregateId: event.aggregateId,
-              aggregateType: STEP_DISPATCH_AGGREGATE_TYPE,
-              type,
-              payload,
-            });
-            await eraseDispatchKey();
-          };
-          const recordFailure = (stepKind: string, error: string) =>
-            recordOutcome(STEP_DISPATCH_FAILED_TYPE, { stepKind, error, attempt: 1 });
-
           const stream = await ctx.loadAggregate(event.aggregateId);
           if (stream.some(isDispatchOutcome)) {
-            await eraseDispatchKey();
+            await eraseDispatchKey(event.aggregateId);
             // skip: redelivery after an outcome was recorded — only the erase is repeated
             return;
           }
-
-          const parsed = dispatchRequestedPayloadSchema.safeParse(event.payload);
-          if (!parsed.success) {
-            await recordFailure(rawStepKindOf(event.payload), INVALID_DISPATCH_PAYLOAD_ERROR);
-            // skip: invalid payload already recorded via step.dispatch-failed above, nothing left to dispatch
-            return;
-          }
-          const payload = parsed.data;
-
-          const read = await readPayloadFields(payload);
-          if (read.kind === "unreadable") {
-            await recordFailure(payload.stepKind, PAYLOAD_UNREADABLE_ERROR);
-            // skip: unreadable payload already recorded via step.dispatch-failed above
-            return;
-          }
-          if (read.kind === "erased") {
-            await recordFailure(payload.stepKind, PAYLOAD_ERASED_ERROR);
-            // skip: erased payload recorded via step.dispatch-failed above
-            return;
-          }
-
-          const dispatchSpec = buildDispatchSpec(payload, read.fields);
-          if (!dispatchSpec) {
-            await recordFailure(payload.stepKind, INVALID_DISPATCH_PAYLOAD_ERROR);
-            // skip: unparseable payload already recorded via step.dispatch-failed above
-            return;
-          }
-          const result =
-            dispatchSpec.stepKind === "webhook.send"
-              ? await performWebhookDispatch(dispatchSpec.spec, {
-                  tenantId: event.tenantId,
-                  userId: event.metadata.userId || SYSTEM_USER_ID,
-                  secrets: ctx.secrets,
-                  idempotencyKey: event.aggregateId,
-                })
-              : await performMailDispatch(dispatchSpec.spec);
-          if (result.ok) {
-            await recordOutcome(STEP_DISPATCHED_TYPE, {
-              stepKind: payload.stepKind,
-              status: result.status,
-            });
-          } else {
-            if (payload.stepKind === "mail.send") {
-              log.warn("mail dispatch failed", {
-                aggregateId: event.aggregateId,
-                reason: redactEmailAddresses(result.error),
-              });
-              await recordFailure(payload.stepKind, MAIL_DELIVERY_FAILED_ERROR);
-            } else {
-              await recordFailure(payload.stepKind, result.error);
-            }
-          }
+          if (!ctx.jobs) throw new Error(NEEDS_JOB_RUNNER_ERROR);
+          await ctx.jobs.dispatch(
+            StepDispatcherJobs.deliver,
+            { aggregateId: event.aggregateId, _tenantId: event.tenantId },
+            { jobId: deliverJobId(event.aggregateId) },
+          );
         },
       },
     });

package/packages/bundled-features/src/step-dispatcher/webhook-runner.ts

@@ -100,9 +100,21 @@ export const webhookSpecSchema = z.object({
 
 export type WebhookSpec = z.infer<typeof webhookSpecSchema>;
 
+// `retryable`: a repeat can succeed (network error, timeout, 5xx, 408, 429).
+// Everything else is final and must not be retried.
+type WebhookDispatchFailure = {
+  readonly ok: false;
+  readonly error: string;
+  readonly retryable: boolean;
+};
+
 export type WebhookDispatchResult =
   | { readonly ok: true; readonly status: number }
-  | { readonly ok: false; readonly error: string };
+  | WebhookDispatchFailure;
+
+function isRetryableHttpStatus(status: number): boolean {
+  return status >= 500 || status === 408 || status === 429;
+}
 
 let fetchImpl: typeof fetch = globalThis.fetch.bind(globalThis);
 
@@ -118,8 +130,7 @@ export type WebhookDispatchDeps = {
   readonly requestTimeoutMs?: number;
 };
 
-// Bounds a single hanging tenant-controlled receiver to this long. The consumer
-// is sequential and shared, so N hanging requests still delay other tenants by N times this.
+// Bounds a single hanging tenant-controlled receiver to this long per attempt.
 export const WEBHOOK_REQUEST_TIMEOUT_MS = 10_000;
 
 // Never includes the secret name or value — spec.auth.secret is a
@@ -146,28 +157,30 @@ function setHeader(headers: Record<string, string>, name: string, value: string)
 async function buildWebhookHeaders(
   spec: WebhookSpec,
   deps: WebhookDispatchDeps,
-): Promise<{ ok: true; headers: Record<string, string> } | { ok: false; error: string }> {
+): Promise<{ ok: true; headers: Record<string, string> } | WebhookDispatchFailure> {
   const headers: Record<string, string> = { ...spec.headers };
   if (!hasHeader(headers, "content-type")) headers["content-type"] = "application/json";
   if (!hasHeader(headers, WEBHOOK_IDEMPOTENCY_KEY_HEADER)) {
     headers[WEBHOOK_IDEMPOTENCY_KEY_HEADER] = deps.idempotencyKey;
   }
   if (!spec.auth) return { ok: true, headers };
-  if (!deps.secrets) return { ok: false, error: WEBHOOK_AUTH_SECRET_UNAVAILABLE_ERROR };
+  if (!deps.secrets)
+    return { ok: false, error: WEBHOOK_AUTH_SECRET_UNAVAILABLE_ERROR, retryable: false };
   // A throwing get (corrupt envelope, rotated key, KMS error) must become a
-  // dispatch-failed event; rethrowing would retry and dead-letter the whole
-  // step-dispatcher consumer for every tenant.
+  // dispatch-failed event; rethrowing would burn the job's retries on a
+  // failure a retry cannot fix.
   let secret: string;
   try {
     const revealed = await deps.secrets.get(deps.tenantId, webhookAuthSecretKey(spec.auth.secret), {
       userId: deps.userId,
       handlerName: "step-dispatcher:webhook.send",
     });
-    if (!revealed) return { ok: false, error: WEBHOOK_AUTH_SECRET_UNAVAILABLE_ERROR };
+    if (!revealed)
+      return { ok: false, error: WEBHOOK_AUTH_SECRET_UNAVAILABLE_ERROR, retryable: false };
     secret = revealed.reveal();
   } catch {
     log.warn("webhook auth secret lookup failed", { tenantId: deps.tenantId });
-    return { ok: false, error: WEBHOOK_AUTH_SECRET_UNAVAILABLE_ERROR };
+    return { ok: false, error: WEBHOOK_AUTH_SECRET_UNAVAILABLE_ERROR, retryable: false };
   }
   if (spec.auth.kind === "bearer") {
     setHeader(headers, "authorization", `Bearer ${secret}`);
@@ -182,7 +195,7 @@ async function resolveWebhookFetchTarget(
   url: URL,
   headers: Record<string, string>,
 ): Promise<
-  { ok: true; fetchUrl: string | URL; requestInit: RequestInit } | { ok: false; error: string }
+  { ok: true; fetchUrl: string | URL; requestInit: RequestInit } | WebhookDispatchFailure
 > {
   if (isHostAllowlisted(url.hostname, readAllowedPrivateWebhookHostsFromEnv()))
     return { ok: true, fetchUrl: rawUrl, requestInit: { headers } };
@@ -194,7 +207,11 @@ async function resolveWebhookFetchTarget(
     const reason = err instanceof Error ? err.message : String(err);
     if (err instanceof BlockedHostError || err instanceof HostResolutionError) {
       log.warn("webhook host unreachable", { host: url.hostname, reason });
-      return { ok: false, error: "webhook host is not reachable or not allowed" };
+      return {
+        ok: false,
+        error: "webhook host is not reachable or not allowed",
+        retryable: false,
+      };
     }
     throw err;
   }
@@ -214,10 +231,14 @@ export async function performWebhookDispatch(
   try {
     url = new URL(spec.url);
   } catch {
-    return { ok: false, error: "invalid url" };
+    return { ok: false, error: "invalid url", retryable: false };
   }
   if (url.protocol !== "http:" && url.protocol !== "https:") {
-    return { ok: false, error: `unsupported url scheme "${url.protocol}"` };
+    return {
+      ok: false,
+      error: `unsupported url scheme "${url.protocol}"`,
+      retryable: false,
+    };
   }
 
   const headers = await buildWebhookHeaders(spec, deps);
@@ -237,7 +258,11 @@ export async function performWebhookDispatch(
     // Only the status is used; an unread body would pin the socket until the timeout fires.
     await res.body?.cancel().catch(() => {});
     if (!res.ok) {
-      return { ok: false, error: `HTTP ${res.status}: ${res.statusText}` };
+      return {
+        ok: false,
+        error: `HTTP ${res.status}: ${res.statusText}`,
+        retryable: isRetryableHttpStatus(res.status),
+      };
     }
     return { ok: true, status: res.status };
   } catch (err) {
@@ -245,6 +270,7 @@ export async function performWebhookDispatch(
     return {
       ok: false,
       error: `webhook request failed (${err instanceof Error ? err.name : "unknown error"})`,
+      retryable: true,
     };
   }
 }

package/packages/bundled-features/src/tenant-lifecycle/stages.ts

@@ -224,7 +224,7 @@ async function tombstoneTenantRow(ctx: DestructionStageCtx): Promise<StageRunOut
       );
     }
   }
-  await tenantCrud.update(
+  const tombstone = await tenantCrud.update(
     {
       id: ctx.tenantId,
       changes: {
@@ -237,6 +237,11 @@ async function tombstoneTenantRow(ctx: DestructionStageCtx): Promise<StageRunOut
     db,
     { skipOptimisticLock: true, writePurpose: WritePurposes.erasure },
   );
+  if (!tombstone.isSuccess) {
+    throw new Error(
+      `tenant-lifecycle: failed to mark tenant ${ctx.tenantId} destroyed: ${tombstone.error.message}`,
+    );
+  }
   invalidateTenantLifecycleGate(ctx.tenantId);
   return STAGE_DONE;
 }

package/samples/apps/use-all-bundled/feature-manifest.json

@@ -2391,7 +2391,7 @@
     },
     {
       "name": "step-dispatcher",
-      "description": "Internal system feature that drains deferred Tier-2 side-effects (currently `webhook.send` and `mail.send`) after their originating transaction commits. Listens via `r.multiStreamProjection` on the `kumiko:system:step.dispatch-requested` system event, performs the actual HTTP or mail delivery, then appends `kumiko:system:step.dispatched` or `kumiko:system:step.dispatch-failed` back onto the same stream so the outcome is recorded in the event log without a separate status table. Mount this feature explicitly via `createStepDispatcherFeature()` in your app's feature list alongside any features that use `r.step.webhook.send` or `r.step.mail.send`. Requires the `secrets` feature (`createSecretsFeature()`) to be mounted — `webhook.send` auth resolves per-tenant through it, under `step-dispatcher:webhook-auth.<name>`. Every `webhook.send` request carries `Idempotency-Key: <dispatch stream id>`, stable across redeliveries of the same dispatch request; an explicit `Idempotency-Key` in `headers` takes precedence.",
+      "description": "Internal system feature that drains deferred Tier-2 side-effects (currently `webhook.send` and `mail.send`) after their originating transaction commits. An `r.multiStreamProjection` on the `kumiko:system:step.dispatch-requested` system event enqueues one `step-dispatcher:job:deliver` BullMQ job per dispatch (Redis carries only the stream id and tenant, never the request payload). The job performs the actual HTTP or mail delivery and appends `kumiko:system:step.dispatched` or `kumiko:system:step.dispatch-failed` back onto the same stream, so the outcome is recorded in the event log without a separate status table. Transient failures (network error, timeout, 5xx, 408, 429, any mail adapter failure) are retried 3 times with exponential backoff; the outcome is recorded after the final attempt and the payload key is erased only then. Permanent failures (invalid url, blocked host, other 4xx, unavailable auth secret) are recorded immediately. At most 2 deliveries per tenant run at once (`maxPerTenant` with `onLimit: \"defer\"`), so a slow receiver of one tenant does not delay other tenants. Requires a job runner (Redis) and a worker consuming the job's default `worker` lane; without a job runner the projection fails loudly. Mount this feature explicitly via `createStepDispatcherFeature()` in your app's feature list alongside any features that use `r.step.webhook.send` or `r.step.mail.send`. Requires the `secrets` feature (`createSecretsFeature()`) to be mounted — `webhook.send` auth resolves per-tenant through it, under `step-dispatcher:webhook-auth.<name>`. Every `webhook.send` request carries `Idempotency-Key: <dispatch stream id>`, stable across retries and redeliveries of the same dispatch request; an explicit `Idempotency-Key` in `headers` takes precedence.",
       "toggleableDefault": null,
       "requires": [
         "secrets"