@cosmicdrift/kumiko-samples 0.390.1 → 0.391.1

package/packages/bundled-features/src/readiness/screens.ts

@@ -0,0 +1,191 @@
+import type {
+  EditRelatedListSection,
+  ListColumnSpec,
+  ProjectionDetailScreenDefinition,
+  RowAction,
+} from "@cosmicdrift/kumiko-framework/ui-types";
+import { ReadinessQueries, ReadinessWrites } from "./constants.ts";
+
+export const SYSTEM_CHECK_SCREEN_ID = "system-check";
+export const SYSTEM_CHECK_RUN_SCREEN_ID = "system-check-run";
+export const SYSTEM_CHECK_SCREEN_QN = `readiness:screen:${SYSTEM_CHECK_SCREEN_ID}`;
+export const SYSTEM_CHECK_NAV_LABEL = "readiness:nav.systemCheck";
+
+export const ScreenTextKeys = {
+  runAction: "readiness.screen.action.run",
+  runPaidAction: "readiness.screen.action.run-paid",
+  latestTitle: "readiness.screen.latest.title",
+  latestDescription: "readiness.screen.latest.description",
+  latestEmpty: "readiness.screen.latest.empty",
+  historyTitle: "readiness.screen.history.title",
+  historyDescription: "readiness.screen.history.description",
+  historyEmpty: "readiness.screen.history.empty",
+  runResultsTitle: "readiness.screen.run.results-title",
+  metricOk: "readiness.screen.metric.ok",
+  metricFailed: "readiness.screen.metric.failed",
+  metricSkipped: "readiness.screen.metric.skipped",
+  columnArea: "readiness.screen.column.area",
+  columnCheck: "readiness.screen.column.check",
+  columnStatus: "readiness.screen.column.status",
+  columnDuration: "readiness.screen.column.duration",
+  columnResult: "readiness.screen.column.result",
+  columnStarted: "readiness.screen.column.started",
+  columnStartedBy: "readiness.screen.column.started-by",
+  columnOutcome: "readiness.screen.column.outcome",
+} as const;
+
+export const RESULT_STATUS_KEY_PREFIX = "readiness.result-status.";
+export const RUN_OUTCOME_KEY_PREFIX = "readiness.outcome.";
+
+const ACCESS = { roles: ["TenantAdmin", "SystemAdmin"] } as const;
+
+const RESULT_COLUMNS: readonly ListColumnSpec[] = [
+  { field: "area", label: ScreenTextKeys.columnArea, renderer: { format: "i18nText" } },
+  { field: "label", label: ScreenTextKeys.columnCheck, renderer: { format: "i18nText" } },
+  {
+    field: "status",
+    label: ScreenTextKeys.columnStatus,
+    renderer: {
+      format: "enumOption",
+      keyPrefix: RESULT_STATUS_KEY_PREFIX,
+      tones: { ok: "ok", failed: "bad", skipped: "neutral" },
+    },
+  },
+  {
+    field: "durationMs",
+    label: ScreenTextKeys.columnDuration,
+    valueType: "number",
+    renderer: { format: "number" },
+    hideOnNarrow: true,
+  },
+  {
+    field: "message",
+    label: ScreenTextKeys.columnResult,
+    renderer: { format: "i18nText" },
+    linkScreenField: "fixScreen",
+  },
+];
+
+function resultsSection(
+  query: string,
+  title: string,
+  extra: Partial<EditRelatedListSection>,
+): EditRelatedListSection {
+  return {
+    kind: "relatedList",
+    title,
+    query,
+    columns: RESULT_COLUMNS,
+    emptyState: { title: ScreenTextKeys.latestEmpty },
+    ...extra,
+  };
+}
+
+const RESULT_METRICS = [
+  { field: "okCount", label: ScreenTextKeys.metricOk, tone: "ok", icon: "check-circle" },
+  { field: "failedCount", label: ScreenTextKeys.metricFailed, tone: "negative", icon: "x-circle" },
+  { field: "skippedCount", label: ScreenTextKeys.metricSkipped, icon: "circle-stop" },
+] as const;
+
+// The paid button sends hasPaidProbes as includePaid: it is only visible while that field is true.
+const RUN_ACTIONS: readonly RowAction[] = [
+  {
+    kind: "writeHandler",
+    id: "run-check",
+    label: ScreenTextKeys.runAction,
+    handler: ReadinessWrites.runProbes,
+    payload: { pick: [] },
+    style: "primary",
+    icon: "shield-check",
+  },
+  {
+    kind: "writeHandler",
+    id: "run-paid-check",
+    label: ScreenTextKeys.runPaidAction,
+    handler: ReadinessWrites.runProbes,
+    payload: { map: { includePaid: "hasPaidProbes" } },
+    visible: { field: "hasPaidProbes", eq: true },
+    style: "secondary",
+    icon: "sparkles",
+  },
+];
+
+export function systemCheckScreen(dormant: boolean): ProjectionDetailScreenDefinition {
+  return {
+    id: SYSTEM_CHECK_SCREEN_ID,
+    type: "projectionDetail",
+    query: ReadinessQueries.latestSummary,
+    singleton: true,
+    live: true,
+    dormant,
+    access: ACCESS,
+    actions: RUN_ACTIONS,
+    primaryAction: "run-check",
+    metrics: RESULT_METRICS,
+    metricsStyle: "tiles",
+    layout: {
+      width: "full",
+      sections: [
+        resultsSection(ReadinessQueries.latestResults, ScreenTextKeys.latestTitle, {
+          id: "latest",
+          description: ScreenTextKeys.latestDescription,
+        }),
+        {
+          id: "history",
+          kind: "relatedList",
+          title: ScreenTextKeys.historyTitle,
+          description: ScreenTextKeys.historyDescription,
+          query: ReadinessQueries.runList,
+          entity: "probe-run",
+          columns: [
+            {
+              field: "requestedAt",
+              label: ScreenTextKeys.columnStarted,
+              renderer: { format: "timestamp", dateStyle: "medium", timeStyle: "short" },
+            },
+            { field: "triggeredByName", label: ScreenTextKeys.columnStartedBy },
+            {
+              field: "outcome",
+              label: ScreenTextKeys.columnOutcome,
+              renderer: {
+                format: "enumOption",
+                keyPrefix: RUN_OUTCOME_KEY_PREFIX,
+                tones: { ok: "ok", failed: "bad", running: "neutral" },
+              },
+            },
+            {
+              field: "durationMs",
+              label: ScreenTextKeys.columnDuration,
+              valueType: "number",
+              renderer: { format: "number" },
+            },
+          ],
+          rowClick: { entity: "probe-run" },
+          emptyState: { title: ScreenTextKeys.historyEmpty },
+        },
+      ],
+    },
+  };
+}
+
+export const systemCheckRunScreen: ProjectionDetailScreenDefinition = {
+  id: SYSTEM_CHECK_RUN_SCREEN_ID,
+  type: "projectionDetail",
+  query: ReadinessQueries.runDetail,
+  idParam: "runId",
+  detailFor: "probe-run",
+  live: true,
+  dormant: true,
+  listScreenId: SYSTEM_CHECK_SCREEN_ID,
+  access: ACCESS,
+  metrics: RESULT_METRICS,
+  metricsStyle: "tiles",
+  layout: {
+    width: "full",
+    sections: [
+      resultsSection(ReadinessQueries.runResults, ScreenTextKeys.runResultsTitle, {
+        parentParam: "runId",
+      }),
+    ],
+  },
+};

package/packages/bundled-features/src/readiness/visible-results.ts

@@ -0,0 +1,106 @@
+import type { SessionUser } from "@cosmicdrift/kumiko-framework/engine";
+import * as z from "zod";
+import { PROBE_RUN_STATUS } from "./schema/probe-run.ts";
+
+const probeTextSchema = z.object({
+  key: z.string(),
+  params: z.record(z.string(), z.union([z.string(), z.number()])).optional(),
+});
+
+const probeResultRowSchema = z.object({
+  probe: z.string(),
+  scope: z.enum(["tenant", "system"]),
+  area: z.string(),
+  label: probeTextSchema,
+  targetId: z.string().optional(),
+  ownerUserId: z.string().optional(),
+  status: z.enum(["ok", "failed", "skipped"]),
+  durationMs: z.number().nullable(),
+  message: probeTextSchema,
+  fixScreen: z.string().optional(),
+});
+
+const storedResultsSchema = z.object({ rows: z.array(probeResultRowSchema) });
+
+export type StoredProbeResult = z.infer<typeof probeResultRowSchema>;
+
+// Same shape as a dashboard text, so the result table renders it with the `i18nText` format.
+export type ProbeText = {
+  readonly i18nKey: string;
+  readonly i18nParams?: Readonly<Record<string, string | number>>;
+};
+
+export type VisibleProbeResult = Omit<StoredProbeResult, "label" | "message" | "fixScreen"> & {
+  readonly id: string;
+  readonly label: ProbeText;
+  readonly message: ProbeText;
+  // Only on failed rows: the result table links the row to the screen that fixes it.
+  readonly fixScreen?: string;
+};
+
+export type RunOutcome = "ok" | "failed" | "running";
+
+export type ResultCounts = {
+  readonly okCount: number;
+  readonly failedCount: number;
+  readonly skippedCount: number;
+  readonly totalCount: number;
+};
+
+// A fresh run still holds the column default `{}`; anything unparsable counts as no results.
+export function parseStoredResults(stored: unknown): readonly StoredProbeResult[] {
+  const parsed = storedResultsSchema.safeParse(stored);
+  return parsed.success ? parsed.data.rows : [];
+}
+
+export function resultId(result: StoredProbeResult): string {
+  return `${result.probe}#${result.targetId ?? ""}`;
+}
+
+function canSee(result: StoredProbeResult, user: SessionUser): boolean {
+  if (result.scope === "system" && !user.roles.includes("SystemAdmin")) return false;
+  return result.ownerUserId === undefined || result.ownerUserId === user.id;
+}
+
+function toProbeText(text: StoredProbeResult["label"]): ProbeText {
+  return text.params === undefined
+    ? { i18nKey: text.key }
+    : { i18nKey: text.key, i18nParams: text.params };
+}
+
+function toVisibleResult(result: StoredProbeResult): VisibleProbeResult {
+  const { label, message, fixScreen, ...rest } = result;
+  return {
+    ...rest,
+    id: resultId(result),
+    label: toProbeText(label),
+    message: toProbeText(message),
+    ...(result.status === "failed" && fixScreen !== undefined && { fixScreen }),
+  };
+}
+
+export function visibleResults(stored: unknown, user: SessionUser): readonly VisibleProbeResult[] {
+  return parseStoredResults(stored)
+    .filter((result) => canSee(result, user))
+    .map(toVisibleResult);
+}
+
+export function countResults(results: readonly Pick<StoredProbeResult, "status">[]): ResultCounts {
+  const count = (status: StoredProbeResult["status"]) =>
+    results.filter((result) => result.status === status).length;
+  return {
+    okCount: count("ok"),
+    failedCount: count("failed"),
+    skippedCount: count("skipped"),
+    totalCount: results.length,
+  };
+}
+
+export function isRunActive(status: string): boolean {
+  return status === PROBE_RUN_STATUS.Pending || status === PROBE_RUN_STATUS.Running;
+}
+
+export function outcomeOf(status: string, counts: ResultCounts): RunOutcome {
+  if (isRunActive(status)) return "running";
+  return status === PROBE_RUN_STATUS.Done && counts.failedCount === 0 ? "ok" : "failed";
+}

package/packages/bundled-features/src/tenant/changes.json

@@ -1,4 +1,16 @@
 [
+  {
+    "version": "0.391.0",
+    "type": "fix",
+    "title": "The stream-tenant backfill archives membership streams a projection replay must not see",
+    "detail": "Before the membership projection is rebuilt, the backfill reads the membership lifecycle from the event log and archives two kinds of streams. Streams of a user whose every stream is archived are archived with it, so a rebuild does not bring back memberships of users that no longer exist. When a second stream of a (userId, tenantId) pair was created while another one was still alive, the stream with the newest event stays and the others are archived; a pair that was removed and added again is left alone. Before, the rebuild failed on read_tenant_memberships_unique on every boot and the backfill stayed pending. The archived streams are logged as a warning and returned in membershipStreamsArchived. If a collision still remains, the failure says so and the step runs again on the next boot. Membership rows of archived users disappear from read_tenant_memberships with the rebuild."
+  },
+  {
+    "version": "0.391.0",
+    "type": "fix",
+    "title": "seedTenantMembership reuses the live membership stream of a pair",
+    "detail": "seedTenantMembership looked only at read_tenant_memberships. After the row was lost it created a second stream for the same pair, which made every later rebuild of the projection collide. It now also checks the event log and returns the id of the pair's live stream without writing anything. The seed does not bring the lost row back; only a rebuild of the membership projection does, and nothing triggers one on its own, so until then the user has no membership in that tenant."
+  },
   {
     "version": "0.390.0",
     "type": "improvement",

package/packages/bundled-features/src/tenant/membership-stream-history.ts

@@ -0,0 +1,224 @@
+// Membership lifecycle as the event log tells it, keyed by the (userId, tenantId) pair of each
+// stream's created event. The projection's unique index allows one live row per pair, so a replay
+// fails as soon as a second stream of a pair is created while another one is alive. Such streams
+// came from seeds that only consulted the projection after its rows were lost.
+
+import {
+  type DbRunner,
+  entityEventName,
+  toPostgresArrayLiteral,
+  unsafeReadRetrying,
+} from "@cosmicdrift/kumiko-framework/db";
+import { parseTenantId } from "@cosmicdrift/kumiko-framework/engine";
+import { archiveStream } from "@cosmicdrift/kumiko-framework/event-store";
+
+const MEMBERSHIP_AGGREGATE = "tenant-membership";
+const USER_AGGREGATE = "user";
+const CREATED = entityEventName(MEMBERSHIP_AGGREGATE, "created");
+const ROW_REMOVING_TYPES = [
+  entityEventName(MEMBERSHIP_AGGREGATE, "deleted"),
+  entityEventName(MEMBERSHIP_AGGREGATE, "forgotten"),
+];
+const LIFECYCLE_TYPES = [
+  CREATED,
+  entityEventName(MEMBERSHIP_AGGREGATE, "updated"),
+  ...ROW_REMOVING_TYPES,
+];
+const ARCHIVED_BY = "stream-tenant-backfill";
+
+type MembershipEventRow = {
+  readonly event_id: string;
+  readonly aggregate_id: string;
+  readonly stream_tenant_id: string;
+  readonly type: string;
+  readonly user_id: string;
+  readonly member_tenant_id: string;
+  readonly user_archived: boolean;
+};
+
+type MembershipStream = {
+  readonly aggregateId: string;
+  readonly streamTenantIds: Set<string>;
+  readonly userArchived: boolean;
+  lastEventId: bigint;
+  alive: boolean;
+};
+
+export type ArchivedMembershipStream = {
+  readonly aggregateId: string;
+  readonly reason: "user_archived" | "duplicate";
+  // The stream of the same pair that stays; set for duplicates.
+  readonly keptAggregateId?: string;
+};
+
+// Archived streams are left out: they do not replay and cannot be written to.
+async function readMembershipEvents(
+  db: DbRunner,
+  pair?: { readonly userId: string; readonly tenantId: string },
+): Promise<readonly MembershipEventRow[]> {
+  const params: unknown[] = [
+    MEMBERSHIP_AGGREGATE,
+    CREATED,
+    toPostgresArrayLiteral(LIFECYCLE_TYPES),
+    USER_AGGREGATE,
+  ];
+  let pairSql = "";
+  if (pair) {
+    params.push(pair.tenantId, pair.userId);
+    // Only created events carry the pair; later events of the stream are matched by aggregate.
+    // The aggregate type is inlined so the planner can use the partial index
+    // events_membership_payload_tenant_idx under a generic plan as well.
+    pairSql = ` AND e."aggregate_id" IN (
+         SELECT p."aggregate_id" FROM "kumiko_events" p
+          WHERE p."aggregate_type" = '${MEMBERSHIP_AGGREGATE}' AND p."type" = $2
+            AND p."payload"->>'tenantId' = $5 AND p."payload"->>'userId' = $6)`;
+  }
+  return unsafeReadRetrying<MembershipEventRow>(
+    db,
+    // A user counts as archived only while none of its events sits outside an archived stream.
+    `WITH archived_users AS (
+       SELECT DISTINCT u."aggregate_id"::text AS aggregate_id FROM "kumiko_archived_streams" u
+        WHERE u."aggregate_type" = $4
+          AND NOT EXISTS (
+            SELECT 1 FROM "kumiko_events" ue
+             WHERE ue."aggregate_id" = u."aggregate_id" AND ue."aggregate_type" = $4
+               AND NOT EXISTS (SELECT 1 FROM "kumiko_archived_streams" ua
+                                WHERE ua."tenant_id" = ue."tenant_id" AND ua."aggregate_id" = ue."aggregate_id"))
+     )
+     SELECT e."id"::text AS event_id, e."aggregate_id"::text AS aggregate_id,
+            e."tenant_id"::text AS stream_tenant_id, e."type" AS type,
+            c."user_id" AS user_id, c."member_tenant_id" AS member_tenant_id,
+            c."user_id" IN (SELECT aggregate_id FROM archived_users) AS user_archived
+       FROM "kumiko_events" e
+       JOIN LATERAL (
+         SELECT f."payload"->>'userId' AS user_id, f."payload"->>'tenantId' AS member_tenant_id
+           FROM "kumiko_events" f
+          WHERE f."aggregate_id" = e."aggregate_id" AND f."aggregate_type" = $1 AND f."type" = $2
+          ORDER BY f."id" ASC LIMIT 1
+       ) c ON c."user_id" IS NOT NULL AND c."member_tenant_id" IS NOT NULL
+      WHERE e."aggregate_type" = $1 AND e."type" = ANY($3::text[])${pairSql}
+        AND NOT EXISTS (SELECT 1 FROM "kumiko_archived_streams" a
+                         WHERE a."tenant_id" = e."tenant_id" AND a."aggregate_id" = e."aggregate_id")
+      ORDER BY e."id" ASC`,
+    params,
+  );
+}
+
+type PairHistory = {
+  readonly streams: Map<string, MembershipStream>;
+  // A created event arrived while another stream of the pair was alive: replay collides.
+  collides: boolean;
+};
+
+function pairKeyOf(row: MembershipEventRow): string {
+  return `${row.user_id}\u0000${row.member_tenant_id}`;
+}
+
+function groupByPair(rows: readonly MembershipEventRow[]): Map<string, PairHistory> {
+  const pairs = new Map<string, PairHistory>();
+  for (const row of rows) {
+    const key = pairKeyOf(row);
+    const pair = pairs.get(key) ?? { streams: new Map(), collides: false };
+    pairs.set(key, pair);
+    let stream = pair.streams.get(row.aggregate_id);
+    if (!stream) {
+      stream = {
+        aggregateId: row.aggregate_id,
+        streamTenantIds: new Set(),
+        userArchived: row.user_archived,
+        lastEventId: 0n,
+        alive: false,
+      };
+      pair.streams.set(row.aggregate_id, stream);
+    }
+    stream.streamTenantIds.add(row.stream_tenant_id);
+    stream.lastEventId = BigInt(row.event_id);
+    if (row.type === CREATED) {
+      if ([...pair.streams.values()].some((other) => other !== stream && other.alive)) {
+        pair.collides = true;
+      }
+      stream.alive = true;
+    } else if (ROW_REMOVING_TYPES.includes(row.type)) {
+      stream.alive = false;
+    }
+  }
+  return pairs;
+}
+
+// The stream written last carries the state the live row was built from; an alive one beats a removed one.
+function streamToKeep(streams: readonly MembershipStream[]): MembershipStream | undefined {
+  const newestFirst = [...streams].sort((a, b) => (a.lastEventId > b.lastEventId ? -1 : 1));
+  return newestFirst.find((s) => s.alive) ?? newestFirst[0];
+}
+
+export async function findLiveMembershipStreamId(
+  db: DbRunner,
+  pair: { readonly userId: string; readonly tenantId: string },
+): Promise<string | undefined> {
+  const history = groupByPair(await readMembershipEvents(db, pair))
+    .values()
+    .next().value;
+  if (!history) return undefined;
+  const kept = streamToKeep([...history.streams.values()]);
+  return kept?.alive ? kept.aggregateId : undefined;
+}
+
+/**
+ * Archives the membership streams a projection replay must not see: those of users whose own
+ * stream is archived, and the older streams of a (user, tenant) pair whose streams collide.
+ */
+export async function archiveUnreplayableMembershipStreams(
+  db: DbRunner,
+): Promise<readonly ArchivedMembershipStream[]> {
+  const archived: ArchivedMembershipStream[] = [];
+  for (const history of groupByPair(await readMembershipEvents(db)).values()) {
+    const streams = [...history.streams.values()];
+    // Every stream of a pair belongs to the same user.
+    if (streams.some((s) => s.userArchived)) {
+      for (const stream of streams) {
+        await archiveMembershipStream(db, stream, "user stream archived");
+        archived.push({ aggregateId: stream.aggregateId, reason: "user_archived" });
+      }
+      continue;
+    }
+    if (!history.collides) continue;
+    const kept = streamToKeep(streams);
+    if (!kept) continue;
+    for (const stream of streams.filter((s) => s !== kept)) {
+      await archiveMembershipStream(
+        db,
+        stream,
+        `duplicate of membership stream ${kept.aggregateId}`,
+      );
+      archived.push({
+        aggregateId: stream.aggregateId,
+        reason: "duplicate",
+        keptAggregateId: kept.aggregateId,
+      });
+    }
+  }
+  return archived;
+}
+
+async function archiveMembershipStream(
+  db: DbRunner,
+  stream: MembershipStream,
+  reason: string,
+): Promise<void> {
+  // The replay filter matches archive markers per (stream tenant, aggregate).
+  for (const streamTenantId of stream.streamTenantIds) {
+    const tenantId = parseTenantId(streamTenantId);
+    if (tenantId === null) {
+      throw new Error(
+        `membership stream ${stream.aggregateId} sits on non-uuid tenant "${streamTenantId}"`,
+      );
+    }
+    await archiveStream(db, {
+      tenantId,
+      aggregateId: stream.aggregateId,
+      aggregateType: MEMBERSHIP_AGGREGATE,
+      archivedBy: ARCHIVED_BY,
+      reason,
+    });
+  }
+}

package/packages/bundled-features/src/tenant/membership-table.ts

@@ -15,6 +15,8 @@ import { createEntity, createTextField } from "@cosmicdrift/kumiko-framework/eng
 // Single source of truth: `tenantMembershipEntity`. The database table is
 // derived from the EntityDefinition through buildEntityTable, and the
 // unique index is declared via entity.indexes.
+export const MEMBERSHIP_UNIQUE_INDEX_NAME = "read_tenant_memberships_unique";
+
 export const tenantMembershipEntity = createEntity({
   table: "read_tenant_memberships",
   fields: {
@@ -24,9 +26,7 @@ export const tenantMembershipEntity = createEntity({
     // read-side stays byte-compatible and no MSP/consumer needs rewrites.
     roles: createTextField({ required: true, personal: false, reason: "technical_reference" }),
   },
-  indexes: [
-    { unique: true, columns: ["userId", "tenantId"], name: "read_tenant_memberships_unique" },
-  ],
+  indexes: [{ unique: true, columns: ["userId", "tenantId"], name: MEMBERSHIP_UNIQUE_INDEX_NAME }],
 });
 
 export const tenantMembershipsTable = buildEntityTable("tenant-membership", tenantMembershipEntity);

package/packages/bundled-features/src/tenant/seeding.ts

@@ -51,6 +51,7 @@ import {
   createLifecycleHooks,
 } from "@cosmicdrift/kumiko-framework/pipeline";
 import { assertAssignableMembershipRoles } from "./membership-roles.ts";
+import { findLiveMembershipStreamId } from "./membership-stream-history.ts";
 import { tenantMembershipEntity, tenantMembershipsTable } from "./membership-table.ts";
 import { tenantEntity, tenantTable } from "./schema/tenant.ts";
 
@@ -329,19 +330,13 @@ export async function seedTenantMembership(
     reason: "membership seed targets an explicit tenant that may differ from the actor's",
   });
 
-  // Idempotency: duplicate seeds are common across beforeEach-resets where
-  // only certain tables get truncated. A plain executor.create would trip
-  // the (user_id, tenant_id) unique index; the fixture call-site would then
-  // have to juggle try/catch. Lookup-first keeps call-sites clean.
-  const existing = await fetchOne(db, tenantMembershipsTable, {
-    userId: options.userId,
-    tenantId: options.tenantId,
-  });
-  if (existing) {
-    // Same validation as the create-path — a missing/non-string id throws
-    // instead of silently returning `undefined as string`.
-    return { id: extractMembershipId(existing) };
-  }
+  // Idempotent: an existing row or live stream of the pair wins. A second stream next to a stream
+  // whose row got lost would make every projection replay collide on the unique index.
+  const pair = { userId: options.userId, tenantId: options.tenantId };
+  const existing = await fetchOne(db, tenantMembershipsTable, pair);
+  if (existing) return { id: extractMembershipId(existing) };
+  const liveStreamId = await findLiveMembershipStreamId(db, pair);
+  if (liveStreamId) return { id: liveStreamId };
 
   // Keep by.id as audit actor, but force stream tenant to the membership tenant
   // so a cross-tenant `by` override cannot diverge row tenant vs event stream.

package/packages/bundled-features/src/tenant/stream-tenant-backfill.ts

@@ -17,6 +17,11 @@ import {
   type StreamBackfillStepFailure,
   stepFailures,
 } from "../shared/index.ts";
+import {
+  type ArchivedMembershipStream,
+  archiveUnreplayableMembershipStreams,
+} from "./membership-stream-history.ts";
+import { MEMBERSHIP_UNIQUE_INDEX_NAME } from "./membership-table.ts";
 
 const TENANT_AGGREGATE = "tenant";
 const MEMBERSHIP_AGGREGATE = "tenant-membership";
@@ -30,6 +35,7 @@ export type TenantStreamBackfillResult = {
   readonly searchDocsRemoved: number;
   readonly projectionsRebuilt: readonly string[];
   readonly reindexedTenants: number;
+  readonly membershipStreamsArchived: readonly ArchivedMembershipStream[];
   // Per-step failures across the whole run; step names the failing stage.
   readonly failures: readonly StreamBackfillStepFailure[];
 };
@@ -81,6 +87,7 @@ export async function backfillTenantStreamTenants(
     return emptyDerived({ tenants, memberships, searchCleanup, failures });
   }
 
+  const archive = await archiveMembershipStreamsBeforeRebuild(db);
   const rebuilt = await rebuildAndReindex({
     db,
     registry,
@@ -94,7 +101,43 @@ export async function backfillTenantStreamTenants(
     searchDocsRemoved: searchCleanup.cleanedAggregates(),
     projectionsRebuilt: rebuilt.projectionsRebuilt,
     reindexedTenants: rebuilt.reindexedRows,
-    failures: [...failures, ...rebuilt.failures],
+    membershipStreamsArchived: archive.archived,
+    failures: [
+      ...failures,
+      ...archive.failures,
+      ...rebuilt.failures.map(explainMembershipCollision),
+    ],
+  };
+}
+
+async function archiveMembershipStreamsBeforeRebuild(db: DbConnection): Promise<{
+  readonly archived: readonly ArchivedMembershipStream[];
+  readonly failures: readonly StreamBackfillStepFailure[];
+}> {
+  try {
+    return { archived: await archiveUnreplayableMembershipStreams(db), failures: [] };
+  } catch (e) {
+    const error = e instanceof Error ? e.message : String(e);
+    return {
+      archived: [],
+      failures: [{ aggregateId: MEMBERSHIP_PROJECTION, step: "rebuild", error }],
+    };
+  }
+}
+
+function explainMembershipCollision(failure: StreamBackfillStepFailure): StreamBackfillStepFailure {
+  if (
+    failure.aggregateId !== MEMBERSHIP_PROJECTION ||
+    !failure.error.includes(MEMBERSHIP_UNIQUE_INDEX_NAME)
+  ) {
+    return failure;
+  }
+  return {
+    ...failure,
+    error:
+      `two membership streams of one (userId, tenantId) pair still replay alive after the boot ` +
+      `archived the colliding streams it found in the event log. The live table is untouched and ` +
+      `the step runs again on the next boot. Cause: ${failure.error}`,
   };
 }
 
@@ -110,6 +153,7 @@ function emptyDerived(args: {
     searchDocsRemoved: args.searchCleanup.cleanedAggregates(),
     projectionsRebuilt: [],
     reindexedTenants: 0,
+    membershipStreamsArchived: [],
     failures: args.failures,
   };
 }