@humwick/server 0.130.1 → 0.130.3

package/node_modules/@humwick/kernel/kernel-persistence.ts

@@ -1,181 +0,0 @@
-import type { JsonValue, Scope } from "./contract";
-import type { EventReplaySlice } from "./event-log";
-import type { MessageAuthorityOperation } from "./message-authority";
-import type { StorageKind } from "./persistence-schema";
-import type {
-  AcquireLeaseParams,
-  ListInstancesOptions,
-  RenewLeaseParams,
-  WorkflowInstancePage,
-  WorkflowInstanceRow,
-  WorkflowInstanceSeed,
-  WorkflowStepCommitRecord,
-  WorkflowStepJournalRow,
-} from "./workflow-journal";
-import type { WorkflowStepLease } from "./workflow-step-commit";
-
-/** A schema-free field description safe to send to a persistence worker. */
-export interface KernelEntityFieldCatalog {
-  readonly name: string;
-  readonly kind: StorageKind;
-  readonly nullable: boolean;
-  /** Whether a stored SQL NULL decodes as explicit null or an omitted optional field. */
-  readonly nullValue: "null" | "undefined";
-}
-
-/** A schema-free entity catalog. Zod schemas and module definitions never cross the port. */
-export interface KernelEntityCatalog {
-  readonly moduleId: string;
-  readonly entityName: string;
-  readonly tableName: string;
-  readonly key: string;
-  readonly fields: readonly KernelEntityFieldCatalog[];
-  readonly references: Readonly<Record<string, { readonly module: string; readonly entity: string }>>;
-  readonly indexes: readonly (readonly string[])[];
-}
-
-export interface KernelModuleCatalog {
-  readonly moduleId: string;
-  readonly entities: readonly KernelEntityCatalog[];
-}
-
-export type KernelStoredValue = string | number | bigint | null;
-
-export interface KernelEntityStoredRow {
-  readonly values: Readonly<Record<string, KernelStoredValue>>;
-  readonly version: number;
-  readonly createdAt: number;
-  readonly updatedAt: number;
-}
-
-export interface KernelEntityPointRead {
-  readonly entity: KernelEntityCatalog;
-  readonly scope: Scope;
-  readonly key: KernelStoredValue;
-}
-
-export interface KernelEntityListRead {
-  readonly entity: KernelEntityCatalog;
-  readonly scope: Scope;
-  readonly where: Readonly<Record<string, KernelStoredValue>>;
-  readonly after?: KernelStoredValue;
-  readonly limit: number;
-}
-
-export interface KernelEntityMutation {
-  readonly kind: "put" | "delete";
-  readonly entity: KernelEntityCatalog;
-  readonly scope: Scope;
-  readonly key: KernelStoredValue;
-  readonly logicalKey: string | number | bigint | boolean;
-  readonly expectedVersion: number | null;
-  readonly values?: Readonly<Record<string, KernelStoredValue>>;
-}
-
-export interface KernelEventAppend {
-  readonly scope: Scope;
-  readonly moduleId: string;
-  readonly name: string;
-  readonly payload: string;
-  readonly createdAt: number;
-}
-
-export interface KernelWorkflowSeed {
-  readonly seed: WorkflowInstanceSeed;
-  /** Position in the caller's prepared-start list; used to return authoritative deduped ids. */
-  readonly resultIndex: number;
-}
-
-export interface KernelWorkflowControl {
-  readonly operation: "cancel" | "fail" | "reopen";
-  readonly instanceId: string;
-  readonly failure?: JsonValue;
-}
-
-export interface KernelOpCommitEnvelope {
-  /** Stable across an indeterminate-ack retry. */
-  readonly requestKey: string;
-  readonly capturedAt: number;
-  readonly messageAuthority: readonly MessageAuthorityOperation[];
-  readonly entities: readonly KernelEntityMutation[];
-  readonly events: readonly KernelEventAppend[];
-  readonly workflowSeeds: readonly KernelWorkflowSeed[];
-  readonly workflowControls?: readonly KernelWorkflowControl[];
-}
-
-export interface KernelOpCommitResult {
-  readonly workflowInstanceIds: readonly string[];
-}
-
-export interface KernelStepCommitEnvelope {
-  /** Stable across an indeterminate-ack retry. */
-  readonly requestKey: string;
-  readonly capturedAt: number;
-  readonly lease: WorkflowStepLease;
-  readonly messageAuthority: readonly MessageAuthorityOperation[];
-  readonly entities: readonly KernelEntityMutation[];
-  readonly events: readonly KernelEventAppend[];
-  readonly record: WorkflowStepCommitRecord;
-}
-
-export interface KernelJournalOnlyCommitEnvelope {
-  readonly requestKey: string;
-  readonly capturedAt: number;
-  readonly lease: WorkflowStepLease;
-  readonly record: WorkflowStepCommitRecord;
-}
-
-/**
- * Kernel-owned async persistence boundary. Every argument/result is structured-clone-safe data;
- * the interface deliberately has no Database, callback, schema, module definition, auth context,
- * dispatcher, listener, or clock member. Writer methods each represent one complete atomic job.
- */
-export interface KernelPersistence {
-  readonly messageAuthorityAvailable: boolean;
-
-  installModule(catalog: KernelModuleCatalog): Promise<void>;
-  moduleTableExists(tableName: string): Promise<boolean>;
-
-  entityPoint(input: KernelEntityPointRead): Promise<KernelEntityStoredRow | undefined>;
-  entityList(input: KernelEntityListRead): Promise<readonly KernelEntityStoredRow[]>;
-  replayEvents(scope: Scope, since: number, limit: number, moduleId?: string): Promise<EventReplaySlice>;
-  eventHighWater(scope: Scope): Promise<number>;
-  eventHasMatch(scope: Scope, topic: string, correlationId?: string): Promise<boolean>;
-
-  getWorkflowInstance(id: string): Promise<WorkflowInstanceRow | undefined>;
-  getWorkflowInstanceByIdentity(moduleName: string, workflowName: string, idempotencyKey: string): Promise<WorkflowInstanceRow | undefined>;
-  listWorkflowInstances(options?: ListInstancesOptions): Promise<WorkflowInstancePage>;
-  listWorkflowSteps(instanceId: string): Promise<WorkflowStepJournalRow[]>;
-  listResumableWorkflows(moduleName: string, now: number): Promise<string[]>;
-  listDueWorkflows(moduleName: string, now: number): Promise<string[]>;
-  listWaitingWorkflows(moduleName: string, now: number): Promise<string[]>;
-  snapshotWorkflowResidual(moduleName: string, now: number): Promise<{ reclaimableCount: number; nextLiveLeaseExpiry: number | undefined }>;
-
-  seedWorkflow(requestKey: string, seed: WorkflowInstanceSeed, capturedAt: number): Promise<{ instance: WorkflowInstanceRow; created: boolean }>;
-  acquireWorkflowLease(requestKey: string, params: AcquireLeaseParams): Promise<boolean>;
-  /** Naturally idempotent owner-fenced cleanup; an empty recovery needs no durable request ledger. */
-  reclaimDeadWorkflowLeases(moduleName: string, now: number): Promise<void>;
-  renewWorkflowLease(requestKey: string, params: RenewLeaseParams): Promise<boolean>;
-  releaseWorkflowLease(requestKey: string, instanceId: string, token: string, now: number): Promise<boolean>;
-  controlWorkflow(requestKey: string, operation: "cancel" | "pause" | "resume" | "reopen" | "fail", id: string, now: number, failure?: unknown): Promise<boolean>;
-
-  opCommit(envelope: KernelOpCommitEnvelope): Promise<KernelOpCommitResult>;
-  stepCommit(envelope: KernelStepCommitEnvelope): Promise<number>;
-  failedStepCommit(envelope: KernelJournalOnlyCommitEnvelope): Promise<number>;
-  supersededStepCommit(envelope: KernelJournalOnlyCommitEnvelope): Promise<number>;
-}
-
-/** Gateway adapters surface this name when COMMIT may have succeeded but its ACK was lost. */
-export function isWriteIndeterminateError(error: unknown): boolean {
-  return error instanceof Error && error.name === "WriteIndeterminateError";
-}
-
-/** Retry an indeterminate writer acknowledgement once with the exact same stable request key. */
-export async function recoverIndeterminate<T>(write: () => Promise<T>): Promise<T> {
-  try {
-    return await write();
-  } catch (error) {
-    if (!isWriteIndeterminateError(error)) throw error;
-    return write();
-  }
-}

package/node_modules/@humwick/kernel/landing-incident.ts

@@ -1,306 +0,0 @@
-// #3363 — the land boundary and the red-main incident (docs/design/3362-land-gate.md §4), part of the
-// Kernel landing module. Server SQL reads the incident table by name in the claim admission predicate
-// (src/db/land-incident-eligibility.ts), so field names here are a contract with that SQL.
-import { z } from "zod";
-import { CI_MONITOR_EVENTS } from "@humwick/sdk";
-import { type EntityStore, type ModuleCtx, OpError, type OpDecl } from "./contract";
-
-export const boundaryVerdictSchema = z.enum(["pending", "green", "red", "unverified"]);
-export const incidentStateSchema = z.enum(["suspected", "rerunning", "confirmed", "recovering", "resolved", "cleared", "abandoned"]);
-export type IncidentState = z.infer<typeof incidentStateSchema>;
-export const incidentEffectSchema = z.discriminatedUnion("kind", [
-  z.object({ kind: z.literal("rerun"), key: z.string(), runId: z.number().int(), attempt: z.number().int(), cancelled: z.boolean() }),
-  z.object({ kind: z.literal("reopen-task"), key: z.string(), taskId: z.number().int() }),
-  z.object({ kind: z.literal("notify"), key: z.string(), taskId: z.number().int() }),
-  // attempt: the failed CI attempt of the red boundary's run, whose failing tests are the flakes (#3362 §10).
-  z.object({ kind: z.literal("quarantine"), key: z.string(), attempt: z.number().int().optional() }),
-  z.object({ kind: z.literal("auto-revert"), key: z.string() }),
-]);
-export type IncidentEffect = z.infer<typeof incidentEffectSchema>;
-export const boundaryWatchSchema = z.object({ workflow: z.string().min(1), branch: z.string().min(1),
-  jobs: z.array(z.string()).optional(), id: z.string().optional(),
-  events: z.array(z.enum(CI_MONITOR_EVENTS)).optional() });
-
-const boundarySchema = z.object({
-  id: z.string(),
-  laneKey: z.string(),
-  mainFailurePolicy: z.enum(["stop", "advisory"]).optional(),
-  commandId: z.string(),
-  workspaceId: z.string(),
-  taskId: z.number().int().optional(),
-  remote: z.string(),
-  baseRef: z.string(),
-  gateBaseSha: z.string().optional(),
-  watch: boundaryWatchSchema.optional(),
-  pendingEffects: z.array(incidentEffectSchema).optional(),
-  rerunAttempt: z.number().int().optional(),
-  previousBaseSha: z.string(),
-  landedTipSha: z.string(),
-  landedAt: z.number().int(),
-  verdict: boundaryVerdictSchema,
-  ciRunId: z.number().int().optional(),
-  ciAttempt: z.number().int().optional(),
-  incidentId: z.string().optional(),
-  verdictAt: z.number().int().optional(),
-});
-
-const incidentSchema = z.object({
-  id: z.string(),
-  laneKey: z.string(),
-  mainFailurePolicy: z.enum(["stop", "advisory"]).optional(),
-  state: incidentStateSchema,
-  culpritBoundaryIds: z.array(z.string()),
-  redBoundaryId: z.string(),
-  ciRunId: z.number().int(),
-  ciAttempt: z.number().int(),
-  recoveryCommandId: z.string().optional(),
-  recoveryWorkspaceId: z.string().optional(),
-  recoveryBoundAt: z.number().int().optional(),
-  automaticRevertFailure: z.string().optional(),
-  openedAt: z.number().int(),
-  escalatedAt: z.number().int().optional(),
-  resolvedAt: z.number().int().optional(),
-  abandonedBy: z.string().optional(),
-  abandonReason: z.string().optional(),
-  owedEffect: z.enum(["confirm", "quarantine", "none"]).optional(),
-  pendingEffects: z.array(incidentEffectSchema).optional(),
-  updatedAt: z.number().int(),
-});
-
-export type LandBoundary = z.infer<typeof boundarySchema>;
-export type RedMainIncident = z.infer<typeof incidentSchema>;
-
-export const landingIncidentEntities = {
-  boundary: { schema: boundarySchema, indexes: ["laneKey", "verdict", "incidentId", ["laneKey", "landedAt"], ["remote", "landedTipSha"]] },
-  incident: { schema: incidentSchema, indexes: ["state", ["laneKey", "state"]] },
-} as const;
-
-type IncidentStores = {
-  boundary: EntityStore<{ schema: typeof boundarySchema }>;
-  incident: EntityStore<{ schema: typeof incidentSchema }>;
-};
-const stores = (ctx: ModuleCtx) => ctx.entities as unknown as IncidentStores;
-
-export const OPEN_STATES: readonly IncidentState[] = ["suspected", "rerunning", "confirmed", "recovering"];
-const TRANSITIONS: Record<IncidentState, readonly IncidentState[]> = {
-  suspected: ["rerunning", "confirmed", "cleared", "resolved"],
-  rerunning: ["confirmed", "cleared", "resolved"],
-  confirmed: ["recovering", "resolved"],
-  recovering: ["recovering", "resolved"],
-  resolved: [],
-  cleared: [],
-  abandoned: [],
-};
-
-export function landBoundaryId(commandId: string): string {
-  return `boundary:${commandId}`;
-}
-
-async function openIncidentOnLane(ctx: ModuleCtx, laneKey: string): Promise<RedMainIncident | undefined> {
-  for (const state of OPEN_STATES) {
-    const page = await stores(ctx).incident.list({ where: { laneKey, state }, limit: 1 });
-    if (page.items[0]) return page.items[0];
-  }
-  return undefined;
-}
-
-/** A rerunning incident already counts its rerun attempt; the failure is the one before it. */
-const failedAttempt = (incident: RedMainIncident) => incident.state === "suspected" ? incident.ciAttempt : incident.ciAttempt - 1;
-
-async function effectsFor(ctx: ModuleCtx, incident: RedMainIncident, flakeAttempt?: number): Promise<IncidentEffect[]> {
-  const prefix = `red-main:${incident.id}`;
-  if (incident.state === "rerunning") return [{ kind: "rerun", key: `${prefix}:rerun:${incident.ciRunId}:${incident.ciAttempt}`,
-    runId: incident.ciRunId, attempt: incident.ciAttempt, cancelled: false }];
-  if (incident.state === "cleared") return [{ kind: "quarantine", key: `${prefix}:quarantine`, ...(flakeAttempt ? { attempt: flakeAttempt } : {}) }];
-  if (incident.state !== "confirmed") return [];
-  const taskIds = new Set<number>();
-  for (const id of incident.culpritBoundaryIds) {
-    const row = await stores(ctx).boundary.get(id);
-    if (row?.taskId !== undefined) taskIds.add(row.taskId);
-  }
-  return [
-    ...[...taskIds].map((taskId): IncidentEffect => ({ kind: "reopen-task", taskId, key: `${prefix}:reopen:${taskId}` })),
-    ...(incident.mainFailurePolicy === "advisory" ? [] : [{ kind: "auto-revert" as const, key: `${prefix}:auto-revert` }]),
-    ...[...taskIds].map((taskId): IncidentEffect => ({ kind: "notify", taskId, key: `${prefix}:notify:${taskId}` })),
-  ];
-}
-
-const ADMIN = "landing:admin" as const;
-const hidden = { http: false, mcp: false, cli: false } as const;
-
-export const landingIncidentOps = {
-  setIncidentPolicy: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Apply authoritative base failure policy without erasing incident history.",
-    input: z.object({ id: z.string(), policy: z.enum(["stop", "advisory"]) }), output: incidentSchema,
-    handler: async (ctx: ModuleCtx, input: { id: string; policy: "stop" | "advisory" }) => {
-      const incident = await stores(ctx).incident.get(input.id);
-      if (!incident) throw new OpError("not_found", 404, `incident ${input.id} not found`);
-      return stores(ctx).incident.update(input.id, { mainFailurePolicy: input.policy,
-        ...(input.policy === "advisory" ? { pendingEffects: (incident.pendingEffects ?? []).filter(effect => effect.kind !== "auto-revert") } : {}),
-        updatedAt: ctx.clock.now() });
-    },
-  },
-  abandonIncident: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Release an incident's lane stop without claiming green CI; retain the operator and reason.",
-    input: z.object({ id: z.string(), reason: z.string().trim().min(1).max(1000) }), output: incidentSchema,
-    handler: async (ctx: ModuleCtx, input: { id: string; reason: string }) => {
-      const incident = await stores(ctx).incident.get(input.id);
-      if (!incident) throw new OpError("not_found", 404, `incident ${input.id} not found`);
-      if (incident.state === "abandoned") return incident;
-      if (!OPEN_STATES.includes(incident.state)) throw new OpError("conflict", 409, `incident ${input.id} is ${incident.state}`);
-      const now = ctx.clock.now();
-      return stores(ctx).incident.update(input.id, { state: "abandoned", abandonedBy: ctx.principal.actor,
-        abandonReason: input.reason, resolvedAt: now, updatedAt: now, pendingEffects: [], owedEffect: "none" });
-    },
-  },
-  recordBoundary: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Record one land boundary previousBaseSha..landedTipSha, written with the landed sha.",
-    input: boundarySchema.omit({ id: true, verdict: true, ciRunId: true, ciAttempt: true, incidentId: true, verdictAt: true }),
-    output: boundarySchema,
-    handler: async (ctx: ModuleCtx, input: Omit<LandBoundary, "id" | "verdict">) => {
-      const id = landBoundaryId(input.commandId);
-      return (await stores(ctx).boundary.get(id)) ?? stores(ctx).boundary.create({ ...input, id, verdict: "pending" });
-    },
-  },
-  setBoundaryVerdict: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Record the exact-sha main CI verdict of one boundary.",
-    input: z.object({ id: z.string(), verdict: boundaryVerdictSchema, ciRunId: z.number().int().optional(),
-      ciAttempt: z.number().int().optional(), incidentId: z.string().optional() }),
-    output: boundarySchema,
-    handler: async (ctx: ModuleCtx, input: Pick<LandBoundary, "id" | "verdict" | "ciRunId" | "ciAttempt" | "incidentId">) => {
-      const { id, ...patch } = input;
-      return stores(ctx).boundary.update(id, { ...patch, verdictAt: ctx.clock.now() });
-    },
-  },
-  openIncident: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Open the lane's red-main incident; at most one is open per lane.",
-    input: z.object({ laneKey: z.string(), redBoundaryId: z.string(), culpritBoundaryIds: z.array(z.string()).min(1),
-      ciRunId: z.number().int(), ciAttempt: z.number().int() }),
-    output: incidentSchema,
-    handler: async (ctx: ModuleCtx, input: Pick<RedMainIncident, "laneKey" | "redBoundaryId" | "culpritBoundaryIds" | "ciRunId" | "ciAttempt">) => {
-      const open = await openIncidentOnLane(ctx, input.laneKey);
-      if (open) return open;
-      const now = ctx.clock.now();
-      const boundary = await stores(ctx).boundary.get(input.redBoundaryId);
-      return stores(ctx).incident.create({ ...input, ...(boundary?.mainFailurePolicy ? { mainFailurePolicy: boundary.mainFailurePolicy } : {}), id: `incident:${input.redBoundaryId}`, state: "suspected", pendingEffects: [], openedAt: now, updatedAt: now });
-    },
-  },
-  transitionIncident: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Move an incident along its lifecycle; the expected state is a CAS.",
-    input: z.object({ id: z.string(), from: incidentStateSchema, to: incidentStateSchema,
-      ciAttempt: z.number().int().optional(), escalated: z.boolean().optional() }),
-    output: incidentSchema,
-    handler: async (ctx: ModuleCtx, input: { id: string; from: IncidentState; to: IncidentState; ciAttempt?: number; escalated?: boolean }) => {
-      const incident = await stores(ctx).incident.get(input.id);
-      if (!incident) throw new OpError("not_found", 404, `incident ${input.id} not found`);
-      if (incident.state === input.to) return incident;
-      if (incident.state !== input.from || !TRANSITIONS[input.from].includes(input.to)) {
-        throw new OpError("conflict", 409, `incident ${input.id} is ${incident.state}, cannot move ${input.from} -> ${input.to}`);
-      }
-      const now = ctx.clock.now();
-      // A flake explains none of the reds judged against it: they go back to pending and are judged
-      // again in landing order, so a successor's own regression opens its own incident.
-      if (input.to === "cleared") {
-        // Each update drops the row from the filter, so re-reading the first page walks them all.
-        for (let page = await stores(ctx).boundary.list({ where: { incidentId: input.id }, limit: 200 }); page.items.length;
-          page = await stores(ctx).boundary.list({ where: { incidentId: input.id }, limit: 200 })) {
-          for (const row of page.items) {
-            await stores(ctx).boundary.update(row.id, { verdict: "pending", incidentId: undefined, ciRunId: undefined, ciAttempt: undefined, verdictAt: now });
-          }
-        }
-      }
-      // A covering green tree cancels obsolete recovery work, but still quarantines an unconfirmed red (#3545).
-      const pendingEffects = input.to === "resolved"
-        ? (incident.state === "suspected" || incident.state === "rerunning"
-          ? await effectsFor(ctx, { ...incident, state: "cleared" }, failedAttempt(incident)) : [])
-        : [...incident.pendingEffects ?? [], ...await effectsFor(ctx, { ...incident, state: input.to, ciAttempt: input.ciAttempt ?? incident.ciAttempt }, failedAttempt(incident))];
-      return stores(ctx).incident.update(input.id, {
-        state: input.to, updatedAt: now,
-        ...(input.ciAttempt !== undefined ? { ciAttempt: input.ciAttempt } : {}),
-        ...(input.escalated ? { escalatedAt: now } : {}),
-        pendingEffects, owedEffect: "none",
-        ...(input.to === "resolved" || input.to === "cleared" ? { resolvedAt: now } : {}),
-      });
-    },
-  },
-  ensureIncidentEffects: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Upgrade pre-enable incident effects without losing unfinished work.",
-    input: z.object({ id: z.string() }), output: incidentSchema,
-    handler: async (ctx: ModuleCtx, input: { id: string }) => {
-      const incident = await stores(ctx).incident.get(input.id);
-      if (!incident) throw new OpError("not_found", 404, `incident ${input.id} not found`);
-      if (incident.pendingEffects !== undefined) return incident;
-      const pendingEffects = incident.owedEffect === "none" ? [] : await effectsFor(ctx, incident);
-      return stores(ctx).incident.update(input.id, { pendingEffects, owedEffect: "none" });
-    },
-  },
-  settleIncidentEffect: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Acknowledge one durable incident effect by its idempotency key.",
-    input: z.object({ id: z.string(), key: z.string() }), output: incidentSchema,
-    handler: async (ctx: ModuleCtx, input: { id: string; key: string }) => {
-      const incident = await stores(ctx).incident.get(input.id);
-      if (!incident) throw new OpError("not_found", 404, `incident ${input.id} not found`);
-      return stores(ctx).incident.update(input.id, {
-        pendingEffects: (incident.pendingEffects ?? []).filter((effect) => effect.key !== input.key), updatedAt: ctx.clock.now(),
-      });
-    },
-  },
-  failAutomaticRevert: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Record a terminal automatic recovery failure for the coordinator escalation.",
-    input: z.object({ id: z.string(), reason: z.string() }), output: incidentSchema,
-    handler: async (ctx: ModuleCtx, input: { id: string; reason: string }) => {
-      const incident = await stores(ctx).incident.get(input.id);
-      if (!incident) throw new OpError("not_found", 404, `incident ${input.id} not found`);
-      if (incident.automaticRevertFailure) return incident;
-      return stores(ctx).incident.update(input.id, { automaticRevertFailure: input.reason.slice(0, 4096), updatedAt: ctx.clock.now() });
-    },
-  },
-  queueBoundaryRerun: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Persist a boundary infrastructure retry before issuing it.",
-    input: z.object({ id: z.string(), effect: incidentEffectSchema }), output: boundarySchema,
-    handler: async (ctx: ModuleCtx, input: { id: string; effect: IncidentEffect }) => {
-      const row = await stores(ctx).boundary.get(input.id);
-      if (!row) throw new OpError("not_found", 404, `boundary ${input.id} not found`);
-      if (input.effect.kind !== "rerun") return row;
-      if (row.ciRunId === input.effect.runId && (row.rerunAttempt ?? 0) >= input.effect.attempt) return row;
-      return stores(ctx).boundary.update(input.id, { ciRunId: input.effect.runId,
-        rerunAttempt: input.effect.attempt, pendingEffects: [input.effect] });
-    },
-  },
-  settleBoundaryEffect: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Acknowledge a boundary infrastructure retry by key.",
-    input: z.object({ id: z.string(), key: z.string() }), output: boundarySchema,
-    handler: async (ctx: ModuleCtx, input: { id: string; key: string }) => {
-      const row = await stores(ctx).boundary.get(input.id);
-      if (!row) throw new OpError("not_found", 404, `boundary ${input.id} not found`);
-      return stores(ctx).boundary.update(input.id, { pendingEffects: (row.pendingEffects ?? []).filter((effect) => effect.key !== input.key) });
-    },
-  },
-  bindRecovery: {
-    kind: "mutation", idempotent: true, grant: ADMIN, expose: hidden,
-    summary: "Bind one land command as the incident's only exempt recovery land; replaces any earlier binding.",
-    input: z.object({ id: z.string(), recoveryCommandId: z.string(), recoveryWorkspaceId: z.string() }),
-    output: incidentSchema,
-    handler: async (ctx: ModuleCtx, input: { id: string; recoveryCommandId: string; recoveryWorkspaceId: string }) => {
-      const incident = await stores(ctx).incident.get(input.id);
-      if (!incident) throw new OpError("not_found", 404, `incident ${input.id} not found`);
-      if (incident.state !== "confirmed" && incident.state !== "recovering") {
-        throw new OpError("conflict", 409, `incident ${input.id} is ${incident.state}; only a confirmed incident takes a recovery land`);
-      }
-      const now = ctx.clock.now();
-      return stores(ctx).incident.update(input.id, { state: "recovering", recoveryCommandId: input.recoveryCommandId,
-        recoveryWorkspaceId: input.recoveryWorkspaceId, recoveryBoundAt: now, updatedAt: now });
-    },
-  },
-} satisfies Record<string, OpDecl>;