@kici-dev/orchestrator 0.1.2 → 0.1.5

package/dist/cluster/join-token.d.ts

@@ -46,25 +46,33 @@ export declare class JoinTokenManager {
         expiryMs?: number;
     }): Promise<string>;
     /**
-     * Validate a token: check hash exists in DB, not expired, not consumed.
-     * Returns routing info and derived keys on success.
+     * Atomically validate and consume a token in one DB round-trip.
+     *
+     * Single UPDATE with `WHERE token_hash = ? AND consumed_at IS NULL AND
+     * expires_at > NOW()` — only one caller can win the claim across a
+     * shared-DB multi-coordinator mesh. The winner gets `{ routing, keys }`;
+     * every other concurrent caller gets `TOKEN_ALREADY_USED_MESSAGE` and is
+     * expected to fall into the idempotent recovery branch in
+     * `peer-handler.ts` (which serialises credential issuance via the
+     * `peer_credentials_active_uniq` partial unique index).
+     *
+     * On a 0-row claim, a follow-up SELECT disambiguates not-found / expired
+     * / already-used so callers can branch on the specific reason — the
+     * recovery path in peer-handler.ts keys on the "already used" string
+     * specifically. The follow-up only fires on the unhappy path.
      */
-    validateToken(token: string): Promise<{
+    validateAndConsumeToken(token: string, consumedBy: string): Promise<{
         routing: TokenRouting;
         keys: DerivedKeys;
     }>;
-    /**
-     * Mark token as consumed (one-time use).
-     */
-    consumeToken(validationHash: string, consumedBy: string): Promise<void>;
 }
 /**
- * Narrow detector for the "already been used" error thrown by validateToken().
- * Used by peer-handler.ts to branch into the idempotent recovery path on
- * legitimate mesh-join races (sibling peer-clients on the same peer identity
- * racing on a shared join token across a multi-coordinator shared-DB mesh).
- * Other validation failures (expired, not found, bad parse) MUST NOT be
- * treated as recoverable.
+ * Narrow detector for the "already been used" error thrown by
+ * validateAndConsumeToken(). Used by peer-handler.ts to branch into the
+ * idempotent recovery path on legitimate mesh-join races (sibling
+ * peer-clients on the same peer identity racing on a shared join token
+ * across a multi-coordinator shared-DB mesh). Other validation failures
+ * (expired, not found, bad parse) MUST NOT be treated as recoverable.
  */
 export declare function isTokenAlreadyUsedError(err: unknown): boolean;
 /**

package/dist/cluster/peer-client.d.ts

@@ -70,6 +70,14 @@ export interface PeerClientOptions {
         restartRequired?: string[];
         fieldsChanged?: string[];
     }>;
+    /**
+     * Callback invoked once the remote peer accepts our auth handshake,
+     * carrying the remote peer's instanceId. Used by callers that initially
+     * register this client in `sub.peerClients` keyed by a placeholder (URL
+     * or stale id) to re-key the map by the canonical instanceId so later
+     * Platform-mediated discovery dedupes against the same client.
+     */
+    onAuthenticated?: (targetInstanceId: string) => void;
 }
 export declare class PeerClient {
     private ws;
@@ -103,6 +111,7 @@ export declare class PeerClient {
     private readonly onPeerLeaving?;
     private readonly onAgentTokenRevoke?;
     private readonly onPeerConfigReload?;
+    private readonly onAuthenticated?;
     constructor(options: PeerClientOptions);
     /** Current connection state. */
     get state(): PeerConnectionState;

package/dist/config/cluster-id.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Orchestrator cluster-id resolution.
+ *
+ * Every orchestrator DB carries a stable UUID identifier
+ * (`cluster_meta.cluster_id`) seeded once on first boot by the initial
+ * schema migration. HA-cluster coords share the same orchestrator DB,
+ * so they share the same `cluster_id`; two genuinely-different
+ * orchestrator clusters carry distinct values.
+ *
+ * The orchestrator publishes `cluster_id` on `source.register` so
+ * Platform can warn when two unrelated clusters in one org accidentally
+ * share a `cluster_name`. This module is read-only — the row is seeded
+ * by the schema migration and never written from application code.
+ */
+import type { Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+/**
+ * Read the cluster id from `cluster_meta`. Throws when no row exists —
+ * the row is seeded by the initial schema migration, so a missing row
+ * indicates the DB was opened before migrations ran.
+ */
+export declare function getClusterId(db: Kysely<Database>): Promise<string>;
+//# sourceMappingURL=cluster-id.d.ts.map

package/dist/config/cluster-name.d.ts

@@ -0,0 +1,39 @@
+import type { Kysely } from 'kysely';
+import { type ClusterName } from '@kici-dev/engine/protocol/cluster-name';
+import type { Database } from '../db/types.js';
+/**
+ * Source attribution for the resolved cluster name. Useful for operator
+ * tooling (`kici-admin cluster-name get`) to explain where the current
+ * value came from.
+ */
+export type ClusterNameSource = 'stored' | 'env-seeded' | 'auto-generated';
+export interface ResolveResult {
+    clusterName: ClusterName;
+    source: ClusterNameSource;
+}
+/**
+ * Read the cluster name from `cluster_meta`. Returns null if no row
+ * exists yet (orch has never resolved its name).
+ */
+export declare function readClusterName(db: Kysely<Database>): Promise<ClusterName | null>;
+/**
+ * Look up the cluster name. Throws if no row exists — callers that hit
+ * this before the boot resolver has run should be considered bugs.
+ */
+export declare function getClusterName(db: Kysely<Database>): Promise<ClusterName>;
+/**
+ * Persist a validated cluster name. Upsert semantics: replaces the
+ * existing row if one exists. Called by both the boot resolver and the
+ * kici-admin CLI's `cluster-name set`.
+ */
+export declare function setClusterName(db: Kysely<Database>, name: string): Promise<ClusterName>;
+/**
+ * Boot-time resolution: returns the existing row, seeds from env, or
+ * auto-generates. Idempotent: a re-run after first boot just returns
+ * the stored value.
+ *
+ * `randomSource` is dependency-injected so tests can drive deterministic
+ * suffixes; production callers pass `node:crypto.randomBytes`.
+ */
+export declare function resolveAndPersistClusterName(db: Kysely<Database>, env?: NodeJS.ProcessEnv, randomSource?: (size: number) => Uint8Array): Promise<ResolveResult>;
+//# sourceMappingURL=cluster-name.d.ts.map

package/dist/config.d.ts

@@ -85,6 +85,8 @@ declare const configSchema: z.ZodObject<{
     eventRouterRetryBaseBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterRetryMaxBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterRetryScanIntervalMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    testMode: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
+    testEventFailFirstN: z.ZodOptional<z.ZodString>;
     eventLogMaxPayloadBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     logLevel: z.ZodDefault<z.ZodEnum<{
         error: "error";
@@ -103,6 +105,7 @@ declare const configSchema: z.ZodObject<{
     agentMaxReconnectDelayMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     skipS3SentinelValidation: z.ZodPipe<z.ZodDefault<z.ZodString>, z.ZodTransform<boolean, string>>;
     otelExporterOtlpEndpoint: z.ZodOptional<z.ZodString>;
+    clusterName: z.ZodOptional<z.ZodString>;
     cluster: z.ZodPrefault<z.ZodObject<{
         instanceId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
         address: z.ZodOptional<z.ZodString>;
@@ -191,6 +194,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     eventRouterRetryBaseBackoffMs: number;
     eventRouterRetryMaxBackoffMs: number;
     eventRouterRetryScanIntervalMs: number;
+    testMode: boolean;
     eventLogMaxPayloadBytes: number;
     logLevel: "error" | "debug" | "info" | "warn";
     nodeEnv: "development" | "production" | "test";
@@ -240,9 +244,11 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     secretKeyOld?: string | undefined;
     secretKeyFileOld?: string | undefined;
     bootstrapAdminToken?: string | undefined;
+    testEventFailFirstN?: string | undefined;
     internalProviderRepoPath?: string | undefined;
     internalProviderCloneUrl?: string | undefined;
     otelExporterOtlpEndpoint?: string | undefined;
+    clusterName?: string | undefined;
 }>;
 export declare function loadConfig(): AppConfig;
 export {};

package/dist/dashboard/handler.d.ts

@@ -11,12 +11,13 @@
  * - run.rerun.request: re-runs a completed workflow run
  * - run.cancel.request: cancels a running workflow run
  */
-import type { Kysely } from 'kysely';
+import { type Kysely } from 'kysely';
 import { type ColdStore } from '@kici-dev/shared';
-import type { DashboardRunDetailRequest, DashboardStepLogsRequest, DashboardPayloadRequest, DashboardOrchLogsRequest, DashboardEventLogListRequest, DashboardEventLogDetailRequest, DashboardEventLogPayloadStreamRequest, DashboardAccessLogListRequest, RunRerunRequest, RunCancelRequest, ManualScheduleRequest } from '@kici-dev/engine';
+import type { DashboardRunDetailRequest, DashboardStepLogsRequest, DashboardPayloadRequest, DashboardOrchLogsRequest, DashboardEventLogListRequest, DashboardEventLogDetailRequest, DashboardEventLogPayloadStreamRequest, DashboardAccessLogListRequest, DashboardEventDlqListRequest, DashboardEventDlqCountRequest, DashboardEventDlqRetryRequest, DashboardEventDlqDiscardRequest, RunRerunRequest, RunCancelRequest, ManualScheduleRequest } from '@kici-dev/engine';
 import type { Database } from '../db/types.js';
 import type { LogStorage } from '../reporting/log-storage.js';
 import type { AccessLogWriter } from '../audit/access-log.js';
+import type { EventStore } from '../events/event-store.js';
 interface DashboardHandlerDeps {
     db: Kysely<Database>;
     logStorage: LogStorage;
@@ -66,6 +67,13 @@ interface DashboardHandlerDeps {
     onManualSchedule: (registrationId: string, triggeredBy: string | null) => Promise<{
         newRunId: string;
     }>;
+    /**
+     * Event store for the per-org DLQ surface. Optional — when absent the
+     * `handleEventDlq*` methods reply with a structured `error` so the
+     * dashboard renders an empty-state without crashing. In real
+     * deployments the store is wired by `bootstrapOrchestrator`.
+     */
+    eventStore?: EventStore | null;
 }
 export declare class DashboardHandler {
     private readonly db;
@@ -79,6 +87,7 @@ export declare class DashboardHandler {
     private readonly onCancel;
     private readonly onManualSchedule;
     private readonly coldStore;
+    private readonly eventStore;
     constructor(deps: DashboardHandlerDeps);
     /**
      * Update the bound orgId + routingKey. Called from server.ts after resolving
@@ -125,6 +134,13 @@ export declare class DashboardHandler {
      * fallback rather than the source of truth.
      */
     private contextOrFallback;
+    /**
+     * Defense-in-depth dashboard-write policy gate for the DLQ handlers.
+     * Returns true when allowed; false (with a `denied` access_log row and
+     * an `operation_disabled` envelope on the wire) when the orch policy
+     * has the operation switched off.
+     */
+    private enforcePolicy;
     /**
      * Write an access_log row for a handler invocation. The caller resolves
      * the run-owning org via `resolveOrgForRun` (or the registration / event-
@@ -225,6 +241,42 @@ export declare class DashboardHandler {
      * ~80 yields per stream — negligible.
      */
     handleEventLogPayloadStream(msg: DashboardEventLogPayloadStreamRequest): Promise<void>;
+    /**
+     * Handle a dashboard.event-dlq.list request.
+     *
+     * Returns a paginated page of DLQ rows (events whose dispatch attempts
+     * exhausted the retry budget). The page is keyed off `dlq_at DESC` and
+     * paginates with a single `before` ISO cursor.
+     *
+     * Records one `event_dlq.list.read` access_log entry using the user
+     * actor on the wire — mirrors the HTTP admin route's audit shape but
+     * with the calling user's identity instead of the bearer-token role.
+     */
+    handleEventDlqList(msg: DashboardEventDlqListRequest): Promise<void>;
+    /**
+     * Handle a dashboard.event-dlq.count request.
+     *
+     * Returns the DLQ depth for the sidebar badge. Polled ~30s by the
+     * dashboard. No access_log row — the count surface is intentionally
+     * un-audited (it's a frequent badge poll, not a triage action).
+     */
+    handleEventDlqCount(msg: DashboardEventDlqCountRequest): Promise<void>;
+    /**
+     * Handle a dashboard.event-dlq.retry request.
+     *
+     * Clears the DLQ flag on the row and issues a `pg_notify` so a healthy
+     * node picks the event up immediately rather than waiting for the
+     * leader-only retry scanner's next tick. Notify failure is non-fatal —
+     * the scanner will catch up.
+     */
+    handleEventDlqRetry(msg: DashboardEventDlqRetryRequest): Promise<void>;
+    /**
+     * Handle a dashboard.event-dlq.discard request.
+     *
+     * Permanently deletes the DLQ row. No retry, no archive — used when the
+     * payload is corrupt or the routing target has been removed.
+     */
+    handleEventDlqDiscard(msg: DashboardEventDlqDiscardRequest): Promise<void>;
     /**
      * Handle a run.manual_schedule.request.
      * Delegates to the onManualSchedule callback which invokes handleManualSchedule.

package/dist/db/migrations/019_generic_sources_change_notify.d.ts

@@ -0,0 +1,25 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add a Postgres trigger on `generic_webhook_sources` that emits
+ * `pg_notify('generic_sources_change', routing_key)` on every INSERT,
+ * UPDATE, and DELETE. Soft-deletes (UPDATE that sets `deleted_at`) ride
+ * the same UPDATE path — the listener treats a row with `deleted_at IS
+ * NOT NULL` (or a row that no longer exists for that routing_key) as a
+ * de-registration signal.
+ *
+ * The consumer is `GenericSourcesChangeListener`
+ * (`packages/orchestrator/src/webhook/generic-sources-listener.ts`),
+ * which runs in every orchestrator peer, LISTENs on the channel, and
+ * mutates the local `ProviderRegistry` in place
+ * (`registerProviderBundleForSource` for INSERT/UPDATE; `unregister` for
+ * DELETE / soft-delete). Without this round-trip, peers other than the
+ * one that handled `kici-admin source add generic` would 404 incoming
+ * webhooks until restart.
+ *
+ * Mirrors the existing GitHub-app `sources` trigger
+ * (`notify_sources_change()` + `sources_change_trigger`, defined in
+ * `001_initial.ts`).
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=019_generic_sources_change_notify.d.ts.map

package/dist/db/migrations/020_org_settings_dashboard_write_policy.d.ts

@@ -0,0 +1,17 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `org_settings.dashboard_write_policy jsonb NOT NULL DEFAULT '{}'`.
+ *
+ * Stores the per-customer policy that decides which Platform-routed
+ * dashboard.* write operations the orchestrator accepts. An empty
+ * JSONB object means everything is enabled (permissive default). To
+ * disable an operation, set its key to `false`, e.g.
+ * `{"secrets.set": false, "variables.set": false}`. Unknown / missing
+ * keys are treated as `true` by the resolver in
+ * `@kici-dev/engine/protocol/dashboard-write-operations`.
+ *
+ * Idempotent: a re-run on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=020_org_settings_dashboard_write_policy.d.ts.map

package/dist/db/migrations/021_check_run_tracking.d.ts

@@ -0,0 +1,26 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `check_run_tracking` table for HA-safe check-run state persistence.
+ *
+ * The `CommitStatusReporter` previously held all of its check-run bookkeeping
+ * in per-process `Map`s. Under HA (Raft leader switch / coord crash) those
+ * maps reset to empty on the replacement coord, leaving GitHub check runs
+ * stuck in `queued` forever.
+ *
+ * Single composite-keyed table per (provider, owner, repo, sha, check_name):
+ *
+ *   - `check_run_id`            replaces in-memory `checkRunIds`.
+ *   - `build_creation_state`    replaces `pendingBuildCreations` Promise map.
+ *   - `step_progress_json`      replaces `stepProgress` array map.
+ *   - `in_progress_sent_at`     replaces `inProgressSent` boolean map.
+ *   - `run_id`                  indexed; replaces `runIdToKeys` reverse map.
+ *
+ * Progress-timer debounce state (the `progressTimers` Map) is NOT persisted —
+ * timers are recreated on demand when an update arrives and the row's
+ * `updated_at` is older than the debounce window.
+ *
+ * Idempotent: a re-run on a DB that already has the table is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=021_check_run_tracking.d.ts.map

package/dist/db/migrations/022_scaler_manager_state.d.ts

@@ -0,0 +1,29 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add three tables persisting `ScalerManager` per-coord state:
+ *
+ *  - `scaler_spawning_agents`: agents spawned via Docker/Podman/Firecracker
+ *    that have not yet registered via WS. Carries the `bound_job_id` so a
+ *    replacement coord still issues the eager-dispatch hop on register.
+ *
+ *  - `scaler_agent_jobs`: agentId → (runId, jobId) correlation for
+ *    scaler-lifecycle event routing. Inserted on `correlateAgentToJob`,
+ *    deleted on disconnect / job completion.
+ *
+ *  - `scaler_reservations`: outstanding resource reservations keyed by
+ *    agentId. Per-scaler / global usage counters are derived state —
+ *    recomputed on coord boot as `SUM(...) GROUP BY scaler_name` so the
+ *    cap-check critical section stays correct.
+ *
+ * Without these tables, a coord crash mid-spawn orphans the agent (eager
+ * dispatch lost), strands the reservation (resource leak until backend
+ * GC eventually disconnects the WS, minutes later), and loses every
+ * scaler-lifecycle event emitted before correlation (execution-tracker
+ * sees a hole in the run timeline).
+ *
+ * Idempotent: a re-run on a DB that already has any of these tables
+ * leaves the existing one alone.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=022_scaler_manager_state.d.ts.map

package/dist/db/migrations/023_dispatch_queue_recovery_deadline.d.ts

@@ -0,0 +1,26 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `dispatch_queue.recovery_deadline TIMESTAMPTZ` and
+ * `dispatch_queue.recovery_agent_id TEXT` columns persisting per-job
+ * recovery state for HA-safe agent-disconnect handling.
+ *
+ * `Dispatcher.recoveringJobs` previously held the timer handle, agent
+ * ID, and deadline entirely in process memory. A coord crash between
+ * starting the timer and it firing dropped the timer; the replacement
+ * coord saw the row as `status='recovering'` but had no record of when
+ * the deadline expired or which agent owned the job. Jobs lingered
+ * forever.
+ *
+ * The new columns make the deadline durable. A leader-gated sweep
+ * (`Dispatcher.sweepExpiredRecoveries`) scans
+ * `WHERE status='recovering' AND recovery_deadline < now()` and marks
+ * each row failed; on coord boot, `Dispatcher.recoverState()`
+ * hydrates the in-memory Map by re-creating timers from the persisted
+ * deadlines.
+ *
+ * Idempotent: re-running on a DB that already has either column is a
+ * no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=023_dispatch_queue_recovery_deadline.d.ts.map

package/dist/db/types.d.ts

@@ -41,6 +41,10 @@ export interface Database {
     access_log: AccessLogTable;
     cold_store_chunk_counts: ColdStoreChunkCountsTable;
     cold_store_chunks: ColdStoreChunksTable;
+    check_run_tracking: CheckRunTrackingTable;
+    scaler_spawning_agents: ScalerSpawningAgentsTable;
+    scaler_agent_jobs: ScalerAgentJobsTable;
+    scaler_reservations: ScalerReservationsTable;
 }
 /**
  * Cluster metadata table
@@ -132,6 +136,20 @@ export interface DispatchQueueTable {
     /** Routing key (e.g. "github:12345") so dispatch can pick the right
      *  per-app provider bundle in multi-app setups. Required (NOT NULL). */
     routing_key: string;
+    /**
+     * For jobs in `status='recovering'`, the moment the recovery grace
+     * period elapses. Populated when an agent disconnects with this job
+     * in-flight; cleared when the agent reconnects + claims the job, OR
+     * when the leader-gated sweep transitions the row to `failed`.
+     * NULL for all other statuses.
+     */
+    recovery_deadline: ColumnType<Date | null, Date | null | undefined, Date | null>;
+    /**
+     * Companion to `recovery_deadline`: the agent id that owned the job
+     * before disconnect. Used to validate that a reconnecting agent is
+     * the rightful claimant. NULL for non-recovering rows.
+     */
+    recovery_agent_id: ColumnType<string | null, string | null | undefined, string | null>;
 }
 /**
  * Deduplication cache table
@@ -1108,6 +1126,15 @@ export interface OrgSettingsTable {
      * URLs are accepted; arbitrary `http://` registries are rejected at dispatch.
      */
     allow_http_npm_registries: ColumnType<boolean, boolean | undefined, boolean>;
+    /**
+     * Per-operation policy controlling which dashboard.* writes the orch
+     * accepts when routed through Platform. JSONB shape:
+     * `{ [operation]: boolean }` where operation matches the engine enum
+     * `DashboardWriteOperation`. Empty object = all enabled (permissive).
+     * Resolver in `@kici-dev/engine/protocol/dashboard-write-operations`
+     * treats missing keys as `true`.
+     */
+    dashboard_write_policy: ColumnType<Record<string, boolean>, Record<string, boolean> | string | undefined, Record<string, boolean> | string>;
     /** When this setting was created */
     created_at: Generated<Date>;
     /** When this setting was last updated */
@@ -1263,4 +1290,114 @@ export interface ColdStoreChunksTable {
 export type ColdStoreChunksRow = Selectable<ColdStoreChunksTable>;
 export type NewColdStoreChunksRow = Insertable<ColdStoreChunksTable>;
 export type ColdStoreChunksUpdate = Updateable<ColdStoreChunksTable>;
+/**
+ * Check-run tracking table (check_run_tracking).
+ *
+ * HA-safe persistence for the per-coord state previously held in
+ * `CommitStatusReporter`'s six in-memory `Map`s. Replacement coord on a
+ * Raft leader switch reads this table to recover check-run IDs, build
+ * creation state, step-progress entries, in-progress-sent timestamps, and
+ * the run-id reverse index used for cleanup. A coord crash mid-check-run
+ * no longer leaves a GitHub check stuck in `queued` forever.
+ */
+export interface CheckRunTrackingTable {
+    /** Provider type (e.g. 'github'). */
+    provider: string;
+    /** Repo owner / namespace. */
+    owner: string;
+    /** Repo name. */
+    repo: string;
+    /** Git commit SHA the check run is anchored to. */
+    sha: string;
+    /** Check-run name (e.g. 'kici/build', 'kici/build/job/test', 'kici/build/setup'). */
+    check_name: string;
+    /**
+     * GitHub Checks API check-run ID. Populated by `checks.create()`; nullable
+     * during the in-flight build-creation window (`build_creation_state =
+     * 'pending'` before the create finishes).
+     */
+    check_run_id: ColumnType<number | null, number | null | undefined, number | null>;
+    /**
+     * Build check-run creation state: 'pending' while a `setBuildPending`
+     * create is in flight, 'completed' once `setBuildComplete` has reconciled.
+     * Replaces the in-memory `pendingBuildCreations` Promise map.
+     */
+    build_creation_state: ColumnType<string | null, string | null | undefined, string | null>;
+    /**
+     * Step-progress entries as JSONB array. Each entry shape is
+     * `{ name: string, status: string, durationMs?: number }`. Replaces the
+     * in-memory `stepProgress` map.
+     */
+    step_progress_json: ColumnType<unknown, string | unknown, string | unknown>;
+    /**
+     * Timestamp the first in-progress transition was sent to GitHub. NULL
+     * before the first running step. Replaces the in-memory `inProgressSent`
+     * boolean map (presence-as-truth).
+     */
+    in_progress_sent_at: ColumnType<Date | null, Date | null | undefined, Date | null>;
+    /**
+     * KiCI run identifier this check-run belongs to. Indexed (partial, NOT
+     * NULL) to power `cleanupRun(runId)` without scanning the table.
+     */
+    run_id: ColumnType<string | null, string | null | undefined, string | null>;
+    /** When this row was first inserted. */
+    created_at: Generated<Date>;
+    /** When this row was last updated. */
+    updated_at: Generated<Date>;
+}
+export type CheckRunTrackingRow = Selectable<CheckRunTrackingTable>;
+export type NewCheckRunTrackingRow = Insertable<CheckRunTrackingTable>;
+export type CheckRunTrackingUpdate = Updateable<CheckRunTrackingTable>;
+/**
+ * Scaler spawning-agents table (scaler_spawning_agents).
+ *
+ * One row per agent that has been spawned via a scaler backend
+ * (container / bare-metal / firecracker) but has not yet registered via
+ * WS. Persists `bound_job_id` so a replacement coord still issues the
+ * eager-dispatch hop when the agent eventually registers. GC'd by a
+ * leader-gated sweep that drops rows older than the spawn-timeout.
+ */
+export interface ScalerSpawningAgentsTable {
+    agent_id: string;
+    scaler_name: string;
+    label_set: ColumnType<string[], string | string[], string | string[]>;
+    run_id: ColumnType<string | null, string | null | undefined, string | null>;
+    job_id: ColumnType<string | null, string | null | undefined, string | null>;
+    bound_job_id: ColumnType<string | null, string | null | undefined, string | null>;
+    spawned_at: Generated<Date>;
+}
+export type ScalerSpawningAgentRow = Selectable<ScalerSpawningAgentsTable>;
+export type NewScalerSpawningAgentRow = Insertable<ScalerSpawningAgentsTable>;
+/**
+ * Scaler agent-jobs table (scaler_agent_jobs).
+ *
+ * agentId → (runId, jobId) correlation used to route scaler-lifecycle
+ * events (spawn / boot / ready / kill) to the execution tracker. Row
+ * inserted in `correlateAgentToJob`, deleted on agent disconnect / job
+ * completion.
+ */
+export interface ScalerAgentJobsTable {
+    agent_id: string;
+    run_id: string;
+    job_id: string;
+    correlated_at: Generated<Date>;
+}
+export type ScalerAgentJobRow = Selectable<ScalerAgentJobsTable>;
+export type NewScalerAgentJobRow = Insertable<ScalerAgentJobsTable>;
+/**
+ * Scaler reservations table (scaler_reservations).
+ *
+ * One row per outstanding resource reservation. `perScalerUsage` /
+ * `globalUsage` are derived state — recomputed from `SUM(...)` on coord
+ * boot so the cap-check critical section is correct under HA.
+ */
+export interface ScalerReservationsTable {
+    agent_id: string;
+    scaler_name: string;
+    cpu_units: number;
+    mem_bytes: ColumnType<string, string | number, string | number>;
+    reserved_at: Generated<Date>;
+}
+export type ScalerReservationRow = Selectable<ScalerReservationsTable>;
+export type NewScalerReservationRow = Insertable<ScalerReservationsTable>;
 //# sourceMappingURL=types.d.ts.map

package/dist/events/event-store.d.ts

@@ -120,13 +120,20 @@ export declare class EventStore {
     /**
      * List events currently in the DLQ. Most recent first. Used by the
      * dashboard admin DLQ page.
+     *
+     * @param sourceRoutingKey - When provided, restrict the result to
+     *   events whose `source_routing_key` matches. Used by routing-key
+     *   -scoped admin tokens so the operator only sees their slice.
      */
-    listDlq(limit: number, beforeDlqAt?: Date): Promise<StoredEvent[]>;
+    listDlq(limit: number, beforeDlqAt?: Date, sourceRoutingKey?: string): Promise<StoredEvent[]>;
     /**
      * Count events currently in the DLQ. Used by the dashboard summary row
      * and the `kici_orch_event_dlq_depth` gauge.
+     *
+     * @param sourceRoutingKey - When provided, restrict the count to
+     *   events whose `source_routing_key` matches.
      */
-    countDlq(): Promise<number>;
+    countDlq(sourceRoutingKey?: string): Promise<number>;
     /**
      * Reset a DLQ event so it gets retried on the next scanner tick. Used by
      * the dashboard "Retry" action.

package/dist/events/trust-store.d.ts

@@ -41,6 +41,13 @@ export declare class TrustStore {
         repo: string;
         routingKey: string;
     }, allowedEvents?: string[]): Promise<string>;
+    /**
+     * Look up a single trust entry by ID. Returns null if the row is
+     * missing. Callers (the admin HTTP route, in particular) use this
+     * to read a row's source/target routing keys before applying a
+     * scope check.
+     */
+    getById(id: string): Promise<TrustEntry | null>;
     /**
      * Remove a trust relationship by ID.
      */

package/dist/events/types.d.ts

@@ -23,6 +23,20 @@ export interface EventRouterConfig {
     retryMaxBackoffMs: number;
     /** Interval at which the leader-only retry scanner ticks (default: 10_000) */
     retryScanIntervalMs: number;
+    /**
+     * **Test-only.** Per-event-name fault injection: when `attempts <= N`,
+     * the EventRouter throws a synthetic dispatch error to drive the retry /
+     * DLQ path. Used by the fault-injection E2E to prove the lease + retry
+     * loop dispatches a real run when the inner dispatch eventually
+     * succeeds, and lands the row in the DLQ when N exceeds
+     * `maxDispatchAttempts`.
+     *
+     * Only honoured when `KICI_TEST_MODE=1` is set at config-load time —
+     * production deployments never see this knob even if the env var is
+     * planted by accident. Source the value from
+     * `KICI_TEST_EVENT_FAIL_FIRST_N` (a JSON object literal).
+     */
+    debugFailFirstNAttemptsByEvent?: Record<string, number>;
 }
 /**
  * Reason an event landed in the DLQ.
@@ -54,6 +68,25 @@ export interface StoredEvent {
     dlqAt: Date | null;
     dlqReason: DlqReason | null;
 }
+/**
+ * Parse the `KICI_TEST_EVENT_FAIL_FIRST_N` JSON payload, gated by
+ * `KICI_TEST_MODE`. Returns the parsed map or `undefined` when:
+ *
+ * - `testMode` is false (the master switch). The JSON value is ignored
+ *   entirely; production deployments never see fault-injection even if
+ *   the per-event env var is accidentally planted.
+ * - the JSON value is absent / empty.
+ * - the JSON value is malformed or carries a wrong-shape entry. We
+ *   refuse to fall back to a partial map: a typo on one line silently
+ *   skipping that test would be confusing in CI logs.
+ *
+ * The accepted shape is `{ "<eventName>": <number> }`; non-string keys
+ * and non-number values are rejected. The numeric value is the inclusive
+ * upper bound of attempts to fail (so `1` fails the first attempt and
+ * succeeds on retry; `99` exceeds `maxDispatchAttempts` and lands the
+ * row in the DLQ).
+ */
+export declare function parseFaultInjectionMap(testMode: boolean, raw: string | undefined): Record<string, number> | undefined;
 /**
  * Default event router configuration.
  */