@kici-dev/orchestrator 0.1.14 → 0.1.16

package/dist/cluster/peer-client.d.ts

@@ -10,7 +10,7 @@
  * Authentication uses ECDH key exchange followed by join token (first connect)
  * or HMAC credential proof (reconnection).
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 type PeerConnectionState = 'disconnected' | 'connecting' | 'handshaking' | 'authenticating' | 'connected';
 export interface PeerClientOptions {
@@ -78,7 +78,20 @@ export interface PeerClientOptions {
      * Platform-mediated discovery dedupes against the same client.
      */
     onAuthenticated?: (targetInstanceId: string) => void;
+    /**
+     * Callback when a peer.logs.collect.request is received from the peer. Builds
+     * this node's subtree bundle and streams it back via the supplied `send`
+     * (peer.logs.collect.chunk frames, or a peer.logs.collect.error on failure).
+     * If undefined, incoming collect requests are ignored.
+     */
+    onLogsCollectRequest?: PeerLogsCollectResponder;
 }
+/**
+ * Builds a node's subtree bundle in response to a peer.logs.collect.request and
+ * streams it back through `send`. Shared by PeerClient (outgoing-dialed peers)
+ * and the peer handler (incoming-dialed peers).
+ */
+export type PeerLogsCollectResponder = (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
 export declare class PeerClient {
     private ws;
     private _state;
@@ -91,6 +104,8 @@ export declare class PeerClient {
     private readonly ackWaiters;
     private readonly cacheWaiters;
     private readonly configReloadWaiters;
+    /** Correlates peer.logs.collect.request with the peer's chunked subtree response. */
+    private readonly logsCollectWaiters;
     private readonly url;
     private readonly joinToken?;
     private readonly credentialFile;
@@ -112,6 +127,7 @@ export declare class PeerClient {
     private readonly onAgentTokenRevoke?;
     private readonly onPeerConfigReload?;
     private readonly onAuthenticated?;
+    private readonly onLogsCollectRequest?;
     constructor(options: PeerClientOptions);
     /** Current connection state. */
     get state(): PeerConnectionState;
@@ -150,6 +166,12 @@ export declare class PeerClient {
      *   if the peer doesn't reply within the timeout.
      */
     sendConfigReloadAndWait(msg: PeerConfigReload, timeoutMs?: number): Promise<PeerConfigReloadResponse | null>;
+    /**
+     * Send a peer.logs.collect.request to the connected peer and await its
+     * reassembled subtree-bundle ZIP. Rejects on timeout, an error frame, or
+     * peer disconnect.
+     */
+    sendLogsCollectAndWait(msg: PeerLogsCollectRequest, timeoutMs: number): Promise<Buffer>;
     /**
      * Send a cache upload request to the coordinator and wait for a response
      * with a pre-signed URL.

package/dist/cluster/peer-handler.d.ts

@@ -6,7 +6,7 @@
  * proof (reconnection), registers the peer in PeerRegistry, and routes
  * messages bidirectionally. Sends periodic heartbeats to the connecting peer.
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 import type { PeerCredentialStore } from './peer-credentials.js';
 import { type JoinTokenManager } from './join-token.js';
@@ -78,6 +78,13 @@ export interface PeerHandlerDeps {
         restartRequired?: string[];
         fieldsChanged?: string[];
     }>;
+    /**
+     * Callback when a peer.logs.collect.request arrives from an incoming-dialed
+     * peer. Builds this node's subtree bundle and streams it back through `send`
+     * (peer.logs.collect.chunk frames, or a peer.logs.collect.error on failure).
+     * If undefined, incoming collect requests are ignored.
+     */
+    onLogsCollectRequest?: (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
 }
 /**
  * Create a handler function for incoming peer WebSocket connections.
@@ -90,6 +97,7 @@ export declare function createPeerHandler(deps: PeerHandlerDeps): {
     sendToPeer: (targetInstanceId: string, msg: PeerToPeerMessage) => boolean;
     sendAndWaitAck: (targetInstanceId: string, msg: JobReroute, timeoutMs?: number) => Promise<boolean>;
     sendConfigReloadAndWait: (targetInstanceId: string, msg: PeerConfigReload, timeoutMs?: number) => Promise<PeerConfigReloadResponse | null>;
+    sendLogsCollectAndWait: (targetInstanceId: string, msg: PeerLogsCollectRequest, timeoutMs: number) => Promise<Buffer>;
     getConnectionCount: () => number;
     broadcastHeartbeat: (inventory: Omit<PeerHeartbeat, "type">) => void;
     broadcastAgentTokenRevoke: (msg: PeerAgentTokenRevoke) => void;

package/dist/cluster/peer-registry.d.ts

@@ -21,6 +21,12 @@ export interface PeerAgentInfo {
      * in the required label set.
      */
     mandatoryLabels: string[];
+    /**
+     * Name of the scaler backend that spawned this agent, or null for static
+     * (stateful) agents not bound to any scaler. Surfaced in diagnostics so the
+     * dashboard can group a peer's agents under the correct scaler row.
+     */
+    scalerName?: string | null;
 }
 export interface PeerInfo {
     instanceId: string;

package/dist/config/schema.d.ts

@@ -107,6 +107,8 @@ export declare const sharedConfigSchema: z.ZodObject<{
     cacheTtlDays: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
+    userCacheQuotaBytes: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
+    userCacheTtlMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
     eventRouter: z.ZodOptional<z.ZodObject<{
         maxChainDepth: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
         rateLimitPerWorkflowPerMinute: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
@@ -189,6 +191,8 @@ export declare const appConfigSchema: z.ZodObject<{
     cacheTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterMaxChainDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterRateLimitPerWorkflowPerMinute: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterEventTtlSeconds: z.ZodDefault<z.ZodCoercedNumber<unknown>>;

package/dist/config.d.ts

@@ -36,6 +36,7 @@ declare const configSchema: z.ZodObject<{
     queueBackpressureThreshold: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     workerConcurrency: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     concurrencyWaitTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    dispatchAckTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheStorageType: z.ZodOptional<z.ZodEnum<{
         s3: "s3";
         filesystem: "filesystem";
@@ -46,6 +47,7 @@ declare const configSchema: z.ZodObject<{
     cacheStorageS3Region: z.ZodOptional<z.ZodString>;
     cacheStorageS3Endpoint: z.ZodOptional<z.ZodString>;
     cacheStorageS3ExternalEndpoint: z.ZodOptional<z.ZodString>;
+    cacheStorageS3UploadEndpoint: z.ZodOptional<z.ZodString>;
     cacheStorageS3ForcePathStyle: z.ZodOptional<z.ZodPipe<z.ZodEnum<{
         true: "true";
         false: "false";
@@ -56,6 +58,8 @@ declare const configSchema: z.ZodObject<{
     cacheTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     webhookPayloadDir: z.ZodOptional<z.ZodString>;
     dataDir: z.ZodOptional<z.ZodString>;
     scalerConfigPath: z.ZodOptional<z.ZodString>;
@@ -149,12 +153,17 @@ export type AppConfig = z.infer<typeof configSchema> & {
         region?: string;
         endpoint?: string;
         externalEndpoint?: string;
+        uploadEndpoint?: string;
         forcePathStyle?: boolean;
         logBucket?: string;
         /** Filesystem backend: absolute base directory for cached blobs. */
         fsBasePath?: string;
         /** Filesystem backend: base URL the agent uses to reach this orchestrator. */
         fsBaseUrl?: string;
+        /** Per-org byte quota for the user-facing cache (UserCache). */
+        userCacheQuotaBytes?: number;
+        /** Per-entry TTL (ms) for the user-facing cache (UserCache). */
+        userCacheTtlMs?: number;
     };
 };
 /**
@@ -177,10 +186,13 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     queueBackpressureThreshold: number;
     workerConcurrency: number;
     concurrencyWaitTimeoutMs: number;
+    dispatchAckTimeoutMs: number;
     cacheStorageS3Prefix: string;
     cacheTtlDays: number;
     cacheBuildTimeoutMs: number;
     cacheMaxTarballBytes: number;
+    userCacheQuotaBytes: number;
+    userCacheTtlMs: number;
     staleDetectorScanIntervalMs: number;
     staleDetectorThresholdMultiplier: number;
     jobHeartbeatIntervalMs: number;
@@ -234,6 +246,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     cacheStorageS3Region?: string | undefined;
     cacheStorageS3Endpoint?: string | undefined;
     cacheStorageS3ExternalEndpoint?: string | undefined;
+    cacheStorageS3UploadEndpoint?: string | undefined;
     cacheStorageS3ForcePathStyle?: boolean | undefined;
     cacheStorageFsPath?: string | undefined;
     cacheStorageFsBaseUrl?: string | undefined;

package/dist/dashboard/handler.d.ts

@@ -13,7 +13,7 @@
  */
 import { type Kysely } from 'kysely';
 import { type ColdStore } from '@kici-dev/shared';
-import type { DashboardRunDetailRequest, DashboardStepLogsRequest, DashboardPayloadRequest, DashboardOrchLogsRequest, DashboardEventLogListRequest, DashboardEventLogDetailRequest, DashboardEventLogPayloadStreamRequest, DashboardAccessLogListRequest, DashboardEventDlqListRequest, DashboardEventDlqCountRequest, DashboardEventDlqRetryRequest, DashboardEventDlqDiscardRequest, RunRerunRequest, RunCancelRequest, ManualScheduleRequest } from '@kici-dev/engine';
+import type { DashboardRunDetailRequest, DashboardRunsListRequest, DashboardRunsListResponse, DashboardRunsFiltersRequest, DashboardRunsFiltersResponse, DashboardSourcesListRequest, DashboardSourcesListResponse, DashboardStepLogsRequest, DashboardPayloadRequest, DashboardOrchLogsRequest, DashboardEventLogListRequest, DashboardEventLogDetailRequest, DashboardEventLogPayloadStreamRequest, DashboardAccessLogListRequest, DashboardEventDlqListRequest, DashboardEventDlqCountRequest, DashboardEventDlqRetryRequest, DashboardEventDlqDiscardRequest, RunRerunRequest, RunCancelRequest, ManualScheduleRequest } from '@kici-dev/engine';
 import type { Database } from '../db/types.js';
 import type { LogStorage } from '../reporting/log-storage.js';
 import type { AccessLogWriter } from '../audit/access-log.js';
@@ -155,6 +155,97 @@ export declare class DashboardHandler {
      * builds a nested job/step tree, and sends the response.
      */
     handleRunDetail(msg: DashboardRunDetailRequest): Promise<void>;
+    /**
+     * Resolve every routing key owned by an org by unioning both source
+     * tables. The orchestrator is single-org but multi-routing-key: one org
+     * owns one GitHub-app source plus N generic sources, each with its own
+     * `routing_key`. `execution_runs` has no `org_id` column — org ownership
+     * is expressed purely through `routing_key → sources.customer_id` /
+     * `generic_webhook_sources.customer_id`, so scoping a run query to an org
+     * means scoping it to the union of that org's routing keys.
+     *
+     * Predicates mirror the org-context resolver in `server.ts` and
+     * `resolveOrgForRoutingKey` above:
+     * - `sources`: `customer_id = orgId` excluding the `__default__` sentinel.
+     * - `generic_webhook_sources`: `customer_id = orgId` and not soft-deleted
+     *   (`deleted_at is null`).
+     */
+    private resolveOrgRoutingKeys;
+    /**
+     * Resolve a per-routing-key source identity map for a set of routing keys,
+     * unioning both source tables. Uses the identical name/subtype/provider
+     * derivation as `handleSourcesList` so a run summary's `source` block
+     * matches what the sources page shows for the same routing key. Routing
+     * keys with no live source row are simply absent from the map (the caller
+     * falls back to the run's own provider).
+     */
+    private resolveSourceIdentities;
+    /**
+     * Look up the page's jobs in one query to derive per-run `jobCount` plus the
+     * compile-job markers (`hadCompileJob` / `compileJobId`). The compile job is
+     * the synthetic `__build__*` job KiCI inserts for the compile phase — the
+     * same definition the Platform run-list route uses
+     * (`job_name LIKE '__build__%'`). Returns a per-run aggregate keyed by
+     * `run_id`; runs with no job rows are absent (the caller omits the fields).
+     */
+    private resolveRunJobAggregates;
+    /**
+     * Handle a dashboard.runs.list request.
+     *
+     * Returns a page of run summaries from `execution_runs`, scoped to ALL
+     * routing keys owned by this orchestrator's bound org (`this.orgId`). The
+     * orchestrator is single-org but multi-routing-key, and the bound
+     * `this.routingKey` is just ONE of the org's keys (resolved
+     * non-deterministically at startup), so filtering on it alone would drop
+     * runs that arrived under a sibling routing key. We resolve the org's full
+     * routing-key set from `sources` + `generic_webhook_sources` and filter on
+     * the union. Attribution still uses the handler-bound `orgId` /
+     * `routingKey` directly — the same inline-ctx pattern as the org-scoped
+     * event-log / access-log handlers.
+     *
+     * Used by the operator console (`support-read` break-glass) and the
+     * dashboard's run-list view. The access_log row carries the wire actor
+     * (including the platform_operator reason) so the customer can audit any
+     * operator read.
+     */
+    handleRunsList(msg: DashboardRunsListRequest): Promise<DashboardRunsListResponse>;
+    /**
+     * Handle a dashboard.runs.filters request.
+     *
+     * Returns the distinct filter-option values the customer runs page renders
+     * in its filter controls — statuses / workflows / branches / repositories /
+     * triggerTypes / sources — scoped to ALL routing keys owned by this
+     * orchestrator's bound org (same scoping as `handleRunsList`). Distinct
+     * values are derived in-memory from a single column projection over
+     * `execution_runs`; `sources` reuses `resolveSourceIdentities` so the list
+     * matches what the sources page shows for the same routing key.
+     *
+     * `triggerTypes` is populated from `execution_runs.provider` — the
+     * orchestrator has no dedicated trigger-event column, so `provider`
+     * (`github` / `generic` / `internal`) is the closest available
+     * discriminator the customer filter can offer.
+     *
+     * Attribution uses the handler-bound `orgId` / `routingKey` directly — the
+     * same inline-ctx pattern as `handleRunsList`. The access_log row carries
+     * the wire actor (including the platform_operator reason) so the customer
+     * can audit any operator read.
+     */
+    handleRunsFilters(msg: DashboardRunsFiltersRequest): Promise<DashboardRunsFiltersResponse>;
+    /**
+     * Handle a dashboard.sources.list request.
+     *
+     * Returns the org's webhook source summaries by unioning both source
+     * tables (GitHub-app `sources` + live `generic_webhook_sources`), scoped
+     * to this orchestrator's bound org (`this.orgId`). Source counts are tiny
+     * so the page is returned unpaginated (`msg.limit` is ignored). Secret-
+     * bearing columns (`config`, `git_config`, `verification_config`) are
+     * never projected — `git_config` is read only to derive the subtype.
+     *
+     * Attribution uses the handler-bound `orgId` / `routingKey` directly —
+     * the same inline-ctx pattern as `handleRunsList`. The access_log row
+     * carries the wire actor so the customer can audit any operator read.
+     */
+    handleSourcesList(msg: DashboardSourcesListRequest): Promise<DashboardSourcesListResponse>;
     /**
      * Handle a dashboard.step.logs request.
      * Looks up the step's log_path and reads content from LogStorage.

package/dist/db/migrations/026_event_log_lockfile_corrupt.d.ts

@@ -0,0 +1,11 @@
+import { type Kysely } from 'kysely';
+/**
+ * Extend the event_log.status CHECK constraint with 'lockfile_corrupt' so the
+ * orchestrator can log a delivery whose lock file was present but unparseable.
+ * Orchestrator-only: the Platform event_log uses a separate, narrower status set.
+ *
+ * Idempotent: the DROP ... IF EXISTS / re-ADD pair re-runs cleanly.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=026_event_log_lockfile_corrupt.d.ts.map

package/dist/db/migrations/027_workflow_timeout.d.ts

@@ -0,0 +1,20 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `workflow_timeout_ms integer` to `execution_runs`.
+ *
+ * Persists the workflow-level wall-clock timeout (in milliseconds) read from
+ * the lock workflow at run creation. NULL means no workflow-level cap is
+ * configured. The WorkflowDeadlineDetector reads this column to find runs
+ * whose `started_at + workflow_timeout_ms` has passed and cancels them with
+ * the distinct workflow-timeout reason.
+ *
+ * Typed INTEGER to match the existing `*_ms` columns (`duration_ms`,
+ * `concurrency_timeout_ms`), which keeps the pg read representation a plain
+ * number rather than the BIGINT-as-string shape. INTEGER caps the timeout at
+ * ~24.8 days, far beyond any sane workflow wall-clock budget.
+ *
+ * Idempotent: re-running on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=027_workflow_timeout.d.ts.map

package/dist/db/migrations/028_org_settings_user_cache.d.ts

@@ -0,0 +1,4 @@
+import { type Kysely } from 'kysely';
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=028_org_settings_user_cache.d.ts.map

package/dist/db/migrations/029_dispatch_queue_attempts.d.ts

@@ -0,0 +1,16 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `dispatch_queue.dispatch_attempts INT NOT NULL DEFAULT 0`.
+ *
+ * Counts how many times a job has been returned to `pending` for
+ * re-dispatch after a failed delivery attempt (agent sent job.reject, or
+ * a scaler-managed agent disconnected before the job started). The
+ * dispatcher fails the job permanently once the counter reaches
+ * MAX_DISPATCH_ATTEMPTS, bounding requeue loops; `expires_at` remains the
+ * time-based backstop.
+ *
+ * Idempotent: re-running on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=029_dispatch_queue_attempts.d.ts.map

package/dist/db/migrations/030_held_runs_env_set_null.d.ts

@@ -0,0 +1,13 @@
+import { type Kysely } from 'kysely';
+/**
+ * held_runs.environment_id becomes nullable with ON DELETE SET NULL so
+ * terminal held-run history survives environment deletion (a null
+ * environment_id means the environment was since deleted). Pending held
+ * runs still block deletion — enforced in EnvironmentStore.delete.
+ *
+ * Idempotent: dropping the NOT NULL and the constraint are both no-ops on a
+ * re-run, and the constraint is re-created with the SET NULL action.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=030_held_runs_env_set_null.d.ts.map

package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts

@@ -0,0 +1,19 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `dispatch_queue.ack_deadline TIMESTAMPTZ` and
+ * `dispatch_queue.ack_agent_id TEXT` persisting the per-dispatch
+ * acknowledgment deadline for HA-safe lost-dispatch detection.
+ *
+ * The dispatcher stamps both when a job.dispatch is sent and clears them
+ * when the agent answers (job.ack / job.reject / job.status running). A
+ * `dispatched` row past its deadline is requeued — by the owning coord's
+ * in-memory timer, or by the leader-gated sweep
+ * (`Dispatcher.sweepExpiredAckDeadlines`) when the owning coord crashed.
+ * On coord boot, `Dispatcher.recoverState()` re-arms timers from the
+ * persisted deadlines.
+ *
+ * Idempotent: re-running on a DB that already has either column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=031_dispatch_queue_ack_deadline.d.ts.map

package/dist/db/migrations/032_org_settings_dispatch_ack_timeout.d.ts

@@ -0,0 +1,14 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `org_settings.dispatch_ack_timeout_ms BIGINT` (nullable).
+ *
+ * Per-org override of the dispatch-acknowledgment deadline. NULL falls
+ * back to the cluster-wide default (`KICI_DISPATCH_ACK_TIMEOUT_MS`,
+ * default 10s). Operators raise it on high-latency networks via
+ * `kici-admin org-settings`.
+ *
+ * Idempotent: a re-run on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=032_org_settings_dispatch_ack_timeout.d.ts.map

package/dist/db/migrations/033_org_settings_approval.d.ts

@@ -0,0 +1,19 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add the two approval-policy columns to `org_settings`:
+ *
+ * - `approval_expiry_seconds INTEGER NOT NULL DEFAULT 86400` — how long a held
+ *   approval element waits before it expires (and its run/job/step is
+ *   rejected). One day by default. An SDK `requireApproval` `timeout` overrides
+ *   this per element; otherwise this per-org value applies.
+ * - `allow_self_approval BOOLEAN NOT NULL DEFAULT true` — whether the user who
+ *   triggered a run may also approve its held elements. Operators turn it off
+ *   to enforce four-eyes review.
+ *
+ * Both are cluster-configurable per org via `kici-admin org-settings approval`
+ * and the orchestrator admin route. Idempotent: a re-run on a DB that already
+ * has the columns is a no-op (each column is guarded independently).
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=033_org_settings_approval.d.ts.map

package/dist/db/migrations/034_held_runs_generalize.d.ts

@@ -0,0 +1,24 @@
+import { type Kysely } from 'kysely';
+/**
+ * Generalize `held_runs` from an environment-only hold into the unified
+ * "held element" model that backs per-element approvals, and add the
+ * `held_run_approvals` table that records each approver's decision.
+ *
+ * New `held_runs` columns (all idempotent, column-exists guarded):
+ * - `hold_scope text NOT NULL DEFAULT 'job'` — 'workflow' | 'job' | 'step'
+ *   (engine `HoldScope`). Existing rows held a single job, so they default to
+ *   'job'.
+ * - `step_index integer` — nullable; set only for step-scoped holds.
+ * - `trigger_source text NOT NULL DEFAULT 'environment'` — 'environment' |
+ *   'explicit' (engine `TriggerSource`). Existing holds came from environment
+ *   protection, so they default to 'environment'.
+ * - `approval_requirement jsonb` — the normalized `ApprovalRequirement`
+ *   (clauses + expiresAt + reason) the hold must satisfy. Nullable for legacy
+ *   rows that predate the approval model.
+ *
+ * New `held_run_approvals` table: one row per approver decision, FK to
+ * `held_runs.id` (uuid) with ON DELETE CASCADE.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=034_held_runs_generalize.d.ts.map

package/dist/db/types.d.ts

@@ -1,5 +1,5 @@
 import type { ColumnType, Generated, Insertable, Selectable, Updateable } from 'kysely';
-import type { InitFailure } from '@kici-dev/engine';
+import type { ApprovalRequirement, ApproverClause, InitFailure } from '@kici-dev/engine';
 /**
  * PostgreSQL-only database types.
  * Column names use snake_case matching the actual database column names.
@@ -19,6 +19,7 @@ export interface Database {
     environment_variables: EnvironmentVariablesTable;
     environment_source_overrides: EnvironmentSourceOverridesTable;
     held_runs: HeldRunsTable;
+    held_run_approvals: HeldRunApprovalsTable;
     admin_tokens: AdminTokenTable;
     agent_tokens: AgentTokenTable;
     config_versions: ConfigVersionTable;
@@ -158,6 +159,17 @@ export interface DispatchQueueTable {
      * real provisioning cause; cleared on dispatch. NULL when none recorded.
      */
     last_provisioning_error: ColumnType<string | null, string | null | undefined, string | null>;
+    /** Times this job was returned to pending for re-dispatch (job.reject / pre-start agent loss). */
+    dispatch_attempts: Generated<number>;
+    /**
+     * Deadline by which the dispatched job's agent must answer the
+     * job.dispatch (job.ack / job.reject / job.status running). Stamped when
+     * the dispatch is sent, cleared on any answer; `dispatched` rows past the
+     * deadline are requeued by the owning coord's timer or the leader sweep.
+     */
+    ack_deadline: ColumnType<Date | null, Date | null | undefined, Date | null>;
+    /** Agent the dispatch was sent to (for ack-timeout disconnect + logging). */
+    ack_agent_id: ColumnType<string | null, string | null | undefined, string | null>;
 }
 /**
  * Deduplication cache table
@@ -266,6 +278,12 @@ export interface ExecutionRunTable {
      * provider context, agent spawn). NULL for normal runs.
      */
     init_failure: ColumnType<InitFailure | null, unknown, unknown>;
+    /**
+     * Whole-run wall-clock timeout in ms from the workflow lock; null when
+     * unset. Read by the WorkflowDeadlineDetector. INTEGER in Postgres, matching
+     * the other `*_ms` columns, so pg returns a plain number.
+     */
+    workflow_timeout_ms: ColumnType<number | null, number | null | undefined, number | null>;
     /** When this record was created */
     created_at: Generated<Date>;
     /**
@@ -595,8 +613,8 @@ export interface HeldRunsTable {
     run_id: string;
     /** Job ID within the run */
     job_id: string;
-    /** Environment ID (FK to environments.id) */
-    environment_id: string;
+    /** Environment ID (FK to environments.id); null once the environment is deleted */
+    environment_id: string | null;
     /** Hold type: 'approval' | 'wait_timer' | 'concurrency' */
     hold_type: string;
     /** Hold status: 'pending' | 'approved' | 'rejected' | 'expired' | 'released' */
@@ -613,10 +631,49 @@ export interface HeldRunsTable {
     expires_at: Date;
     /** When this hold was resolved */
     resolved_at: Date | null;
+    /**
+     * Hold granularity: 'workflow' | 'job' | 'step' (engine `HoldScope`).
+     * Existing environment holds are job-scoped, hence the 'job' default.
+     */
+    hold_scope: Generated<string>;
+    /** Step index within the job for step-scoped holds; null otherwise. */
+    step_index: number | null;
+    /**
+     * What created the hold: 'environment' (mandatory env policy) | 'explicit'
+     * (SDK `requireApproval`). Engine `TriggerSource`.
+     */
+    trigger_source: Generated<string>;
+    /**
+     * Normalized `ApprovalRequirement` (clauses + expiresAt + reason) the hold
+     * must satisfy. Null for legacy rows that predate the approval model.
+     */
+    approval_requirement: ColumnType<ApprovalRequirement | null, ApprovalRequirement | string | null | undefined, ApprovalRequirement | string | null>;
 }
 export type HeldRun = Selectable<HeldRunsTable>;
 export type NewHeldRun = Insertable<HeldRunsTable>;
 export type HeldRunUpdate = Updateable<HeldRunsTable>;
+/**
+ * One approver's recorded decision on a held element. Multiple rows accumulate
+ * until the hold's `ApprovalRequirement` clauses are all satisfied (approve) or
+ * any single reject lands.
+ */
+export interface HeldRunApprovalsTable {
+    /** UUID primary key */
+    id: Generated<string>;
+    /** FK to held_runs.id (ON DELETE CASCADE) */
+    held_run_id: string;
+    /** The approver's user id (Keycloak sub) */
+    approver_user_id: string;
+    /** 'approve' | 'reject' (engine `ApprovalDecision`) */
+    decision: string;
+    /** Which requirement clauses this decision satisfied (for attribution). */
+    clauses_satisfied: ColumnType<ApproverClause[] | null, ApproverClause[] | string | null | undefined, ApproverClause[] | string | null>;
+    /** When the decision was recorded */
+    created_at: Generated<Date>;
+}
+export type HeldRunApproval = Selectable<HeldRunApprovalsTable>;
+export type NewHeldRunApproval = Insertable<HeldRunApprovalsTable>;
+export type HeldRunApprovalUpdate = Updateable<HeldRunApprovalsTable>;
 /**
  * Secret audit log table
  * Immutable log of secret access and denial events.
@@ -1157,6 +1214,35 @@ export interface OrgSettingsTable {
      * treats missing keys as `true`.
      */
     dashboard_write_policy: ColumnType<Record<string, boolean>, Record<string, boolean> | string | undefined, Record<string, boolean> | string>;
+    /**
+     * Per-org byte quota for the user-facing cache (UserCache). NULL = use the
+     * cluster-wide default (`KICI_USER_CACHE_QUOTA_BYTES`, 5 GiB). Postgres
+     * BIGINT — pg returns a string on select; accept a number on insert/update.
+     */
+    user_cache_quota_bytes: ColumnType<string | null, number | null | undefined, number | null>;
+    /**
+     * Per-entry TTL (ms) for the user-facing cache (UserCache). NULL = use the
+     * cluster-wide default (`KICI_USER_CACHE_TTL_MS`, 7 days). Postgres BIGINT —
+     * pg returns a string on select; accept a number on insert/update.
+     */
+    user_cache_ttl_ms: ColumnType<string | null, number | null | undefined, number | null>;
+    /**
+     * Per-org dispatch-acknowledgment deadline (ms); null = cluster default
+     * (config.dispatchAckTimeoutMs / KICI_DISPATCH_ACK_TIMEOUT_MS). Postgres
+     * BIGINT — pg returns a string on select; accept a number on insert/update.
+     */
+    dispatch_ack_timeout_ms: ColumnType<string | null, number | null | undefined, number | null>;
+    /**
+     * Per-org expiry (seconds) for a held approval element before it is rejected
+     * and its run/job/step fails. NOT NULL, default 86400 (one day). An SDK
+     * `requireApproval` `timeout` overrides this per element.
+     */
+    approval_expiry_seconds: ColumnType<number, number | undefined, number>;
+    /**
+     * Whether the user who triggered a run may also approve its held elements.
+     * NOT NULL, default true. Operators turn it off to enforce four-eyes review.
+     */
+    allow_self_approval: ColumnType<boolean, boolean | undefined, boolean>;
     /** When this setting was created */
     created_at: Generated<Date>;
     /** When this setting was last updated */

package/dist/diagnostics/bundle-writer.d.ts

@@ -6,8 +6,8 @@
  * This is the primary support tool -- operators run debug-bundle and share
  * the ZIP for troubleshooting.
  */
-import archiver from 'archiver';
 import type { DiagnosticDeps } from './types.js';
+export { redactConfig, addLogsToArchive } from '@kici-dev/shared';
 export interface BundleOptions {
     /** Where to write the ZIP. */
     outputPath: string;
@@ -26,11 +26,6 @@ export interface BundleOptions {
     /** URL for recent runs endpoint. */
     recentRunsUrl?: string;
 }
-/**
- * Redact config values using allowlist approach.
- * Only known-safe fields are preserved; everything else becomes "****".
- */
-export declare function redactConfig(obj: unknown, parentKey?: string): unknown;
 /**
  * Create a debug bundle ZIP file.
  *
@@ -43,15 +38,4 @@ export declare function redactConfig(obj: unknown, parentKey?: string): unknown;
  * - logs/summary.json: log statistics
  */
 export declare function createDebugBundle(options: BundleOptions): Promise<string>;
-/**
- * Add log files from logDir to the archive, respecting MAX_LOG_BYTES cap
- * and the logWindow time filter. Matches any `*.log` file in the directory,
- * so the per-instance filename pattern produced by
- * `buildLogFilename()` is picked up without additional configuration.
- *
- * Exported for reuse by the `kici-admin debug-bundle` CLI command, which
- * runs outside the orchestrator process but still needs to include the
- * same log files in its locally-assembled bundle.
- */
-export declare function addLogsToArchive(archive: archiver.Archiver, logDir: string, logWindowHours: number): Promise<void>;
 //# sourceMappingURL=bundle-writer.d.ts.map

package/dist/diagnostics/checks/firecracker-network.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Firecracker host-bridge diagnostic. Emits one row per configured Firecracker
+ * scaler backend: the bridge interface + its gateway addr + its nft table must
+ * all be present, else the scaler cannot spawn microVMs.
+ */
+import type { DiagnosticDeps, DiagnosticResult } from '../types.js';
+import { type BridgeHealth, type FirecrackerBridgeConfig } from '../../firecracker/host-network.js';
+interface CheckOpts {
+    verify?: (cfg: FirecrackerBridgeConfig) => Promise<BridgeHealth>;
+}
+export declare function checkFirecrackerNetwork(deps: DiagnosticDeps, opts?: CheckOpts): Promise<DiagnosticResult[]>;
+export {};
+//# sourceMappingURL=firecracker-network.d.ts.map