@kici-dev/orchestrator 0.1.14 → 0.1.15

package/dist/cluster/peer-registry.d.ts

@@ -21,6 +21,12 @@ export interface PeerAgentInfo {
      * in the required label set.
      */
     mandatoryLabels: string[];
+    /**
+     * Name of the scaler backend that spawned this agent, or null for static
+     * (stateful) agents not bound to any scaler. Surfaced in diagnostics so the
+     * dashboard can group a peer's agents under the correct scaler row.
+     */
+    scalerName?: string | null;
 }
 export interface PeerInfo {
     instanceId: string;

package/dist/config/schema.d.ts

@@ -107,6 +107,8 @@ export declare const sharedConfigSchema: z.ZodObject<{
     cacheTtlDays: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
+    userCacheQuotaBytes: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
+    userCacheTtlMs: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
     eventRouter: z.ZodOptional<z.ZodObject<{
         maxChainDepth: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
         rateLimitPerWorkflowPerMinute: z.ZodOptional<z.ZodCoercedNumber<unknown>>;
@@ -189,6 +191,8 @@ export declare const appConfigSchema: z.ZodObject<{
     cacheTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterMaxChainDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterRateLimitPerWorkflowPerMinute: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     eventRouterEventTtlSeconds: z.ZodDefault<z.ZodCoercedNumber<unknown>>;

package/dist/config.d.ts

@@ -36,6 +36,7 @@ declare const configSchema: z.ZodObject<{
     queueBackpressureThreshold: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     workerConcurrency: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     concurrencyWaitTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    dispatchAckTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheStorageType: z.ZodOptional<z.ZodEnum<{
         s3: "s3";
         filesystem: "filesystem";
@@ -46,6 +47,7 @@ declare const configSchema: z.ZodObject<{
     cacheStorageS3Region: z.ZodOptional<z.ZodString>;
     cacheStorageS3Endpoint: z.ZodOptional<z.ZodString>;
     cacheStorageS3ExternalEndpoint: z.ZodOptional<z.ZodString>;
+    cacheStorageS3UploadEndpoint: z.ZodOptional<z.ZodString>;
     cacheStorageS3ForcePathStyle: z.ZodOptional<z.ZodPipe<z.ZodEnum<{
         true: "true";
         false: "false";
@@ -56,6 +58,8 @@ declare const configSchema: z.ZodObject<{
     cacheTtlDays: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     webhookPayloadDir: z.ZodOptional<z.ZodString>;
     dataDir: z.ZodOptional<z.ZodString>;
     scalerConfigPath: z.ZodOptional<z.ZodString>;
@@ -149,12 +153,17 @@ export type AppConfig = z.infer<typeof configSchema> & {
         region?: string;
         endpoint?: string;
         externalEndpoint?: string;
+        uploadEndpoint?: string;
         forcePathStyle?: boolean;
         logBucket?: string;
         /** Filesystem backend: absolute base directory for cached blobs. */
         fsBasePath?: string;
         /** Filesystem backend: base URL the agent uses to reach this orchestrator. */
         fsBaseUrl?: string;
+        /** Per-org byte quota for the user-facing cache (UserCache). */
+        userCacheQuotaBytes?: number;
+        /** Per-entry TTL (ms) for the user-facing cache (UserCache). */
+        userCacheTtlMs?: number;
     };
 };
 /**
@@ -177,10 +186,13 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     queueBackpressureThreshold: number;
     workerConcurrency: number;
     concurrencyWaitTimeoutMs: number;
+    dispatchAckTimeoutMs: number;
     cacheStorageS3Prefix: string;
     cacheTtlDays: number;
     cacheBuildTimeoutMs: number;
     cacheMaxTarballBytes: number;
+    userCacheQuotaBytes: number;
+    userCacheTtlMs: number;
     staleDetectorScanIntervalMs: number;
     staleDetectorThresholdMultiplier: number;
     jobHeartbeatIntervalMs: number;
@@ -234,6 +246,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     cacheStorageS3Region?: string | undefined;
     cacheStorageS3Endpoint?: string | undefined;
     cacheStorageS3ExternalEndpoint?: string | undefined;
+    cacheStorageS3UploadEndpoint?: string | undefined;
     cacheStorageS3ForcePathStyle?: boolean | undefined;
     cacheStorageFsPath?: string | undefined;
     cacheStorageFsBaseUrl?: string | undefined;

package/dist/dashboard/handler.d.ts

@@ -13,7 +13,7 @@
  */
 import { type Kysely } from 'kysely';
 import { type ColdStore } from '@kici-dev/shared';
-import type { DashboardRunDetailRequest, DashboardStepLogsRequest, DashboardPayloadRequest, DashboardOrchLogsRequest, DashboardEventLogListRequest, DashboardEventLogDetailRequest, DashboardEventLogPayloadStreamRequest, DashboardAccessLogListRequest, DashboardEventDlqListRequest, DashboardEventDlqCountRequest, DashboardEventDlqRetryRequest, DashboardEventDlqDiscardRequest, RunRerunRequest, RunCancelRequest, ManualScheduleRequest } from '@kici-dev/engine';
+import type { DashboardRunDetailRequest, DashboardRunsListRequest, DashboardRunsListResponse, DashboardRunsFiltersRequest, DashboardRunsFiltersResponse, DashboardSourcesListRequest, DashboardSourcesListResponse, DashboardStepLogsRequest, DashboardPayloadRequest, DashboardOrchLogsRequest, DashboardEventLogListRequest, DashboardEventLogDetailRequest, DashboardEventLogPayloadStreamRequest, DashboardAccessLogListRequest, DashboardEventDlqListRequest, DashboardEventDlqCountRequest, DashboardEventDlqRetryRequest, DashboardEventDlqDiscardRequest, RunRerunRequest, RunCancelRequest, ManualScheduleRequest } from '@kici-dev/engine';
 import type { Database } from '../db/types.js';
 import type { LogStorage } from '../reporting/log-storage.js';
 import type { AccessLogWriter } from '../audit/access-log.js';
@@ -155,6 +155,97 @@ export declare class DashboardHandler {
      * builds a nested job/step tree, and sends the response.
      */
     handleRunDetail(msg: DashboardRunDetailRequest): Promise<void>;
+    /**
+     * Resolve every routing key owned by an org by unioning both source
+     * tables. The orchestrator is single-org but multi-routing-key: one org
+     * owns one GitHub-app source plus N generic sources, each with its own
+     * `routing_key`. `execution_runs` has no `org_id` column — org ownership
+     * is expressed purely through `routing_key → sources.customer_id` /
+     * `generic_webhook_sources.customer_id`, so scoping a run query to an org
+     * means scoping it to the union of that org's routing keys.
+     *
+     * Predicates mirror the org-context resolver in `server.ts` and
+     * `resolveOrgForRoutingKey` above:
+     * - `sources`: `customer_id = orgId` excluding the `__default__` sentinel.
+     * - `generic_webhook_sources`: `customer_id = orgId` and not soft-deleted
+     *   (`deleted_at is null`).
+     */
+    private resolveOrgRoutingKeys;
+    /**
+     * Resolve a per-routing-key source identity map for a set of routing keys,
+     * unioning both source tables. Uses the identical name/subtype/provider
+     * derivation as `handleSourcesList` so a run summary's `source` block
+     * matches what the sources page shows for the same routing key. Routing
+     * keys with no live source row are simply absent from the map (the caller
+     * falls back to the run's own provider).
+     */
+    private resolveSourceIdentities;
+    /**
+     * Look up the page's jobs in one query to derive per-run `jobCount` plus the
+     * compile-job markers (`hadCompileJob` / `compileJobId`). The compile job is
+     * the synthetic `__build__*` job KiCI inserts for the compile phase — the
+     * same definition the Platform run-list route uses
+     * (`job_name LIKE '__build__%'`). Returns a per-run aggregate keyed by
+     * `run_id`; runs with no job rows are absent (the caller omits the fields).
+     */
+    private resolveRunJobAggregates;
+    /**
+     * Handle a dashboard.runs.list request.
+     *
+     * Returns a page of run summaries from `execution_runs`, scoped to ALL
+     * routing keys owned by this orchestrator's bound org (`this.orgId`). The
+     * orchestrator is single-org but multi-routing-key, and the bound
+     * `this.routingKey` is just ONE of the org's keys (resolved
+     * non-deterministically at startup), so filtering on it alone would drop
+     * runs that arrived under a sibling routing key. We resolve the org's full
+     * routing-key set from `sources` + `generic_webhook_sources` and filter on
+     * the union. Attribution still uses the handler-bound `orgId` /
+     * `routingKey` directly — the same inline-ctx pattern as the org-scoped
+     * event-log / access-log handlers.
+     *
+     * Used by the operator console (`support-read` break-glass) and the
+     * dashboard's run-list view. The access_log row carries the wire actor
+     * (including the platform_operator reason) so the customer can audit any
+     * operator read.
+     */
+    handleRunsList(msg: DashboardRunsListRequest): Promise<DashboardRunsListResponse>;
+    /**
+     * Handle a dashboard.runs.filters request.
+     *
+     * Returns the distinct filter-option values the customer runs page renders
+     * in its filter controls — statuses / workflows / branches / repositories /
+     * triggerTypes / sources — scoped to ALL routing keys owned by this
+     * orchestrator's bound org (same scoping as `handleRunsList`). Distinct
+     * values are derived in-memory from a single column projection over
+     * `execution_runs`; `sources` reuses `resolveSourceIdentities` so the list
+     * matches what the sources page shows for the same routing key.
+     *
+     * `triggerTypes` is populated from `execution_runs.provider` — the
+     * orchestrator has no dedicated trigger-event column, so `provider`
+     * (`github` / `generic` / `internal`) is the closest available
+     * discriminator the customer filter can offer.
+     *
+     * Attribution uses the handler-bound `orgId` / `routingKey` directly — the
+     * same inline-ctx pattern as `handleRunsList`. The access_log row carries
+     * the wire actor (including the platform_operator reason) so the customer
+     * can audit any operator read.
+     */
+    handleRunsFilters(msg: DashboardRunsFiltersRequest): Promise<DashboardRunsFiltersResponse>;
+    /**
+     * Handle a dashboard.sources.list request.
+     *
+     * Returns the org's webhook source summaries by unioning both source
+     * tables (GitHub-app `sources` + live `generic_webhook_sources`), scoped
+     * to this orchestrator's bound org (`this.orgId`). Source counts are tiny
+     * so the page is returned unpaginated (`msg.limit` is ignored). Secret-
+     * bearing columns (`config`, `git_config`, `verification_config`) are
+     * never projected — `git_config` is read only to derive the subtype.
+     *
+     * Attribution uses the handler-bound `orgId` / `routingKey` directly —
+     * the same inline-ctx pattern as `handleRunsList`. The access_log row
+     * carries the wire actor so the customer can audit any operator read.
+     */
+    handleSourcesList(msg: DashboardSourcesListRequest): Promise<DashboardSourcesListResponse>;
     /**
      * Handle a dashboard.step.logs request.
      * Looks up the step's log_path and reads content from LogStorage.

package/dist/db/migrations/026_event_log_lockfile_corrupt.d.ts

@@ -0,0 +1,11 @@
+import { type Kysely } from 'kysely';
+/**
+ * Extend the event_log.status CHECK constraint with 'lockfile_corrupt' so the
+ * orchestrator can log a delivery whose lock file was present but unparseable.
+ * Orchestrator-only: the Platform event_log uses a separate, narrower status set.
+ *
+ * Idempotent: the DROP ... IF EXISTS / re-ADD pair re-runs cleanly.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=026_event_log_lockfile_corrupt.d.ts.map

package/dist/db/migrations/027_workflow_timeout.d.ts

@@ -0,0 +1,20 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `workflow_timeout_ms integer` to `execution_runs`.
+ *
+ * Persists the workflow-level wall-clock timeout (in milliseconds) read from
+ * the lock workflow at run creation. NULL means no workflow-level cap is
+ * configured. The WorkflowDeadlineDetector reads this column to find runs
+ * whose `started_at + workflow_timeout_ms` has passed and cancels them with
+ * the distinct workflow-timeout reason.
+ *
+ * Typed INTEGER to match the existing `*_ms` columns (`duration_ms`,
+ * `concurrency_timeout_ms`), which keeps the pg read representation a plain
+ * number rather than the BIGINT-as-string shape. INTEGER caps the timeout at
+ * ~24.8 days, far beyond any sane workflow wall-clock budget.
+ *
+ * Idempotent: re-running on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=027_workflow_timeout.d.ts.map

package/dist/db/migrations/028_org_settings_user_cache.d.ts

@@ -0,0 +1,4 @@
+import { type Kysely } from 'kysely';
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=028_org_settings_user_cache.d.ts.map

package/dist/db/migrations/029_dispatch_queue_attempts.d.ts

@@ -0,0 +1,16 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `dispatch_queue.dispatch_attempts INT NOT NULL DEFAULT 0`.
+ *
+ * Counts how many times a job has been returned to `pending` for
+ * re-dispatch after a failed delivery attempt (agent sent job.reject, or
+ * a scaler-managed agent disconnected before the job started). The
+ * dispatcher fails the job permanently once the counter reaches
+ * MAX_DISPATCH_ATTEMPTS, bounding requeue loops; `expires_at` remains the
+ * time-based backstop.
+ *
+ * Idempotent: re-running on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=029_dispatch_queue_attempts.d.ts.map

package/dist/db/migrations/030_held_runs_env_set_null.d.ts

@@ -0,0 +1,13 @@
+import { type Kysely } from 'kysely';
+/**
+ * held_runs.environment_id becomes nullable with ON DELETE SET NULL so
+ * terminal held-run history survives environment deletion (a null
+ * environment_id means the environment was since deleted). Pending held
+ * runs still block deletion — enforced in EnvironmentStore.delete.
+ *
+ * Idempotent: dropping the NOT NULL and the constraint are both no-ops on a
+ * re-run, and the constraint is re-created with the SET NULL action.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=030_held_runs_env_set_null.d.ts.map

package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts

@@ -0,0 +1,19 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `dispatch_queue.ack_deadline TIMESTAMPTZ` and
+ * `dispatch_queue.ack_agent_id TEXT` persisting the per-dispatch
+ * acknowledgment deadline for HA-safe lost-dispatch detection.
+ *
+ * The dispatcher stamps both when a job.dispatch is sent and clears them
+ * when the agent answers (job.ack / job.reject / job.status running). A
+ * `dispatched` row past its deadline is requeued — by the owning coord's
+ * in-memory timer, or by the leader-gated sweep
+ * (`Dispatcher.sweepExpiredAckDeadlines`) when the owning coord crashed.
+ * On coord boot, `Dispatcher.recoverState()` re-arms timers from the
+ * persisted deadlines.
+ *
+ * Idempotent: re-running on a DB that already has either column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=031_dispatch_queue_ack_deadline.d.ts.map

package/dist/db/migrations/032_org_settings_dispatch_ack_timeout.d.ts

@@ -0,0 +1,14 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `org_settings.dispatch_ack_timeout_ms BIGINT` (nullable).
+ *
+ * Per-org override of the dispatch-acknowledgment deadline. NULL falls
+ * back to the cluster-wide default (`KICI_DISPATCH_ACK_TIMEOUT_MS`,
+ * default 10s). Operators raise it on high-latency networks via
+ * `kici-admin org-settings`.
+ *
+ * Idempotent: a re-run on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=032_org_settings_dispatch_ack_timeout.d.ts.map

package/dist/db/types.d.ts

@@ -158,6 +158,17 @@ export interface DispatchQueueTable {
      * real provisioning cause; cleared on dispatch. NULL when none recorded.
      */
     last_provisioning_error: ColumnType<string | null, string | null | undefined, string | null>;
+    /** Times this job was returned to pending for re-dispatch (job.reject / pre-start agent loss). */
+    dispatch_attempts: Generated<number>;
+    /**
+     * Deadline by which the dispatched job's agent must answer the
+     * job.dispatch (job.ack / job.reject / job.status running). Stamped when
+     * the dispatch is sent, cleared on any answer; `dispatched` rows past the
+     * deadline are requeued by the owning coord's timer or the leader sweep.
+     */
+    ack_deadline: ColumnType<Date | null, Date | null | undefined, Date | null>;
+    /** Agent the dispatch was sent to (for ack-timeout disconnect + logging). */
+    ack_agent_id: ColumnType<string | null, string | null | undefined, string | null>;
 }
 /**
  * Deduplication cache table
@@ -266,6 +277,12 @@ export interface ExecutionRunTable {
      * provider context, agent spawn). NULL for normal runs.
      */
     init_failure: ColumnType<InitFailure | null, unknown, unknown>;
+    /**
+     * Whole-run wall-clock timeout in ms from the workflow lock; null when
+     * unset. Read by the WorkflowDeadlineDetector. INTEGER in Postgres, matching
+     * the other `*_ms` columns, so pg returns a plain number.
+     */
+    workflow_timeout_ms: ColumnType<number | null, number | null | undefined, number | null>;
     /** When this record was created */
     created_at: Generated<Date>;
     /**
@@ -595,8 +612,8 @@ export interface HeldRunsTable {
     run_id: string;
     /** Job ID within the run */
     job_id: string;
-    /** Environment ID (FK to environments.id) */
-    environment_id: string;
+    /** Environment ID (FK to environments.id); null once the environment is deleted */
+    environment_id: string | null;
     /** Hold type: 'approval' | 'wait_timer' | 'concurrency' */
     hold_type: string;
     /** Hold status: 'pending' | 'approved' | 'rejected' | 'expired' | 'released' */
@@ -1157,6 +1174,24 @@ export interface OrgSettingsTable {
      * treats missing keys as `true`.
      */
     dashboard_write_policy: ColumnType<Record<string, boolean>, Record<string, boolean> | string | undefined, Record<string, boolean> | string>;
+    /**
+     * Per-org byte quota for the user-facing cache (UserCache). NULL = use the
+     * cluster-wide default (`KICI_USER_CACHE_QUOTA_BYTES`, 5 GiB). Postgres
+     * BIGINT — pg returns a string on select; accept a number on insert/update.
+     */
+    user_cache_quota_bytes: ColumnType<string | null, number | null | undefined, number | null>;
+    /**
+     * Per-entry TTL (ms) for the user-facing cache (UserCache). NULL = use the
+     * cluster-wide default (`KICI_USER_CACHE_TTL_MS`, 7 days). Postgres BIGINT —
+     * pg returns a string on select; accept a number on insert/update.
+     */
+    user_cache_ttl_ms: ColumnType<string | null, number | null | undefined, number | null>;
+    /**
+     * Per-org dispatch-acknowledgment deadline (ms); null = cluster default
+     * (config.dispatchAckTimeoutMs / KICI_DISPATCH_ACK_TIMEOUT_MS). Postgres
+     * BIGINT — pg returns a string on select; accept a number on insert/update.
+     */
+    dispatch_ack_timeout_ms: ColumnType<string | null, number | null | undefined, number | null>;
     /** When this setting was created */
     created_at: Generated<Date>;
     /** When this setting was last updated */

package/dist/environments/environment-store.d.ts

@@ -1,6 +1,11 @@
 import { type Kysely } from 'kysely';
 import type { Environment as EngineEnvironment } from '@kici-dev/engine';
 import type { Database, Environment } from '../db/types.js';
+/** Thrown by `delete` when pending held runs reference the environment. */
+export declare class EnvironmentDeleteBlockedError extends Error {
+    readonly pendingCount: number;
+    constructor(pendingCount: number);
+}
 /**
  * Map a DB environment row (snake_case) to the engine Environment type (camelCase).
  *
@@ -60,7 +65,15 @@ export declare class EnvironmentStore {
     create(orgId: string, data: EnvironmentCreateInput): Promise<Environment>;
     /** Update an environment. Returns the updated row, or null if not found. */
     update(orgId: string, id: string, updates: EnvironmentUpdateInput): Promise<Environment | null>;
-    /** Delete an environment. Cascade deletes bindings, variables, and overrides. */
+    /**
+     * Delete an environment.
+     *
+     * Bindings, variables, and source overrides cascade away via their FK.
+     * Terminal held-run history survives with a null `environment_id` (the FK
+     * uses ON DELETE SET NULL). Pending held runs still reference the
+     * environment, so deletion is blocked with `EnvironmentDeleteBlockedError`
+     * until they are approved or rejected.
+     */
     delete(orgId: string, id: string): Promise<boolean>;
     /**
      * Match an environment name against org environments.

package/dist/index.d.ts

@@ -3,5 +3,6 @@ export { S3CacheStorage, type S3CacheStorageOptions } from './storage/s3.js';
 export { createCacheStorage } from './storage/index.js';
 export { SourceCache } from './cache/source-cache.js';
 export { DepCache } from './cache/dep-cache.js';
+export { UserCache, DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, type UserCacheRef, type UserCacheRestoreResult, type UserCacheBeginSaveResult, } from './cache/user-cache.js';
 export { PeerCredentialStore, createPeerCredentialStoreFromUrl, type PeerCredential, } from './cluster/peer-credentials.js';
 //# sourceMappingURL=index.d.ts.map