@kici-dev/orchestrator 0.1.13 → 0.1.14

package/dist/db/migrations/025_init_failure.d.ts

@@ -0,0 +1,16 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `init_failure jsonb` columns to `execution_runs` and `execution_jobs`.
+ *
+ * Presence of this column on a row means the run/job never executed a step
+ * because of an init-phase failure; absence (NULL) means a normal run.
+ * Shape on the wire is `InitFailure` from `@kici-dev/engine`. The dashboard
+ * reads this column directly so it can render the right banner without
+ * round-tripping to the orchestrator (which may be offline).
+ *
+ * Idempotent: re-running on a DB that already has either column is a no-op
+ * for that column.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=025_init_failure.d.ts.map

package/dist/db/types.d.ts

@@ -1,4 +1,5 @@
 import type { ColumnType, Generated, Insertable, Selectable, Updateable } from 'kysely';
+import type { InitFailure } from '@kici-dev/engine';
 /**
  * PostgreSQL-only database types.
  * Column names use snake_case matching the actual database column names.
@@ -258,6 +259,13 @@ export interface ExecutionRunTable {
     contributor_username: string | null;
     /** Human-readable reason why the run failed (null for non-failed runs). */
     failure_reason: string | null;
+    /**
+     * Structured init-phase failure detail (shape: `InitFailure` from
+     * `@kici-dev/engine`). Non-null means the run never executed a step
+     * because something failed during the init phase (lock-file fetch,
+     * provider context, agent spawn). NULL for normal runs.
+     */
+    init_failure: ColumnType<InitFailure | null, unknown, unknown>;
     /** When this record was created */
     created_at: Generated<Date>;
     /**
@@ -304,6 +312,13 @@ export interface ExecutionJobTable {
     log_bytes: Generated<number>;
     /** Error info if failed */
     error_message: string | null;
+    /**
+     * Structured init-phase failure detail (shape: `InitFailure` from
+     * `@kici-dev/engine`). Non-null means the job never executed a step
+     * because something failed during init (lock-file fetch, provider
+     * context, agent spawn). NULL for normal runs.
+     */
+    init_failure: ColumnType<InitFailure | null, unknown, unknown>;
     /** Labels used for agent routing (e.g. ["kici:os:linux", "kici:arch:x64"]). JSONB. */
     runs_on_labels: string | null;
     /** Last heartbeat received from agent (for stale run detection) */

package/dist/diagnostics/checks/index.d.ts

@@ -11,7 +11,8 @@ import { checkAgentConnectivity } from './agents.js';
 import { checkDiskSpace } from './disk.js';
 import { checkConfigValidity } from './config.js';
 import { checkCertificateExpiry } from './certs.js';
+import { checkScalerProvisioning } from './scaler.js';
 /** All diagnostic checks in display order. */
 export declare const defaultChecks: DiagnosticCheck[];
-export { checkDbConnectivity, checkWsToPlatform, checkAgentConnectivity, checkDiskSpace, checkConfigValidity, checkCertificateExpiry, };
+export { checkDbConnectivity, checkWsToPlatform, checkAgentConnectivity, checkDiskSpace, checkConfigValidity, checkCertificateExpiry, checkScalerProvisioning, };
 //# sourceMappingURL=index.d.ts.map

package/dist/diagnostics/checks/scaler.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Scaler provisioning diagnostic check.
+ *
+ * Reads recent scaler spawn failures (in-process, over a short rolling window)
+ * and emits one row per configured scaler backend instance. Severity is
+ * bound-aware: a warm-pool/unbound failure is a warning (no run impact yet); a
+ * job-bound failure is a failure (a queued run could not get an agent).
+ */
+import type { DiagnosticDeps, DiagnosticResult } from '../types.js';
+/** Rolling window for "recent" spawn failures. */
+export declare const SCALER_FAILURE_WINDOW_MS: number;
+export declare function checkScalerProvisioning(deps: DiagnosticDeps): Promise<DiagnosticResult[]>;
+//# sourceMappingURL=scaler.d.ts.map

package/dist/diagnostics/types.d.ts

@@ -7,6 +7,7 @@
 import type { Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
 import type { AgentRegistry } from '../agent/registry.js';
+import type { ScalerManager } from '../scaler/manager.js';
 /** Result of a single diagnostic check. */
 export interface DiagnosticResult {
     /** Human-readable check name (e.g., "Database connectivity"). */
@@ -32,7 +33,9 @@ export interface DiagnosticDeps {
     config: Record<string, unknown>;
     /** TLS cert path (for expiry check). */
     tlsCertPath?: string;
+    /** Scaler manager for recent spawn-failure health (optional -- no scaler configured). */
+    scalerManager?: ScalerManager;
 }
-/** A diagnostic check function. */
-export type DiagnosticCheck = (deps: DiagnosticDeps) => Promise<DiagnosticResult>;
+/** A diagnostic check function. May return one result or several (e.g. one per scaler backend). */
+export type DiagnosticCheck = (deps: DiagnosticDeps) => Promise<DiagnosticResult | DiagnosticResult[]>;
 //# sourceMappingURL=types.d.ts.map

package/dist/reporting/execution-tracker.d.ts

@@ -13,7 +13,7 @@
  */
 import { type Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
-import { ExecutionRunStatus, ScalerEventType } from '@kici-dev/engine';
+import { ExecutionRunStatus, type InitFailure, ScalerEventType } from '@kici-dev/engine';
 import type { ObserverRegistry } from '../ws/observer-registry.js';
 import type { LogStorage } from './log-storage.js';
 import type { JobQueue } from '../queue/job-queue.js';
@@ -92,7 +92,13 @@ export interface ExecutionTrackerDeps {
      * Set on terminal run states only. Powers the operator-side
      * `kici_org_log_bytes` capacity-planning gauge on the Platform.
      */
-    logBytes?: number) => void;
+    logBytes?: number, 
+    /**
+     * Structured init-failure signal. Set when the run never executed a step
+     * because of an init-phase failure. Forwarded to Platform's execution.status
+     * forward and persisted in execution_runs.init_failure on both sides.
+     */
+    initFailure?: InitFailure) => void;
     /** Optional callback to forward job status changes to Platform.
      *  Fires on every job state transition (pending->running, running->success, etc.).
      *  Used to populate Platform's execution_jobs projection table. */
@@ -101,7 +107,12 @@ export interface ExecutionTrackerDeps {
      * Total raw log bytes accumulated across the job (sum of per-step totals).
      * Set on terminal job states only.
      */
-    logBytes?: number) => void;
+    logBytes?: number, 
+    /**
+     * Structured init-failure signal. Set for synthetic rejected-* / init-failed-*
+     * jobs that never started. Persisted in execution_jobs.init_failure.
+     */
+    initFailure?: InitFailure) => void;
     /**
      * Optional callback to emit run.event messages to Platform.
      * Fires at orchestrator lifecycle points (dispatch, agent assignment, job start/complete).
@@ -142,6 +153,22 @@ export declare class ExecutionTracker {
     private readonly orgId?;
     private readonly jobQueue?;
     private readonly runs;
+    /**
+     * Per-run async-mutex chain. `onJobStatus` and `addJobsToRun` mutate the same
+     * `run.jobs` Map / `execution_jobs` row; without serialization a job-status
+     * reply that lands mid synthetic→real swap (`addJobsToRun`) clobbers the swap
+     * and the run hangs in `running` forever (see `withRunLock`). The map value is
+     * the tail of the chain for that runId; entries are GC'd when the last holder
+     * releases.
+     */
+    private readonly runLockTails;
+    /**
+     * Tracks which runIds the current async context already holds in
+     * `withRunLock`, so reentrant calls (e.g. onJobStatus → scheduler hook →
+     * dispatchReadyJob → addJobsToRun, all the same runId) bypass re-acquisition
+     * instead of deadlocking. Propagates across awaits via AsyncLocalStorage.
+     */
+    private readonly heldRunLocks;
     /** Tracks which runs are test runs for observer broadcasting. */
     private readonly testRunIds;
     /**
@@ -208,6 +235,22 @@ export declare class ExecutionTracker {
      * which peer owns the downstream dispatch.
      */
     findSyntheticJobId(runId: string, jobName: string): Promise<string | undefined>;
+    /**
+     * Run `fn` while holding a per-run lock, serializing the run-mutating methods
+     * (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
+     * synthetic→real job swap and wedge the run in `running`.
+     *
+     * The lock is **reentrant**: three paths re-enter a locked method within the
+     * same async context for the same runId — onJobStatus → scheduler hook →
+     * dispatchReadyJob → addJobsToRun; onJobStatus → enforceSchedulerInvariant →
+     * onJobStatus; runSchedulerHook → onJobStatus (skip). A non-reentrant mutex
+     * would deadlock on these, so a context that already holds the runId's lock
+     * (tracked via `heldRunLocks`) runs `fn` inline. A genuinely concurrent caller
+     * for the same runId (a separate WS message) is a different async context and
+     * correctly waits. All reentrant paths are same-runId, so there is no
+     * cross-run lock-ordering deadlock.
+     */
+    private withRunLock;
     addJobsToRun(runId: string, jobs: Array<{
         jobId: string;
         jobName: string;
@@ -216,6 +259,7 @@ export declare class ExecutionTracker {
     }>, dispatchedContexts?: string[], 
     /** Synthetic job ID to replace (e.g. needs-pending-deploy-{uuid}). */
     replaceSyntheticId?: string): Promise<void>;
+    private addJobsToRunImpl;
     /**
      * Mark a run as a test run for observer broadcasting.
      * Called by the test pipeline after creating the execution run.
@@ -233,6 +277,7 @@ export declare class ExecutionTracker {
      * fires the onExecutionComplete callback.
      */
     onJobStatus(runId: string, jobId: string, state: string, timestamp: number, agentId?: string, data?: Record<string, unknown>): Promise<void>;
+    private onJobStatusImpl;
     /**
      * Phase 1a: recover run state from the DB when in-memory tracking is empty.
      * Returns the rehydrated RunState or null if the run is unknown to the DB
@@ -313,7 +358,7 @@ export declare class ExecutionTracker {
      * onExecutionStarted but the build subsequently fails. Without this,
      * the execution_runs row would stay in a non-terminal state.
      */
-    onBuildFailed(runId: string): Promise<void>;
+    onBuildFailed(runId: string, initFailure?: InitFailure): Promise<void>;
     /**
      * Create a failed execution run when the build timed out before onExecutionStarted
      * had a chance to insert the row (buildJobTrackedEarly was false).
@@ -321,7 +366,33 @@ export declare class ExecutionTracker {
      * Inserts a minimal execution_runs row with status='failed' directly so the E2E
      * test (and dashboard) can observe the failure instead of a missing run.
      */
-    onBuildFailedBeforeTracking(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, routingKey: string, triggerEvent?: string, commitMessage?: string, failureReason?: string): Promise<void>;
+    onBuildFailedBeforeTracking(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, routingKey: string, triggerEvent?: string, commitMessage?: string, failureReason?: string, initFailure?: InitFailure): Promise<void>;
+    /**
+     * Insert a `failed` execution_runs row directly for an init failure that
+     * occurred BEFORE onExecutionStarted ran (so no in-memory state exists
+     * and no jobs were dispatched). Also writes the structured init_failure
+     * signal and fires onExecutionStatusChange so Platform's projection picks
+     * it up via the normal forward path. Idempotent: if a row already exists
+     * for this runId, the insert is a no-op (ON CONFLICT DO NOTHING).
+     *
+     * Closes the silent pre-run-failure gap — without this helper, secret /
+     * install-secret / all-jobs-rejected early-exits in dispatch-matched-workflow
+     * leave no trace on the dashboard.
+     */
+    recordInitFailureRun(args: {
+        runId: string;
+        workflowName: string;
+        provider: string;
+        repoIdentifier: string;
+        ref: string;
+        sha: string;
+        deliveryId: string | null;
+        providerContext: Record<string, unknown>;
+        routingKey: string;
+        initFailure: InitFailure;
+        triggerEvent?: string;
+        commitMessage?: string;
+    }): Promise<void>;
     /**
      * Mark a run as failed immediately with a reason message.
      *
@@ -330,7 +401,7 @@ export declare class ExecutionTracker {
      * Instead of leaving the run in 'running' for OrphanRecovery to catch after 5 min,
      * this fails it right away.
      */
-    failRun(runId: string, reason: string): Promise<void>;
+    failRun(runId: string, reason: string, initFailure?: InitFailure): Promise<void>;
     /**
      * Update step status within a job.
      *

package/dist/scaler/failure-tracker.d.ts

@@ -0,0 +1,46 @@
+/**
+ * In-process bounded record of recent scaler spawn failures.
+ *
+ * The scaler manager records every `scaler.failed` event here at the same site
+ * it increments the fleet-wide Prometheus counter. The diagnose scaler check
+ * reads recent failures grouped per backend instance to produce its rows. This
+ * is memory-only and bounded — the recent-failures window resets on restart,
+ * which is acceptable for an on-demand operator view over a short window.
+ */
+/** A single recorded scaler spawn failure. */
+export interface ScalerFailureRecord {
+    /** Scaler instance name (the configured scaler `name`); the diagnose row key. */
+    backendName: string;
+    /** Backend type: 'container' | 'bare-metal' | 'firecracker' | 'unknown'. */
+    backendType: string;
+    /** True when the failed spawn was bound to a queued job (a run was affected). */
+    bound: boolean;
+    /** Captured error string from the scaler event detail. */
+    detail: string;
+    /** Event timestamp in epoch milliseconds. */
+    timestampMs: number;
+}
+/** Per-backend summary of recent failures within a window. */
+export interface BackendFailureSummary {
+    backendType: string;
+    boundCount: number;
+    unboundCount: number;
+    /** Detail of the most recent failure in the window. */
+    lastError: string;
+    /** Timestamp of the most recent failure in the window. */
+    lastAtMs: number;
+}
+export declare class ScalerFailureTracker {
+    private readonly records;
+    private readonly maxEntries;
+    constructor(maxEntries?: number);
+    /** Record a failure, evicting the oldest entry when over capacity. */
+    record(rec: ScalerFailureRecord): void;
+    /**
+     * Group failures newer than `nowMs - windowMs` by backend instance name.
+     * `nowMs` is injected so callers control the clock (and tests are
+     * deterministic).
+     */
+    recentByBackend(windowMs: number, nowMs: number): Map<string, BackendFailureSummary>;
+}
+//# sourceMappingURL=failure-tracker.d.ts.map

package/dist/scaler/manager.d.ts

@@ -7,6 +7,7 @@
  * and manages the agent lifecycle from spawn to destroy.
  */
 import type { ResourceRequest } from '@kici-dev/engine';
+import type { BackendFailureSummary } from './failure-tracker.js';
 import type { ScalerBackend, ScalerConfig, ScaleResult, ScalerEvent, ResourceCap, ValidationResult } from './types.js';
 import type { ScalerStateStore, ScalerStateRecovery } from './scaler-state-store.js';
 /**
@@ -80,6 +81,8 @@ export interface ScalerStatus {
 export declare class ScalerManager {
     private readonly backends;
     private readonly backendRoles;
+    /** Recent scaler spawn failures, surfaced by `kici-admin diagnose`. */
+    private readonly failureTracker;
     private globalMaxAgents;
     /** Per-scaler resource caps (`{ maxCpu, maxMemoryBytes }`), keyed by scaler name. */
     private readonly resourceCaps;
@@ -320,6 +323,11 @@ export declare class ScalerManager {
     /**
      * Return status summary for metrics and health endpoints.
      */
+    /**
+     * Recent scaler spawn failures grouped per backend instance, for the
+     * diagnose scaler check. `nowMs` is injected by the caller.
+     */
+    recentSpawnFailures(windowMs: number, nowMs: number): Map<string, BackendFailureSummary>;
     getStatus(): ScalerStatus;
     /**
      * Get the backend name managing a specific agent.