@kici-dev/orchestrator 0.1.10 → 0.1.12

package/dist/cluster/coordinator.d.ts

@@ -10,7 +10,7 @@
  * - "Peers report step-by-step progress back to coordinator"
  * - "Cancel mode: graceful -- finish current step, cancel remaining"
  */
-import type { JobReroute, JobProgress, PeerToPeerMessage, ResourceRequest } from '@kici-dev/engine';
+import type { JobReroute, JobProgress, PeerScalerEvent, PeerToPeerMessage, ResourceRequest } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 import type { PeerClient } from './peer-client.js';
 import type { Dispatcher } from '../agent/dispatcher.js';
@@ -139,6 +139,16 @@ export declare class RunCoordinator {
      * `onStepStatus` and the run stayed in `running` forever.
      */
     onPeerJobProgress(msg: JobProgress): void;
+    /**
+     * Handle a scaler provisioning event forwarded by a worker peer.
+     *
+     * Workers have no database, so they relay scaler events for jobs the
+     * worker is provisioning to the coordinator that owns the run. The
+     * coordinator's ExecutionTracker persists the event (provisioning log
+     * line + dispatch-queue last-error column) just as it would for a
+     * locally-emitted scaler event.
+     */
+    onPeerScalerEvent(msg: PeerScalerEvent): void;
     /**
      * Handle peer job completion. Updates ExecutionTracker.
      * The execution tracker's onJobStatus fires the needs-aware scheduler hook,

package/dist/cluster/peer-handler.d.ts

@@ -6,7 +6,7 @@
  * proof (reconnection), registers the peer in PeerRegistry, and routes
  * messages bidirectionally. Sends periodic heartbeats to the connecting peer.
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 import type { PeerCredentialStore } from './peer-credentials.js';
 import { type JoinTokenManager } from './join-token.js';
@@ -41,6 +41,8 @@ export interface PeerHandlerDeps {
     onJobReroute: (msg: JobReroute) => Promise<void>;
     /** Callback when a job progress update is received from peer. */
     onJobProgress: (msg: JobProgress) => void;
+    /** Callback when a scaler provisioning event is forwarded by a worker peer. */
+    onPeerScalerEvent?: (msg: PeerScalerEvent) => void;
     /** Callback when a job cancel request is received from peer. */
     onJobCancel: (msg: PeerJobCancel) => void;
     /** Callback when a log chunk is received from a worker peer. */

package/dist/db/migrations/024_dispatch_queue_provisioning_error.d.ts

@@ -0,0 +1,18 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `dispatch_queue.last_provisioning_error TEXT` recording the most
+ * recent scaler spawn-failure detail for a queued job.
+ *
+ * When the scaler fails to provision an agent for a job (e.g. the agent
+ * process exits with "spawn node ENOENT", or a container backend rejects
+ * the run), a `scaler.failed` event bound to the job writes the failure
+ * detail into this column. The queue-timeout reaper reads it so the job's
+ * eventual failure surfaces the real provisioning cause instead of a bare
+ * "no agent available" timeout. The column is cleared on dispatch and is
+ * NULL whenever no provisioning failure has been recorded.
+ *
+ * Idempotent: re-running on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=024_dispatch_queue_provisioning_error.d.ts.map

package/dist/db/types.d.ts

@@ -150,6 +150,13 @@ export interface DispatchQueueTable {
      * the rightful claimant. NULL for non-recovering rows.
      */
     recovery_agent_id: ColumnType<string | null, string | null | undefined, string | null>;
+    /**
+     * Most recent scaler spawn-failure detail for this queued job (e.g.
+     * "agent process error: spawn node ENOENT"). Written on a `scaler.failed`
+     * event bound to the job; read by the queue-timeout reaper to surface the
+     * real provisioning cause; cleared on dispatch. NULL when none recorded.
+     */
+    last_provisioning_error: ColumnType<string | null, string | null | undefined, string | null>;
 }
 /**
  * Deduplication cache table

package/dist/metrics/prometheus.d.ts

@@ -21,12 +21,22 @@ interface ScalerUsageRow {
 export declare function setScalerUsageBreakdown(rows: ScalerUsageRow[]): void;
 /** Increment the spawn-refusal counter (called by ScalerManager when a request fails the cap check). */
 export declare function incScalerSpawnRefusals(): void;
+/**
+ * Merge current per-label counts with explicit `0`s for previously-seen labels
+ * whose queue has drained. Mutates `everSeen` to include the current labels.
+ *
+ * Exported for direct unit testing of the sticky-zero invariant: a label that
+ * appears in one snapshot and is absent from the next must report `0` rather
+ * than vanish, otherwise the by-label gauge series freezes at its last value.
+ */
+export declare function applyStickyQueueLabels(everSeen: Set<string>, byLabel: Record<string, number>): Record<string, number>;
 /**
  * Update the dispatch-queue depth breakdown used by the Prometheus gauges.
  *
  * Called by the orchestrator's periodic depth refresher after polling the
  * JobQueue. Normalizes the input so the gauge callback can emit explicit
- * zero series for known statuses even when the queue is empty.
+ * zero series for known statuses even when the queue is empty, and keeps
+ * emitting `0` for any per-label series whose queue has drained.
  */
 export declare function setDispatchQueueDepthBreakdown(snapshot: {
     byStatus: Partial<Record<'pending' | 'dispatched', number>>;
@@ -113,6 +123,23 @@ export declare const logBytesStoredTotal: import("@opentelemetry/api").Counter<i
  * - result: attempted | success | failed
  */
 export declare const scalerConfigReloadsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+/**
+ * `bound` label for `kici_orch_scaler_spawn_failures_total`.
+ * - `true`  — the failed spawn was bound to a queued job (a run is affected).
+ * - `false` — a warm-pool / unbound pre-spawn (no run impact yet; fleet-health signal).
+ */
+export declare const ScalerSpawnFailureBound: {
+    readonly Bound: "true";
+    readonly Unbound: "false";
+};
+export type ScalerSpawnFailureBound = (typeof ScalerSpawnFailureBound)[keyof typeof ScalerSpawnFailureBound];
+/**
+ * Total scaler agent spawn failures.
+ * Labels:
+ * - backend: scaler backend type (bare-metal | container | firecracker | unknown)
+ * - bound: true (job-bound spawn) | false (warm-pool/unbound)
+ */
+export declare const scalerSpawnFailuresTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 /**
  * Total number of config reload operations.
  * Labels:

package/dist/queue/job-queue.d.ts

@@ -7,6 +7,8 @@ export interface ExpiredJobInfo {
     id: string;
     runId: string;
     jobName: string;
+    /** Most recent scaler spawn-failure detail, if any was recorded. */
+    lastProvisioningError: string | null;
 }
 /**
  * Point-in-time breakdown of dispatch_queue depth used for Prometheus gauges

package/dist/reporting/execution-tracker.d.ts

@@ -13,7 +13,7 @@
  */
 import { type Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
-import { ExecutionRunStatus } from '@kici-dev/engine';
+import { ExecutionRunStatus, ScalerEventType } from '@kici-dev/engine';
 import type { ObserverRegistry } from '../ws/observer-registry.js';
 import type { LogStorage } from './log-storage.js';
 import type { JobQueue } from '../queue/job-queue.js';
@@ -509,7 +509,7 @@ export declare class ExecutionTracker {
      */
     emitScalerEvent(runId: string, jobId: string, event: {
         agentId: string;
-        eventType: string;
+        eventType: ScalerEventType;
         detail: string;
         timestampMs: number;
     }): void;

package/dist/scaler/index.d.ts

@@ -5,7 +5,7 @@
  * Supports Docker, bare-metal, and Firecracker backends with YAML configuration,
  * label-set matching, and warm pool management.
  */
-export type { ResourceLimits, LabelSetConfig, ManagedAgent, ScaleResult, ValidationResult, ScalerBackend, ScalerConfig, ScalerEntry, WarmPoolConfig, FirecrackerNetworkConfig, } from './types.js';
+export type { ResourceLimits, LabelSetConfig, ManagedAgent, ScaleResult, ValidationResult, ScalerBackend, ScalerConfig, ScalerEntry, ScalerEvent, WarmPoolConfig, FirecrackerNetworkConfig, } from './types.js';
 export { loadScalerConfig, scalerFileSchema, firecrackerNetworkSchema, parseMemoryString, } from './config.js';
 export { normalizeLabelSet, labelSetsMatch, detectLabelSetOverlaps, findBackendForLabels, } from './label-matcher.js';
 export { ContainerScalerBackend, detectRuntime } from './container-backend.js';

package/dist/scaler/manager.d.ts

@@ -192,7 +192,7 @@ export declare class ScalerManager {
      * back to scaler defaults). Used by per-scaler / per-orchestrator / per-machine
      * resource caps; the cap math is wired up by ScalerManager itself.
      */
-    requestScale(labels: string[], jobId: string, excludeLabels?: string[], resources?: ResourceRequest): Promise<ScaleResult>;
+    requestScale(labels: string[], jobId: string, runId: string, excludeLabels?: string[], resources?: ResourceRequest): Promise<ScaleResult>;
     /**
      * Run a critical section under the reservation lock. Serializes concurrent
      * `requestScale()` calls so the check+reserve sequence is atomic.
@@ -381,8 +381,21 @@ export declare class ScalerManager {
     private createEventEmitter;
     /**
      * Handle a scaler event from a backend.
-     * If the agent is correlated to a job, relay immediately.
-     * Otherwise, buffer until correlation is established.
+     *
+     * For spawn FAILURES the bound spawning entry is the usual resolver, since
+     * correlation isn't set until a job is dispatched post-registration. The
+     * correlation map wins only in the rarer post-registration failure window
+     * (e.g. a bare-metal child 'error' firing after the agent already
+     * registered, by which point the spawning entry is gone). Warm-pool /
+     * unbound spawns resolve to neither. When attributed, the event is relayed
+     * immediately; otherwise it is buffered until correlateAgentToJob() flushes
+     * it.
+     *
+     * Every `scaler.failed` also increments the fleet-wide spawn-failure counter
+     * and emits a structured warn, regardless of whether it could be attributed.
+     * The backend label is resolved from the spawning entry first, then the
+     * managed-agent index — so a late failure after the spawning entry is gone
+     * still buckets under the real backend instead of "unknown".
      */
     private handleScalerEvent;
 }

package/dist/scaler/types.d.ts

@@ -4,8 +4,11 @@
  * Provides the foundational types that all scaler backends, the ScalerManager,
  * and the configuration layer depend on.
  */
+import { z } from 'zod';
+import { ScalerEventType } from '@kici-dev/engine';
 import type { ResourceRequest, ResourceSpec, ScalerBackendType } from '@kici-dev/engine';
 export type { ResourceRequest, ResourceSpec } from '@kici-dev/engine';
+export { ScalerEventType } from '@kici-dev/engine';
 /**
  * Resource limits for spawned agents.
  *
@@ -353,18 +356,13 @@ export interface WarmPoolConfig {
      */
     idleTimeoutSeconds: number;
 }
-/**
- * Standardized scaler lifecycle event types emitted by all backends.
- * These are stored in the event_log table for timeline rendering.
- */
-type ScalerEventType = 'scaler.provisioning' | 'scaler.network' | 'scaler.ready' | 'scaler.failed' | 'agent.connecting';
 /**
  * A single scaler lifecycle event.
  * Each backend emits these at key provisioning milestones.
  */
 export interface ScalerEvent {
     agentId: string;
-    eventType: ScalerEventType;
+    eventType: z.infer<typeof ScalerEventType>;
     /** Backend-specific detail text (e.g. "pulling image nginx:latest", "booting microVM") */
     detail: string;
     timestampMs: number;
@@ -382,6 +380,12 @@ export type ScalerEventCallback = (event: ScalerEvent) => void;
 export interface LogCapture {
     /** Async iterable yielding individual log lines from the agent process. */
     lines(): AsyncIterable<string>;
+    /**
+     * Return the most recent buffered output lines (bounded ring buffer),
+     * joined oldest→newest. Used to enrich a `scaler.failed` event when a
+     * spawn dies before WS registration. Empty string when nothing captured.
+     */
+    tail(): string;
     /** Stop capturing and destroy underlying streams. Safe to call multiple times. */
     close(): void;
 }