@kici-dev/orchestrator 0.1.27 → 0.2.0

package/dist/scaler/config.d.ts

@@ -9,6 +9,13 @@
  */
 import { z } from 'zod';
 import type { ScalerConfig } from './types.js';
+/**
+ * Default cap on concurrent `backend.spawn` operations per backend when a
+ * scaler entry does not set `maxConcurrentSpawns`. Bounds the provisioning rate
+ * (image pull + create + start) so a burst of queued jobs cannot storm the
+ * container socket / registry.
+ */
+export declare const DEFAULT_MAX_CONCURRENT_SPAWNS = 8;
 /**
  * Zod schema for network policy configuration on a label set.
  * Controls RFC1918 and internet access for spawned agents.
@@ -67,8 +74,8 @@ export declare const labelSetConfigSchema: z.ZodObject<{
     containerSocket: z.ZodDefault<z.ZodBoolean>;
     env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
     backpressureMode: z.ZodOptional<z.ZodEnum<{
-        pause: "pause";
         drop: "drop";
+        pause: "pause";
     }>>;
     networkPolicy: z.ZodOptional<z.ZodObject<{
         allowlist: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -90,6 +97,7 @@ export declare const firecrackerNetworkSchema: z.ZodObject<{
     gateway: z.ZodDefault<z.ZodString>;
     netmask: z.ZodDefault<z.ZodString>;
     table: z.ZodDefault<z.ZodString>;
+    autoProvisionHost: z.ZodDefault<z.ZodBoolean>;
 }, z.core.$strip>;
 /**
  * Zod schema for the complete scaler configuration file.
@@ -160,11 +168,12 @@ export declare const scalerFileSchema: z.ZodObject<{
     scalers: z.ZodArray<z.ZodObject<{
         name: z.ZodString;
         type: z.ZodEnum<{
-            container: "container";
             "bare-metal": "bare-metal";
+            container: "container";
             firecracker: "firecracker";
         }>;
         maxAgents: z.ZodNumber;
+        maxConcurrentSpawns: z.ZodDefault<z.ZodNumber>;
         labelSets: z.ZodArray<z.ZodObject<{
             labels: z.ZodArray<z.ZodString>;
             image: z.ZodOptional<z.ZodString>;
@@ -212,8 +221,8 @@ export declare const scalerFileSchema: z.ZodObject<{
             containerSocket: z.ZodDefault<z.ZodBoolean>;
             env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
             backpressureMode: z.ZodOptional<z.ZodEnum<{
-                pause: "pause";
                 drop: "drop";
+                pause: "pause";
             }>>;
             networkPolicy: z.ZodOptional<z.ZodObject<{
                 allowlist: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -228,9 +237,9 @@ export declare const scalerFileSchema: z.ZodObject<{
         host: z.ZodOptional<z.ZodString>;
         socketPath: z.ZodOptional<z.ZodString>;
         runtime: z.ZodDefault<z.ZodEnum<{
+            auto: "auto";
             docker: "docker";
             podman: "podman";
-            auto: "auto";
         }>>;
         orchestratorUrl: z.ZodOptional<z.ZodString>;
         extraHosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -241,6 +250,17 @@ export declare const scalerFileSchema: z.ZodObject<{
             idleTimeoutSeconds: z.ZodDefault<z.ZodNumber>;
         }, z.core.$strip>>;
         mandatoryLabels: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        platform: z.ZodOptional<z.ZodObject<{
+            os: z.ZodEnum<{
+                linux: "linux";
+                macos: "macos";
+                windows: "windows";
+            }>;
+            arch: z.ZodEnum<{
+                arm64: "arm64";
+                x64: "x64";
+            }>;
+        }, z.core.$strict>>;
         roles: z.ZodPipe<z.ZodOptional<z.ZodArray<z.ZodEnum<{
             all: "all";
             builder: "builder";
@@ -274,6 +294,7 @@ export declare const scalerFileSchema: z.ZodObject<{
         gateway: z.ZodDefault<z.ZodString>;
         netmask: z.ZodDefault<z.ZodString>;
         table: z.ZodDefault<z.ZodString>;
+        autoProvisionHost: z.ZodDefault<z.ZodBoolean>;
     }, z.core.$strip>>;
 }, z.core.$strip>;
 /**

package/dist/scaler/container-backend.d.ts

@@ -44,6 +44,12 @@ export interface ContainerScalerBackendOptions {
     tokenStore?: AgentTokenStore;
     /** TTL for ephemeral agent tokens in ms. Default: 1 hour. */
     tokenTtlMs?: number;
+    /**
+     * Live resolver for the ephemeral agent-token TTL (ms). When set, called per
+     * spawn so the leader can serve the fleet-wide
+     * `cluster_settings.agent_token_ttl_ms` override; falls back to `tokenTtlMs`.
+     */
+    tokenTtlProvider?: () => Promise<number>;
     /** Agent roles for this scaler. undefined = all, [] = execution only. */
     roles?: string[];
 }
@@ -67,6 +73,8 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     private readonly tokenStore?;
     /** TTL for ephemeral agent tokens in ms */
     private readonly tokenTtlMs;
+    /** Live per-spawn resolver for the ephemeral agent-token TTL (ms). */
+    private readonly tokenTtlProvider?;
     /** Agent roles for this scaler. undefined = all, [] = execution only. */
     private readonly roles;
     /** ID of the isolated bridge network (set after creation/discovery) */
@@ -113,7 +121,7 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     get logsSource(): string;
     get labelSets(): LabelSetConfig[];
     getActiveCount(): number;
-    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext): Promise<ManagedAgent>;
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
     getScalerContext(agentId: string): Record<string, unknown> | undefined;
     destroy(managedId: string): Promise<void>;
     /**

package/dist/scaler/firecracker-backend.d.ts

@@ -19,7 +19,8 @@
 import { type ToolRequirement } from '@kici-dev/shared';
 import type { IpAllocator, IpAllocationResult } from './ip-allocator.js';
 import type { AgentTokenStore } from '../agent/token-store.js';
-import type { ScalerBackend, ScalerEntry, ManagedAgent, LabelSetConfig, ScalerEventCallback, ValidationResult, EffectiveLimits } from './types.js';
+import { type FirecrackerBridgeConfig, type BridgeHealth, type ExecOptions } from '../firecracker/host-network.js';
+import type { ScalerBackend, ScalerEntry, ManagedAgent, LabelSetConfig, ScalerEventCallback, ValidationResult, EffectiveLimits, SpawnContext } from './types.js';
 /**
  * Infrastructure interfaces that share the `kici-` prefix but must NEVER be
  * deleted by the orphan sweep. Bridges (kici-br0/kici-br1) carry all VM
@@ -73,6 +74,12 @@ export interface FirecrackerScalerBackendOptions {
     tokenStore?: AgentTokenStore;
     /** TTL for ephemeral agent tokens in ms. Default: 1 hour. */
     tokenTtlMs?: number;
+    /**
+     * Live resolver for the ephemeral agent-token TTL (ms). When set, called per
+     * spawn so the leader can serve the fleet-wide
+     * `cluster_settings.agent_token_ttl_ms` override; falls back to `tokenTtlMs`.
+     */
+    tokenTtlProvider?: () => Promise<number>;
     /** Agent roles for this scaler. undefined = all, [] = execution only. */
     roles?: string[];
     /**
@@ -83,6 +90,16 @@ export interface FirecrackerScalerBackendOptions {
      * @default false
      */
     requireSudo?: boolean;
+    /**
+     * When true, `ensureHostReady()` verifies and provisions this host bridge on
+     * startup (self-heal). Threaded from `firecracker.autoProvisionHost`. When
+     * false the method is a no-op. @default false (cores pass the real value)
+     */
+    autoProvisionHost?: boolean;
+    /** Injectable bridge verifier (tests). @default host-network verifyBridge */
+    verifyBridgeFn?: (cfg: FirecrackerBridgeConfig, opts: ExecOptions) => Promise<BridgeHealth>;
+    /** Injectable bridge provisioner (tests). @default host-network provisionBridge */
+    provisionBridgeFn?: (cfg: FirecrackerBridgeConfig, opts: ExecOptions) => Promise<void>;
 }
 export declare class FirecrackerScalerBackend implements ScalerBackend {
     readonly type: "firecracker";
@@ -109,8 +126,12 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     private readonly table;
     private readonly tokenStore?;
     private readonly tokenTtlMs;
+    private readonly tokenTtlProvider?;
     private readonly roles;
     private readonly requireSudo;
+    private readonly autoProvisionHost;
+    private readonly verifyBridgeFn;
+    private readonly provisionBridgeFn;
     /** Tracks all managed VM agents by ManagedAgent.id */
     private readonly agents;
     /** Periodic orphan sweep timer (null when stopped). */
@@ -124,7 +145,7 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     static getRequiredTools(entry: ScalerEntry): ToolRequirement[];
     get labelSets(): LabelSetConfig[];
     getActiveCount(): number;
-    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits): Promise<ManagedAgent>;
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, _spawnContext?: SpawnContext, _signal?: AbortSignal): Promise<ManagedAgent>;
     getScalerContext(agentId: string): Record<string, unknown> | undefined;
     /**
      * Bridge config for host-network diagnostics (read-only snapshot).
@@ -138,6 +159,12 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
         bridgeCidr: string;
         table: string;
     };
+    /**
+     * Verify and (if needed) provision this backend's host bridge on startup.
+     * No-op unless autoProvisionHost is set. Throws on a real provision failure;
+     * ScalerManager.ensureHostsReady catches per-backend and degrades this scaler.
+     */
+    ensureHostReady(): Promise<void>;
     destroy(managedId: string): Promise<void>;
     shutdownAll(): Promise<void>;
     /**

package/dist/scaler/index.d.ts

@@ -5,7 +5,7 @@
  * Supports Docker, bare-metal, and Firecracker backends with YAML configuration,
  * label-set matching, and warm pool management.
  */
-export type { ResourceLimits, LabelSetConfig, ManagedAgent, ScaleResult, ValidationResult, ScalerBackend, ScalerConfig, ScalerEntry, ScalerEvent, WarmPoolConfig, FirecrackerNetworkConfig, } from './types.js';
+export type { ResourceLimits, LabelSetConfig, ManagedAgent, ScaleResult, ValidationResult, ScalerBackend, ScalerConfig, ScalerEntry, ScalerEvent, ScalerRedispatchTrigger, WarmPoolConfig, FirecrackerNetworkConfig, } from './types.js';
 export { loadScalerConfig, scalerFileSchema, firecrackerNetworkSchema, parseMemoryString, } from './config.js';
 export { normalizeLabelSet, labelSetsMatch, detectLabelSetOverlaps, findBackendForLabels, } from './label-matcher.js';
 export { ContainerScalerBackend, detectRuntime } from './container-backend.js';

package/dist/scaler/manager.d.ts

@@ -10,6 +10,27 @@ import type { ResourceRequest } from '@kici-dev/engine';
 import type { BackendFailureSummary } from './failure-tracker.js';
 import type { ScalerBackend, ScalerConfig, ScaleResult, ScalerEvent, ResourceCap, ValidationResult } from './types.js';
 import type { ScalerStateStore, ScalerStateRecovery } from './scaler-state-store.js';
+/**
+ * Counting semaphore that throttles the number of concurrent async operations.
+ *
+ * Used to bound how many `backend.spawn` provisioning operations (image pull +
+ * container create + start) run at once per backend, so a burst of in-cap jobs
+ * cannot storm the container socket / registry.
+ *
+ * Slots are handed directly to the next waiter on release (rather than bumping a
+ * counter the woken task re-decrements), so a task arriving in the microtask gap
+ * between a release and a woken waiter cannot jump the queue and transiently
+ * exceed the cap. The slot is always released in `run`'s `finally`, so a failed
+ * or throwing operation never leaks a slot and wedges the queue.
+ */
+export declare class SpawnSemaphore {
+    private available;
+    private readonly waiters;
+    constructor(max: number);
+    private acquire;
+    private release;
+    run<T>(fn: () => Promise<T>): Promise<T>;
+}
 /**
  * Resolved per-job resource amounts (cpus + bytes) for both `requests` and
  * `limits`. The scaler manager produces this from the job's declared resources
@@ -115,6 +136,10 @@ export declare class ScalerManager {
     private readonly globalUsage;
     /** Per-scaler machine-pool name (set when scalers reference a pool). */
     private readonly scalerMachinePools;
+    /** Per-scaler cap on concurrent `backend.spawn` operations (provisioning-rate throttle). */
+    private readonly maxConcurrentSpawns;
+    /** Per-scaler spawn throttles, created lazily from `maxConcurrentSpawns`. */
+    private readonly spawnSemaphores;
     /** Outstanding reservations keyed by agentId; used to release on disconnect. */
     private readonly reservations;
     /**
@@ -160,6 +185,23 @@ export declare class ScalerManager {
      * External callback for relaying correlated scaler events (e.g. to execution tracker).
      */
     private readonly onScalerEvent?;
+    /** Coordinator drain predicate; when true, requestScale() no-ops (no fresh
+     *  agents spawned for the held Pending backlog). Defaults to never-draining. */
+    private readonly isDraining;
+    /**
+     * Optional callback fired (debounced) whenever scaler capacity frees — an
+     * ephemeral agent released its reservation on disconnect, a spawn failed, or a
+     * managed agent completed its job. Wired by the composition roots to the
+     * dispatcher's capacity-freed re-drive so jobs that got an `at-capacity`
+     * verdict get re-offered to the scaler the moment a slot opens, instead of
+     * sitting pending until they time out. Assigned after construction because it
+     * references the dispatcher, which is built after the manager.
+     */
+    onCapacityFreed?: () => void;
+    /** Trailing-debounce timer coalescing a burst of releases into one re-drive. */
+    private capacityFreedTimer;
+    /** Trailing debounce window for {@link notifyCapacityFreed}. */
+    private static readonly CAPACITY_FREED_DEBOUNCE_MS;
     private readonly warmPool;
     /**
      * Optional DB-backed state store. When wired (production path), every
@@ -169,6 +211,10 @@ export declare class ScalerManager {
      * Unit tests can omit the store and operate from in-memory Maps only.
      */
     private readonly stateStore?;
+    /** Cluster-wide default deadline (ms) for a single `backend.spawn`. */
+    private readonly spawnTimeoutMs;
+    /** Optional per-org spawn-deadline resolver (production path; DB-backed). */
+    private readonly resolveSpawnTimeoutMs?;
     constructor(deps: {
         config: ScalerConfig;
         backends: Array<{
@@ -194,17 +240,64 @@ export declare class ScalerManager {
             /** Orchestrator instance id (used in ledger rows for ownership). */
             instanceId: string;
         };
+        /** Coordinator drain predicate. When true, requestScale() declines to spawn
+         *  fresh capacity for the held Pending backlog. Defaults to never-draining. */
+        isDraining?: () => boolean;
+        /**
+         * Cluster-wide default deadline (ms) for a single `backend.spawn`. Always
+         * supplied (from config.scalerSpawnTimeoutMs); the per-org resolver, when
+         * present, overrides it per tenant.
+         */
+        spawnTimeoutMs: number;
+        /**
+         * Optional per-org resolver for the spawn deadline: given the job's org
+         * (jobConfig.cacheOrgId, undefined for warm-pool spawns), returns the
+         * effective timeout, falling back to the cluster default internally. Wired
+         * in orchestrator-core (has DB); omitted by the worker + unit tests → the
+         * cluster default spawnTimeoutMs is used.
+         */
+        resolveSpawnTimeoutMs?: (orgId: string | undefined) => Promise<number>;
     });
     /** Per-scaler mandatoryLabels (taint-style opt-in gate). */
     private readonly scalerMandatoryLabels;
+    /** Declared structured platform per scaler (undefined = host-derive / linux default). */
+    private readonly scalerPlatform;
+    /**
+     * Warn when a pool declares a plain platform-ish label (matched by the legacy
+     * denylist) but omits the structured `platform` field. The denylist is a
+     * migration shim; declaring `platform` is the canonical way to taint a pool.
+     */
+    private warnOnUnstructuredPlatformLabels;
+    /**
+     * Resolve the pool's platform for label + taint derivation. Precedence:
+     *   1. A declared `platform` field always wins (even on a linux host).
+     *   2. Otherwise a bare-metal pool derives its platform from the host OS/arch.
+     *   3. Otherwise (container / firecracker, undeclared) returns null — those
+     *      pools run linux agents and are not auto-tainted; only an explicit
+     *      declared platform or a legacy denylist label taints them.
+     * Returns null when a bare-metal host's OS/arch is not a supported enum value
+     * (the caller falls back to the raw host-label derivation for labels).
+     */
+    private resolveScalerPlatform;
+    /**
+     * Effective taint gate for a scaler: its configured mandatoryLabels plus the
+     * platform-taint labels derived from the pool's resolved structured platform,
+     * unioned with the legacy plain-label denylist shim. A pool declaring a
+     * non-default platform (`windows`, `macos`, `arm64`, …) only accepts jobs that
+     * request that platform. Container / firecracker pools resolve to linux and
+     * carry no taint; calling it uniformly is safe.
+     */
+    private effectiveMandatoryLabels;
     /**
      * Build enriched scaler entries with auto-labels injected into each label set.
      * This ensures label matching accounts for auto-injected labels (kici:role:*,
      * kici:os:*, kici:arch:*, kici:agent:*, kici:scaler:*) that agents receive
      * at spawn time.
      *
-     * Mandatory labels are surfaced unchanged from the per-scaler config so the
-     * label matcher can apply the gate alongside subset matching.
+     * The os/arch labels derive from the pool's resolved structured platform
+     * (declared field wins, else host-derive for bare-metal, else linux), and the
+     * mandatory labels combine the per-scaler config gate with the platform taint,
+     * so the label matcher applies the gate alongside subset matching.
      */
     private getEnrichedScalerEntries;
     /**
@@ -214,12 +307,28 @@ export declare class ScalerManager {
      * back to scaler defaults). Used by per-scaler / per-orchestrator / per-machine
      * resource caps; the cap math is wired up by ScalerManager itself.
      */
-    requestScale(labels: string[], jobId: string, runId: string, excludeLabels?: string[], resources?: ResourceRequest): Promise<ScaleResult>;
+    /**
+     * Run a single backend spawn under the resolved spawn deadline. Returns the
+     * ManagedAgent on success. On deadline: aborts the signal (so the backend's
+     * threaded container-runtime/HTTP calls cancel and its own catch cleans up)
+     * AND rejects, so the caller's semaphore `run()` finally releases the slot
+     * even if the backend's unwind stalls in a non-abortable spot. The backing
+     * spawn promise gets a no-op catch attached so a late abort-rejection doesn't
+     * surface as an unhandled rejection.
+     */
+    private runSpawnWithTimeout;
+    requestScale(labels: string[], jobId: string, runId: string, excludeLabels?: string[], resources?: ResourceRequest, orgId?: string): Promise<ScaleResult>;
     /**
      * Run a critical section under the reservation lock. Serializes concurrent
      * `requestScale()` calls so the check+reserve sequence is atomic.
      */
     private runWithReservationLock;
+    /**
+     * Lazily resolve the per-backend spawn throttle, sized by the scaler's
+     * `maxConcurrentSpawns` (falling back to the cluster default). One semaphore
+     * per backend name, since each backend has its own container socket.
+     */
+    private spawnSemaphoreFor;
     /**
      * Resolve the effective `{ requests, limits }` for a job at this scaler.
      *
@@ -259,6 +368,15 @@ export declare class ScalerManager {
      * references a pool.
      */
     private releaseAll;
+    /**
+     * Signal that scaler capacity freed. Trailing-debounced by
+     * {@link CAPACITY_FREED_DEBOUNCE_MS} so a burst of simultaneous releases (and
+     * the onJobComplete → onAgentDisconnect pair for a single-use agent) coalesces
+     * into one re-drive pass that runs after the last release settles. No-op when
+     * no callback is wired. The timer is `unref`'d so it never keeps the process
+     * alive, and cleared on shutdown.
+     */
+    private notifyCapacityFreed;
     /**
      * Subtract the given `requests` from the per-scaler and global in-memory
      * counters. Used both by `releaseAll()` and by the rollback path when a
@@ -284,7 +402,9 @@ export declare class ScalerManager {
      *   Used by the caller to eagerly dispatch the bound job before the
      *   agent's idle timer fires, skipping the generic queue drain race.
      * - `mandatoryLabels` (always populated for scaler-managed agents): the
-     *   spawning scaler's `mandatoryLabels` gate. Threaded into the
+     *   spawning scaler's effective `mandatoryLabels` gate — its configured
+     *   labels plus the platform taint derived from the pool's declared OS/arch.
+     *   Threaded into the
      *   AgentRegistry so the queue-drain path (`onAgentAvailable` →
      *   `dequeueForLabels`) and the eager-dispatch path
      *   (`dispatchBoundJob` → `dequeueById`) both reject queued jobs whose
@@ -359,6 +479,29 @@ export declare class ScalerManager {
      * Firecracker backend's `getBridgeConfig()`).
      */
     getBackend(name: string): ScalerBackend | undefined;
+    /**
+     * Whether some configured backend could spawn capacity for this label set —
+     * the same `findBackendForLabels` match `requestScale` makes, asked without
+     * spawning anything.
+     *
+     * Read by the queue-expiry sweep to tell a fleet/label problem apart from a
+     * capacity one: a job whose labels no agent AND no backend can serve could
+     * never have run (`unroutable`), while a job whose backend matched but whose
+     * spawn kept failing is a provisioning/capacity problem (`timed_out_stale`),
+     * and mislabelling the latter would tell an operator to fix a `runsOn` that
+     * is perfectly correct.
+     *
+     * Exact labels only — the scaler's own routing (`requestScale`) matches on
+     * exact labels, so a regex `runsOn` matcher is not expressible here. On a
+     * scaler-configured orchestrator that makes the answer conservative for a
+     * pattern-only `runsOn`: a backend whose labelSets carry no mandatory labels
+     * matches the empty exact set, so such a job is reported routable and settles
+     * `timed_out_stale` rather than `unroutable`. That loses precision, never
+     * safety — the job is still terminal and the run still fails, which is the
+     * invariant that matters. Tightening it means teaching scaler routing about
+     * patterns, which would change where jobs actually spawn.
+     */
+    hasBackendForLabels(labels: string[], excludeLabels?: string[]): boolean;
     /**
      * Get scaler-specific configuration metadata for a managed agent.
      * Returns undefined for non-scaler-managed (static) agents.
@@ -370,10 +513,25 @@ export declare class ScalerManager {
      */
     start(): void;
     /**
-     * Remove spawning entries older than 5 minutes.
-     * These represent agents that were spawned but never registered via WS
-     * (e.g., process/container crashed on startup). Without cleanup, these
-     * entries would leak in spawningAgents forever.
+     * Provision/heal every backend's host prerequisites before spawning starts.
+     * Awaits each backend's optional ensureHostReady, catching per-backend so one
+     * scaler's host-prep failure degrades only that scaler (its spawns will fail
+     * with clear errors + the firecracker-network diagnostic reports it) rather
+     * than aborting orchestrator startup.
+     */
+    ensureHostsReady(): Promise<void>;
+    /**
+     * Remove spawning entries whose `backend.spawn` started more than 5 minutes
+     * ago but never registered via WS (e.g., process/container crashed on
+     * startup). Without cleanup, these entries would leak in spawningAgents
+     * forever.
+     *
+     * The window is measured from `spawnStartedAt` (when the throttled spawn
+     * actually began), NOT from `spawnedAt` (enqueue time): a spawn still waiting
+     * behind the per-backend spawn semaphore has not started and must not be
+     * reaped mid-queue during a large burst. Recovered orphan entries carry
+     * `spawnStartedAt` set to their persisted enqueue time, so they are subject to
+     * the normal window.
      */
     private pruneStaleSpawningEntries;
     private persistSpawningAgent;

package/dist/scaler/pending-scale-sweeper.d.ts

@@ -0,0 +1,35 @@
+export interface PendingScaleSweeperOptions {
+    /**
+     * Re-drive callback — the dispatcher's `retryPendingScaleRequests` bound to
+     * the `sweep` trigger. Returns the number of jobs re-driven (logged when > 0).
+     */
+    redrive: () => Promise<number>;
+    /** Sweep interval in milliseconds. */
+    intervalMs: number;
+}
+/**
+ * Leader-gated periodic backstop for the scaler capacity-freed re-drive.
+ *
+ * The capacity-freed hook (`ScalerManager.onCapacityFreed`) handles the common
+ * case with near-zero latency, but it can miss edge cases: a spawn that failed
+ * silently, a reservation freed on a non-leader coordinator, or a lost event.
+ * This sweeper re-runs the SAME `retryPendingScaleRequests` on a timer so a
+ * pending at-capacity job is never stranded past one interval.
+ *
+ * Modeled on `HostRosterReaper` / `EventRetryScanner`: started/stopped via the
+ * Raft leadership callbacks, a single timer cleared on `stop()`. Leader-only so
+ * multiple coordinators don't each drive the same queue.
+ */
+export declare class PendingScaleSweeper {
+    private readonly redrive;
+    private readonly intervalMs;
+    private timer;
+    private isLeader;
+    constructor(opts: PendingScaleSweeperOptions);
+    onBecomeLeader(): void;
+    onLoseLeadership(): void;
+    stop(): void;
+    /** One sweep pass. Public for tests. Never throws (logs and keeps ticking). */
+    tick(): Promise<void>;
+}
+//# sourceMappingURL=pending-scale-sweeper.d.ts.map

package/dist/scaler/types.d.ts

@@ -6,7 +6,7 @@
  */
 import { z } from 'zod';
 import { ScalerEventType } from '@kici-dev/engine';
-import type { ResourceRequest, ResourceSpec, ScalerBackendType } from '@kici-dev/engine';
+import type { ResourceRequest, ResourceSpec, ScalerBackendType, ScalerPlatform } from '@kici-dev/engine';
 export type { ResourceRequest, ResourceSpec } from '@kici-dev/engine';
 export { ScalerEventType } from '@kici-dev/engine';
 /**
@@ -73,6 +73,22 @@ export interface NetworkPolicy {
     /** Block all outbound traffic except allowlisted ranges */
     denyAll?: boolean;
 }
+/**
+ * Image pull policy for container-backend agent images.
+ *
+ * - `IfNotPresent` (default): pull only when the image is not already local.
+ *   KiCI agent images are pinned + immutable, so re-pulling on every spawn only
+ *   storms the registry/socket.
+ * - `Always`: re-pull on every spawn. Set this on a label set that tracks a
+ *   moving tag (e.g. `:latest`) or otherwise needs a fresh image each spawn.
+ * - `Never`: never pull; fail if the image is absent.
+ */
+export declare const ImagePullPolicy: z.ZodEnum<{
+    Always: "Always";
+    IfNotPresent: "IfNotPresent";
+    Never: "Never";
+}>;
+export type ImagePullPolicy = z.infer<typeof ImagePullPolicy>;
 /**
  * Configuration for a single label-set mapping within a scaler backend.
  * Maps an exact set of labels to the agent provisioning details.
@@ -82,8 +98,8 @@ export interface LabelSetConfig {
     labels: string[];
     /** Container image (Docker backend) */
     image?: string;
-    /** Image pull policy: Always (default), IfNotPresent, or Never */
-    imagePullPolicy?: 'Always' | 'IfNotPresent' | 'Never';
+    /** Image pull policy: IfNotPresent (default), Always, or Never */
+    imagePullPolicy?: ImagePullPolicy;
     /** Path to agent binary (bare-metal backend) */
     binaryPath?: string;
     /**
@@ -154,7 +170,23 @@ export type ScaleResult = {
 } | {
     action: 'failed';
     error: string;
+}
+/** Scale-up declined without attempting a spawn (e.g. coordinator draining). */
+ | {
+    action: 'skipped';
+    reason: string;
 };
+/**
+ * What triggered a pending-scale re-drive (`Dispatcher.retryPendingScaleRequests`):
+ * the near-zero-latency capacity-freed `hook` fired from `ScalerManager`, or the
+ * leader-gated periodic `sweep` backstop. Used as the `trigger` label on the
+ * `kici_orch_scaler_redispatch_total` counter.
+ */
+export declare const ScalerRedispatchTrigger: z.ZodEnum<{
+    hook: "hook";
+    sweep: "sweep";
+}>;
+export type ScalerRedispatchTrigger = z.infer<typeof ScalerRedispatchTrigger>;
 /**
  * Result of a configuration validation or reload operation.
  */
@@ -184,7 +216,8 @@ export interface ScalerBackend {
      * True for bare-metal and Firecracker (local processes / local VMs) and for
      * container backends using a local runtime socket; false when the backend
      * provisions elsewhere (remote container runtime, future cloud backends).
-     * Drives the static spawning-host display on the diagnostics page.
+     * Drives the static spawning-host display on the dashboard Infrastructure
+     * page.
      */
     readonly spawnsOnLocalHost: boolean;
     /** Current count of agents managed by this backend (including spawning) */
@@ -205,7 +238,14 @@ export interface ScalerBackend {
      * @returns The managed agent tracking object
      * @throws If the label set is not supported by this backend
      */
-    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext): Promise<ManagedAgent>;
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, 
+    /**
+     * Optional abort signal from the ScalerManager spawn-timeout wrapper. When
+     * aborted, backends thread it into any long-running provisioning I/O
+     * (container-runtime requests, HTTP calls) so a hung provision cancels
+     * instead of pinning the per-backend spawn-semaphore slot.
+     */
+    signal?: AbortSignal): Promise<ManagedAgent>;
     /**
      * Destroy a specific managed agent.
      * Docker: docker rm -f; Bare-metal: SIGTERM -> SIGKILL
@@ -224,6 +264,14 @@ export interface ScalerBackend {
      * Used by the orchestrator to enrich job.context before forwarding to Platform.
      */
     getScalerContext?(agentId: string): Record<string, unknown> | undefined;
+    /**
+     * Provision or heal this backend's host prerequisites before it spawns.
+     * Called once by ScalerManager during startup. Optional — only backends with
+     * host-side setup (Firecracker's bridge) implement it; container/bare-metal
+     * omit it. May throw; the manager catches per-backend and degrades that
+     * scaler rather than aborting orchestrator startup.
+     */
+    ensureHostReady?(): Promise<void>;
     /**
      * Reload configuration (called on SIGHUP).
      * Returns validation errors if new config is invalid.
@@ -250,6 +298,12 @@ export interface FirecrackerNetworkConfig {
     netmask?: string;
     /** nft table name for this coordinator's host bridge @default 'kici' */
     table?: string;
+    /**
+     * When true (default), the orchestrator verifies and provisions this host
+     * bridge on startup (self-heal), so a fresh Firecracker host needs no manual
+     * `kici-admin firecracker provision`. @default true
+     */
+    autoProvisionHost?: boolean;
 }
 /**
  * Parsed and validated scaler configuration from YAML.
@@ -293,8 +347,15 @@ export interface ScalerEntry {
     name: string;
     /** Backend type */
     type: Exclude<ScalerBackendType, 'kubernetes'>;
-    /** Maximum concurrent agents for this scaler */
+    /** Maximum concurrent agents for this scaler (population cap) */
     maxAgents: number;
+    /**
+     * Maximum concurrent `backend.spawn` operations for this scaler
+     * (provisioning-rate throttle). Orthogonal to `maxAgents`: excess in-cap
+     * spawns queue and drain at this rate rather than storming the socket.
+     * Defaults to `DEFAULT_MAX_CONCURRENT_SPAWNS` at config-load.
+     */
+    maxConcurrentSpawns: number;
     /** Label-set to image/binary mappings */
     labelSets: LabelSetConfig[];
     /** Container runtime host (e.g. 'tcp://192.168.1.10:2376'). Works for both Docker and Podman remote. */
@@ -319,6 +380,14 @@ export interface ScalerEntry {
      * `labelSets`. Default: `[]` (no gating).
      */
     mandatoryLabels?: string[];
+    /**
+     * Structured platform of this pool's agents. When set, the auto-injected
+     * `kici:os:*` / `kici:arch:*` labels AND the mandatory platform taint both
+     * derive from this single field. Optional: when omitted, bare-metal pools
+     * derive the platform from the host OS/arch, and container / firecracker
+     * pools default to linux.
+     */
+    readonly platform?: ScalerPlatform;
     /**
      * Agent roles this scaler handles. scaler-entry level, not per-label-set.
      * - undefined (not set): handles all job types including build and init (backward compat,)