@kici-dev/orchestrator 0.1.13 → 0.1.15

package/dist/pipeline/inline-eval.d.ts

@@ -5,11 +5,12 @@
  * in a sandboxed VM context at dispatch time, eliminating the init-job
  * round-trip for pure functions.
  */
+import type { LockJob } from '@kici-dev/engine';
 /**
  * Evaluate an inline expression that returns a string.
  *
  * @param expression - Serialized arrow function, e.g. '(event) => event.ref.split("/").pop()'
- * @param event - Normalized webhook event payload
+ * @param event - Normalized event envelope (SimulatedEvent shape; raw provider payload at event.payload)
  * @returns The string result
  * @throws TypeError if result is not a string
  * @throws Error on timeout or sandbox violation
@@ -19,10 +20,24 @@ export declare function evaluateInlineString(expression: string, event: object):
  * Evaluate an inline expression that returns a Record<string, string>.
  *
  * @param expression - Serialized arrow function, e.g. '(event) => ({ NODE_ENV: event.env })'
- * @param event - Normalized webhook event payload
+ * @param event - Normalized event envelope (SimulatedEvent shape; raw provider payload at event.payload)
  * @returns The record result
  * @throws TypeError if result is not a plain object
  * @throws Error on timeout or sandbox violation
  */
 export declare function evaluateInlineRecord(expression: string, event: object): Record<string, string>;
+/**
+ * Evaluate a lock job's inline (pure-function) dynamic fields against the
+ * normalized event envelope (SimulatedEvent shape: { type, action,
+ * targetBranch, sourceBranch, payload, … }). The raw provider webhook body is
+ * nested at `event.payload` — the same shape rules and step contexts see.
+ *
+ * Throws a job-attributed Error when any expression throws; inline evaluation
+ * failures are immediate dispatch failures (no init-job fallback).
+ */
+export declare function evaluateInlineFields(lockJob: LockJob, event: object): {
+    inlineEnvironmentName: string | undefined;
+    inlineEnv: Record<string, string> | undefined;
+    inlineConcurrencyGroup: string | undefined;
+};
 //# sourceMappingURL=inline-eval.d.ts.map

package/dist/pipeline/process-webhook.d.ts

@@ -16,8 +16,26 @@
  * Internal helpers are pure phase functions returning typed results; the only
  * top-level export is `processWebhook`, callable from server.ts / app.ts.
  */
+import type { SimulatedEvent } from '@kici-dev/engine';
 import type { WebhookInfo } from '../webhook/handler.js';
+import type { ProviderBundle } from '../provider-registry.js';
+import type { TrustResolution } from '../security/trust-resolver.js';
 import { type ProcessingDeps } from './processor.js';
+interface TrustOutcome {
+    trustResolution: TrustResolution | undefined;
+    /** Default 'base' for PR events; trust resolution may override to 'head'. */
+    lockFileSource: 'head' | 'base';
+}
+export declare function resolveTrustForPR(args: {
+    info: WebhookInfo;
+    deps: ProcessingDeps;
+    bundle: ProviderBundle;
+    event: SimulatedEvent;
+    payload: Record<string, unknown>;
+    resolvedOrgId: string;
+    repoIdentifier: string;
+    credentials: Record<string, unknown>;
+}): Promise<TrustOutcome>;
 /**
  * Process a webhook through the complete pipeline.
  *
@@ -35,4 +53,5 @@ import { type ProcessingDeps } from './processor.js';
  *  10. Forward Platform trace + record event log
  */
 export declare function processWebhook(info: WebhookInfo, deps: ProcessingDeps): Promise<void>;
+export {};
 //# sourceMappingURL=process-webhook.d.ts.map

package/dist/pipeline/processor.d.ts

@@ -34,6 +34,7 @@ import type { LogStorage } from '../reporting/log-storage.js';
 import type { SecretResolver } from '../secrets/secret-resolver.js';
 import type { ContributorCache } from '../security/contributor-cache.js';
 import type { LockFile as FullLockFile, LockWorkflow, SimulatedEvent, WebhookNormalizer } from '@kici-dev/engine';
+import { LockFileParseError } from '@kici-dev/engine';
 import type { EventRouter } from '../events/event-router.js';
 import type { RegistrationStore } from '../registration/registration-store.js';
 import type { RegistrationIndex } from '../registration/registration-index.js';
@@ -158,7 +159,7 @@ export declare function resolveLockFileWithFallback(args: {
     deliveryId: string;
 }): Promise<{
     lockFile: FullLockFile | null;
-    resolvedVia: 'inbound' | 'fallback' | 'miss';
+    resolvedVia: 'inbound' | 'fallback' | 'miss' | 'corrupt';
     fallbackRoutingKey?: string;
     /** The winning provider bundle when resolvedVia='fallback'. Used by the dispatch
      *  site to swap repoUrlBuilder and cloneTokenProvider (Layer 4 cross-provider fix). */
@@ -166,6 +167,10 @@ export declare function resolveLockFileWithFallback(args: {
     /** The winning registration's providerContext when resolvedVia='fallback'.
      *  Carries installationId etc. for clone token issuance. */
     fallbackCredentials?: Record<string, unknown>;
+    /** Set when resolvedVia='corrupt': the parse error seen while attempting to
+     *  resolve a lock file. A valid fallback always wins over a corrupt inbound,
+     *  so this is only surfaced when NOTHING resolved. */
+    corruptError?: LockFileParseError;
 }>;
 /**
  * Resolve the customer/org ID for a routing key.

package/dist/pipeline/test-pipeline.d.ts

@@ -25,6 +25,8 @@ import type { BuildCoordinator } from '../cache/index.js';
 import type { DepCache } from '../cache/index.js';
 import type { PendingBuildTracker } from '../cache/index.js';
 import type { SecretResolver } from '../secrets/secret-resolver.js';
+import type { EnvironmentStore } from '../environments/environment-store.js';
+import type { VariableStore } from '../environments/variable-store.js';
 import type { LogStorage } from '../reporting/log-storage.js';
 import type { Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
@@ -49,6 +51,10 @@ export interface TestTriggerInput {
     uploadId?: string;
     /** Fixture secret context mappings (optional). */
     secrets?: Record<string, string>;
+    /** Base64 X25519+AES-GCM blob of the developer's local secrets ({flat, contexts}). */
+    encryptedSecrets?: string;
+    /** Base64 ephemeral CLI public key that encrypted `encryptedSecrets`. */
+    encryptedSecretsKey?: string;
     /** Direct workflow run -- bypass triggers (optional). */
     workflowName?: string;
     /** Resolved overlay metadata for tarball download + decryption. */
@@ -97,6 +103,10 @@ export interface TestPipelineDeps {
     logStorage?: LogStorage;
     /** Database connection for environment protection checks. Optional. */
     db?: Kysely<Database>;
+    /** Environment store for resolving environment ids in test dispatch parity. Optional. */
+    environmentStore?: EnvironmentStore;
+    /** Variable store for resolving environment variables in test dispatch parity. Optional. */
+    variableStore?: VariableStore;
 }
 /**
  * Process a test trigger through the existing pipeline.

package/dist/providers/github/lock-file.d.ts

@@ -7,7 +7,7 @@
  * NOTE: This is the raw fetcher without caching. The orchestrator wraps
  * this with an LRU cache (LockFileCache) for production use.
  */
-import type { LockFileFetcher, LockFile } from '@kici-dev/engine';
+import { type LockFileFetcher, type LockFile } from '@kici-dev/engine';
 import { type GitHubAppConfig } from './auth.js';
 /**
  * GitHub-specific implementation of LockFileFetcher.

package/dist/providers/internal/lock-file-fetcher.d.ts

@@ -25,10 +25,11 @@ export declare class InternalLockFileFetcher implements LockFileFetcher {
      * Fetch the lock file from the local filesystem.
      *
      * @param repoIdentifier - Either a file:// URL or a relative path under repoBasePath
-     * @param _ref - Git ref (ignored for filesystem access -- always reads current state)
+     * @param ref - Git ref (ignored for filesystem access -- always reads current state)
      * @param _credentials - Not used (file:// access needs no auth)
      * @returns Parsed LockFile, or null if not found
+     * @throws LockFileParseError when the file is present but unparseable/invalid
      */
-    fetchLockFile(repoIdentifier: string, _ref: string, _credentials: unknown): Promise<LockFile | null>;
+    fetchLockFile(repoIdentifier: string, ref: string, _credentials: unknown): Promise<LockFile | null>;
 }
 //# sourceMappingURL=lock-file-fetcher.d.ts.map

package/dist/queue/job-queue.d.ts

@@ -43,6 +43,13 @@ export declare enum DispatchQueueStatus {
     Expired = "expired",
     Recovering = "recovering"
 }
+/**
+ * Maximum delivery attempts for a single dispatch_queue job. A job whose
+ * `dispatch_attempts` reaches this value is failed permanently instead of
+ * being requeued again. Bounds requeue loops from repeated job.reject /
+ * pre-start agent loss; `expires_at` is the time-based backstop.
+ */
+export declare const MAX_DISPATCH_ATTEMPTS = 5;
 /**
  * Input for enqueuing a job. Callers provide these fields;
  * the queue generates id, status, created_at, and expires_at.
@@ -307,6 +314,34 @@ export declare class JobQueue {
      * the leader-gated sweep (`sweepExpiredRecoveries()`).
      */
     markRecovering(jobId: string, deadline?: Date, agentId?: string): Promise<void>;
+    /**
+     * Stamp the dispatch-acknowledgment deadline for a dispatched job.
+     * Only touches rows still in 'dispatched' for safety.
+     */
+    setAckDeadline(jobId: string, deadline: Date, agentId: string): Promise<void>;
+    /** Clear the ack deadline (agent answered, or the job left 'dispatched'). */
+    clearAckDeadline(jobId: string): Promise<void>;
+    /**
+     * List dispatched rows still awaiting an ack (non-null deadline). Used at
+     * coord boot (`Dispatcher.recoverState()`) to re-arm in-memory timers.
+     */
+    getDispatchedAwaitingAck(): Promise<Array<{
+        id: string;
+        runId: string;
+        agentId: string | null;
+        deadline: Date;
+    }>>;
+    /**
+     * List every dispatched row whose ack deadline is in the past. The caller
+     * (leader-gated `Dispatcher.sweepExpiredAckDeadlines`) requeues each via
+     * the atomic `requeue()` (WHERE status='dispatched'), so racing coords
+     * cannot double-requeue.
+     */
+    listExpiredAckDeadlines(now: Date): Promise<Array<{
+        id: string;
+        runId: string;
+        agentId: string | null;
+    }>>;
     /**
      * List every job currently in `recovering` state with its persisted
      * recovery deadline. Used at coord boot (`Dispatcher.recoverState()`)
@@ -325,7 +360,7 @@ export declare class JobQueue {
     /**
      * Sweep every `recovering` row whose `recovery_deadline` is in the
      * past, marking them `failed`. Returns the rows that flipped so the
-     * caller can fire the per-job `onRecoveryTimeout` hook in process.
+     * caller can fire the per-job `onJobFailedPermanently` hook in process.
      *
      * Intended for the leader-gated `Dispatcher.sweepExpiredRecoveries`
      * tick — running on N coords would still be correct (the WHERE
@@ -343,6 +378,23 @@ export declare class JobQueue {
      * @returns true if the update affected a row (job was still recovering).
      */
     markFailedIfRecovering(jobId: string, _reason: string): Promise<boolean>;
+    /**
+     * Return a dispatched job to the pending queue for re-dispatch, bumping
+     * its attempt counter. Used when an agent explicitly rejects a dispatch
+     * (job.reject) and when a scaler-managed agent disconnects before the
+     * job started. Only flips rows still in 'dispatched' — a job that was
+     * concurrently completed / failed / cancelled is left untouched.
+     *
+     * @returns the post-increment dispatch_attempts, or null when the row
+     *   was not in 'dispatched' state (nothing requeued).
+     */
+    requeue(jobId: string): Promise<number | null>;
+    /**
+     * Get the full QueuedJob row by ID regardless of status. Used by the
+     * dispatcher's redispatch path, which needs runsOnLabels / excludeLabels /
+     * resources to pick an agent or consult the scaler for a requeued job.
+     */
+    getFullJobById(jobId: string): Promise<QueuedJob | null>;
     /**
      * Mark a job as dispatched only if it is still in 'recovering' state.
      * Used when an agent reconnects and claims a recovering job.

package/dist/reporting/execution-tracker.d.ts

@@ -13,7 +13,7 @@
  */
 import { type Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
-import { ExecutionRunStatus, ScalerEventType } from '@kici-dev/engine';
+import { ExecutionRunStatus, type InitFailure, ScalerEventType } from '@kici-dev/engine';
 import type { ObserverRegistry } from '../ws/observer-registry.js';
 import type { LogStorage } from './log-storage.js';
 import type { JobQueue } from '../queue/job-queue.js';
@@ -92,7 +92,13 @@ export interface ExecutionTrackerDeps {
      * Set on terminal run states only. Powers the operator-side
      * `kici_org_log_bytes` capacity-planning gauge on the Platform.
      */
-    logBytes?: number) => void;
+    logBytes?: number, 
+    /**
+     * Structured init-failure signal. Set when the run never executed a step
+     * because of an init-phase failure. Forwarded to Platform's execution.status
+     * forward and persisted in execution_runs.init_failure on both sides.
+     */
+    initFailure?: InitFailure) => void;
     /** Optional callback to forward job status changes to Platform.
      *  Fires on every job state transition (pending->running, running->success, etc.).
      *  Used to populate Platform's execution_jobs projection table. */
@@ -101,7 +107,12 @@ export interface ExecutionTrackerDeps {
      * Total raw log bytes accumulated across the job (sum of per-step totals).
      * Set on terminal job states only.
      */
-    logBytes?: number) => void;
+    logBytes?: number, 
+    /**
+     * Structured init-failure signal. Set for synthetic rejected-* / init-failed-*
+     * jobs that never started. Persisted in execution_jobs.init_failure.
+     */
+    initFailure?: InitFailure) => void;
     /**
      * Optional callback to emit run.event messages to Platform.
      * Fires at orchestrator lifecycle points (dispatch, agent assignment, job start/complete).
@@ -142,6 +153,22 @@ export declare class ExecutionTracker {
     private readonly orgId?;
     private readonly jobQueue?;
     private readonly runs;
+    /**
+     * Per-run async-mutex chain. `onJobStatus` and `addJobsToRun` mutate the same
+     * `run.jobs` Map / `execution_jobs` row; without serialization a job-status
+     * reply that lands mid synthetic→real swap (`addJobsToRun`) clobbers the swap
+     * and the run hangs in `running` forever (see `withRunLock`). The map value is
+     * the tail of the chain for that runId; entries are GC'd when the last holder
+     * releases.
+     */
+    private readonly runLockTails;
+    /**
+     * Tracks which runIds the current async context already holds in
+     * `withRunLock`, so reentrant calls (e.g. onJobStatus → scheduler hook →
+     * dispatchReadyJob → addJobsToRun, all the same runId) bypass re-acquisition
+     * instead of deadlocking. Propagates across awaits via AsyncLocalStorage.
+     */
+    private readonly heldRunLocks;
     /** Tracks which runs are test runs for observer broadcasting. */
     private readonly testRunIds;
     /**
@@ -192,7 +219,9 @@ export declare class ExecutionTracker {
     concurrency?: {
         cancelInProgress?: boolean;
         max?: number;
-    }): Promise<void>;
+    }, 
+    /** Workflow-level wall-clock timeout in ms from the lock file. Sets the run deadline. */
+    workflowTimeoutMs?: number): Promise<void>;
     /**
      * Add additional jobs to an already-started execution run.
      * Used when build jobs are tracked early and regular jobs are dispatched later.
@@ -208,6 +237,22 @@ export declare class ExecutionTracker {
      * which peer owns the downstream dispatch.
      */
     findSyntheticJobId(runId: string, jobName: string): Promise<string | undefined>;
+    /**
+     * Run `fn` while holding a per-run lock, serializing the run-mutating methods
+     * (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
+     * synthetic→real job swap and wedge the run in `running`.
+     *
+     * The lock is **reentrant**: three paths re-enter a locked method within the
+     * same async context for the same runId — onJobStatus → scheduler hook →
+     * dispatchReadyJob → addJobsToRun; onJobStatus → enforceSchedulerInvariant →
+     * onJobStatus; runSchedulerHook → onJobStatus (skip). A non-reentrant mutex
+     * would deadlock on these, so a context that already holds the runId's lock
+     * (tracked via `heldRunLocks`) runs `fn` inline. A genuinely concurrent caller
+     * for the same runId (a separate WS message) is a different async context and
+     * correctly waits. All reentrant paths are same-runId, so there is no
+     * cross-run lock-ordering deadlock.
+     */
+    private withRunLock;
     addJobsToRun(runId: string, jobs: Array<{
         jobId: string;
         jobName: string;
@@ -216,6 +261,7 @@ export declare class ExecutionTracker {
     }>, dispatchedContexts?: string[], 
     /** Synthetic job ID to replace (e.g. needs-pending-deploy-{uuid}). */
     replaceSyntheticId?: string): Promise<void>;
+    private addJobsToRunImpl;
     /**
      * Mark a run as a test run for observer broadcasting.
      * Called by the test pipeline after creating the execution run.
@@ -233,6 +279,7 @@ export declare class ExecutionTracker {
      * fires the onExecutionComplete callback.
      */
     onJobStatus(runId: string, jobId: string, state: string, timestamp: number, agentId?: string, data?: Record<string, unknown>): Promise<void>;
+    private onJobStatusImpl;
     /**
      * Phase 1a: recover run state from the DB when in-memory tracking is empty.
      * Returns the rehydrated RunState or null if the run is unknown to the DB
@@ -313,7 +360,7 @@ export declare class ExecutionTracker {
      * onExecutionStarted but the build subsequently fails. Without this,
      * the execution_runs row would stay in a non-terminal state.
      */
-    onBuildFailed(runId: string): Promise<void>;
+    onBuildFailed(runId: string, initFailure?: InitFailure): Promise<void>;
     /**
      * Create a failed execution run when the build timed out before onExecutionStarted
      * had a chance to insert the row (buildJobTrackedEarly was false).
@@ -321,7 +368,33 @@ export declare class ExecutionTracker {
      * Inserts a minimal execution_runs row with status='failed' directly so the E2E
      * test (and dashboard) can observe the failure instead of a missing run.
      */
-    onBuildFailedBeforeTracking(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, routingKey: string, triggerEvent?: string, commitMessage?: string, failureReason?: string): Promise<void>;
+    onBuildFailedBeforeTracking(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, routingKey: string, triggerEvent?: string, commitMessage?: string, failureReason?: string, initFailure?: InitFailure): Promise<void>;
+    /**
+     * Insert a `failed` execution_runs row directly for an init failure that
+     * occurred BEFORE onExecutionStarted ran (so no in-memory state exists
+     * and no jobs were dispatched). Also writes the structured init_failure
+     * signal and fires onExecutionStatusChange so Platform's projection picks
+     * it up via the normal forward path. Idempotent: if a row already exists
+     * for this runId, the insert is a no-op (ON CONFLICT DO NOTHING).
+     *
+     * Closes the silent pre-run-failure gap — without this helper, secret /
+     * install-secret / all-jobs-rejected early-exits in dispatch-matched-workflow
+     * leave no trace on the dashboard.
+     */
+    recordInitFailureRun(args: {
+        runId: string;
+        workflowName: string;
+        provider: string;
+        repoIdentifier: string;
+        ref: string;
+        sha: string;
+        deliveryId: string | null;
+        providerContext: Record<string, unknown>;
+        routingKey: string;
+        initFailure: InitFailure;
+        triggerEvent?: string;
+        commitMessage?: string;
+    }): Promise<void>;
     /**
      * Mark a run as failed immediately with a reason message.
      *
@@ -330,7 +403,7 @@ export declare class ExecutionTracker {
      * Instead of leaving the run in 'running' for OrphanRecovery to catch after 5 min,
      * this fails it right away.
      */
-    failRun(runId: string, reason: string): Promise<void>;
+    failRun(runId: string, reason: string, initFailure?: InitFailure): Promise<void>;
     /**
      * Update step status within a job.
      *

package/dist/routes/admin-environments.d.ts

@@ -6,6 +6,7 @@
  *   PATCH  /api/v1/admin/environments/:name/policy                       — update policy fields
  *   GET    /api/v1/admin/environments?orgId=<id>                         — list environments
  *   GET    /api/v1/admin/environments/:name?orgId=<id>                   — show env + vars + bindings
+ *   DELETE /api/v1/admin/environments/:name?orgId=<id>                   — delete env (cascades bindings/variables/overrides; pending held runs block with 409)
  *   POST   /api/v1/admin/environments/templates                          — create/update a template
  *   GET    /api/v1/admin/environments/:name/variables?orgId=<id>         — list org-level variables
  *   PUT    /api/v1/admin/environments/:name/variables/:key?orgId=<id>    — upsert variable

package/dist/scaler/bare-metal-backend.d.ts

@@ -39,6 +39,7 @@ export interface BareMetalScalerBackendOptions {
 }
 export declare class BareMetalScalerBackend implements ScalerBackend {
     readonly type: "bare-metal";
+    readonly spawnsOnLocalHost = true;
     readonly logsSource = "bare-metal";
     readonly maxAgents: number;
     private _labelSets;

package/dist/scaler/container-backend.d.ts

@@ -8,7 +8,7 @@
  */
 import { type ToolRequirement } from '@kici-dev/shared';
 import type { AgentTokenStore } from '../agent/token-store.js';
-import type { ScalerBackend, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
+import type { ScalerBackend, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, SpawnContext, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
 /**
  * Result of runtime detection.
  */
@@ -49,6 +49,7 @@ export interface ContainerScalerBackendOptions {
 }
 export declare class ContainerScalerBackend implements ScalerBackend {
     readonly type: "container";
+    readonly spawnsOnLocalHost: boolean;
     readonly maxAgents: number;
     private _labelSets;
     private readonly name;
@@ -112,7 +113,7 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     get logsSource(): string;
     get labelSets(): LabelSetConfig[];
     getActiveCount(): number;
-    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits): Promise<ManagedAgent>;
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext): Promise<ManagedAgent>;
     getScalerContext(agentId: string): Record<string, unknown> | undefined;
     destroy(managedId: string): Promise<void>;
     /**

package/dist/scaler/failure-tracker.d.ts

@@ -0,0 +1,46 @@
+/**
+ * In-process bounded record of recent scaler spawn failures.
+ *
+ * The scaler manager records every `scaler.failed` event here at the same site
+ * it increments the fleet-wide Prometheus counter. The diagnose scaler check
+ * reads recent failures grouped per backend instance to produce its rows. This
+ * is memory-only and bounded — the recent-failures window resets on restart,
+ * which is acceptable for an on-demand operator view over a short window.
+ */
+/** A single recorded scaler spawn failure. */
+export interface ScalerFailureRecord {
+    /** Scaler instance name (the configured scaler `name`); the diagnose row key. */
+    backendName: string;
+    /** Backend type: 'container' | 'bare-metal' | 'firecracker' | 'unknown'. */
+    backendType: string;
+    /** True when the failed spawn was bound to a queued job (a run was affected). */
+    bound: boolean;
+    /** Captured error string from the scaler event detail. */
+    detail: string;
+    /** Event timestamp in epoch milliseconds. */
+    timestampMs: number;
+}
+/** Per-backend summary of recent failures within a window. */
+export interface BackendFailureSummary {
+    backendType: string;
+    boundCount: number;
+    unboundCount: number;
+    /** Detail of the most recent failure in the window. */
+    lastError: string;
+    /** Timestamp of the most recent failure in the window. */
+    lastAtMs: number;
+}
+export declare class ScalerFailureTracker {
+    private readonly records;
+    private readonly maxEntries;
+    constructor(maxEntries?: number);
+    /** Record a failure, evicting the oldest entry when over capacity. */
+    record(rec: ScalerFailureRecord): void;
+    /**
+     * Group failures newer than `nowMs - windowMs` by backend instance name.
+     * `nowMs` is injected so callers control the clock (and tests are
+     * deterministic).
+     */
+    recentByBackend(windowMs: number, nowMs: number): Map<string, BackendFailureSummary>;
+}
+//# sourceMappingURL=failure-tracker.d.ts.map

package/dist/scaler/firecracker-backend.d.ts

@@ -82,6 +82,7 @@ export interface FirecrackerScalerBackendOptions {
 }
 export declare class FirecrackerScalerBackend implements ScalerBackend {
     readonly type: "firecracker";
+    readonly spawnsOnLocalHost = true;
     readonly maxAgents: number;
     readonly logsSource = "firecracker-serial";
     /** AbortControllers for file tailing per managed VM (keyed by agent ID) */
@@ -211,6 +212,22 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * Promisified execFile wrapper with 30s timeout.
      */
     private execAsync;
+    /**
+     * Remove a VM's jailer chroot directory.
+     *
+     * On rootless nodes (requireSudo), the jailer chowns the chroot contents to
+     * the jailer uid/gid, leaving the inner directories owned by that uid with
+     * mode 0755 — so the orchestrator process (a different, non-root uid) has no
+     * write permission on them and a plain `rm` fails with EACCES. destroy() and
+     * both cleanupOrphans passes treat removal as best-effort and swallow that
+     * failure, so each spawn would otherwise leak a multi-GiB chroot until the
+     * data disk fills and the orchestrator crash-loops on ENOSPC (it can no
+     * longer write its own files to start, so the in-process orphan sweep never
+     * runs to recover). Reclaim ownership via the same sudo-wrapped `chown` path
+     * used for `ip` before the `rm` runs as ourselves. On root nodes (requireSudo
+     * false) the chown is skipped and `rm` works directly.
+     */
+    private removeChrootDir;
     /**
      * Get the API socket path for an agent's Firecracker VM.
      */

package/dist/scaler/manager.d.ts

@@ -7,6 +7,7 @@
  * and manages the agent lifecycle from spawn to destroy.
  */
 import type { ResourceRequest } from '@kici-dev/engine';
+import type { BackendFailureSummary } from './failure-tracker.js';
 import type { ScalerBackend, ScalerConfig, ScaleResult, ScalerEvent, ResourceCap, ValidationResult } from './types.js';
 import type { ScalerStateStore, ScalerStateRecovery } from './scaler-state-store.js';
 /**
@@ -58,6 +59,8 @@ export interface ScalerStatus {
         type: string;
         activeCount: number;
         maxAgents: number;
+        /** Whether this backend spawns its agents on the orchestrator's own host. */
+        spawnsOnLocalHost: boolean;
         /** Label sets this backend can provision (each entry is a string[] of labels) */
         labelSets: string[][];
         /** Sum of `requests` reserved by this scaler's active agents. */
@@ -80,6 +83,8 @@ export interface ScalerStatus {
 export declare class ScalerManager {
     private readonly backends;
     private readonly backendRoles;
+    /** Recent scaler spawn failures, surfaced by `kici-admin diagnose`. */
+    private readonly failureTracker;
     private globalMaxAgents;
     /** Per-scaler resource caps (`{ maxCpu, maxMemoryBytes }`), keyed by scaler name. */
     private readonly resourceCaps;
@@ -320,6 +325,11 @@ export declare class ScalerManager {
     /**
      * Return status summary for metrics and health endpoints.
      */
+    /**
+     * Recent scaler spawn failures grouped per backend instance, for the
+     * diagnose scaler check. `nowMs` is injected by the caller.
+     */
+    recentSpawnFailures(windowMs: number, nowMs: number): Map<string, BackendFailureSummary>;
     getStatus(): ScalerStatus;
     /**
      * Get the backend name managing a specific agent.

package/dist/scaler/nftables.d.ts

@@ -41,13 +41,35 @@ export declare function validateNftablesAvailability(opts?: NftOptions): Promise
  * Idempotent -- safe to call multiple times.
  */
 export declare function ensureKiciTable(opts?: NftOptions): Promise<void>;
+/** A single nft rule operation: the verb decides chain placement. */
+export interface NftRuleOp {
+    /** 'insert' prepends to the chain head; 'add' appends to the tail. */
+    verb: 'insert' | 'add';
+    /** Tokens after `nft <verb> rule ip kici forward`. */
+    tokens: string[];
+}
+/**
+ * Build the per-identifier isolation rule operations.
+ *
+ * Placement matters because nftables is first-match-wins and the `kici`
+ * forward chain is shared: it accumulates rules for every live agent plus
+ * any host-level baseline (e.g. wildcard `iifname "kici-*"` RFC1918 drops
+ * installed at host bootstrap). Accepts therefore go in via `insert`
+ * (chain head) so they beat any pre-existing drop that also matches the
+ * traffic; an appended accept after a wildcard drop is dead code. Drops go
+ * in via `add` (tail) — they only need to beat the chain's accept policy.
+ *
+ * Effective per-identifier order: gateway + allowlist accepts (head),
+ * RFC1918 / metadata / denyAll drops (tail).
+ */
+export declare function buildIsolationRuleOps(matchClause: string[], gatewayIp: string, networkPolicy?: NetworkPolicy): NftRuleOp[];
 /**
  * Add network isolation rules for a specific network interface.
  * Used by Firecracker and container backends.
  *
- * Rule order (top to bottom):
- * 1. Accept gateway traffic (MUST be first -- inserted, not appended)
- * 2. Accept allowlisted CIDRs (if any)
+ * Rule placement (see buildIsolationRuleOps):
+ * 1. Accept gateway traffic (inserted at chain head)
+ * 2. Accept allowlisted CIDRs (inserted at chain head)
  * 3. Drop RFC1918 ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16)
  * 4. Drop cloud metadata (169.254.0.0/16)
  * 5. Drop all remaining traffic (only if denyAll is true)

package/dist/scaler/types.d.ts

@@ -50,6 +50,20 @@ export interface EffectiveLimits {
     cpus?: number;
     memBytes?: number;
 }
+/**
+ * Identity of the work a spawn was provisioned for, passed to
+ * `ScalerBackend.spawn()`. Backends surface it on the provisioned resource
+ * (container labels today) so an operator inspecting the backend — e.g.
+ * `podman ps` — can tell which job/run each agent serves, and so tests can
+ * select the exact container a trigger produced instead of guessing among
+ * concurrent kici-managed containers. Absent for unbound spawns (warm pool).
+ */
+export interface SpawnContext {
+    /** Execution job id the spawn is bound to. */
+    boundJobId?: string;
+    /** Execution run id the bound job belongs to. */
+    runId?: string;
+}
 /**
  * Network policy controlling RFC1918 and internet access for agents in this label set.
  */
@@ -165,6 +179,14 @@ export interface ScalerBackend {
     readonly labelSets: LabelSetConfig[];
     /** Per-backend maximum agents */
     readonly maxAgents: number;
+    /**
+     * Whether this backend spawns its agents on the orchestrator's own host.
+     * True for bare-metal and Firecracker (local processes / local VMs) and for
+     * container backends using a local runtime socket; false when the backend
+     * provisions elsewhere (remote container runtime, future cloud backends).
+     * Drives the static spawning-host display on the diagnostics page.
+     */
+    readonly spawnsOnLocalHost: boolean;
     /** Current count of agents managed by this backend (including spawning) */
     getActiveCount(): number;
     /**
@@ -177,10 +199,13 @@ export interface ScalerBackend {
      *   for this spawn. Computed by ScalerManager from the job/label-set/scaler
      *   default chain. When omitted (or both fields zero), the backend falls
      *   back to its label-set / default limits the same way it always has.
+     * @param spawnContext - Identity of the bound job/run this spawn serves
+     *   (omitted for unbound spawns, e.g. warm pool). Backends surface it on
+     *   the provisioned resource for operator inspection.
      * @returns The managed agent tracking object
      * @throws If the label set is not supported by this backend
      */
-    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits): Promise<ManagedAgent>;
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext): Promise<ManagedAgent>;
     /**
      * Destroy a specific managed agent.
      * Docker: docker rm -f; Bare-metal: SIGTERM -> SIGKILL