@kici-dev/orchestrator 0.1.14 → 0.1.15

package/dist/pipeline/inline-eval.d.ts

@@ -5,11 +5,12 @@
  * in a sandboxed VM context at dispatch time, eliminating the init-job
  * round-trip for pure functions.
  */
+import type { LockJob } from '@kici-dev/engine';
 /**
  * Evaluate an inline expression that returns a string.
  *
  * @param expression - Serialized arrow function, e.g. '(event) => event.ref.split("/").pop()'
- * @param event - Normalized webhook event payload
+ * @param event - Normalized event envelope (SimulatedEvent shape; raw provider payload at event.payload)
  * @returns The string result
  * @throws TypeError if result is not a string
  * @throws Error on timeout or sandbox violation
@@ -19,10 +20,24 @@ export declare function evaluateInlineString(expression: string, event: object):
  * Evaluate an inline expression that returns a Record<string, string>.
  *
  * @param expression - Serialized arrow function, e.g. '(event) => ({ NODE_ENV: event.env })'
- * @param event - Normalized webhook event payload
+ * @param event - Normalized event envelope (SimulatedEvent shape; raw provider payload at event.payload)
  * @returns The record result
  * @throws TypeError if result is not a plain object
  * @throws Error on timeout or sandbox violation
  */
 export declare function evaluateInlineRecord(expression: string, event: object): Record<string, string>;
+/**
+ * Evaluate a lock job's inline (pure-function) dynamic fields against the
+ * normalized event envelope (SimulatedEvent shape: { type, action,
+ * targetBranch, sourceBranch, payload, … }). The raw provider webhook body is
+ * nested at `event.payload` — the same shape rules and step contexts see.
+ *
+ * Throws a job-attributed Error when any expression throws; inline evaluation
+ * failures are immediate dispatch failures (no init-job fallback).
+ */
+export declare function evaluateInlineFields(lockJob: LockJob, event: object): {
+    inlineEnvironmentName: string | undefined;
+    inlineEnv: Record<string, string> | undefined;
+    inlineConcurrencyGroup: string | undefined;
+};
 //# sourceMappingURL=inline-eval.d.ts.map

package/dist/pipeline/process-webhook.d.ts

@@ -16,8 +16,26 @@
  * Internal helpers are pure phase functions returning typed results; the only
  * top-level export is `processWebhook`, callable from server.ts / app.ts.
  */
+import type { SimulatedEvent } from '@kici-dev/engine';
 import type { WebhookInfo } from '../webhook/handler.js';
+import type { ProviderBundle } from '../provider-registry.js';
+import type { TrustResolution } from '../security/trust-resolver.js';
 import { type ProcessingDeps } from './processor.js';
+interface TrustOutcome {
+    trustResolution: TrustResolution | undefined;
+    /** Default 'base' for PR events; trust resolution may override to 'head'. */
+    lockFileSource: 'head' | 'base';
+}
+export declare function resolveTrustForPR(args: {
+    info: WebhookInfo;
+    deps: ProcessingDeps;
+    bundle: ProviderBundle;
+    event: SimulatedEvent;
+    payload: Record<string, unknown>;
+    resolvedOrgId: string;
+    repoIdentifier: string;
+    credentials: Record<string, unknown>;
+}): Promise<TrustOutcome>;
 /**
  * Process a webhook through the complete pipeline.
  *
@@ -35,4 +53,5 @@ import { type ProcessingDeps } from './processor.js';
  *  10. Forward Platform trace + record event log
  */
 export declare function processWebhook(info: WebhookInfo, deps: ProcessingDeps): Promise<void>;
+export {};
 //# sourceMappingURL=process-webhook.d.ts.map

package/dist/pipeline/processor.d.ts

@@ -34,6 +34,7 @@ import type { LogStorage } from '../reporting/log-storage.js';
 import type { SecretResolver } from '../secrets/secret-resolver.js';
 import type { ContributorCache } from '../security/contributor-cache.js';
 import type { LockFile as FullLockFile, LockWorkflow, SimulatedEvent, WebhookNormalizer } from '@kici-dev/engine';
+import { LockFileParseError } from '@kici-dev/engine';
 import type { EventRouter } from '../events/event-router.js';
 import type { RegistrationStore } from '../registration/registration-store.js';
 import type { RegistrationIndex } from '../registration/registration-index.js';
@@ -158,7 +159,7 @@ export declare function resolveLockFileWithFallback(args: {
     deliveryId: string;
 }): Promise<{
     lockFile: FullLockFile | null;
-    resolvedVia: 'inbound' | 'fallback' | 'miss';
+    resolvedVia: 'inbound' | 'fallback' | 'miss' | 'corrupt';
     fallbackRoutingKey?: string;
     /** The winning provider bundle when resolvedVia='fallback'. Used by the dispatch
      *  site to swap repoUrlBuilder and cloneTokenProvider (Layer 4 cross-provider fix). */
@@ -166,6 +167,10 @@ export declare function resolveLockFileWithFallback(args: {
     /** The winning registration's providerContext when resolvedVia='fallback'.
      *  Carries installationId etc. for clone token issuance. */
     fallbackCredentials?: Record<string, unknown>;
+    /** Set when resolvedVia='corrupt': the parse error seen while attempting to
+     *  resolve a lock file. A valid fallback always wins over a corrupt inbound,
+     *  so this is only surfaced when NOTHING resolved. */
+    corruptError?: LockFileParseError;
 }>;
 /**
  * Resolve the customer/org ID for a routing key.

package/dist/pipeline/test-pipeline.d.ts

@@ -25,6 +25,8 @@ import type { BuildCoordinator } from '../cache/index.js';
 import type { DepCache } from '../cache/index.js';
 import type { PendingBuildTracker } from '../cache/index.js';
 import type { SecretResolver } from '../secrets/secret-resolver.js';
+import type { EnvironmentStore } from '../environments/environment-store.js';
+import type { VariableStore } from '../environments/variable-store.js';
 import type { LogStorage } from '../reporting/log-storage.js';
 import type { Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
@@ -49,6 +51,10 @@ export interface TestTriggerInput {
     uploadId?: string;
     /** Fixture secret context mappings (optional). */
     secrets?: Record<string, string>;
+    /** Base64 X25519+AES-GCM blob of the developer's local secrets ({flat, contexts}). */
+    encryptedSecrets?: string;
+    /** Base64 ephemeral CLI public key that encrypted `encryptedSecrets`. */
+    encryptedSecretsKey?: string;
     /** Direct workflow run -- bypass triggers (optional). */
     workflowName?: string;
     /** Resolved overlay metadata for tarball download + decryption. */
@@ -97,6 +103,10 @@ export interface TestPipelineDeps {
     logStorage?: LogStorage;
     /** Database connection for environment protection checks. Optional. */
     db?: Kysely<Database>;
+    /** Environment store for resolving environment ids in test dispatch parity. Optional. */
+    environmentStore?: EnvironmentStore;
+    /** Variable store for resolving environment variables in test dispatch parity. Optional. */
+    variableStore?: VariableStore;
 }
 /**
  * Process a test trigger through the existing pipeline.

package/dist/providers/github/lock-file.d.ts

@@ -7,7 +7,7 @@
  * NOTE: This is the raw fetcher without caching. The orchestrator wraps
  * this with an LRU cache (LockFileCache) for production use.
  */
-import type { LockFileFetcher, LockFile } from '@kici-dev/engine';
+import { type LockFileFetcher, type LockFile } from '@kici-dev/engine';
 import { type GitHubAppConfig } from './auth.js';
 /**
  * GitHub-specific implementation of LockFileFetcher.

package/dist/providers/internal/lock-file-fetcher.d.ts

@@ -25,10 +25,11 @@ export declare class InternalLockFileFetcher implements LockFileFetcher {
      * Fetch the lock file from the local filesystem.
      *
      * @param repoIdentifier - Either a file:// URL or a relative path under repoBasePath
-     * @param _ref - Git ref (ignored for filesystem access -- always reads current state)
+     * @param ref - Git ref (ignored for filesystem access -- always reads current state)
      * @param _credentials - Not used (file:// access needs no auth)
      * @returns Parsed LockFile, or null if not found
+     * @throws LockFileParseError when the file is present but unparseable/invalid
      */
-    fetchLockFile(repoIdentifier: string, _ref: string, _credentials: unknown): Promise<LockFile | null>;
+    fetchLockFile(repoIdentifier: string, ref: string, _credentials: unknown): Promise<LockFile | null>;
 }
 //# sourceMappingURL=lock-file-fetcher.d.ts.map

package/dist/queue/job-queue.d.ts

@@ -43,6 +43,13 @@ export declare enum DispatchQueueStatus {
     Expired = "expired",
     Recovering = "recovering"
 }
+/**
+ * Maximum delivery attempts for a single dispatch_queue job. A job whose
+ * `dispatch_attempts` reaches this value is failed permanently instead of
+ * being requeued again. Bounds requeue loops from repeated job.reject /
+ * pre-start agent loss; `expires_at` is the time-based backstop.
+ */
+export declare const MAX_DISPATCH_ATTEMPTS = 5;
 /**
  * Input for enqueuing a job. Callers provide these fields;
  * the queue generates id, status, created_at, and expires_at.
@@ -307,6 +314,34 @@ export declare class JobQueue {
      * the leader-gated sweep (`sweepExpiredRecoveries()`).
      */
     markRecovering(jobId: string, deadline?: Date, agentId?: string): Promise<void>;
+    /**
+     * Stamp the dispatch-acknowledgment deadline for a dispatched job.
+     * Only touches rows still in 'dispatched' for safety.
+     */
+    setAckDeadline(jobId: string, deadline: Date, agentId: string): Promise<void>;
+    /** Clear the ack deadline (agent answered, or the job left 'dispatched'). */
+    clearAckDeadline(jobId: string): Promise<void>;
+    /**
+     * List dispatched rows still awaiting an ack (non-null deadline). Used at
+     * coord boot (`Dispatcher.recoverState()`) to re-arm in-memory timers.
+     */
+    getDispatchedAwaitingAck(): Promise<Array<{
+        id: string;
+        runId: string;
+        agentId: string | null;
+        deadline: Date;
+    }>>;
+    /**
+     * List every dispatched row whose ack deadline is in the past. The caller
+     * (leader-gated `Dispatcher.sweepExpiredAckDeadlines`) requeues each via
+     * the atomic `requeue()` (WHERE status='dispatched'), so racing coords
+     * cannot double-requeue.
+     */
+    listExpiredAckDeadlines(now: Date): Promise<Array<{
+        id: string;
+        runId: string;
+        agentId: string | null;
+    }>>;
     /**
      * List every job currently in `recovering` state with its persisted
      * recovery deadline. Used at coord boot (`Dispatcher.recoverState()`)
@@ -325,7 +360,7 @@ export declare class JobQueue {
     /**
      * Sweep every `recovering` row whose `recovery_deadline` is in the
      * past, marking them `failed`. Returns the rows that flipped so the
-     * caller can fire the per-job `onRecoveryTimeout` hook in process.
+     * caller can fire the per-job `onJobFailedPermanently` hook in process.
      *
      * Intended for the leader-gated `Dispatcher.sweepExpiredRecoveries`
      * tick — running on N coords would still be correct (the WHERE
@@ -343,6 +378,23 @@ export declare class JobQueue {
      * @returns true if the update affected a row (job was still recovering).
      */
     markFailedIfRecovering(jobId: string, _reason: string): Promise<boolean>;
+    /**
+     * Return a dispatched job to the pending queue for re-dispatch, bumping
+     * its attempt counter. Used when an agent explicitly rejects a dispatch
+     * (job.reject) and when a scaler-managed agent disconnects before the
+     * job started. Only flips rows still in 'dispatched' — a job that was
+     * concurrently completed / failed / cancelled is left untouched.
+     *
+     * @returns the post-increment dispatch_attempts, or null when the row
+     *   was not in 'dispatched' state (nothing requeued).
+     */
+    requeue(jobId: string): Promise<number | null>;
+    /**
+     * Get the full QueuedJob row by ID regardless of status. Used by the
+     * dispatcher's redispatch path, which needs runsOnLabels / excludeLabels /
+     * resources to pick an agent or consult the scaler for a requeued job.
+     */
+    getFullJobById(jobId: string): Promise<QueuedJob | null>;
     /**
      * Mark a job as dispatched only if it is still in 'recovering' state.
      * Used when an agent reconnects and claims a recovering job.

package/dist/reporting/execution-tracker.d.ts

@@ -219,7 +219,9 @@ export declare class ExecutionTracker {
     concurrency?: {
         cancelInProgress?: boolean;
         max?: number;
-    }): Promise<void>;
+    }, 
+    /** Workflow-level wall-clock timeout in ms from the lock file. Sets the run deadline. */
+    workflowTimeoutMs?: number): Promise<void>;
     /**
      * Add additional jobs to an already-started execution run.
      * Used when build jobs are tracked early and regular jobs are dispatched later.

package/dist/routes/admin-environments.d.ts

@@ -6,6 +6,7 @@
  *   PATCH  /api/v1/admin/environments/:name/policy                       — update policy fields
  *   GET    /api/v1/admin/environments?orgId=<id>                         — list environments
  *   GET    /api/v1/admin/environments/:name?orgId=<id>                   — show env + vars + bindings
+ *   DELETE /api/v1/admin/environments/:name?orgId=<id>                   — delete env (cascades bindings/variables/overrides; pending held runs block with 409)
  *   POST   /api/v1/admin/environments/templates                          — create/update a template
  *   GET    /api/v1/admin/environments/:name/variables?orgId=<id>         — list org-level variables
  *   PUT    /api/v1/admin/environments/:name/variables/:key?orgId=<id>    — upsert variable

package/dist/scaler/bare-metal-backend.d.ts

@@ -39,6 +39,7 @@ export interface BareMetalScalerBackendOptions {
 }
 export declare class BareMetalScalerBackend implements ScalerBackend {
     readonly type: "bare-metal";
+    readonly spawnsOnLocalHost = true;
     readonly logsSource = "bare-metal";
     readonly maxAgents: number;
     private _labelSets;

package/dist/scaler/container-backend.d.ts

@@ -8,7 +8,7 @@
  */
 import { type ToolRequirement } from '@kici-dev/shared';
 import type { AgentTokenStore } from '../agent/token-store.js';
-import type { ScalerBackend, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
+import type { ScalerBackend, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, SpawnContext, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
 /**
  * Result of runtime detection.
  */
@@ -49,6 +49,7 @@ export interface ContainerScalerBackendOptions {
 }
 export declare class ContainerScalerBackend implements ScalerBackend {
     readonly type: "container";
+    readonly spawnsOnLocalHost: boolean;
     readonly maxAgents: number;
     private _labelSets;
     private readonly name;
@@ -112,7 +113,7 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     get logsSource(): string;
     get labelSets(): LabelSetConfig[];
     getActiveCount(): number;
-    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits): Promise<ManagedAgent>;
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext): Promise<ManagedAgent>;
     getScalerContext(agentId: string): Record<string, unknown> | undefined;
     destroy(managedId: string): Promise<void>;
     /**

package/dist/scaler/firecracker-backend.d.ts

@@ -82,6 +82,7 @@ export interface FirecrackerScalerBackendOptions {
 }
 export declare class FirecrackerScalerBackend implements ScalerBackend {
     readonly type: "firecracker";
+    readonly spawnsOnLocalHost = true;
     readonly maxAgents: number;
     readonly logsSource = "firecracker-serial";
     /** AbortControllers for file tailing per managed VM (keyed by agent ID) */
@@ -211,6 +212,22 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * Promisified execFile wrapper with 30s timeout.
      */
     private execAsync;
+    /**
+     * Remove a VM's jailer chroot directory.
+     *
+     * On rootless nodes (requireSudo), the jailer chowns the chroot contents to
+     * the jailer uid/gid, leaving the inner directories owned by that uid with
+     * mode 0755 — so the orchestrator process (a different, non-root uid) has no
+     * write permission on them and a plain `rm` fails with EACCES. destroy() and
+     * both cleanupOrphans passes treat removal as best-effort and swallow that
+     * failure, so each spawn would otherwise leak a multi-GiB chroot until the
+     * data disk fills and the orchestrator crash-loops on ENOSPC (it can no
+     * longer write its own files to start, so the in-process orphan sweep never
+     * runs to recover). Reclaim ownership via the same sudo-wrapped `chown` path
+     * used for `ip` before the `rm` runs as ourselves. On root nodes (requireSudo
+     * false) the chown is skipped and `rm` works directly.
+     */
+    private removeChrootDir;
     /**
      * Get the API socket path for an agent's Firecracker VM.
      */

package/dist/scaler/manager.d.ts

@@ -59,6 +59,8 @@ export interface ScalerStatus {
         type: string;
         activeCount: number;
         maxAgents: number;
+        /** Whether this backend spawns its agents on the orchestrator's own host. */
+        spawnsOnLocalHost: boolean;
         /** Label sets this backend can provision (each entry is a string[] of labels) */
         labelSets: string[][];
         /** Sum of `requests` reserved by this scaler's active agents. */

package/dist/scaler/nftables.d.ts

@@ -41,13 +41,35 @@ export declare function validateNftablesAvailability(opts?: NftOptions): Promise
  * Idempotent -- safe to call multiple times.
  */
 export declare function ensureKiciTable(opts?: NftOptions): Promise<void>;
+/** A single nft rule operation: the verb decides chain placement. */
+export interface NftRuleOp {
+    /** 'insert' prepends to the chain head; 'add' appends to the tail. */
+    verb: 'insert' | 'add';
+    /** Tokens after `nft <verb> rule ip kici forward`. */
+    tokens: string[];
+}
+/**
+ * Build the per-identifier isolation rule operations.
+ *
+ * Placement matters because nftables is first-match-wins and the `kici`
+ * forward chain is shared: it accumulates rules for every live agent plus
+ * any host-level baseline (e.g. wildcard `iifname "kici-*"` RFC1918 drops
+ * installed at host bootstrap). Accepts therefore go in via `insert`
+ * (chain head) so they beat any pre-existing drop that also matches the
+ * traffic; an appended accept after a wildcard drop is dead code. Drops go
+ * in via `add` (tail) — they only need to beat the chain's accept policy.
+ *
+ * Effective per-identifier order: gateway + allowlist accepts (head),
+ * RFC1918 / metadata / denyAll drops (tail).
+ */
+export declare function buildIsolationRuleOps(matchClause: string[], gatewayIp: string, networkPolicy?: NetworkPolicy): NftRuleOp[];
 /**
  * Add network isolation rules for a specific network interface.
  * Used by Firecracker and container backends.
  *
- * Rule order (top to bottom):
- * 1. Accept gateway traffic (MUST be first -- inserted, not appended)
- * 2. Accept allowlisted CIDRs (if any)
+ * Rule placement (see buildIsolationRuleOps):
+ * 1. Accept gateway traffic (inserted at chain head)
+ * 2. Accept allowlisted CIDRs (inserted at chain head)
  * 3. Drop RFC1918 ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16)
  * 4. Drop cloud metadata (169.254.0.0/16)
  * 5. Drop all remaining traffic (only if denyAll is true)

package/dist/scaler/types.d.ts

@@ -50,6 +50,20 @@ export interface EffectiveLimits {
     cpus?: number;
     memBytes?: number;
 }
+/**
+ * Identity of the work a spawn was provisioned for, passed to
+ * `ScalerBackend.spawn()`. Backends surface it on the provisioned resource
+ * (container labels today) so an operator inspecting the backend — e.g.
+ * `podman ps` — can tell which job/run each agent serves, and so tests can
+ * select the exact container a trigger produced instead of guessing among
+ * concurrent kici-managed containers. Absent for unbound spawns (warm pool).
+ */
+export interface SpawnContext {
+    /** Execution job id the spawn is bound to. */
+    boundJobId?: string;
+    /** Execution run id the bound job belongs to. */
+    runId?: string;
+}
 /**
  * Network policy controlling RFC1918 and internet access for agents in this label set.
  */
@@ -165,6 +179,14 @@ export interface ScalerBackend {
     readonly labelSets: LabelSetConfig[];
     /** Per-backend maximum agents */
     readonly maxAgents: number;
+    /**
+     * Whether this backend spawns its agents on the orchestrator's own host.
+     * True for bare-metal and Firecracker (local processes / local VMs) and for
+     * container backends using a local runtime socket; false when the backend
+     * provisions elsewhere (remote container runtime, future cloud backends).
+     * Drives the static spawning-host display on the diagnostics page.
+     */
+    readonly spawnsOnLocalHost: boolean;
     /** Current count of agents managed by this backend (including spawning) */
     getActiveCount(): number;
     /**
@@ -177,10 +199,13 @@ export interface ScalerBackend {
      *   for this spawn. Computed by ScalerManager from the job/label-set/scaler
      *   default chain. When omitted (or both fields zero), the backend falls
      *   back to its label-set / default limits the same way it always has.
+     * @param spawnContext - Identity of the bound job/run this spawn serves
+     *   (omitted for unbound spawns, e.g. warm pool). Backends surface it on
+     *   the provisioned resource for operator inspection.
      * @returns The managed agent tracking object
      * @throws If the label set is not supported by this backend
      */
-    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits): Promise<ManagedAgent>;
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext): Promise<ManagedAgent>;
     /**
      * Destroy a specific managed agent.
      * Docker: docker rm -f; Bare-metal: SIGTERM -> SIGKILL