@kici-dev/orchestrator 0.0.0 → 0.1.1

package/dist/scaler/machine-ledger.d.ts

@@ -0,0 +1,160 @@
+/**
+ * File-backed, cross-process resource ledger for named machine pools.
+ *
+ * Multiple orchestrators on the same host that reference the same `machinePool`
+ * name share a JSON file under `KICI_MACHINE_LEDGER_DIR ?? /var/lib/kici/scaler-ledger`.
+ * Each scaler reservation appends a row; releases delete the matching row;
+ * and a periodic reaper prunes rows whose `(pid, bootId)` no longer exists.
+ *
+ * ## Concurrency
+ *
+ * Every read-modify-write goes through `withLock(poolName, fn)`, which uses
+ * an atomic POSIX `mkdir` on a sibling lockdir as the cross-process mutex.
+ * `mkdir` succeeds for exactly one caller when the directory does not exist,
+ * so it works as a portable `flock` substitute without any native dependency.
+ *
+ * The plan considered `proper-lockfile` and `fs-ext` first; neither is a
+ * transitive dep today (`pnpm why proper-lockfile` and `pnpm why fs-ext`
+ * both empty), so we use the documented `mkdir` fallback. This avoids
+ * adding a native binding for the sake of a single locking primitive.
+ *
+ * ## Crash safety
+ *
+ * Reservations carry `{ pid, bootId, instanceId, agentId, cpus, memBytes }`.
+ * `bootId` is read once at orchestrator startup from
+ * `/proc/sys/kernel/random/boot_id`; combined with `pid`, it lets the reaper
+ * detect "this PID is dead AND was on a previous boot" reliably:
+ *
+ * - Same boot: `process.kill(pid, 0)` answers liveness.
+ * - Different boot: any pre-reboot row is unconditionally stale.
+ *
+ * If an orchestrator dies without releasing, the next reservation request
+ * from any orchestrator on the host calls `reapStale()` first, so there's
+ * no leak window beyond the next spawn attempt.
+ */
+import type { ResourceCap } from './types.js';
+/** Wire format for the per-pool ledger JSON file. */
+export interface LedgerFile {
+    version: number;
+    cap: ResourceCap;
+    reservations: ReservationRow[];
+}
+/** A single reservation row in the ledger. */
+export interface ReservationRow {
+    /** Orchestrator instance id (from `KICI_CLUSTER_INSTANCE_ID` or generated). */
+    instanceId: string;
+    /** Process id that owns this reservation. */
+    pid: number;
+    /** Linux boot id at the time the row was created. */
+    bootId: string;
+    /** Agent id this reservation is associated with. */
+    agentId: string;
+    /** Reserved cpus (cores). */
+    cpus: number;
+    /** Reserved memory in bytes. */
+    memBytes: number;
+    /** When the row was inserted (epoch ms). */
+    reservedAt: number;
+}
+/**
+ * File-backed machine pool ledger.
+ *
+ * One instance is shared across all scaler entries that reference any pool
+ * (the pool name is passed per-call). The ledger does not cache the JSON:
+ * every operation goes through the lock, reads the file, applies the
+ * mutation, and writes it back — keeping the wire format the only source
+ * of truth across processes.
+ */
+export declare class MachineLedger {
+    private readonly bootId;
+    private dirPath;
+    private readonly explicitDir;
+    private readonly instanceId;
+    /** Per-process knownCaps cache; the on-disk file's `cap` must match the caller's expectation. */
+    private readonly expectedCaps;
+    /** Reaper interval handle. */
+    private reaperTimer;
+    constructor(opts: {
+        explicitDir?: string;
+        instanceId: string;
+    });
+    /**
+     * Lazy directory resolution so construction stays synchronous. Called from
+     * every operation; resolves once and caches.
+     */
+    private ensureDir;
+    /** Register the expected cap for a pool. The on-disk file's cap must match. */
+    registerPool(name: string, cap: ResourceCap): void;
+    /**
+     * Path to the JSON file for `poolName`. Sibling lockdir is `<file>.lock`.
+     */
+    private pathFor;
+    /**
+     * Atomic mkdir-based lock. Held for the duration of `fn`.
+     */
+    private withLock;
+    /**
+     * If the lock directory has been held for more than 30 seconds, assume the
+     * holder is dead and release it. Without this, an orchestrator that gets
+     * SIGKILL'd while holding the lock would leave it stuck forever.
+     */
+    private maybeReleaseStaleLock;
+    /**
+     * Read the ledger file (or initialize an empty one when missing).
+     */
+    private readLedger;
+    /** Atomic write via temp-file + rename. */
+    private writeLedger;
+    /**
+     * Try to add a reservation. Returns true if the cap allowed it (and the row
+     * was persisted), false if any cap would be exceeded after summing.
+     *
+     * Reaps stale rows from dead processes before checking the cap so an
+     * orphaned reservation doesn't permanently squat capacity.
+     */
+    tryReserve(poolName: string, agentId: string, cpus: number, memBytes: number): Promise<boolean>;
+    /**
+     * Release the reservation for `agentId`. Idempotent.
+     */
+    release(poolName: string, agentId: string): Promise<void>;
+    /**
+     * Reap stale (dead-process) rows for the given pool. Called periodically.
+     */
+    reapStale(poolName: string): Promise<void>;
+    /**
+     * Mutates `ledger.reservations` in place to drop rows that point at dead
+     * processes. Live-pid heuristic:
+     * - Different bootId → unconditionally dead (host rebooted since).
+     * - Same bootId, OWN instance: probe via `process.kill(pid, 0)` (always
+     *   works inside this process's PID namespace).
+     * - Same bootId, OTHER instance: keep the row. We cannot safely
+     *   `process.kill(pid, 0)` against a peer orchestrator's PID — when the
+     *   two orchestrators share a host but live in different PID namespaces
+     *   (e.g. one runs natively and the other in a container), `kill` returns
+     *   ESRCH for any peer pid and the reservation would be falsely reaped,
+     *   silently breaking the cross-process cap. Peer cleanup happens via
+     *   `releaseAllForInstance()` on graceful shutdown; SIGKILL'd peers leave
+     *   their rows in place until the host reboots (which the bootId check
+     *   then reaps unconditionally).
+     */
+    private dropDeadReservations;
+    /** Return current usage (sum of `requests` across all live rows). */
+    getUsage(poolName: string): Promise<{
+        cpus: number;
+        memBytes: number;
+    }>;
+    /**
+     * Start the periodic reaper. Idempotent. The reaper iterates every
+     * registered pool every 30s.
+     */
+    start(intervalMs?: number): void;
+    /** Stop the periodic reaper. */
+    stop(): void;
+    private reapAllPools;
+    /**
+     * Best-effort: release every reservation owned by this instance across all
+     * registered pools. Called on graceful shutdown.
+     */
+    releaseAllForInstance(): Promise<void>;
+}
+//# sourceMappingURL=machine-ledger.d.ts.map

package/dist/scaler/manager.d.ts

@@ -0,0 +1,338 @@
+/**
+ * ScalerManager: central coordinator for all scaler backends.
+ *
+ * Sits between the orchestrator's Dispatcher and individual scaler backends.
+ * Receives "no agent available" signals, routes to the correct backend,
+ * enforces global limits, tracks spawning agents to prevent over-provisioning,
+ * and manages the agent lifecycle from spawn to destroy.
+ */
+import type { ResourceRequest } from '@kici-dev/engine';
+import type { ScalerBackend, ScalerConfig, ScaleResult, ScalerEvent, ResourceCap, ValidationResult } from './types.js';
+/**
+ * Resolved per-job resource amounts (cpus + bytes) for both `requests` and
+ * `limits`. The scaler manager produces this from the job's declared resources
+ * combined with the scaler's `defaults.resources`, applying the request<->limit
+ * mirroring rule. Caps aggregate `requests`; backends use `limits`.
+ */
+export interface ResolvedResources {
+    requests: {
+        cpus: number;
+        memBytes: number;
+    };
+    limits: {
+        cpus: number;
+        memBytes: number;
+    };
+}
+/**
+ * Status summary for metrics and health endpoints.
+ */
+export interface ScalerStatus {
+    globalMaxAgents: number;
+    globalActiveCount: number;
+    spawningCount: number;
+    warmPoolCount: number;
+    /** Sum of `requests.cpus` / `requests.memBytes` reserved across all scalers. */
+    globalUsage: {
+        cpus: number;
+        memBytes: number;
+    };
+    /** Per-orchestrator resource cap, if configured. */
+    globalResourceCap?: ResourceCap;
+    backends: Array<{
+        name: string;
+        type: string;
+        activeCount: number;
+        maxAgents: number;
+        /** Label sets this backend can provision (each entry is a string[] of labels) */
+        labelSets: string[][];
+        /** Sum of `requests` reserved by this scaler's active agents. */
+        usage: {
+            cpus: number;
+            memBytes: number;
+        };
+        /** Per-scaler resource cap, if configured. */
+        resourceCap?: ResourceCap;
+        /** Machine-pool reference, if any. */
+        machinePool?: string;
+        /**
+         * Labels a job MUST declare in `runsOn` to be allowed on this backend.
+         * Empty array = no gate. Surfaced in heartbeat-side scaler capacity
+         * summaries so cross-peer routing applies the same gate.
+         */
+        mandatoryLabels: string[];
+    }>;
+}
+export declare class ScalerManager {
+    private readonly backends;
+    private readonly backendRoles;
+    private globalMaxAgents;
+    /** Per-scaler resource caps (`{ maxCpu, maxMemoryBytes }`), keyed by scaler name. */
+    private readonly resourceCaps;
+    /** Per-scaler default resources (used when neither job nor label-set declares them). */
+    private readonly scalerDefaults;
+    /** Per-scaler usage counters (sum of `requests` for active + spawning agents). */
+    private readonly perScalerUsage;
+    /** Orchestrator-wide cap on summed `requests`. */
+    private globalResourceCap;
+    /** Orchestrator-wide usage counter. */
+    private readonly globalUsage;
+    /** Per-scaler machine-pool name (set when scalers reference a pool). */
+    private readonly scalerMachinePools;
+    /** Outstanding reservations keyed by agentId; used to release on disconnect. */
+    private readonly reservations;
+    /**
+     * Serialization queue for the check+reserve critical section. A simple
+     * promise chain is sufficient because every reservation runs on the same
+     * Node.js event loop and the work inside the critical section is purely
+     * synchronous (`tryReserveAll`).
+     */
+    private reservationLock;
+    /**
+     * File-backed cross-process ledger for named machine pools (optional).
+     * Lazily initialized in the constructor when at least one scaler entry
+     * references a pool.
+     */
+    private readonly machineLedger;
+    /** Per-scaler URL overrides from config, keyed by scaler name */
+    private readonly scalerUrls;
+    /**
+     * Tracks agents being spawned but not yet registered via WS.
+     * Keyed by pre-generated agentId.
+     */
+    private readonly spawningAgents;
+    /**
+     * Maps registered agentId to backendName for lifecycle events.
+     */
+    private readonly managedAgentIndex;
+    /**
+     * Active log forwarders for scaler-managed agents (container/bare-metal).
+     * Keyed by agentId. Each forwarder consumes a LogCapture stream.
+     */
+    private readonly logForwarders;
+    /**
+     * Correlation map: agentId -> { runId, jobId }.
+     * Populated by correlateAgentToJob() after job dispatch to a scaler-managed agent.
+     */
+    private readonly agentJobCorrelation;
+    /**
+     * Buffer for scaler events emitted before job correlation is established.
+     * Keyed by agentId. Flushed when correlateAgentToJob() is called.
+     */
+    private readonly eventBuffer;
+    /**
+     * External callback for relaying correlated scaler events (e.g. to execution tracker).
+     */
+    private readonly onScalerEvent?;
+    private readonly warmPool;
+    constructor(deps: {
+        config: ScalerConfig;
+        backends: Array<{
+            name: string;
+            backend: ScalerBackend;
+        }>;
+        /** Callback for relaying scaler events with runId/jobId context. */
+        onScalerEvent?: (runId: string, jobId: string, event: ScalerEvent) => void;
+        /**
+         * Optional machine-ledger options. When `machinePools` are configured,
+         * the manager initializes a `MachineLedger` keyed off this directory and
+         * the orchestrator's instance id; reservations sum across orchestrators
+         * on the same host.
+         */
+        machineLedger?: {
+            /** Override the on-disk ledger directory. Falls back to `KICI_MACHINE_LEDGER_DIR`. */
+            dir?: string;
+            /** Orchestrator instance id (used in ledger rows for ownership). */
+            instanceId: string;
+        };
+    });
+    /** Per-scaler mandatoryLabels (taint-style opt-in gate). */
+    private readonly scalerMandatoryLabels;
+    /**
+     * Build enriched scaler entries with auto-labels injected into each label set.
+     * This ensures label matching accounts for auto-injected labels (kici:role:*,
+     * kici:os:*, kici:arch:*, kici:agent:*, kici:scaler:*) that agents receive
+     * at spawn time.
+     *
+     * Mandatory labels are surfaced unchanged from the per-scaler config so the
+     * label matcher can apply the gate alongside subset matching.
+     */
+    private getEnrichedScalerEntries;
+    /**
+     * Main entry point: called by the Dispatcher when no agent is available.
+     *
+     * `resources` is the per-job request/limit declaration (or `undefined` to fall
+     * back to scaler defaults). Used by per-scaler / per-orchestrator / per-machine
+     * resource caps; the cap math is wired up by ScalerManager itself.
+     */
+    requestScale(labels: string[], jobId: string, excludeLabels?: string[], resources?: ResourceRequest): Promise<ScaleResult>;
+    /**
+     * Run a critical section under the reservation lock. Serializes concurrent
+     * `requestScale()` calls so the check+reserve sequence is atomic.
+     */
+    private runWithReservationLock;
+    /**
+     * Resolve the effective `{ requests, limits }` for a job at this scaler.
+     *
+     * Layered defaults (highest priority first):
+     * 1. Job's own `resources` (after request<->limit mirroring).
+     * 2. Label-set's `resources`.
+     * 3. Scaler's `defaults.resources` (already merged from global defaults at construction).
+     * 4. Zero (`{ cpus: 0, memBytes: 0 }`).
+     *
+     * Returns both sides fully resolved as numeric `{ cpus, memBytes }` pairs.
+     */
+    private resolveEffective;
+    /**
+     * Attempt to reserve `requests` against the per-scaler and per-orchestrator
+     * caps. Returns false (no state change) if any cap would be exceeded.
+     *
+     * Caller must hold the reservation lock.
+     */
+    private tryReserveAll;
+    /**
+     * Push the current `perScalerUsage` and `globalUsage` snapshot into the
+     * Prometheus observable gauges. Called on every successful reservation /
+     * release so the gauge callback always reflects the latest state without
+     * doing any work itself (gauge callbacks are synchronous and MUST NOT
+     * touch this map directly — they just read whatever was last published).
+     *
+     * Note: pool-level usage is intentionally not emitted here. The on-disk
+     * machine-pool ledger requires async I/O + a file lock to read accurately,
+     * and OTel observable-gauge callbacks must be synchronous. Operators
+     * dashboarding pool utilization can derive a lower bound from the sum of
+     * `kici_orch_scaler_cpus_used` rows whose scaler belongs to the pool.
+     */
+    private publishUsageMetrics;
+    /**
+     * Release the reservation tracked for an agent. Idempotent. Also releases
+     * the cross-process pool reservation (best-effort) when the scaler
+     * references a pool.
+     */
+    private releaseAll;
+    /**
+     * Subtract the given `requests` from the per-scaler and global in-memory
+     * counters. Used both by `releaseAll()` and by the rollback path when a
+     * machine-pool reservation refuses after the in-memory reservation
+     * already succeeded.
+     */
+    private releaseInMemory;
+    /**
+     * Resolve the scaler-level default resources, merging the global
+     * `defaults.resources` underneath any scaler-specific override.
+     *
+     * Currently scaler entries themselves do not carry their own `defaults` (the
+     * label-set level does). The merged result is stored per-scaler so that
+     * future scaler-level defaults can be added without touching the resolver.
+     */
+    private resolveScalerDefaults;
+    /**
+     * Called from agent-handler.ts when an agent registers via WS.
+     * Correlates the registered agent to a spawned tracking entry.
+     *
+     * Returns:
+     * - `boundJobId` (optional): the queued jobId this agent was spawned for.
+     *   Used by the caller to eagerly dispatch the bound job before the
+     *   agent's idle timer fires, skipping the generic queue drain race.
+     * - `mandatoryLabels` (always populated for scaler-managed agents): the
+     *   spawning scaler's `mandatoryLabels` gate. Threaded into the
+     *   AgentRegistry so the queue-drain path (`onAgentAvailable` →
+     *   `dequeueForLabels`) and the eager-dispatch path
+     *   (`dispatchBoundJob` → `dequeueById`) both reject queued jobs whose
+     *   `runsOn` does not include every gate label.
+     *
+     * Returns `null` for static agents and warm-pool replenishment spawns
+     * (no spawning entry exists).
+     */
+    onAgentRegistered(agentId: string, labels: string[]): {
+        boundJobId?: string;
+        mandatoryLabels: string[];
+    } | null;
+    /**
+     * Called from agent-handler.ts when an agent disconnects.
+     */
+    onAgentDisconnected(agentId: string): void;
+    /**
+     * Look up the scaler backend name (e.g. `container`, `firecracker`,
+     * `bare-metal`) for a registered agent. Returns null if the agent is
+     * not scaler-managed (i.e. a static / stateful agent).
+     *
+     * Used by AgentMetricsAggregator to inject a `scaler` label on each
+     * `kici_agent_*` series so dashboards can split per-scaler instead of
+     * per-agent_id (which is too high-cardinality and doesn't tell the
+     * operator which pool is loaded).
+     */
+    getBackendName(agentId: string): string | null;
+    /**
+     * Called from agent-handler.ts when an agent sends config.ack.
+     * For Firecracker agents, clears MMDS data (belt-and-suspenders with in-VM iptables).
+     * For non-Firecracker agents, this is a no-op.
+     */
+    onConfigAck(agentId: string): void;
+    /**
+     * Called when a scaler-managed agent completes a job.
+     */
+    onJobComplete(agentId: string): void;
+    /**
+     * Correlate an agentId to a runId/jobId after the dispatcher assigns a job
+     * to a scaler-managed agent. Flushes any buffered pre-dispatch events.
+     */
+    correlateAgentToJob(agentId: string, runId: string, jobId: string): void;
+    /**
+     * Sum of all backends' active count + spawning agents.
+     */
+    getGlobalActiveCount(): number;
+    /**
+     * Stop warm pool, shutdown all backends, clear tracking maps.
+     */
+    shutdownAll(): Promise<void>;
+    /**
+     * Validate and reload configuration.
+     */
+    reload(newConfig: ScalerConfig): Promise<ValidationResult>;
+    /**
+     * Return status summary for metrics and health endpoints.
+     */
+    getStatus(): ScalerStatus;
+    /**
+     * Get the backend name managing a specific agent.
+     * Returns null if the agent is not scaler-managed (standalone).
+     */
+    getBackendForAgent(agentId: string): string | null;
+    /**
+     * Get scaler-specific configuration metadata for a managed agent.
+     * Returns undefined for non-scaler-managed (static) agents.
+     * Used to enrich job.context before forwarding to Platform.
+     */
+    getScalerContextForAgent(agentId: string): Record<string, unknown> | undefined;
+    /**
+     * Start the warm pool idle check interval and the machine-pool ledger reaper.
+     */
+    start(): void;
+    /**
+     * Remove spawning entries older than 5 minutes.
+     * These represent agents that were spawned but never registered via WS
+     * (e.g., process/container crashed on startup). Without cleanup, these
+     * entries would leak in spawningAgents forever.
+     */
+    private pruneStaleSpawningEntries;
+    private generateAgentId;
+    /**
+     * Start log forwarding for a scaler-managed agent if its backend supports LogCapture.
+     * Fire-and-forget: the forward() promise runs for the agent's lifetime.
+     */
+    private startLogForwarding;
+    private getOrchestratorUrl;
+    /**
+     * Create a per-agent event emitter closure to pass to backend.spawn().
+     * The closure captures agentId and routes events through handleScalerEvent().
+     */
+    private createEventEmitter;
+    /**
+     * Handle a scaler event from a backend.
+     * If the agent is correlated to a job, relay immediately.
+     * Otherwise, buffer until correlation is established.
+     */
+    private handleScalerEvent;
+}
+//# sourceMappingURL=manager.d.ts.map

package/dist/scaler/nftables.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Programmatic nftables rule management for agent network isolation.
+ *
+ * Manages RFC1918 + cloud metadata blocking rules per-interface (Firecracker/container)
+ * or per-UID (bare-metal). All operations use `nft` CLI via child_process.execFile.
+ *
+ * Table layout:
+ *   table ip kici {
+ *     chain forward { type filter hook forward priority 0; policy accept; }
+ *     chain output  { type filter hook output priority 0; policy accept; }
+ *   }
+ */
+import type { NetworkPolicy } from './types.js';
+/**
+ * Match mode for nftables isolation rules.
+ * - 'iifname': Match on input interface name (Firecracker TAP devices)
+ * - 'saddr': Match on source IP address (container backends)
+ */
+export type NftMatchMode = 'iifname' | 'saddr';
+/**
+ * Options for nft command execution.
+ */
+interface NftOptions {
+    /**
+     * Wrap the `nft` invocation with `sudo -n` so non-root orchestrators (e.g.
+     * Pi user-mode systemd) can manage rules. Operators must have a NOPASSWD
+     * sudoers entry for /usr/sbin/nft. Default false.
+     */
+    requireSudo?: boolean;
+}
+/**
+ * Validate that nftables is available and the process has NET_ADMIN capability.
+ * Attempts `nft list tables` -- if it fails:
+ *   - ENOENT: nft binary not installed
+ *   - EPERM: nft binary present but NET_ADMIN capability missing
+ * Throws with a clear error message in both cases.
+ */
+export declare function validateNftablesAvailability(opts?: NftOptions): Promise<void>;
+/**
+ * Ensure the `kici` nftables table and required chains exist.
+ * Idempotent -- safe to call multiple times.
+ */
+export declare function ensureKiciTable(opts?: NftOptions): Promise<void>;
+/**
+ * Add network isolation rules for a specific network interface.
+ * Used by Firecracker and container backends.
+ *
+ * Rule order (top to bottom):
+ * 1. Accept gateway traffic (MUST be first -- inserted, not appended)
+ * 2. Accept allowlisted CIDRs (if any)
+ * 3. Drop RFC1918 ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16)
+ * 4. Drop cloud metadata (169.254.0.0/16)
+ * 5. Drop all remaining traffic (only if denyAll is true)
+ *
+ * @param identifier - Network interface name or source IP to match (e.g., "veth-abc123" or "172.30.0.5")
+ * @param gatewayIp - Gateway IP that must remain accessible (e.g., "10.0.0.1")
+ * @param networkPolicy - Optional policy with allowlist and denyAll settings
+ * @param matchMode - How to match traffic: 'iifname' for interface name (default), 'saddr' for source IP
+ */
+export declare function addIsolationRules(identifier: string, gatewayIp: string, networkPolicy?: NetworkPolicy, matchMode?: NftMatchMode, opts?: NftOptions): Promise<void>;
+/**
+ * Remove all isolation rules matching a specific interface from the kici forward chain.
+ * Called during agent cleanup/destruction.
+ *
+ * Lists all rules with handles, finds those matching the interface name, and deletes them.
+ * Best-effort: logs errors but does not throw (cleanup should not block destruction).
+ *
+ * @param interfaceName - Network interface whose rules should be removed
+ */
+export declare function removeIsolationRules(interfaceName: string, opts?: NftOptions): Promise<void>;
+/**
+ * Re-export NftOptions type for use by callers (Firecracker / container backends)
+ * that need to pass the requireSudo flag through to these functions.
+ */
+export type { NftOptions };
+/**
+ * Parse nft rule listing output and extract handles for rules matching a given identifier.
+ * Handles lines like: `  iifname "veth-abc" ip daddr 10.0.0.0/8 drop # handle 42`
+ *
+ * @param nftOutput - Raw output from `nft -a list chain ip kici <chain>`
+ * @param identifier - String to search for in each rule line (interface name or UID)
+ * @returns Array of numeric rule handles
+ */
+export declare function parseRuleHandles(nftOutput: string, identifier: string): number[];
+//# sourceMappingURL=nftables.d.ts.map