@norskvideo/norsk-auto-manager 0.1.0

package/lib/src/placement.js

@@ -0,0 +1,468 @@
+"use strict";
+// Pure-function placement engine. Given a job-to-place, the bundle it
+// belongs to, the current inventory snapshot, and the pool configuration,
+// returns a placement decision: place on a specific node, provision a new
+// node in an elastic pool, or fail.
+//
+// Per §6 of the design doc: filter by hard constraints, score by packing
+// strategy, tiebreak by nodeId for determinism. Per-replica pool
+// preferences (replicaOverrides) take priority over the bundle default.
+//
+// Soft constraints (same-AZ preference, hot-spare scoring bonus) are
+// not yet implemented in this slice — they'll layer on top of the score
+// function once the hard-constraint path is settled and tested.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.tierLaunchMode = tierLaunchMode;
+exports.tierReliability = tierReliability;
+exports.defaultPlacementLeadMs = defaultPlacementLeadMs;
+exports.nodeMatchesTier = nodeMatchesTier;
+exports.place = place;
+exports.placeWithTrace = placeWithTrace;
+const inventory_1 = require("./inventory");
+/** The tier's launch mode, defaulting to on-demand. @public */
+function tierLaunchMode(tier) {
+    return tier.launchMode ?? "on-demand";
+}
+/** The tier's reliability — explicit, else interruptible for spot. @public */
+function tierReliability(tier) {
+    return tier.reliability ?? (tierLaunchMode(tier) === "spot" ? "interruptible" : "durable");
+}
+/** Default placement lead time by tier kind. Cluster tiers place
+ *  against existing running nodes so we don't need lead. Cloud tiers
+ *  default to 5 minutes — typical EC2/OCI instance boot + image pull
+ *  budget; operators can override per-tier via PlacementTier. */
+function defaultPlacementLeadMs(kind) {
+    return kind === "local-servers" ? 0 : 5 * 60 * 1000;
+}
+/**
+ * Whether a node belongs to (pool, tier). Elastic nodes carry an explicit
+ * `tierName` (set from `tags["tier"]` at provision). Pre-registered cluster
+ * nodes carry no tier tag and are taken to belong to the pool's `fixed`
+ * tier — so a pool must have at most one fixed tier (enforced in
+ * settingsValidation) for this to be unambiguous.
+ *
+ * @public
+ */
+function nodeMatchesTier(n, pool, tier) {
+    if (n.poolName !== pool.name)
+        return false;
+    if (n.tierName !== undefined)
+        return n.tierName === tier.name;
+    return tier.scaleOut === "fixed";
+}
+/**
+ * Core placement function. Pure — no side effects, no I/O.
+ *
+ * @public
+ */
+function place(input) {
+    return placeWithTrace(input).result;
+}
+/**
+ * Placement with a structured trace of every pool attempted and every
+ * node's filter outcome. AutoManager logs the trace via debuglog when
+ * placing; tests use it to assert on the precise reason a candidate
+ * was rejected.
+ *
+ * @public
+ */
+function placeWithTrace(input) {
+    const { bundle, job } = input;
+    const tried = [];
+    const attempts = [];
+    const pool = input.pools.get(bundle.pool);
+    if (!pool || pool.tiers.length === 0) {
+        return {
+            result: { kind: "failure", reason: { code: "noConfiguredPools", triedPools: [] } },
+            trace: { attempts: [], decision: "failure" },
+        };
+    }
+    // Walk the tiers cheapest-first; first tier that can place or provision
+    // wins. This positional order is the cost cascade (no cost weight in v1).
+    for (const tier of pool.tiers) {
+        tried.push(tier.name);
+        const { matched, filtered } = filterCandidatesTraced(input, pool, tier);
+        const initialCandidates = input.inventory.length;
+        if (matched.length > 0) {
+            const scored = matched.map((n) => ({
+                node: n,
+                score: scoreNode(n, tier.packingStrategy, input),
+            }));
+            // Sort descending by score, lexicographic nodeId tiebreak.
+            scored.sort((a, b) => {
+                if (a.score !== b.score)
+                    return b.score - a.score;
+                return a.node.nodeId.localeCompare(b.node.nodeId);
+            });
+            const winner = scored[0].node;
+            const gpuIndex = pickGpuIndex(winner, job.requirements, tier.packingStrategy);
+            const degraded = placeDegradation(winner, input) ?? primaryInterruptibleDegradation(tier, input);
+            attempts.push({
+                pool: pool.name,
+                tier: tier.name,
+                initialCandidates,
+                filtered,
+                scored: scored.map((s) => ({ nodeId: s.node.nodeId, score: s.score })),
+                outcome: { kind: "placed", nodeId: winner.nodeId, gpuIndex },
+            });
+            return {
+                result: {
+                    kind: "place",
+                    nodeId: winner.nodeId,
+                    gpuIndex,
+                    pool: pool.name,
+                    tier: tier.name,
+                    ...(degraded ? { degraded } : {}),
+                },
+                trace: { attempts, decision: "place" },
+            };
+        }
+        // A tier at its node cap can't grow — the cost cascade spills to the
+        // next tier. (Placing onto an existing node above doesn't add a node,
+        // so the cap only gates provisioning.)
+        const atCap = tier.maxNodes !== undefined && tierNodeCount(input.inventory, pool, tier) >= tier.maxNodes;
+        if (tier.scaleOut === "elastic" && !atCap) {
+            const instanceType = findProvisionableInstanceType(tier, job.requirements);
+            if (instanceType) {
+                const degraded = provisionDegradation(tier, input) ?? primaryInterruptibleDegradation(tier, input);
+                attempts.push({
+                    pool: pool.name,
+                    tier: tier.name,
+                    initialCandidates,
+                    filtered,
+                    scored: [],
+                    outcome: { kind: "provisioned", instanceType },
+                });
+                return {
+                    result: {
+                        kind: "provision",
+                        pool: pool.name,
+                        tier: tier.name,
+                        instanceType,
+                        launchMode: tierLaunchMode(tier),
+                        ...(degraded ? { degraded } : {}),
+                    },
+                    trace: { attempts, decision: "provision" },
+                };
+            }
+            attempts.push({
+                pool: pool.name,
+                tier: tier.name,
+                initialCandidates,
+                filtered,
+                scored: [],
+                outcome: { kind: "no-instance-type" },
+            });
+            continue;
+        }
+        attempts.push({
+            pool: pool.name,
+            tier: tier.name,
+            initialCandidates,
+            filtered,
+            scored: [],
+            outcome: { kind: "exhausted" },
+        });
+    }
+    return {
+        result: { kind: "failure", reason: { code: "noCapacityInAnyPool", triedPools: tried } },
+        trace: { attempts, decision: "failure" },
+    };
+}
+// ---------- filter ----------
+function filterCandidatesTraced(input, pool, tier) {
+    const { bundle, job, replicaIndex, inventory, recoveryContext } = input;
+    const matched = [];
+    const filtered = [];
+    for (const n of inventory) {
+        const reason = firstFilterMiss(n, input, pool, tier, recoveryContext);
+        if (reason === undefined) {
+            matched.push(n);
+        }
+        else {
+            // Skip "wrong-pool"/"wrong-tier" entries from the trace by default —
+            // for a large cross-pool inventory the "this isn't in my pool/tier"
+            // entries dominate the trace and aren't useful. Keep all other reasons.
+            if (reason !== "wrong-pool" && reason !== "wrong-tier")
+                filtered.push({ nodeId: n.nodeId, reason });
+        }
+    }
+    // Suppressed unused-import lint when no consumer references these
+    // utilities directly any more.
+    void bundle;
+    void job;
+    void replicaIndex;
+    return { matched, filtered };
+}
+function firstFilterMiss(n, input, pool, tier, recoveryContext) {
+    const { bundle, job, replicaIndex, inventory } = input;
+    if (n.poolName !== pool.name)
+        return "wrong-pool";
+    if (!nodeMatchesTier(n, pool, tier))
+        return "wrong-tier";
+    if (recoveryContext?.excludeNodes.has(n.nodeId))
+        return "excluded-recovery";
+    if (!n.inventory.reachable)
+        return "unreachable";
+    if (n.inventory.cordoned)
+        return "cordoned";
+    if (!matchesAllCapabilities(n.inventory, job.requirements.requiredCapabilities))
+        return "missing-capability";
+    if ((0, inventory_1.freeCapacity)(n.inventory) < job.requirements.requiredCapacity)
+        return "insufficient-capacity";
+    if ((0, inventory_1.freeCores)(n.inventory) < (job.requirements.requiredCores ?? 0))
+        return "insufficient-cores";
+    if (!matchesGpu(n.inventory.gpus, job.requirements))
+        return "no-fitting-gpu";
+    if (!satisfiesIntraReplica(n, bundle, job, replicaIndex, inventory))
+        return "fails-intra-replica";
+    if (!satisfiesIntraReplicaAz(n, bundle, replicaIndex, inventory))
+        return "fails-intra-az";
+    // Resilience anti-affinity (backup vs primary) is NOT a hard filter — it
+    // is best-effort, applied as a scoring penalty so the backup still
+    // places when no compliant node has capacity (then flagged degraded).
+    return undefined;
+}
+function matchesAllCapabilities(inv, reqs) {
+    for (const req of reqs) {
+        if ((0, inventory_1.freeCapability)(inv, req.name) < req.count)
+            return false;
+        if (req.attributeMatches) {
+            const cap = inv.capabilities.find((c) => c.name === req.name);
+            if (!cap)
+                return false;
+            for (const [k, v] of Object.entries(req.attributeMatches)) {
+                if (cap.attributes?.[k] !== v)
+                    return false;
+            }
+        }
+    }
+    return true;
+}
+function matchesGpu(gpus, req) {
+    if (req.requiredGpuCapacity === undefined || req.requiredGpuCapacity <= 0) {
+        return true;
+    }
+    return gpus.some((g) => gpuFits(g, req));
+}
+function gpuFits(g, req) {
+    if (g.exclusivelyReserved)
+        return false;
+    if (req.requiredGpuModel !== undefined && g.model !== req.requiredGpuModel)
+        return false;
+    const free = g.totalCapacity - g.reservedCapacity;
+    return free >= (req.requiredGpuCapacity ?? 0);
+}
+function satisfiesIntraReplica(n, bundle, job, replicaIndex, inventory) {
+    // coLocateWith: every named job in this same replica must be running on this node.
+    if (job.coLocateWith && job.coLocateWith.length > 0) {
+        for (const peer of job.coLocateWith) {
+            const peerNode = findRunningJob(inventory, bundle.bundleId, replicaIndex, peer);
+            // If peer not yet placed, no constraint to check yet.
+            if (peerNode && peerNode.nodeId !== n.nodeId)
+                return false;
+        }
+    }
+    // separateFrom: every named peer must be on a different node.
+    if (job.separateFrom && job.separateFrom.length > 0) {
+        for (const peer of job.separateFrom) {
+            const peerNode = findRunningJob(inventory, bundle.bundleId, replicaIndex, peer);
+            if (peerNode && peerNode.nodeId === n.nodeId)
+                return false;
+        }
+    }
+    return true;
+}
+function satisfiesIntraReplicaAz(n, bundle, replicaIndex, inventory) {
+    const sameAz = bundle.intraReplicaPlacement?.sameAz ?? "soft";
+    if (sameAz !== "hard")
+        return true;
+    const peerAzs = peerAzsInSameReplica(inventory, bundle.bundleId, replicaIndex);
+    if (peerAzs.length === 0)
+        return true;
+    // Hard sameAz with peers in known AZs: this node's AZ must match one of theirs.
+    return n.az !== undefined && peerAzs.includes(n.az);
+}
+function peerAzsInSameReplica(inventory, bundleId, replicaIndex) {
+    const azs = new Set();
+    for (const n of inventory) {
+        if (n.az !== undefined &&
+            n.runningJobs.some((rj) => rj.bundleId === bundleId && rj.replicaIndex === replicaIndex)) {
+            azs.add(n.az);
+        }
+    }
+    return [...azs];
+}
+// ---------- resilience anti-affinity (backup vs primary) ----------
+/** True when this placement is a backup whose policy carries anti-affinity. */
+function backupPolicy(input) {
+    return input.replicaIndex > 0 ? input.bundle.resiliencePolicy : undefined;
+}
+/** Failure domains the primary (replica 0) currently occupies. */
+function primaryDomains(inventory, bundleId) {
+    const nodeIds = new Set();
+    const azs = new Set();
+    const clouds = new Set();
+    for (const n of inventory) {
+        if (n.runningJobs.some((rj) => rj.bundleId === bundleId && rj.replicaIndex === 0)) {
+            nodeIds.add(n.nodeId);
+            if (n.az !== undefined)
+                azs.add(n.az);
+            if (n.cloud !== undefined)
+                clouds.add(n.cloud);
+        }
+    }
+    return { nodeIds, azs, clouds };
+}
+/** Which `forbid` flags placing the backup on `n` would violate. */
+function resilienceViolations(n, policy, primary) {
+    const violated = [];
+    if (policy.sameNode === "forbid" && primary.nodeIds.has(n.nodeId))
+        violated.push("sameNode");
+    if (policy.sameAz === "forbid" && n.az !== undefined && primary.azs.has(n.az))
+        violated.push("sameAz");
+    if (policy.sameCloud === "forbid" && n.cloud !== undefined && primary.clouds.has(n.cloud))
+        violated.push("sameCloud");
+    return violated;
+}
+/** Degradation for placing a backup on an existing node `n`, if any. */
+function placeDegradation(n, input) {
+    const policy = backupPolicy(input);
+    if (!policy)
+        return undefined;
+    const violated = resilienceViolations(n, policy, primaryDomains(input.inventory, input.bundle.bundleId));
+    return violated.length > 0 ? { violated } : undefined;
+}
+/**
+ * Degradation for provisioning a backup into `tier`. Only the cloud domain
+ * is knowable pre-boot (it is the tier's provider); node/AZ are decided
+ * once the instance starts, so they aren't evaluated here.
+ */
+function provisionDegradation(tier, input) {
+    const policy = backupPolicy(input);
+    if (!policy || policy.sameCloud !== "forbid")
+        return undefined;
+    const primary = primaryDomains(input.inventory, input.bundle.bundleId);
+    return primary.clouds.has(tier.kind) ? { violated: ["sameCloud"] } : undefined;
+}
+/** Degradation for placing the *primary* (replica 0) onto interruptible
+ *  capacity with no durable backup configured — one interruption and the
+ *  workload is gone. (Backups have their own anti-affinity degradation.) */
+function primaryInterruptibleDegradation(tier, input) {
+    if (input.replicaIndex !== 0 || input.bundle.resiliencePolicy)
+        return undefined;
+    if (tierReliability(tier) !== "interruptible")
+        return undefined;
+    return { violated: [], interruptiblePrimary: true };
+}
+/** Count of nodes currently in (pool, tier) — used for the tier node cap. */
+function tierNodeCount(inventory, pool, tier) {
+    let n = 0;
+    for (const node of inventory)
+        if (nodeMatchesTier(node, pool, tier))
+            n++;
+    return n;
+}
+function findRunningJob(inventory, bundleId, replicaIndex, jobName) {
+    return inventory.find((n) => n.runningJobs.some((rj) => rj.bundleId === bundleId &&
+        rj.replicaIndex === replicaIndex &&
+        rj.jobName === jobName));
+}
+// ---------- score ----------
+//
+// Score = base (pack/spread on capacity) + soft-constraint bonuses.
+// Bonuses are deliberately small relative to a fully-laden vs empty
+// difference (which is 1.0 for binpack), but large enough to break
+// ties between similar candidates. Hot-spare bonus dominates because
+// recovery placements should aggressively prefer prewarmed slots.
+const HOT_SPARE_BONUS = 1.0;
+const INTRA_SAME_AZ_BONUS = 0.1;
+// A resilience `forbid` violation is best-effort, not a hard reject: a
+// large per-violation penalty so the backup lands outside the primary's
+// failure domains whenever a node there has capacity, but still places
+// (and is flagged degraded) when none does. Dwarfs the [-1,1] capacity
+// score and the small AZ/hot-spare bonuses so a compliant node always wins.
+const RESILIENCE_VIOLATION_PENALTY = 100;
+function scoreNode(n, strategy, input) {
+    const used = (0, inventory_1.usedCapacityFraction)(n.inventory);
+    let score = strategy === "binpack" ? used : -used;
+    score += softBonus(n, input);
+    return score;
+}
+function softBonus(n, input) {
+    const { bundle, replicaIndex, inventory, recoveryContext } = input;
+    let bonus = 0;
+    if (recoveryContext?.preferHotSpares && n.isHotSpare) {
+        bonus += HOT_SPARE_BONUS;
+    }
+    // Soft sameAz (intra-replica): bonus if node's AZ matches a peer in the same replica.
+    const intraAz = bundle.intraReplicaPlacement?.sameAz ?? "soft";
+    if (intraAz === "soft" && n.az !== undefined) {
+        const peerAzs = peerAzsInSameReplica(inventory, bundle.bundleId, replicaIndex);
+        if (peerAzs.includes(n.az))
+            bonus += INTRA_SAME_AZ_BONUS;
+    }
+    // Resilience anti-affinity (backup vs primary): strong best-effort penalty
+    // for each `forbid` flag this node would violate, so the backup prefers a
+    // node outside the primary's failure domains.
+    const policy = backupPolicy(input);
+    if (policy) {
+        const primary = primaryDomains(inventory, bundle.bundleId);
+        bonus -= resilienceViolations(n, policy, primary).length * RESILIENCE_VIOLATION_PENALTY;
+    }
+    return bonus;
+}
+// ---------- GPU pick ----------
+function pickGpuIndex(n, req, strategy) {
+    if (req.requiredGpuCapacity === undefined || req.requiredGpuCapacity <= 0) {
+        return undefined;
+    }
+    const eligible = n.inventory.gpus.filter((g) => gpuFits(g, req));
+    if (eligible.length === 0)
+        return undefined;
+    // Pack: pick the GPU with least free capacity that still fits. Spread:
+    // pick the most free. Same packing intent as node-level scoring.
+    const sorted = eligible.slice().sort((a, b) => {
+        const freeA = a.totalCapacity - a.reservedCapacity;
+        const freeB = b.totalCapacity - b.reservedCapacity;
+        if (freeA !== freeB) {
+            return strategy === "binpack" ? freeA - freeB : freeB - freeA;
+        }
+        return a.index - b.index;
+    });
+    return sorted[0].index;
+}
+// ---------- elastic provision ----------
+function findProvisionableInstanceType(tier, req) {
+    for (const it of tier.candidateInstanceTypes) {
+        if (instanceTypeSatisfies(it, req))
+            return it.instanceType;
+    }
+    return undefined;
+}
+function instanceTypeSatisfies(it, req) {
+    if (it.totalCapacity < req.requiredCapacity)
+        return false;
+    if (it.totalCores < (req.requiredCores ?? 0))
+        return false;
+    for (const cap of req.requiredCapabilities) {
+        const found = it.capabilities.find((c) => c.name === cap.name);
+        if (!found || found.count < cap.count)
+            return false;
+        if (cap.attributeMatches) {
+            for (const [k, v] of Object.entries(cap.attributeMatches)) {
+                if (found.attributes?.[k] !== v)
+                    return false;
+            }
+        }
+    }
+    if (req.requiredGpuCapacity !== undefined && req.requiredGpuCapacity > 0) {
+        const gpus = it.gpus ?? [];
+        const fits = gpus.some((g) => (req.requiredGpuModel === undefined || g.model === req.requiredGpuModel) &&
+            g.totalCapacity >= (req.requiredGpuCapacity ?? 0));
+        if (!fits)
+            return false;
+    }
+    return true;
+}
+//# sourceMappingURL=placement.js.map

package/lib/src/settingsValidation.d.ts

@@ -0,0 +1,34 @@
+import type { AutoSettings } from "./automanager";
+/** @public */
+export interface SettingsValidationError {
+    code: SettingsValidationErrorCode;
+    /** Dotted path to the offending field. */
+    path: string;
+    message: string;
+}
+/** @public */
+export type SettingsValidationErrorCode = "invalidType" | "missingRequired" | "outOfRange" | "invalidEnumValue" | "unknownPool" | "duplicatePool" | "incompatibleConfig";
+/**
+ * Validate an `AutoSettings` value at boot. Returns all errors found
+ * (does not short-circuit). Callbacks are not validated — TypeScript
+ * structural typing already covers them as well as a runtime check
+ * usefully can. The point of this validator is to catch
+ * configuration mistakes operators make (typos in pool names, bad
+ * enum values, negative timeouts), not to second-guess the type
+ * system.
+ *
+ * @public
+ */
+export declare function validateAutoSettings(settings: AutoSettings): SettingsValidationError[];
+/**
+ * Thrown by `AutoManager.run` when `validateAutoSettings` finds
+ * configuration errors. The `.errors` property carries the structured
+ * details; `.message` contains a human-readable summary.
+ *
+ * @public
+ */
+export declare class AutoSettingsValidationError extends Error {
+    readonly errors: SettingsValidationError[];
+    constructor(errors: SettingsValidationError[]);
+}
+//# sourceMappingURL=settingsValidation.d.ts.map