@norskvideo/norsk-auto-manager 0.1.0

package/src/placement.ts

@@ -0,0 +1,804 @@
+// Pure-function placement engine. Given a job-to-place, the bundle it
+// belongs to, the current inventory snapshot, and the pool configuration,
+// returns a placement decision: place on a specific node, provision a new
+// node in an elastic pool, or fail.
+//
+// Per §6 of the design doc: filter by hard constraints, score by packing
+// strategy, tiebreak by nodeId for determinism. Per-replica pool
+// preferences (replicaOverrides) take priority over the bundle default.
+//
+// Soft constraints (same-AZ preference, hot-spare scoring bonus) are
+// not yet implemented in this slice — they'll layer on top of the score
+// function once the hard-constraint path is settled and tested.
+
+import {
+  Bundle,
+  BundleId,
+  BundleJobSpec,
+  Capability,
+  CapabilityRequirement,
+  GpuResource,
+  JobRequirements,
+  NodeId,
+  NodeInventory,
+  ResiliencePolicy,
+} from "./types";
+import {
+  freeCapability,
+  freeCapacity,
+  freeCores,
+  usedCapacityFraction,
+} from "./inventory";
+
+/** @public */
+export interface NodeView {
+  nodeId: NodeId;
+  poolName: string;
+  /**
+   * Which tier within the pool this node belongs to. Set from
+   * `tags["tier"]` at provision time for elastic tiers. Pre-registered
+   * cluster nodes carry no tier tag (undefined) and are treated as
+   * belonging to their pool's `fixed` tier — see `nodeMatchesTier`.
+   */
+  tierName?: string;
+  inventory: NodeInventory;
+  /** Jobs currently running on this node (or pending placement onto it). */
+  runningJobs: RunningJob[];
+  /** Topology key for the resilience sameAz rule. Cluster pools can leave undefined. */
+  az?: string;
+  /**
+   * Cloud / failure-domain key for the resilience sameCloud rule — the
+   * node's provider (e.g. "aws" / "oci" / "cluster"). Undefined when
+   * unknown; sameCloud then can't be evaluated against this node.
+   */
+  cloud?: string;
+  /** Hot spare flag (Phase C). Currently unused by hard-constraint filters. */
+  isHotSpare: boolean;
+}
+
+/** @public */
+export interface RunningJob {
+  bundleId: BundleId;
+  replicaIndex: number;
+  jobName: string;
+}
+
+/**
+ * A pool is an ordered list of capacity tiers. Placement walks the tiers
+ * in order (cheapest first) and takes the first that can satisfy the job
+ * — the tier order *is* the cost cascade (no cost weight in v1). The
+ * homogeneous single-cloud pool of before is just a one-tier pool.
+ *
+ * @public
+ */
+export interface PlacementPool {
+  name: string;
+  /** Ordered cheapest → most expensive. Placement tries them in order. */
+  tiers: PlacementTier[];
+}
+
+/**
+ * One homogeneous capacity segment within a pool: a single cloud/cluster
+ * kind + region with its own scale-out behaviour and candidate shapes.
+ *
+ * @public
+ */
+export interface PlacementTier {
+  /** Unique within the pool. Tagged onto provisioned nodes as `tags["tier"]`. */
+  name: string;
+  /**
+   * Cloud / local-servers kind. Used by AutoManager to decide which gRPC
+   * call to make when provisioning (createAwsNode vs createOciNode vs
+   * local-servers-tier placement, which uses startJob against an existing
+   * registered server).
+   */
+  kind: "aws" | "oci" | "local-servers";
+  /**
+   * For aws/oci elastic tiers: which region/availability-domain to
+   * provision into. Cluster tiers leave undefined.
+   */
+  region?: string;
+  packingStrategy: "binpack" | "spread";
+  scaleOut: "elastic" | "fixed";
+  /**
+   * For elastic tiers: shapes that can be provisioned on demand. For
+   * fixed tiers: typically empty (the tier consists of registered
+   * cluster nodes already in `inventory`).
+   */
+  candidateInstanceTypes: InstanceTypeOption[];
+  /**
+   * How long ahead of a scheduled job's `startDateTime` AutoManager
+   * should kick off placement, so the worker is ready to run by the
+   * requested time. Cluster tiers place against an existing running
+   * node (~instant) so the default is 0. Cloud tiers have to boot a
+   * fresh instance — defaults to 5 minutes; operators set per-tier
+   * when their boot time differs. AutoManager picks the maximum across
+   * the bundle pool's tiers (worst case — guarantees readiness
+   * regardless of which tier the placement engine settles on).
+   */
+  placementLeadMs?: number;
+  /**
+   * How nodes in this tier are purchased. `"spot"` requests interruptible
+   * spot-market capacity; `"reserved"` is an on-demand launch the operator
+   * knows is covered by a standing reservation (same AWS call as on-demand,
+   * but capped — see `maxNodes`). Defaults to `"on-demand"`.
+   */
+  launchMode?: LaunchMode;
+  /**
+   * Failure-domain reliability of this tier's capacity. Defaults to
+   * `"interruptible"` for spot, `"durable"` otherwise. Read by the
+   * resilience engine — an interruptible-tier primary with no durable
+   * backup is flagged degraded.
+   */
+  reliability?: Reliability;
+  /**
+   * Cap on the number of nodes this tier may hold (tier-wide, across
+   * bundles). The cost cascade spills to the next tier once the cap is
+   * reached. Used mainly for reserved tiers — exceeding the reservation
+   * count just bills as on-demand, defeating the point. Unset ⇒ no cap.
+   */
+  maxNodes?: number;
+}
+
+/** @public */
+export type LaunchMode = "on-demand" | "spot" | "reserved";
+
+/** @public */
+export type Reliability = "durable" | "interruptible";
+
+/** The tier's launch mode, defaulting to on-demand. @public */
+export function tierLaunchMode(tier: PlacementTier): LaunchMode {
+  return tier.launchMode ?? "on-demand";
+}
+
+/** The tier's reliability — explicit, else interruptible for spot. @public */
+export function tierReliability(tier: PlacementTier): Reliability {
+  return tier.reliability ?? (tierLaunchMode(tier) === "spot" ? "interruptible" : "durable");
+}
+
+/** Default placement lead time by tier kind. Cluster tiers place
+ *  against existing running nodes so we don't need lead. Cloud tiers
+ *  default to 5 minutes — typical EC2/OCI instance boot + image pull
+ *  budget; operators can override per-tier via PlacementTier. */
+export function defaultPlacementLeadMs(kind: PlacementTier["kind"]): number {
+  return kind === "local-servers" ? 0 : 5 * 60 * 1000;
+}
+
+/**
+ * Whether a node belongs to (pool, tier). Elastic nodes carry an explicit
+ * `tierName` (set from `tags["tier"]` at provision). Pre-registered cluster
+ * nodes carry no tier tag and are taken to belong to the pool's `fixed`
+ * tier — so a pool must have at most one fixed tier (enforced in
+ * settingsValidation) for this to be unambiguous.
+ *
+ * @public
+ */
+export function nodeMatchesTier(
+  n: NodeView,
+  pool: PlacementPool,
+  tier: PlacementTier
+): boolean {
+  if (n.poolName !== pool.name) return false;
+  if (n.tierName !== undefined) return n.tierName === tier.name;
+  return tier.scaleOut === "fixed";
+}
+
+/** @public */
+export interface InstanceTypeOption {
+  instanceType: string;
+  totalCapacity: number;
+  totalCores: number;
+  capabilities: Capability[];
+  /** GPUs we could provision a node with — for feasibility check. */
+  gpus?: { totalCapacity: number; model?: string }[];
+}
+
+/** @public */
+export interface RecoveryContext {
+  /** NodeIds the placement engine must skip (e.g., recently failed). */
+  excludeNodes: Set<NodeId>;
+  /** Reserved for Phase C — biases scoring toward hot spares. */
+  preferHotSpares: boolean;
+}
+
+/** @public */
+export interface PlacementInput {
+  bundle: Bundle;
+  job: BundleJobSpec;
+  replicaIndex: number;
+  inventory: NodeView[];
+  pools: Map<string, PlacementPool>;
+  recoveryContext?: RecoveryContext;
+}
+
+/** @public */
+/** Which anti-affinity flag(s) a backup placement couldn't honour. */
+export type ResilienceFlag = "sameNode" | "sameAz" | "sameCloud";
+
+/**
+ * Set on a backup placement that landed inside a failure domain its
+ * resilience policy asked to avoid (best-effort: no compliant node had
+ * capacity). The engine places it anyway and surfaces this so the
+ * operator knows the bundle's DR posture is currently reduced.
+ *
+ * @public
+ */
+export interface ResilienceDegradation {
+  violated: ResilienceFlag[];
+  /**
+   * Set when the primary landed on interruptible (spot) capacity with no
+   * durable backup — a single interruption takes the workload down. Surfaced
+   * so the operator can add a durable backup.
+   */
+  interruptiblePrimary?: boolean;
+}
+
+/** @public */
+export type PlacementResult =
+  | {
+      kind: "place";
+      nodeId: NodeId;
+      gpuIndex?: number;
+      pool: string;
+      tier: string;
+      degraded?: ResilienceDegradation;
+    }
+  | {
+      kind: "provision";
+      pool: string;
+      tier: string;
+      instanceType: string;
+      launchMode: LaunchMode;
+      degraded?: ResilienceDegradation;
+    }
+  | { kind: "failure"; reason: PlacementFailureReason };
+
+/** @public */
+export interface PlacementFailureReason {
+  code: "noCapacityInAnyPool" | "noConfiguredPools";
+  triedPools: string[];
+}
+
+/**
+ * Structured trace of a placement decision — one entry per pool
+ * attempted, with per-node filter outcomes and resulting scores. Used
+ * for "why did my job land here?" debugging.
+ *
+ * @public
+ */
+export interface PlacementTrace {
+  attempts: PoolAttempt[];
+  /** Final decision kind, mirroring PlacementResult.kind. */
+  decision: "place" | "provision" | "failure";
+}
+
+/** @public */
+export interface PoolAttempt {
+  pool: string;
+  /** Which tier of the pool this attempt covers. */
+  tier: string;
+  /** Total nodes considered in this pool before any filtering. */
+  initialCandidates: number;
+  /** Why each non-matching node was filtered out. Indexed by nodeId. */
+  filtered: { nodeId: NodeId; reason: FilterReason }[];
+  /** Nodes that survived all filters, with their assigned scores. */
+  scored: { nodeId: NodeId; score: number }[];
+  /** What this pool attempt produced. */
+  outcome:
+    | { kind: "placed"; nodeId: NodeId; gpuIndex?: number }
+    | { kind: "provisioned"; instanceType: string }
+    | { kind: "exhausted" }
+    | { kind: "no-instance-type" }
+    | { kind: "missing-pool" };
+}
+
+/** @public */
+export type FilterReason =
+  | "wrong-pool"
+  | "wrong-tier"
+  | "excluded-recovery"
+  | "unreachable"
+  | "cordoned"
+  | "missing-capability"
+  | "insufficient-capacity"
+  | "insufficient-cores"
+  | "no-fitting-gpu"
+  | "fails-intra-replica"
+  | "fails-intra-az";
+
+/**
+ * Core placement function. Pure — no side effects, no I/O.
+ *
+ * @public
+ */
+export function place(input: PlacementInput): PlacementResult {
+  return placeWithTrace(input).result;
+}
+
+/**
+ * Placement with a structured trace of every pool attempted and every
+ * node's filter outcome. AutoManager logs the trace via debuglog when
+ * placing; tests use it to assert on the precise reason a candidate
+ * was rejected.
+ *
+ * @public
+ */
+export function placeWithTrace(
+  input: PlacementInput
+): { result: PlacementResult; trace: PlacementTrace } {
+  const { bundle, job } = input;
+  const tried: string[] = [];
+  const attempts: PoolAttempt[] = [];
+
+  const pool = input.pools.get(bundle.pool);
+  if (!pool || pool.tiers.length === 0) {
+    return {
+      result: { kind: "failure", reason: { code: "noConfiguredPools", triedPools: [] } },
+      trace: { attempts: [], decision: "failure" },
+    };
+  }
+
+  // Walk the tiers cheapest-first; first tier that can place or provision
+  // wins. This positional order is the cost cascade (no cost weight in v1).
+  for (const tier of pool.tiers) {
+    tried.push(tier.name);
+
+    const { matched, filtered } = filterCandidatesTraced(input, pool, tier);
+    const initialCandidates = input.inventory.length;
+
+    if (matched.length > 0) {
+      const scored = matched.map((n) => ({
+        node: n,
+        score: scoreNode(n, tier.packingStrategy, input),
+      }));
+      // Sort descending by score, lexicographic nodeId tiebreak.
+      scored.sort((a, b) => {
+        if (a.score !== b.score) return b.score - a.score;
+        return a.node.nodeId.localeCompare(b.node.nodeId);
+      });
+      const winner = scored[0].node;
+      const gpuIndex = pickGpuIndex(winner, job.requirements, tier.packingStrategy);
+      const degraded = placeDegradation(winner, input) ?? primaryInterruptibleDegradation(tier, input);
+      attempts.push({
+        pool: pool.name,
+        tier: tier.name,
+        initialCandidates,
+        filtered,
+        scored: scored.map((s) => ({ nodeId: s.node.nodeId, score: s.score })),
+        outcome: { kind: "placed", nodeId: winner.nodeId, gpuIndex },
+      });
+      return {
+        result: {
+          kind: "place",
+          nodeId: winner.nodeId,
+          gpuIndex,
+          pool: pool.name,
+          tier: tier.name,
+          ...(degraded ? { degraded } : {}),
+        },
+        trace: { attempts, decision: "place" },
+      };
+    }
+
+    // A tier at its node cap can't grow — the cost cascade spills to the
+    // next tier. (Placing onto an existing node above doesn't add a node,
+    // so the cap only gates provisioning.)
+    const atCap = tier.maxNodes !== undefined && tierNodeCount(input.inventory, pool, tier) >= tier.maxNodes;
+
+    if (tier.scaleOut === "elastic" && !atCap) {
+      const instanceType = findProvisionableInstanceType(tier, job.requirements);
+      if (instanceType) {
+        const degraded = provisionDegradation(tier, input) ?? primaryInterruptibleDegradation(tier, input);
+        attempts.push({
+          pool: pool.name,
+          tier: tier.name,
+          initialCandidates,
+          filtered,
+          scored: [],
+          outcome: { kind: "provisioned", instanceType },
+        });
+        return {
+          result: {
+            kind: "provision",
+            pool: pool.name,
+            tier: tier.name,
+            instanceType,
+            launchMode: tierLaunchMode(tier),
+            ...(degraded ? { degraded } : {}),
+          },
+          trace: { attempts, decision: "provision" },
+        };
+      }
+      attempts.push({
+        pool: pool.name,
+        tier: tier.name,
+        initialCandidates,
+        filtered,
+        scored: [],
+        outcome: { kind: "no-instance-type" },
+      });
+      continue;
+    }
+    attempts.push({
+      pool: pool.name,
+      tier: tier.name,
+      initialCandidates,
+      filtered,
+      scored: [],
+      outcome: { kind: "exhausted" },
+    });
+  }
+
+  return {
+    result: { kind: "failure", reason: { code: "noCapacityInAnyPool", triedPools: tried } },
+    trace: { attempts, decision: "failure" },
+  };
+}
+
+// ---------- filter ----------
+
+function filterCandidatesTraced(
+  input: PlacementInput,
+  pool: PlacementPool,
+  tier: PlacementTier
+): { matched: NodeView[]; filtered: { nodeId: NodeId; reason: FilterReason }[] } {
+  const { bundle, job, replicaIndex, inventory, recoveryContext } = input;
+  const matched: NodeView[] = [];
+  const filtered: { nodeId: NodeId; reason: FilterReason }[] = [];
+
+  for (const n of inventory) {
+    const reason = firstFilterMiss(n, input, pool, tier, recoveryContext);
+    if (reason === undefined) {
+      matched.push(n);
+    } else {
+      // Skip "wrong-pool"/"wrong-tier" entries from the trace by default —
+      // for a large cross-pool inventory the "this isn't in my pool/tier"
+      // entries dominate the trace and aren't useful. Keep all other reasons.
+      if (reason !== "wrong-pool" && reason !== "wrong-tier")
+        filtered.push({ nodeId: n.nodeId, reason });
+    }
+  }
+  // Suppressed unused-import lint when no consumer references these
+  // utilities directly any more.
+  void bundle;
+  void job;
+  void replicaIndex;
+
+  return { matched, filtered };
+}
+
+function firstFilterMiss(
+  n: NodeView,
+  input: PlacementInput,
+  pool: PlacementPool,
+  tier: PlacementTier,
+  recoveryContext: RecoveryContext | undefined
+): FilterReason | undefined {
+  const { bundle, job, replicaIndex, inventory } = input;
+  if (n.poolName !== pool.name) return "wrong-pool";
+  if (!nodeMatchesTier(n, pool, tier)) return "wrong-tier";
+  if (recoveryContext?.excludeNodes.has(n.nodeId)) return "excluded-recovery";
+  if (!n.inventory.reachable) return "unreachable";
+  if (n.inventory.cordoned) return "cordoned";
+  if (!matchesAllCapabilities(n.inventory, job.requirements.requiredCapabilities))
+    return "missing-capability";
+  if (freeCapacity(n.inventory) < job.requirements.requiredCapacity)
+    return "insufficient-capacity";
+  if (freeCores(n.inventory) < (job.requirements.requiredCores ?? 0))
+    return "insufficient-cores";
+  if (!matchesGpu(n.inventory.gpus, job.requirements)) return "no-fitting-gpu";
+  if (!satisfiesIntraReplica(n, bundle, job, replicaIndex, inventory))
+    return "fails-intra-replica";
+  if (!satisfiesIntraReplicaAz(n, bundle, replicaIndex, inventory))
+    return "fails-intra-az";
+  // Resilience anti-affinity (backup vs primary) is NOT a hard filter — it
+  // is best-effort, applied as a scoring penalty so the backup still
+  // places when no compliant node has capacity (then flagged degraded).
+  return undefined;
+}
+
+function matchesAllCapabilities(
+  inv: NodeInventory,
+  reqs: CapabilityRequirement[]
+): boolean {
+  for (const req of reqs) {
+    if (freeCapability(inv, req.name) < req.count) return false;
+    if (req.attributeMatches) {
+      const cap = inv.capabilities.find((c) => c.name === req.name);
+      if (!cap) return false;
+      for (const [k, v] of Object.entries(req.attributeMatches)) {
+        if (cap.attributes?.[k] !== v) return false;
+      }
+    }
+  }
+  return true;
+}
+
+function matchesGpu(gpus: GpuResource[], req: JobRequirements): boolean {
+  if (req.requiredGpuCapacity === undefined || req.requiredGpuCapacity <= 0) {
+    return true;
+  }
+  return gpus.some((g) => gpuFits(g, req));
+}
+
+function gpuFits(g: GpuResource, req: JobRequirements): boolean {
+  if (g.exclusivelyReserved) return false;
+  if (req.requiredGpuModel !== undefined && g.model !== req.requiredGpuModel)
+    return false;
+  const free = g.totalCapacity - g.reservedCapacity;
+  return free >= (req.requiredGpuCapacity ?? 0);
+}
+
+function satisfiesIntraReplica(
+  n: NodeView,
+  bundle: Bundle,
+  job: BundleJobSpec,
+  replicaIndex: number,
+  inventory: NodeView[]
+): boolean {
+  // coLocateWith: every named job in this same replica must be running on this node.
+  if (job.coLocateWith && job.coLocateWith.length > 0) {
+    for (const peer of job.coLocateWith) {
+      const peerNode = findRunningJob(inventory, bundle.bundleId, replicaIndex, peer);
+      // If peer not yet placed, no constraint to check yet.
+      if (peerNode && peerNode.nodeId !== n.nodeId) return false;
+    }
+  }
+  // separateFrom: every named peer must be on a different node.
+  if (job.separateFrom && job.separateFrom.length > 0) {
+    for (const peer of job.separateFrom) {
+      const peerNode = findRunningJob(inventory, bundle.bundleId, replicaIndex, peer);
+      if (peerNode && peerNode.nodeId === n.nodeId) return false;
+    }
+  }
+  return true;
+}
+
+function satisfiesIntraReplicaAz(
+  n: NodeView,
+  bundle: Bundle,
+  replicaIndex: number,
+  inventory: NodeView[]
+): boolean {
+  const sameAz = bundle.intraReplicaPlacement?.sameAz ?? "soft";
+  if (sameAz !== "hard") return true;
+  const peerAzs = peerAzsInSameReplica(inventory, bundle.bundleId, replicaIndex);
+  if (peerAzs.length === 0) return true;
+  // Hard sameAz with peers in known AZs: this node's AZ must match one of theirs.
+  return n.az !== undefined && peerAzs.includes(n.az);
+}
+
+function peerAzsInSameReplica(
+  inventory: NodeView[],
+  bundleId: BundleId,
+  replicaIndex: number
+): string[] {
+  const azs = new Set<string>();
+  for (const n of inventory) {
+    if (
+      n.az !== undefined &&
+      n.runningJobs.some(
+        (rj) => rj.bundleId === bundleId && rj.replicaIndex === replicaIndex
+      )
+    ) {
+      azs.add(n.az);
+    }
+  }
+  return [...azs];
+}
+
+// ---------- resilience anti-affinity (backup vs primary) ----------
+
+/** True when this placement is a backup whose policy carries anti-affinity. */
+function backupPolicy(input: PlacementInput): ResiliencePolicy | undefined {
+  return input.replicaIndex > 0 ? input.bundle.resiliencePolicy : undefined;
+}
+
+interface PrimaryDomains {
+  nodeIds: Set<NodeId>;
+  azs: Set<string>;
+  clouds: Set<string>;
+}
+
+/** Failure domains the primary (replica 0) currently occupies. */
+function primaryDomains(inventory: NodeView[], bundleId: BundleId): PrimaryDomains {
+  const nodeIds = new Set<NodeId>();
+  const azs = new Set<string>();
+  const clouds = new Set<string>();
+  for (const n of inventory) {
+    if (n.runningJobs.some((rj) => rj.bundleId === bundleId && rj.replicaIndex === 0)) {
+      nodeIds.add(n.nodeId);
+      if (n.az !== undefined) azs.add(n.az);
+      if (n.cloud !== undefined) clouds.add(n.cloud);
+    }
+  }
+  return { nodeIds, azs, clouds };
+}
+
+/** Which `forbid` flags placing the backup on `n` would violate. */
+function resilienceViolations(
+  n: NodeView,
+  policy: ResiliencePolicy,
+  primary: PrimaryDomains
+): ResilienceFlag[] {
+  const violated: ResilienceFlag[] = [];
+  if (policy.sameNode === "forbid" && primary.nodeIds.has(n.nodeId)) violated.push("sameNode");
+  if (policy.sameAz === "forbid" && n.az !== undefined && primary.azs.has(n.az)) violated.push("sameAz");
+  if (policy.sameCloud === "forbid" && n.cloud !== undefined && primary.clouds.has(n.cloud))
+    violated.push("sameCloud");
+  return violated;
+}
+
+/** Degradation for placing a backup on an existing node `n`, if any. */
+function placeDegradation(n: NodeView, input: PlacementInput): ResilienceDegradation | undefined {
+  const policy = backupPolicy(input);
+  if (!policy) return undefined;
+  const violated = resilienceViolations(n, policy, primaryDomains(input.inventory, input.bundle.bundleId));
+  return violated.length > 0 ? { violated } : undefined;
+}
+
+/**
+ * Degradation for provisioning a backup into `tier`. Only the cloud domain
+ * is knowable pre-boot (it is the tier's provider); node/AZ are decided
+ * once the instance starts, so they aren't evaluated here.
+ */
+function provisionDegradation(tier: PlacementTier, input: PlacementInput): ResilienceDegradation | undefined {
+  const policy = backupPolicy(input);
+  if (!policy || policy.sameCloud !== "forbid") return undefined;
+  const primary = primaryDomains(input.inventory, input.bundle.bundleId);
+  return primary.clouds.has(tier.kind) ? { violated: ["sameCloud"] } : undefined;
+}
+
+/** Degradation for placing the *primary* (replica 0) onto interruptible
+ *  capacity with no durable backup configured — one interruption and the
+ *  workload is gone. (Backups have their own anti-affinity degradation.) */
+function primaryInterruptibleDegradation(
+  tier: PlacementTier,
+  input: PlacementInput
+): ResilienceDegradation | undefined {
+  if (input.replicaIndex !== 0 || input.bundle.resiliencePolicy) return undefined;
+  if (tierReliability(tier) !== "interruptible") return undefined;
+  return { violated: [], interruptiblePrimary: true };
+}
+
+/** Count of nodes currently in (pool, tier) — used for the tier node cap. */
+function tierNodeCount(inventory: NodeView[], pool: PlacementPool, tier: PlacementTier): number {
+  let n = 0;
+  for (const node of inventory) if (nodeMatchesTier(node, pool, tier)) n++;
+  return n;
+}
+
+function findRunningJob(
+  inventory: NodeView[],
+  bundleId: BundleId,
+  replicaIndex: number,
+  jobName: string
+): NodeView | undefined {
+  return inventory.find((n) =>
+    n.runningJobs.some(
+      (rj) =>
+        rj.bundleId === bundleId &&
+        rj.replicaIndex === replicaIndex &&
+        rj.jobName === jobName
+    )
+  );
+}
+
+// ---------- score ----------
+//
+// Score = base (pack/spread on capacity) + soft-constraint bonuses.
+// Bonuses are deliberately small relative to a fully-laden vs empty
+// difference (which is 1.0 for binpack), but large enough to break
+// ties between similar candidates. Hot-spare bonus dominates because
+// recovery placements should aggressively prefer prewarmed slots.
+
+const HOT_SPARE_BONUS = 1.0;
+const INTRA_SAME_AZ_BONUS = 0.1;
+// A resilience `forbid` violation is best-effort, not a hard reject: a
+// large per-violation penalty so the backup lands outside the primary's
+// failure domains whenever a node there has capacity, but still places
+// (and is flagged degraded) when none does. Dwarfs the [-1,1] capacity
+// score and the small AZ/hot-spare bonuses so a compliant node always wins.
+const RESILIENCE_VIOLATION_PENALTY = 100;
+
+function scoreNode(
+  n: NodeView,
+  strategy: "binpack" | "spread",
+  input: PlacementInput
+): number {
+  const used = usedCapacityFraction(n.inventory);
+  let score = strategy === "binpack" ? used : -used;
+  score += softBonus(n, input);
+  return score;
+}
+
+function softBonus(n: NodeView, input: PlacementInput): number {
+  const { bundle, replicaIndex, inventory, recoveryContext } = input;
+  let bonus = 0;
+
+  if (recoveryContext?.preferHotSpares && n.isHotSpare) {
+    bonus += HOT_SPARE_BONUS;
+  }
+
+  // Soft sameAz (intra-replica): bonus if node's AZ matches a peer in the same replica.
+  const intraAz = bundle.intraReplicaPlacement?.sameAz ?? "soft";
+  if (intraAz === "soft" && n.az !== undefined) {
+    const peerAzs = peerAzsInSameReplica(inventory, bundle.bundleId, replicaIndex);
+    if (peerAzs.includes(n.az)) bonus += INTRA_SAME_AZ_BONUS;
+  }
+
+  // Resilience anti-affinity (backup vs primary): strong best-effort penalty
+  // for each `forbid` flag this node would violate, so the backup prefers a
+  // node outside the primary's failure domains.
+  const policy = backupPolicy(input);
+  if (policy) {
+    const primary = primaryDomains(inventory, bundle.bundleId);
+    bonus -= resilienceViolations(n, policy, primary).length * RESILIENCE_VIOLATION_PENALTY;
+  }
+
+  return bonus;
+}
+
+// ---------- GPU pick ----------
+
+function pickGpuIndex(
+  n: NodeView,
+  req: JobRequirements,
+  strategy: "binpack" | "spread"
+): number | undefined {
+  if (req.requiredGpuCapacity === undefined || req.requiredGpuCapacity <= 0) {
+    return undefined;
+  }
+  const eligible = n.inventory.gpus.filter((g) => gpuFits(g, req));
+  if (eligible.length === 0) return undefined;
+  // Pack: pick the GPU with least free capacity that still fits. Spread:
+  // pick the most free. Same packing intent as node-level scoring.
+  const sorted = eligible.slice().sort((a, b) => {
+    const freeA = a.totalCapacity - a.reservedCapacity;
+    const freeB = b.totalCapacity - b.reservedCapacity;
+    if (freeA !== freeB) {
+      return strategy === "binpack" ? freeA - freeB : freeB - freeA;
+    }
+    return a.index - b.index;
+  });
+  return sorted[0].index;
+}
+
+// ---------- elastic provision ----------
+
+function findProvisionableInstanceType(
+  tier: PlacementTier,
+  req: JobRequirements
+): string | undefined {
+  for (const it of tier.candidateInstanceTypes) {
+    if (instanceTypeSatisfies(it, req)) return it.instanceType;
+  }
+  return undefined;
+}
+
+function instanceTypeSatisfies(
+  it: InstanceTypeOption,
+  req: JobRequirements
+): boolean {
+  if (it.totalCapacity < req.requiredCapacity) return false;
+  if (it.totalCores < (req.requiredCores ?? 0)) return false;
+  for (const cap of req.requiredCapabilities) {
+    const found = it.capabilities.find((c) => c.name === cap.name);
+    if (!found || found.count < cap.count) return false;
+    if (cap.attributeMatches) {
+      for (const [k, v] of Object.entries(cap.attributeMatches)) {
+        if (found.attributes?.[k] !== v) return false;
+      }
+    }
+  }
+  if (req.requiredGpuCapacity !== undefined && req.requiredGpuCapacity > 0) {
+    const gpus = it.gpus ?? [];
+    const fits = gpus.some(
+      (g) =>
+        (req.requiredGpuModel === undefined || g.model === req.requiredGpuModel) &&
+        g.totalCapacity >= (req.requiredGpuCapacity ?? 0)
+    );
+    if (!fits) return false;
+  }
+  return true;
+}