@norskvideo/norsk-auto-manager 0.1.0

package/lib/src/automanager.js

@@ -0,0 +1,2746 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AutoManager = exports.NODE_SERVICE_ROLE = exports.MIGRATION_TARGET_ROLE = void 0;
+exports.getJobIdForNode = getJobIdForNode;
+exports.currentNodeIdForJob = currentNodeIdForJob;
+const util = __importStar(require("util"));
+const uuid = __importStar(require("uuid"));
+const clock_1 = require("./clock");
+const types_1 = require("./types");
+const validation_1 = require("./validation");
+const settingsValidation_1 = require("./settingsValidation");
+const conversions_1 = require("./conversions");
+const placement_1 = require("./placement");
+const norsk_manager_sdk_1 = require("@norskvideo/norsk-manager-sdk");
+/** Fixed role for the migration-target instance. No concurrent migrations per Job. @public */
+exports.MIGRATION_TARGET_ROLE = "migrationTarget";
+/**
+ * Fixed role assigned to runtime jobs that exist to back a
+ * NodeService. Worker reads `WorkerJob.role === NODE_SERVICE_ROLE`
+ * to recognise the instance as a service rather than a regular
+ * workload, and adds the service's `provides[]` capabilities to its
+ * inventory once the job reaches `ready`.
+ *
+ * @public
+ */
+exports.NODE_SERVICE_ROLE = "nodeService";
+const DEFAULT_REJECTION_BACKOFF_MS = 5000;
+const DEFAULT_REJECTION_ESCALATION_COUNT = 3;
+const DEFAULT_FAILURE_BACKOFF_MS = 30000;
+const DEFAULT_MAX_RESTARTS_PER_HOUR = 6;
+const RESTART_WINDOW_MS = 60 * 60 * 1000;
+// Minimum spacing between consecutive recovery placement attempts for
+// the same JobId. Stops the "provisionJob → JobFailed → provisionJob"
+// tight loop that JobStore state-transition events naturally drive
+// when worker-side launches keep failing fast. 5s is short enough that
+// transient failures still recover quickly but long enough that even a
+// pathological failure pattern doesn't burn through `maxRestartsPerHour`
+// in seconds. Operator-tunable later via AutoSettings.
+const MIN_RECOVERY_INTERVAL_MS = 5000;
+// Minimum spacing between any `placeAndProvision` entries for the same
+// JobId. Tighter than MIN_RECOVERY_INTERVAL_MS — the window we need is
+// just "time for daemon's provisionJob to land + JobStarted to round-
+// trip back into AutoManager's nodeMap". A second is comfortably more
+// than typical local-host dispatch latency.
+const MIN_PLACEMENT_INTERVAL_MS = 1000;
+const DEFAULT_JOB_STARTUP_GRACE_MS = 120000;
+const DEFAULT_HOT_SPARE_RECONCILE_INTERVAL_MS = 60000;
+const DEFAULT_NODE_SERVICE_RECONCILE_INTERVAL_MS = 60000;
+// Phase E phase-deadline defaults. Tuned to the design's expectation
+// that target launch + workflow init typically completes well under
+// two minutes, source-stop is workflow-driven and quick, and apply
+// is a handful of wire hops.
+const DEFAULT_MIGRATION_LAUNCHING_DEADLINE_MS = 120000;
+const DEFAULT_MIGRATION_TARGET_READY_DEADLINE_MS = 120000;
+const DEFAULT_MIGRATION_SOURCE_STOP_DEADLINE_MS = 30000;
+const DEFAULT_MIGRATION_APPLYING_DEADLINE_MS = 10000;
+class AutoManager {
+    /** @public */
+    static async run(settings) {
+        // Phase D1 — fail fast on configuration mistakes.
+        const validationErrors = (0, settingsValidation_1.validateAutoSettings)(settings);
+        if (validationErrors.length > 0) {
+            throw new settingsValidation_1.AutoSettingsValidationError(validationErrors);
+        }
+        debuglog("Trying to connect", { url: settings.url });
+        const fire = (state) => settings.onConnectionStateChange?.(state);
+        try {
+            const norsk = await norsk_manager_sdk_1.NorskManager.connect({
+                url: settings.url,
+                signal: settings.signal,
+                onAttemptingToConnect: () => fire("idle"),
+                onConnecting: () => fire("connecting"),
+                onReady: () => fire("ready"),
+                onFailedToConnect: () => fire("transientFailure"),
+                onShutdown: () => fire("shutdown"),
+            });
+            const autoManager = new AutoManager(norsk, settings);
+            const promise = autoManager.runInternal();
+            return [autoManager, promise];
+        }
+        catch (err) {
+            debuglog("Exception in connection");
+            throw err;
+        }
+    }
+    /** @public */
+    async terminateNode(nodeId) {
+        await this.norsk.terminateNode(nodeId);
+    }
+    /**
+     * Phase C8 — cordon a node. AutoManager projects this onto every
+     * placement decision: cordoned nodes are filtered out, so no new
+     * placements land on them. Existing jobs continue running.
+     * Idempotent.
+     * @public
+     */
+    cordonNode(nodeId) {
+        if (this.cordonedNodes.has(nodeId))
+            return;
+        this.cordonedNodes.add(nodeId);
+        debuglog("Node cordoned", { nodeId });
+        // Reconcile in case a cordoned node was holding a spare slot —
+        // its un-spare-ness now opens the underflow path. Same for any
+        // NodeServices placed on the now-cordoned node.
+        this.reconcileHotSpares();
+        this.reconcileNodeServices();
+    }
+    /**
+     * Phase C8 — un-cordon a node, restoring it to the placement pool.
+     * Idempotent.
+     * @public
+     */
+    uncordonNode(nodeId) {
+        if (!this.cordonedNodes.delete(nodeId))
+            return;
+        debuglog("Node uncordoned", { nodeId });
+        this.reconcileHotSpares();
+        this.reconcileNodeServices();
+    }
+    /**
+     * Phase C8 — whether a node is currently cordoned by AutoManager.
+     * @public
+     */
+    isCordoned(nodeId) {
+        return this.cordonedNodes.has(nodeId);
+    }
+    /**
+     * Phase C8 — drain a node. Cordons it (so no new placements land
+     * here) and terminates the node — Manager emits `nodeStopping`
+     * for every job hosted there, and the existing band-aware recovery
+     * path re-places them onto other nodes (excluding this cordoned
+     * one). Bronze jobs are dropped; gold prefers hot spares.
+     *
+     * v1 takes the immediate-terminate path; the design's "graceful
+     * shuffle without recovery gap" needs Phase E job migration to
+     * land first.
+     * @public
+     */
+    async drainNode(nodeId) {
+        this.cordonNode(nodeId);
+        await this.norsk.terminateNode(nodeId);
+    }
+    /**
+     * Shut down this AutoManager. Cancels periodic intervals and closes
+     * the underlying NorskManager — which causes the event-loop iterator
+     * to terminate, resolving the `Promise<void>` returned from `run()`.
+     * Idempotent.
+     * @public
+     */
+    close() {
+        if (this.closed)
+            return;
+        this.closed = true;
+        this.cleanupInterval?.cancel();
+        this.cleanupInterval = undefined;
+        this.hotSpareReconcileInterval?.cancel();
+        this.hotSpareReconcileInterval = undefined;
+        this.nodeServiceReconcileInterval?.cancel();
+        this.nodeServiceReconcileInterval = undefined;
+        for (const t of this.startupGraceTimers.values())
+            t.cancel();
+        this.startupGraceTimers.clear();
+        for (const t of this.migrationPhaseTimers.values())
+            t.cancel();
+        this.migrationPhaseTimers.clear();
+        for (const t of this.pendingPlacementRetries.values())
+            clearTimeout(t);
+        this.pendingPlacementRetries.clear();
+        this.lastPlacementAttemptAt.clear();
+        this.norsk.close();
+    }
+    /**
+     * Alias for `close()`. Provided for symmetry with the
+     * `using`-style disposal conventions emerging in TypeScript.
+     * @public
+     */
+    dispose() {
+        this.close();
+    }
+    /**
+     * AutoManager v1 — submit a new bundle. The bundle is validated by
+     * Manager (per §4.3) and expanded into N replica jobs server-side;
+     * AutoManager mirrors the resulting state via the bundleUpdated +
+     * jobUpdated events. The returned bundleId matches the one Manager
+     * assigns (typically the same as bundle.bundleId, but Manager has
+     * the final word).
+     *
+     * @public
+     */
+    async createBundle(bundle) {
+        return this.norsk.createBundle((0, conversions_1.toPbBundle)(bundle));
+    }
+    /**
+     * AutoManager v1 — update an existing bundle.
+     * @public
+     */
+    async updateBundle(bundle) {
+        await this.norsk.updateBundle((0, conversions_1.toPbBundle)(bundle));
+    }
+    /**
+     * AutoManager v1 — delete a bundle and all its replica jobs.
+     * @public
+     */
+    async deleteBundle(bundleId) {
+        await this.norsk.deleteBundle(bundleId);
+    }
+    /**
+     * AutoManager v1 — activate a dormant (start_mode = "manual") bundle
+     * job. Manager removes the dormant tag and re-emits jobUpdated;
+     * AutoManager's placement path picks the job up via the standard
+     * flow. See §10.2 of the design.
+     * @public
+     */
+    async startBundleJob(bundleId, replicaIndex, jobName) {
+        await this.norsk.startBundleJob(bundleId, replicaIndex, jobName);
+    }
+    /**
+     * AutoManager v1 — deactivate a bundle job (tag-flip).
+     *
+     * Runtime termination of an in-flight workload and empty-node
+     * release with linger are pre-go-live work tracked as Phase E.5
+     * (§10.2.1). Until they land, calling stopBundleJob on an active
+     * job marks it dormant for future placement but does not stop the
+     * currently-running worker.
+     * @public
+     */
+    async stopBundleJob(bundleId, replicaIndex, jobName) {
+        await this.norsk.stopBundleJob(bundleId, replicaIndex, jobName);
+    }
+    /**
+     * AutoManager v1 — typed view of the jobs Manager has told us about.
+     * `JobWithHistory` carries the state-transition log for each job so
+     * consumers can render lifecycle timelines without a separate
+     * round-trip.
+     * @public
+     */
+    jobs() {
+        return Array.from(this.jobMap.values());
+    }
+    nodes() {
+        debuglog("Getting nodes", this.nodeMap);
+        return Array.from(this.nodeMap.values());
+    }
+    /**
+     * AutoManager v1 — typed view of the bundle definitions Manager has
+     * told us about. Mirror only; AutoManager owns no durable state.
+     * @public
+     */
+    bundles() {
+        return Array.from(this.bundleMap.values());
+    }
+    /**
+     * Phase D4 — snapshot of AutoManager's operational metrics.
+     * Cumulative counters since `startedAt`; per-pool gauges computed
+     * at call time. Cheap to call repeatedly — consumers poll on their
+     * own cadence and compute deltas if they want rates.
+     * @public
+     */
+    metrics() {
+        const now = this.clock.now();
+        const uptimeMs = this.startedAt
+            ? now.getTime() - this.startedAt.getTime()
+            : 0;
+        const poolNames = new Set();
+        for (const p of this.settings.placementPools ?? [])
+            poolNames.add(p.name);
+        // Include any pool name we've seen on a node, in case nodes are
+        // tagged with a pool that isn't in placementPools (rare but worth
+        // surfacing — operator's likely misconfigured).
+        for (const node of this.nodeMap.values()) {
+            const pn = node.nodeMetadata?.tags?.["pool"];
+            if (pn)
+                poolNames.add(pn);
+        }
+        const pools = [];
+        for (const poolName of poolNames) {
+            pools.push(this.computePoolMetrics(poolName));
+        }
+        return {
+            startedAt: this.startedAt,
+            uptimeMs,
+            ...this.metricsCounters,
+            pools,
+            brokenBundleIds: [...this.brokenBundles],
+        };
+    }
+    computePoolMetrics(poolName) {
+        let nodeCount = 0;
+        let liveSpares = 0;
+        let utilSum = 0;
+        let utilN = 0;
+        for (const node of this.nodeMap.values()) {
+            if (this.poolNameForNode(node.nodeId) !== poolName)
+                continue;
+            nodeCount++;
+            if (this.isHotSpareNode(node.nodeId) &&
+                node.nodeState === "running" &&
+                (this.inventoryMap.get(node.nodeId)?.assignedJobs.length ?? 0) === 0) {
+                liveSpares++;
+            }
+            const inv = this.inventoryMap.get(node.nodeId);
+            if (inv && inv.totalCapacity > 0) {
+                utilSum += inv.reservedCapacity / inv.totalCapacity;
+                utilN++;
+            }
+        }
+        let spareTarget = 0;
+        for (const cfg of this.settings.hotSpares ?? []) {
+            if (cfg.pool === poolName)
+                spareTarget += cfg.targetCount;
+        }
+        return {
+            poolName,
+            nodeCount,
+            liveSpares,
+            spareTarget,
+            usedCapacityFraction: utilN > 0 ? utilSum / utilN : 0,
+        };
+    }
+    /**
+     * Look up a single bundle by id. Convenience over `bundles().find(...)`.
+     * @public
+     */
+    bundleById(bundleId) {
+        return this.bundleMap.get(bundleId);
+    }
+    /**
+     * Every job (with history) that belongs to a given bundle. Derived
+     * from `tags.bundleId` on the job entries in jobMap. Returns an empty
+     * array if the bundle is unknown or has no jobs.
+     * @public
+     */
+    jobsForBundle(bundleId) {
+        const out = [];
+        for (const entry of this.jobMap.values()) {
+            if ((entry.job.tags ?? {})["bundleId"] === bundleId) {
+                out.push(entry);
+            }
+        }
+        return out;
+    }
+    /**
+     * The node the given job is currently placed on, or undefined if
+     * the job hasn't been placed (or has been deleted). Useful for
+     * operator UIs that render "job X is on node Y" and for proxy
+     * routers that need to know which worker hosts each job.
+     *
+     * O(1) lookup against the `jobToNode` index — set at placement
+     * time, deleted on jobDeleted, rebuilt from job history on R7
+     * reconnect. Works regardless of whether the node hosts one job
+     * (cloud, today) or many (cluster, or any future cloud N:1)
+     * because the assignment is recorded against the JOB rather than
+     * scanned out of NODE tags.
+     * @public
+     */
+    nodeForJob(jobId) {
+        const nodeId = this.jobToNode.get(jobId);
+        return nodeId !== undefined ? this.nodeMap.get(nodeId) : undefined;
+    }
+    /**
+     * Current placement of every job AutoManager has bound to a node,
+     * newest-first by decision timestamp. Mirrors `nodeForJob` but
+     * returns the full `PlacementDecisionInfo` (so consumers get the
+     * pool, trace, replicaIndex etc., not just the node) — and exposes
+     * it as queryable live state rather than only via the
+     * `onPlacementDecision` event stream.
+     *
+     * Hydrated jobs (placed in a previous AutoManager generation, picked
+     * up on R7 reconnect) get an entry with an empty `trace.attempts`
+     * and a decision inferred from job history. Consumers that *need*
+     * a real trace should subscribe to `onPlacementDecision` for live
+     * decisions; this getter is for "where is everything right now?"
+     * operator views.
+     *
+     * @public
+     */
+    placements() {
+        return [...this.placementsByJob.values()].sort((a, b) => b.at.getTime() - a.at.getTime());
+    }
+    /**
+     * O(1) lookup for the most recent placement decision binding `jobId`
+     * to a node. Returns undefined for jobs that have never been placed
+     * (or whose placement attempts have only failed). @public
+     */
+    placementForJob(jobId) {
+        return this.placementsByJob.get(jobId);
+    }
+    /**
+     * Run the placement engine against AutoManager's current inventory
+     * snapshot and configured pools, without actually placing or
+     * provisioning anything. Powers preview affordances in operator
+     * UIs ("if I submitted this bundle now, where would each replica
+     * land?").
+     *
+     * `jobName` is optional; if omitted the *first* job in the bundle
+     * is used. For multi-job bundles, supply the name to get a decision
+     * specific to that job.
+     *
+     * Returns undefined if the bundle's named job is missing or the
+     * replica index is out of range.
+     * @public
+     */
+    placeDryRun(bundle, replicaIndex, jobName) {
+        if (replicaIndex < 0 || replicaIndex >= (0, types_1.replicaCount)(bundle))
+            return undefined;
+        const job = jobName
+            ? bundle.jobs.find((j) => j.jobName === jobName)
+            : bundle.jobs[0];
+        if (!job)
+            return undefined;
+        const pools = new Map((this.settings.placementPools ?? []).map((p) => [p.name, p]));
+        return (0, placement_1.placeWithTrace)({
+            bundle,
+            job,
+            replicaIndex,
+            inventory: this.inventoriesAsNodeViews(),
+            pools,
+        });
+    }
+    /**
+     * Static validation of a bundle against AutoManager's currently
+     * configured pools. Equivalent to calling the module-level
+     * `validateBundle` with `PoolDescriptor[]` derived from
+     * `AutoSettings.placementPools` — elastic pools project their
+     * `candidateInstanceTypes` into node profiles; cluster pools project
+     * their currently-known live inventory.
+     * @public
+     */
+    validateBundle(bundle) {
+        return (0, validation_1.validateBundle)(bundle, this.poolDescriptorsForValidation());
+    }
+    /**
+     * Phase C4 — packaged admission check. Combines `validateBundle`
+     * (static spec sanity) with `placeDryRun` for every (replica,
+     * jobName) pair. Returns `ok` only when both pass for every
+     * member; otherwise reports the structured failure. Use as a
+     * pre-submit guard in operator-facing tooling.
+     *
+     * Per the design's §3.4 band table, all bands reject at admission
+     * on "no capacity right now" — there's no band differentiation
+     * here. (Gold/silver get *queued* internally on capacity-after-
+     * submit failures via the rejection re-place path, but that's a
+     * runtime concern, not admission.)
+     *
+     * @public
+     */
+    validateBundleAdmission(bundle) {
+        const validation = this.validateBundle(bundle);
+        if (validation.errors.length > 0) {
+            return { kind: "validationErrors", errors: validation.errors };
+        }
+        const unplaceable = [];
+        for (let replicaIndex = 0; replicaIndex < (0, types_1.replicaCount)(bundle); replicaIndex++) {
+            for (const job of bundle.jobs) {
+                const dry = this.placeDryRun(bundle, replicaIndex, job.jobName);
+                if (!dry) {
+                    // placeDryRun returns undefined only for malformed input
+                    // (out-of-range replica, unknown jobName) — validateBundle
+                    // should have caught those already. Treat as unplaceable.
+                    unplaceable.push({
+                        replicaIndex,
+                        jobName: job.jobName,
+                        reason: { code: "noCapacityInAnyPool", triedPools: [] },
+                    });
+                    continue;
+                }
+                if (dry.result.kind === "failure") {
+                    unplaceable.push({
+                        replicaIndex,
+                        jobName: job.jobName,
+                        reason: dry.result.reason,
+                    });
+                }
+            }
+        }
+        if (unplaceable.length > 0) {
+            return { kind: "noCapacity", unplaceable };
+        }
+        return { kind: "ok" };
+    }
+    poolDescriptorsForValidation() {
+        const out = [];
+        for (const pool of this.settings.placementPools ?? []) {
+            // Union the feasibility profiles across the pool's tiers: elastic
+            // tiers contribute the shapes they can provision; fixed (cluster)
+            // tiers contribute the live inventory projected for them.
+            const profiles = [];
+            for (const tier of pool.tiers) {
+                if (tier.scaleOut === "elastic") {
+                    for (const opt of tier.candidateInstanceTypes) {
+                        profiles.push({
+                            totalCapacity: opt.totalCapacity,
+                            totalCores: opt.totalCores,
+                            capabilities: opt.capabilities,
+                        });
+                    }
+                }
+                else {
+                    for (const view of this.inventoriesAsNodeViews()) {
+                        if (!(0, placement_1.nodeMatchesTier)(view, pool, tier))
+                            continue;
+                        profiles.push({
+                            totalCapacity: view.inventory.totalCapacity,
+                            totalCores: view.inventory.totalCores,
+                            capabilities: view.inventory.capabilities,
+                        });
+                    }
+                }
+            }
+            // A pool that can provision is judged on what it can provision (a
+            // miss is a hard error); a purely-fixed pool's misses are warnings —
+            // a worker may join after submission (see validation.ts).
+            const kind = pool.tiers.some((t) => t.scaleOut === "elastic")
+                ? "elastic"
+                : "cluster";
+            const clouds = [...new Set(pool.tiers.map((t) => t.kind))];
+            out.push({ poolName: pool.name, kind, nodeProfiles: profiles, clouds });
+        }
+        return out;
+    }
+    /**
+     * AutoManager v1 — typed inventory snapshot for a single node, or
+     * undefined if the worker hasn't (yet) advertised one.
+     * @public
+     */
+    inventory(nodeId) {
+        return this.inventoryMap.get(nodeId);
+    }
+    /**
+     * AutoManager v1 — typed inventory snapshots for every node we know
+     * about. Iteration order is insertion order.
+     * @public
+     */
+    inventories() {
+        return Array.from(this.inventoryMap.entries()).map(([nodeId, inventory]) => ({
+            nodeId,
+            inventory,
+        }));
+    }
+    // ---- AutoManager v1 event handlers ----
+    handleNodeInventoryUpdated(raw) {
+        if (!raw.nodeId || !raw.inventory)
+            return;
+        const nodeId = raw.nodeId.nodeId;
+        this.inventoryMap.set(nodeId, (0, conversions_1.fromPbNodeInventory)(raw.inventory));
+        // A clean inventory update clears any "unhealthy" / recently-failed
+        // mark — the worker has reported back successfully, treat as recovered.
+        this.unhealthyNodes.delete(nodeId);
+        this.rejectionsPerNode.delete(nodeId);
+        this.recentFailures.delete(nodeId);
+        // Phase C6: inventory change can flip a spare to "consumed" (jobs
+        // assigned > 0). Reconcile so a replacement is provisioned
+        // promptly rather than waiting for the periodic tick.
+        this.reconcileHotSpares();
+        // Phase F: inventory change might newly satisfy / no-longer satisfy
+        // a NodeService's `consumes` — reconcile placements.
+        this.reconcileNodeServices();
+        // Fresh inventory means capacity may have appeared (a node's first
+        // report after Hello — the startup race — a freed reservation, or an
+        // un-cordon). Re-attempt any job that couldn't be placed before.
+        this.reconcilePendingPlacements();
+    }
+    /**
+     * A running job has had its config updated at runtime (operator action,
+     * job-driven, or other side channel). Refresh our in-memory copy so a
+     * subsequent `redeploy` (placeAndProvision triggered by node failure
+     * etc.) hands the new config rather than the launch one.
+     *
+     * Manager is responsible for persisting the new value; AutoManager
+     * just keeps its mirror up to date.
+     */
+    handleJobConfigUpdated(raw) {
+        if (!raw.jobId)
+            return;
+        const jobId = raw.jobId.jobId;
+        const existing = this.jobMap.get(jobId);
+        if (!existing)
+            return;
+        this.jobMap.set(jobId, {
+            ...existing,
+            job: { ...existing.job, managerConfiguration: raw.config },
+        });
+        this.settings.onJobConfigUpdated?.(jobId, raw.config);
+    }
+    handleJobRejected(raw) {
+        if (!raw.nodeId || !raw.jobId)
+            return;
+        const nodeId = raw.nodeId.nodeId;
+        const jobId = raw.jobId.jobId;
+        const typedInventory = raw.inventory
+            ? (0, conversions_1.fromPbNodeInventory)(raw.inventory)
+            : undefined;
+        if (typedInventory) {
+            // Carries the worker's latest view, but does NOT clear unhealthy —
+            // a rejection is the opposite of "I'm fine". Update the cache only.
+            this.inventoryMap.set(nodeId, typedInventory);
+        }
+        this.metricsCounters.jobsRejected++;
+        this.settings.onJobRejected?.({
+            jobId,
+            nodeId,
+            reason: raw.reason,
+            inventory: typedInventory,
+            at: this.clock.now(),
+        });
+        const escalated = this.recordRejection(nodeId);
+        if (escalated) {
+            debuglog("Node escalated to unhealthy", { nodeId });
+            this.settings.onNodeUnhealthy?.(nodeId);
+        }
+        const existing = this.jobMap.get(jobId);
+        const existingReq = existing?.job.autoPlacement?.requirements;
+        if (existing && existingReq) {
+            // Re-place the job, excluding the rejecting node (and any others
+            // currently considered unfit).
+            const exclude = this.currentExcludeSet();
+            exclude.add(nodeId);
+            void this.placeAndProvision(jobId, existingReq, existing.job.tags ?? {}, { excludeNodes: exclude }).catch((err) => this.settings.onError?.(err));
+        }
+    }
+    recordRejection(nodeId) {
+        const now = this.clock.now();
+        const window = this.cfgRejectionBackoffMs();
+        const previous = this.rejectionsPerNode.get(nodeId) ?? [];
+        const recent = previous.filter((d) => now.getTime() - d.getTime() < window);
+        recent.push(now);
+        this.rejectionsPerNode.set(nodeId, recent);
+        const threshold = this.cfgRejectionEscalationCount();
+        if (recent.length >= threshold && !this.unhealthyNodes.has(nodeId)) {
+            this.unhealthyNodes.add(nodeId);
+            return true;
+        }
+        return false;
+    }
+    currentExcludeSet() {
+        const exclude = new Set(this.unhealthyNodes);
+        const now = this.clock.now().getTime();
+        for (const [nodeId, deadline] of this.recentFailures) {
+            if (deadline.getTime() < now) {
+                this.recentFailures.delete(nodeId);
+            }
+            else {
+                exclude.add(nodeId);
+            }
+        }
+        return exclude;
+    }
+    /**
+     * Phase C2 — restart-rate check. Records the current attempt; if the
+     * trailing-hour count crosses the band's `maxRestartsPerHour`, marks
+     * the bundle broken, fires `onBundleBroken`, and returns true
+     * (caller must short-circuit). Returns false when within budget.
+     */
+    shouldBreakOnRestart(jobId, bundleId, band) {
+        const now = this.clock.now();
+        const previous = this.restartHistory.get(jobId) ?? [];
+        const recent = previous.filter((d) => now.getTime() - d.getTime() < RESTART_WINDOW_MS);
+        recent.push(now);
+        this.restartHistory.set(jobId, recent);
+        const limit = this.cfgMaxRestartsPerHour(band);
+        if (recent.length > limit) {
+            if (!this.brokenBundles.has(bundleId)) {
+                this.brokenBundles.add(bundleId);
+                debuglog("Bundle marked broken — restart rate exceeded", {
+                    bundleId,
+                    jobId,
+                    attempts: recent.length,
+                    limit,
+                });
+                this.metricsCounters.bundlesBroken++;
+                const reason = `Restart rate ${recent.length}/h exceeds ceiling ${limit}/h for replica ${jobId}`;
+                this.settings.onBundleBroken?.(bundleId, reason);
+                // Persist the verdict back to the daemon so the poison job is not
+                // re-placed after a restart (the in-process `brokenBundles` flag is
+                // rebuilt-from-scratch on reconnect). Fire-and-forget; the live flag
+                // above already suppresses further placement this session.
+                void this.norsk
+                    .markJobFailed(jobId, reason)
+                    .catch((err) => this.settings.onError?.(err));
+            }
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Whether the named bundle has tripped the restart ceiling and is
+     * being held by AutoManager. Cleared by `bundleUpdated` (operator
+     * pushed a fresh spec) or `bundleDeleted`.
+     * @public
+     */
+    isBundleBroken(bundleId) {
+        return this.brokenBundles.has(bundleId);
+    }
+    // ---- Phase E — migration state machine ---------------------------
+    /**
+     * Initiate a migration of a running job to a target node. AutoManager
+     * provisions a second instance of the same Job on `targetNodeId` with
+     * role="migrationTarget" and drives the cutover handshake. Returns
+     * a stable `migrationId` for tracking.
+     *
+     * Throws if a migration is already in flight for `sourceJobId` (no
+     * concurrent migrations per Job).
+     *
+     * `sourceRole` defaults to whichever role the first node hosting
+     * `sourceJobId` has — adequate for single-instance Jobs. Specify
+     * explicitly for multi-replica bundles.
+     *
+     * @public
+     */
+    async migrateJob(sourceJobId, targetNodeId, sourceRole) {
+        if (this.migrationByJobId.has(sourceJobId)) {
+            throw new Error(`migrateJob: a migration is already in flight for jobId ${sourceJobId}`);
+        }
+        const resolvedSourceRole = sourceRole ?? this.inferSourceRole(sourceJobId);
+        if (!resolvedSourceRole) {
+            throw new Error(`migrateJob: cannot locate a running instance for jobId ${sourceJobId} — pass sourceRole explicitly`);
+        }
+        const migrationId = uuid.v4();
+        const now = this.clock.now();
+        const state = {
+            migrationId,
+            sourceJobId,
+            sourceRole: resolvedSourceRole,
+            targetNodeId,
+            startedAt: now,
+            phase: "launching",
+            phaseHistory: [{ phase: "launching", at: now }],
+        };
+        this.migrationStates.set(migrationId, state);
+        this.migrationByJobId.set(sourceJobId, migrationId);
+        this.settings.onMigrationPhase?.(state);
+        this.armMigrationPhaseDeadline(state);
+        debuglog("Migration started", { migrationId, sourceJobId, targetNodeId });
+        try {
+            await this.norsk.migrateJob(sourceJobId, targetNodeId, migrationId);
+        }
+        catch (err) {
+            this.failMigration(migrationId, `Manager.migrateJob failed: ${String(err)}`);
+            throw err;
+        }
+        return migrationId;
+    }
+    /**
+     * Abort an in-flight migration. AutoManager sends MigrationAbort to
+     * any live source/target instances and marks the migration aborted.
+     * No-op (with warning log) for an unknown migration id.
+     *
+     * @public
+     */
+    async abortMigration(migrationId, reason) {
+        const state = this.migrationStates.get(migrationId);
+        if (!state) {
+            debuglog("abortMigration: unknown migrationId", { migrationId });
+            return;
+        }
+        if (state.phase === "done" || state.phase === "aborted")
+            return;
+        await this.dispatchMigrationAborts(state, reason).catch((err) => this.settings.onError?.(err));
+        this.markMigrationAborted(state, reason);
+    }
+    /** All known migrations (active + done + aborted). @public */
+    migrations() {
+        return Array.from(this.migrationStates.values());
+    }
+    /** Look up a single migration by id. @public */
+    migrationById(migrationId) {
+        return this.migrationStates.get(migrationId);
+    }
+    /** Migration active for the given source jobId, if any. @public */
+    migrationForJob(jobId) {
+        const id = this.migrationByJobId.get(jobId);
+        return id ? this.migrationStates.get(id) : undefined;
+    }
+    /**
+     * Find the role of the (first) node currently running this Job.
+     * Returns undefined if no node hosts the job.
+     */
+    inferSourceRole(jobId) {
+        for (const node of this.nodeMap.values()) {
+            if (getJobIdForNode(node.nodeMetadata) === jobId) {
+                const role = node.nodeMetadata?.tags?.["role"] ?? node.provisional?.role;
+                if (role)
+                    return role;
+            }
+        }
+        return undefined;
+    }
+    advanceMigration(state, phase) {
+        if (state.phase === phase)
+            return;
+        const now = this.clock.now();
+        state.phase = phase;
+        state.phaseHistory.push({ phase, at: now });
+        this.settings.onMigrationPhase?.(state);
+        debuglog("Migration phase", { migrationId: state.migrationId, phase });
+        this.armMigrationPhaseDeadline(state);
+    }
+    /** Phase E deadlines — re-arm the per-phase timer for the migration. */
+    armMigrationPhaseDeadline(state) {
+        // Always cancel any prior timer; terminal phases don't re-arm.
+        this.migrationPhaseTimers.get(state.migrationId)?.cancel();
+        this.migrationPhaseTimers.delete(state.migrationId);
+        if (state.phase === "done" || state.phase === "aborted")
+            return;
+        const deadlineMs = this.cfgMigrationPhaseDeadline(state.phase);
+        if (deadlineMs === undefined)
+            return;
+        const id = state.migrationId;
+        const phase = state.phase;
+        const timer = this.clock.setTimeout(() => {
+            this.migrationPhaseTimers.delete(id);
+            const current = this.migrationStates.get(id);
+            // Only fire if the migration is still in the phase we armed for.
+            if (!current || current.phase !== phase)
+                return;
+            debuglog("Migration phase deadline fired — aborting", {
+                migrationId: id,
+                phase,
+                deadlineMs,
+            });
+            const reason = `phase timeout: ${phase}`;
+            void this.dispatchMigrationAborts(current, reason).catch((err) => this.settings.onError?.(err));
+            this.markMigrationAborted(current, reason);
+        }, deadlineMs);
+        this.migrationPhaseTimers.set(state.migrationId, timer);
+    }
+    cfgMigrationPhaseDeadline(phase) {
+        const m = this.settings.migration;
+        switch (phase) {
+            case "launching":
+                return m?.launchingDeadlineMs ?? DEFAULT_MIGRATION_LAUNCHING_DEADLINE_MS;
+            case "awaitingTargetReady":
+                return m?.awaitingTargetReadyDeadlineMs ?? DEFAULT_MIGRATION_TARGET_READY_DEADLINE_MS;
+            case "awaitingSourceStop":
+                return m?.awaitingSourceStopDeadlineMs ?? DEFAULT_MIGRATION_SOURCE_STOP_DEADLINE_MS;
+            case "applying":
+                return m?.applyingDeadlineMs ?? DEFAULT_MIGRATION_APPLYING_DEADLINE_MS;
+            case "done":
+            case "aborted":
+                return undefined;
+        }
+    }
+    markMigrationDone(state) {
+        this.advanceMigration(state, "done");
+        this.migrationByJobId.delete(state.sourceJobId);
+        this.settings.onMigrationCompleted?.(state.migrationId);
+    }
+    markMigrationAborted(state, reason) {
+        state.abortReason = reason;
+        this.advanceMigration(state, "aborted");
+        this.migrationByJobId.delete(state.sourceJobId);
+        this.settings.onMigrationAborted?.(state.migrationId, reason);
+    }
+    failMigration(migrationId, reason) {
+        const state = this.migrationStates.get(migrationId);
+        if (!state)
+            return;
+        if (state.phase === "done" || state.phase === "aborted")
+            return;
+        this.markMigrationAborted(state, reason);
+    }
+    async dispatchMigrationAborts(state, reason) {
+        // Manager's abortMigration takes both keys and fans out to source
+        // + target workers; AutoManager doesn't need to send anything
+        // else.
+        await this.norsk
+            .abortMigration(state.sourceJobId, state.sourceRole, exports.MIGRATION_TARGET_ROLE, reason)
+            .catch((err) => this.settings.onError?.(err));
+    }
+    // ---- Phase E — event handlers ------------------------------------
+    async handleMigrationTargetReady(raw) {
+        const jobId = raw.jobKey?.jobId?.jobId;
+        if (!jobId)
+            return;
+        const state = this.migrationForJob(jobId);
+        if (!state) {
+            debuglog("MigrationTargetReady for unknown migration", { jobId });
+            return;
+        }
+        if (state.phase !== "launching" && state.phase !== "awaitingTargetReady") {
+            debuglog("MigrationTargetReady in unexpected phase", {
+                migrationId: state.migrationId,
+                phase: state.phase,
+            });
+            return;
+        }
+        this.advanceMigration(state, "awaitingSourceStop");
+        try {
+            await this.norsk.notifyMigrationCutoverReady({
+                jobId: state.sourceJobId,
+                role: state.sourceRole,
+            });
+        }
+        catch (err) {
+            this.settings.onError?.(err);
+            this.failMigration(state.migrationId, `notifyMigrationCutoverReady failed: ${String(err)}`);
+        }
+    }
+    async handleMigrationSourceStoppedOutput(raw) {
+        const jobId = raw.jobKey?.jobId?.jobId;
+        if (!jobId)
+            return;
+        const state = this.migrationForJob(jobId);
+        if (!state)
+            return;
+        if (state.phase !== "awaitingSourceStop") {
+            debuglog("MigrationSourceStoppedOutput in unexpected phase", {
+                migrationId: state.migrationId,
+                phase: state.phase,
+            });
+            return;
+        }
+        const ts = Number(raw.lastOutputTimestampNs);
+        state.cutoverTimestampNs = ts;
+        this.advanceMigration(state, "applying");
+        try {
+            await this.norsk.notifyMigrationCutoverApply({ jobId: state.sourceJobId, role: exports.MIGRATION_TARGET_ROLE }, raw.lastOutputTimestampNs);
+        }
+        catch (err) {
+            this.settings.onError?.(err);
+            this.failMigration(state.migrationId, `notifyMigrationCutoverApply failed: ${String(err)}`);
+            return;
+        }
+        // Source workflow has already stopped output; safe to terminate the
+        // source instance. norsk-mgr-equivalents would do this via the
+        // existing stopJob path, but since AutoManager owns the protocol
+        // we drive the cleanup here.
+        // R10: don't dispatch stopJob with nodeId="" if we can't locate
+        // the source — that just sends a malformed RPC. If the source is
+        // already gone, treat as success; surface other mismatches.
+        const sourceNodeId = this.findNodeForJobKey(state.sourceJobId, state.sourceRole);
+        if (sourceNodeId === undefined) {
+            debuglog("Migration: source instance not located in nodeMap — skipping stopJob", {
+                migrationId: state.migrationId,
+                jobId: state.sourceJobId,
+                role: state.sourceRole,
+            });
+        }
+        else {
+            try {
+                await this.norsk.stopJob(state.sourceJobId, state.sourceRole, sourceNodeId);
+            }
+            catch (err) {
+                debuglog("Migration: source stopJob failed (continuing)", {
+                    migrationId: state.migrationId,
+                    err: String(err),
+                });
+                this.settings.onError?.(err);
+            }
+        }
+        this.markMigrationDone(state);
+    }
+    handleMigrationAborted(raw) {
+        const jobId = raw.jobKey?.jobId?.jobId;
+        if (!jobId)
+            return;
+        const state = this.migrationForJob(jobId);
+        if (!state)
+            return;
+        debuglog("Migration aborted by worker", {
+            migrationId: state.migrationId,
+            reason: raw.reason,
+        });
+        // Dispatch an abort to the other side so it tears down its
+        // migration state too. Don't await — fire-and-forget.
+        void this.dispatchMigrationAborts(state, raw.reason);
+        this.markMigrationAborted(state, raw.reason);
+    }
+    findNodeForJobKey(jobId, role) {
+        for (const node of this.nodeMap.values()) {
+            const tags = node.nodeMetadata?.tags ?? {};
+            if (tags["jobId"] === jobId && tags["role"] === role) {
+                return node.nodeId;
+            }
+            const prov = node.provisional;
+            if (prov && prov.jobId === jobId && prov.role === role) {
+                return node.nodeId;
+            }
+        }
+        return undefined;
+    }
+    // ---- Phase C4 — config resolution ---------------------------------
+    // Precedence for every tunable: settings.placement.X > settings.X
+    // > compiled-in default. Per-band tunables use settings.bands[band]
+    // first, with the same flat/default fallback.
+    cfgRejectionBackoffMs() {
+        return (this.settings.placement?.rejectionBackoffMs ??
+            this.settings.rejectionBackoffMs ??
+            DEFAULT_REJECTION_BACKOFF_MS);
+    }
+    cfgRejectionEscalationCount() {
+        return (this.settings.placement?.rejectionEscalationCount ??
+            this.settings.rejectionEscalationCount ??
+            DEFAULT_REJECTION_ESCALATION_COUNT);
+    }
+    cfgFailureBackoffMs() {
+        return (this.settings.placement?.failureBackoffMs ??
+            this.settings.failureBackoffMs ??
+            DEFAULT_FAILURE_BACKOFF_MS);
+    }
+    cfgJobStartupGraceMs() {
+        return (this.settings.placement?.jobStartupGraceMs ??
+            this.settings.jobStartupGraceMs ??
+            DEFAULT_JOB_STARTUP_GRACE_MS);
+    }
+    cfgMaxRestartsPerHour(band) {
+        // Bronze drops on failure — restart ceiling is moot, but return a
+        // large number so the rate-check path can be called uniformly
+        // without crashing if anyone reaches it.
+        if (band === "bronze")
+            return Number.POSITIVE_INFINITY;
+        const perBand = this.settings.bands?.[band]?.maxRestartsPerHour;
+        return (perBand ??
+            this.settings.maxRestartsPerHour ??
+            DEFAULT_MAX_RESTARTS_PER_HOUR);
+    }
+    // ---- AutoManager v1 placement-driven path ----
+    //
+    // A "modern" job — one with `requirements` set — is a member of a
+    // bundle that Manager has expanded into N replica jobs. The job
+    // carries bundleId / replicaIndex / jobName tags so AutoManager can
+    // recover its bundle context and run the placement engine. The
+    // legacy createJob path (no requirements) is unaffected.
+    async placeAndProvision(jobId, pbRequirements, tags, recovery) {
+        const bundleId = tags["bundleId"];
+        const replicaIndexStr = tags["replicaIndex"];
+        const jobName = tags["jobName"];
+        if (!bundleId || replicaIndexStr === undefined || !jobName) {
+            debuglog("Job has requirements but missing bundle context tags", tags);
+            return;
+        }
+        const replicaIndex = parseInt(replicaIndexStr, 10);
+        // In-flight-placement gate (initial placements only). handleJobChanged
+        // can re-fire `placeAndProvision` for the same jobId while the
+        // daemon's provisionJob is mid-dispatch — `nodesByJob().size` is
+        // still 0 because JobStarted hasn't landed yet. Each re-fire ends
+        // in `INVALID_ARGUMENT: Job already running on node` from the
+        // daemon and a `placementsDispatchFailed++`. Suppress. Recovery
+        // placements bypass this gate; they're driven by failure events
+        // (JobRejected, NodeStopped) and have their own MIN_RECOVERY_INTERVAL_MS
+        // backoff further down — and the bronze-drop / band-aware-recovery
+        // branch needs to run even when an initial placement just dispatched.
+        const now = this.clock.now();
+        if (!recovery) {
+            const lastAttempt = this.lastPlacementAttemptAt.get(jobId);
+            if (lastAttempt && now.getTime() - lastAttempt.getTime() < MIN_PLACEMENT_INTERVAL_MS) {
+                debuglog("Placement attempt suppressed (in-flight)", { jobId, bundleId });
+                return;
+            }
+            this.lastPlacementAttemptAt.set(jobId, now);
+        }
+        const bundle = this.bundleMap.get(bundleId);
+        if (!bundle) {
+            debuglog("Job's bundle not in map yet", { jobId, bundleId });
+            return;
+        }
+        const bundleJobSpec = bundle.jobs.find((j) => j.jobName === jobName);
+        if (!bundleJobSpec) {
+            debuglog("Bundle has no job named", { bundleId, jobName });
+            return;
+        }
+        // Use the bundle's typed requirements (richer / canonical) rather
+        // than re-converting from PB. Keeps placement decisions consistent
+        // with the bundle the user submitted.
+        void pbRequirements;
+        // Band-aware recovery (Phase C1). On a recovery path:
+        //   - bronze: drop without re-placing; fire onJobDropped.
+        //   - gold:   re-place, preferring hot spares.
+        //   - silver: re-place, no spare preference.
+        // Initial placements bypass this entirely.
+        const band = bundleJobSpec.requirements.priorityBand;
+        if (recovery && band === "bronze") {
+            debuglog("Bronze job dropped (no restart on failure)", { jobId, bundleId });
+            this.metricsCounters.jobsDropped++;
+            this.settings.onJobDropped?.(jobId);
+            // Bronze never restarts — this is terminal. Persist the verdict so the
+            // job isn't re-placed after a restart.
+            void this.norsk
+                .markJobFailed(jobId, "bronze job dropped after failure")
+                .catch((err) => this.settings.onError?.(err));
+            return;
+        }
+        // Phase C2 — restart-rate ceiling + broken-bundle short-circuit.
+        // Always run the rate check on every recovery trigger — the
+        // ceiling tracks failure rate, which is real-time, even if we
+        // defer the actual placement below. Without this, the
+        // minimum-interval backoff would coalesce N rapid failures into
+        // a single recorded attempt and the rate-broken trip would never
+        // fire.
+        if (recovery) {
+            if (this.brokenBundles.has(bundleId)) {
+                debuglog("Skipping recovery — bundle is marked broken", { jobId, bundleId });
+                return;
+            }
+            if (this.shouldBreakOnRestart(jobId, bundleId, band)) {
+                return;
+            }
+        }
+        // Minimum-interval backoff between consecutive recovery placement
+        // dispatches for the same job. The failure rate has already been
+        // counted above; this gate only delays the next actual placement
+        // attempt so we don't hammer the daemon/worker at 12 attempts/sec.
+        // Coalescing: if a deferred retry is already pending, further
+        // triggers in the window become no-ops (the timer will see the
+        // latest state when it fires).
+        if (recovery) {
+            const history = this.restartHistory.get(jobId) ?? [];
+            // The just-pushed entry is the last one; we want the entry
+            // before it to determine spacing.
+            const previousAttempt = history[history.length - 2];
+            const now = this.clock.now();
+            if (previousAttempt && now.getTime() - previousAttempt.getTime() < MIN_RECOVERY_INTERVAL_MS) {
+                if (this.pendingPlacementRetries.has(jobId)) {
+                    debuglog("Recovery placement already pending (coalescing)", { jobId, bundleId });
+                    return;
+                }
+                const delay = MIN_RECOVERY_INTERVAL_MS - (now.getTime() - previousAttempt.getTime());
+                debuglog("Recovery placement deferred (backoff)", { jobId, bundleId, delayMs: delay });
+                const timer = setTimeout(() => {
+                    this.pendingPlacementRetries.delete(jobId);
+                    void this.placeAndProvision(jobId, pbRequirements, tags, recovery).catch((err) => this.settings.onError?.(err));
+                }, delay);
+                this.pendingPlacementRetries.set(jobId, timer);
+                return;
+            }
+        }
+        const inventory = this.inventoriesAsNodeViews();
+        const pools = new Map((this.settings.placementPools ?? []).map((p) => [p.name, p]));
+        if (pools.size === 0) {
+            debuglog("placementPools not configured — cannot place", { jobId });
+            this.settings.onError?.(new Error(`placementPools not configured; can't place job ${jobId}. ` +
+                `Add to AutoSettings or this job will never start.`));
+            return;
+        }
+        const { result, trace } = (0, placement_1.placeWithTrace)({
+            bundle,
+            job: bundleJobSpec,
+            replicaIndex,
+            inventory,
+            pools,
+            recoveryContext: recovery
+                ? { excludeNodes: recovery.excludeNodes, preferHotSpares: band === "gold" }
+                : undefined,
+        });
+        debuglog("Placement decision", {
+            jobId,
+            bundleId,
+            replicaIndex,
+            decision: result,
+            trace: summariseTrace(trace),
+        });
+        const at = this.clock.now();
+        this.metricsCounters.placementsAttempted++;
+        if (result.kind === "failure") {
+            this.metricsCounters.placementsFailed++;
+        }
+        // R12: placementsSucceeded counted post-dispatch; see below.
+        const info = { jobId, bundleId, replicaIndex, jobName, result, trace, at };
+        // Mirror into live-state map so consumers can query the current
+        // placement without subscribing from process start. Failures don't
+        // bind a placement, so they don't go in the map — `placementForJob`
+        // returning undefined for "we tried and failed" is the correct
+        // signal; failures still flow through `onPlacementFailed` for
+        // consumers that care.
+        if (result.kind !== "failure") {
+            this.placementsByJob.set(jobId, info);
+        }
+        this.settings.onPlacementDecision?.(info);
+        if (result.kind === "failure") {
+            // A failed placement dispatched nothing, so the in-flight gate
+            // (lastPlacementAttemptAt + MIN_PLACEMENT_INTERVAL_MS) must not
+            // hold this job back: a capacity-increasing event can arrive
+            // moments later and `reconcilePendingPlacements` needs to be free
+            // to retry. The gate exists to suppress duplicate dispatch while a
+            // real placement is in flight — a failure is not in flight.
+            this.lastPlacementAttemptAt.delete(jobId);
+            this.settings.onPlacementFailed?.({
+                jobId,
+                bundleId,
+                replicaIndex,
+                jobName,
+                reason: result.reason,
+                triedPools: result.reason.triedPools,
+                at,
+            });
+            this.settings.onError?.(new Error(`Placement failed for ${jobId}: ${JSON.stringify(result.reason)}`));
+            return;
+        }
+        const role = roleFromReplicaIndex(replicaIndex);
+        // A degraded backup still gets placed/provisioned (best-effort) — surface
+        // the reduced DR posture so an operator can act / re-place later.
+        if (result.degraded) {
+            this.settings.onResilienceDegraded?.({
+                jobId,
+                bundleId,
+                replicaIndex,
+                jobName,
+                violated: result.degraded.violated,
+                ...(result.degraded.interruptiblePrimary ? { interruptiblePrimary: true } : {}),
+                at,
+            });
+        }
+        if (result.kind === "place") {
+            // Existing node already running on the chosen target. Manager's
+            // StartJob RPC binds the job to that node.
+            try {
+                await this.norsk.startJob(jobId, role, result.nodeId);
+                this.jobToNode.set(jobId, result.nodeId);
+                this.metricsCounters.placementsSucceeded++;
+            }
+            catch (err) {
+                this.metricsCounters.placementsDispatchFailed++;
+                this.settings.onError?.(err);
+            }
+            return;
+        }
+        // Provision in the chosen pool's chosen tier.
+        const pool = pools.get(result.pool);
+        if (!pool)
+            return;
+        const tier = pool.tiers.find((t) => t.name === result.tier);
+        if (!tier)
+            return;
+        // `pool` + `tier` tags are load-bearing: poolNameForNode / tierNameForNode
+        // read them back via nodeMetadata.tags to identify which pool+tier an
+        // existing node belongs to. We must set them here, or every placement
+        // after the node starts will filter it out as wrong-pool/wrong-tier.
+        const provisionTags = {
+            "auto-manager": "true",
+            jobId,
+            role,
+            bundleId,
+            replicaIndex: replicaIndexStr,
+            jobName,
+            pool: pool.name,
+            tier: tier.name,
+            // Failure-domain metadata for the resilience engine + observability.
+            reliability: (0, placement_1.tierReliability)(tier),
+        };
+        const nodeId = uuid.v4() + "." + jobId + "." + role;
+        this.updateNodeMap(nodeId, "starting", undefined, undefined, { jobId, role });
+        this.jobToNode.set(jobId, nodeId);
+        this.settings.onNodeStarting?.(nodeId);
+        if (tier.kind === "aws") {
+            try {
+                await this.norsk.createAwsNode({
+                    nodeId,
+                    region: tier.region ?? "",
+                    instanceType: result.instanceType,
+                    tags: provisionTags,
+                    writeFiles: [],
+                    launchMode: result.launchMode,
+                });
+                this.metricsCounters.placementsSucceeded++;
+            }
+            catch (err) {
+                this.metricsCounters.placementsDispatchFailed++;
+                this.settings.onError?.(err);
+            }
+        }
+        else if (tier.kind === "oci") {
+            try {
+                await this.norsk.createOciNode({
+                    nodeId,
+                    availabilityDomain: tier.region ?? "",
+                    architecture: "x86_64",
+                    shape: result.instanceType,
+                    subnet: "",
+                    tags: provisionTags,
+                    writeFiles: [],
+                });
+                this.metricsCounters.placementsSucceeded++;
+            }
+            catch (err) {
+                this.metricsCounters.placementsDispatchFailed++;
+                this.settings.onError?.(err);
+            }
+        }
+        else {
+            // Cluster tiers never auto-scale; place-on-existing only.
+            this.settings.onError?.(new Error(`Cluster tier '${pool.name}/${tier.name}' returned a 'provision' ` +
+                `placement; cluster tiers should only place onto existing nodes.`));
+        }
+    }
+    inventoriesAsNodeViews() {
+        // Reservation (reservedCapacity / cores / capabilities, assignedJobs) is
+        // not populated upstream: the worker advertises raw hardware and the
+        // daemon forwards it verbatim, so the wire values arrive as 0 / []. Left
+        // as-is, every node would look fully free regardless of what's running on
+        // it — placement would over-subscribe, and the capacity-driven reconcile
+        // could never observe a node fill up and later free up. AutoManager is the
+        // one component that knows both which jobs it placed where (`jobToNode`)
+        // and each job's requirement (the bundle's typed spec), so it derives the
+        // reservation here. A non-zero wire reservation, if a producer ever sends
+        // one, is treated as authoritative and the derivation defers to it.
+        const placedJobsByNode = new Map();
+        for (const [jobId, nodeId] of this.jobToNode) {
+            const list = placedJobsByNode.get(nodeId);
+            if (list)
+                list.push(jobId);
+            else
+                placedJobsByNode.set(nodeId, [jobId]);
+        }
+        const out = [];
+        for (const [nodeId, inv] of this.inventoryMap.entries()) {
+            const placedHere = placedJobsByNode.get(nodeId) ?? [];
+            // Prefer the wire's assigned jobs when present; otherwise list the
+            // jobs AutoManager itself placed on this node. Either feeds the
+            // affinity / spread constraints in the placement engine.
+            const runningJobs = inv.assignedJobs.length > 0
+                ? inv.assignedJobs.flatMap((a) => this.runningJobFor(a.jobId))
+                : placedHere.flatMap((jobId) => this.runningJobFor(jobId));
+            // Phase C8: merge the AutoManager-local cordon into the wire-reported
+            // cordoned flag. Either side can mark a node off-limits.
+            const cordoned = this.cordonedNodes.has(nodeId)
+                ? { ...inv, cordoned: true }
+                : inv;
+            // Overlay derived reservation, deferring to a non-zero wire value.
+            const reservation = this.derivedReservation(placedHere);
+            const effectiveInv = {
+                ...cordoned,
+                reservedCapacity: cordoned.reservedCapacity > 0
+                    ? cordoned.reservedCapacity
+                    : reservation.capacity,
+                reservedCores: cordoned.reservedCores > 0 ? cordoned.reservedCores : reservation.cores,
+                reservedCapabilities: cordoned.reservedCapabilities.length > 0
+                    ? cordoned.reservedCapabilities
+                    : reservation.capabilities,
+            };
+            // poolName / tierName are read from the node's NodeMetadata (nodeMap);
+            // nodes we haven't seen there yet resolve to "" / undefined and never
+            // match a pool filter.
+            out.push({
+                nodeId,
+                poolName: this.poolNameForNode(nodeId),
+                tierName: this.tierNameForNode(nodeId),
+                cloud: this.cloudForNode(nodeId),
+                inventory: effectiveInv,
+                runningJobs,
+                az: this.azForNode(nodeId),
+                // Phase C5: spare-ness is advertised via NodeMetadata.tags["spare"].
+                isHotSpare: this.isHotSpareNode(nodeId),
+            });
+        }
+        return out;
+    }
+    /** Map a placed jobId to the RunningJob view the placement engine uses
+     *  for affinity / spread, via the job's bundle-context tags. Empty when
+     *  the job lacks bundle tags. */
+    runningJobFor(jobId) {
+        const tags = this.tagsForJob(jobId);
+        if (!tags)
+            return [];
+        const bundleId = tags["bundleId"];
+        const replicaIndexStr = tags["replicaIndex"];
+        const jobName = tags["jobName"];
+        if (!bundleId || replicaIndexStr === undefined || !jobName)
+            return [];
+        return [{ bundleId, replicaIndex: parseInt(replicaIndexStr, 10), jobName }];
+    }
+    /** Sum the resource requirements of the given placed jobs into a node
+     *  reservation. Capacity / cores / capabilities come from each job's
+     *  bundle spec — the canonical typed requirements the placement engine
+     *  scores against, looked up via the job's bundleId / jobName tags. Jobs
+     *  whose bundle or spec we don't have yet contribute nothing. */
+    derivedReservation(jobIds) {
+        let capacity = 0;
+        let cores = 0;
+        const caps = new Map();
+        for (const jobId of jobIds) {
+            const req = this.requirementsForPlacedJob(jobId);
+            if (!req)
+                continue;
+            capacity += req.requiredCapacity;
+            cores += req.requiredCores ?? 0;
+            for (const c of req.requiredCapabilities) {
+                caps.set(c.name, (caps.get(c.name) ?? 0) + c.count);
+            }
+        }
+        const capabilities = [...caps.entries()].map(([name, count]) => ({ name, count }));
+        return { capacity, cores, capabilities };
+    }
+    /** The canonical typed requirements for a placed job, from its bundle
+     *  spec (the same source `placeAndProvision` scores against). */
+    requirementsForPlacedJob(jobId) {
+        const tags = this.tagsForJob(jobId);
+        if (!tags)
+            return undefined;
+        const bundleId = tags["bundleId"];
+        const jobName = tags["jobName"];
+        if (!bundleId || !jobName)
+            return undefined;
+        return this.bundleMap
+            .get(bundleId)
+            ?.jobs.find((j) => j.jobName === jobName)?.requirements;
+    }
+    /**
+     * Phase C5 — is the named node currently tagged as a hot spare?
+     * Spares are warm-running nodes provisioned (by the C6 reconciler)
+     * with `NodeMetadata.tags["spare"] = "true"`. The optional
+     * `tags["spareBand"]` ("gold" | "silver") dedicates a spare to a
+     * band; if absent, the spare is band-agnostic.
+     * @public
+     */
+    isHotSpareNode(nodeId) {
+        return this.nodeMap.get(nodeId)?.nodeMetadata?.tags?.["spare"] === "true";
+    }
+    /**
+     * Phase C5 — band a spare is dedicated to, if any. `undefined` for a
+     * non-spare or for a band-agnostic spare.
+     * @public
+     */
+    spareBandForNode(nodeId) {
+        if (!this.isHotSpareNode(nodeId))
+            return undefined;
+        const v = this.nodeMap.get(nodeId)?.nodeMetadata?.tags?.["spareBand"];
+        return v === "gold" || v === "silver" || v === "bronze" ? v : undefined;
+    }
+    // ---- Phase C6 — hot-spare reconciliation -------------------------
+    /**
+     * Drive every configured `hotSpares` entry toward its `targetCount`.
+     * Counts live + pending spares matching (pool, band); provisions
+     * short, terminates excess (oldest live first). Cluster pools are
+     * non-elastic — under-target just fires `onSpareUnderflow` and
+     * leaves provisioning to the operator. Idempotent and cheap; safe
+     * to call on every lifecycle event plus the periodic tick.
+     */
+    reconcileHotSpares() {
+        const entries = this.settings.hotSpares ?? [];
+        if (entries.length === 0)
+            return;
+        for (const cfg of entries) {
+            const pool = (this.settings.placementPools ?? []).find((p) => p.name === cfg.pool);
+            if (!pool) {
+                debuglog("hotSpares entry references unknown pool", { entry: cfg });
+                continue;
+            }
+            const matching = this.spareNodesMatching(cfg);
+            const pending = this.pendingSparesMatching(cfg);
+            const effective = matching.length + pending;
+            const target = cfg.targetCount;
+            if (effective < target) {
+                const short = target - effective;
+                this.settings.onSpareUnderflow?.(cfg.pool, effective, target);
+                // Warm the cheapest provisionable tier (first elastic tier in
+                // cost order). A pool with no elastic tier can't grow spares.
+                const provisionTier = pool.tiers.find((t) => t.scaleOut === "elastic");
+                if (!provisionTier) {
+                    debuglog("Pool has no elastic tier — under spare target, no provision", {
+                        cfg,
+                        effective,
+                        target,
+                    });
+                    continue;
+                }
+                for (let i = 0; i < short; i++) {
+                    void this.provisionHotSpare(pool, provisionTier, cfg);
+                }
+            }
+            else if (matching.length > target) {
+                // Only terminate from the truly-live (non-pending) set. Sort
+                // by lastActivity ascending so the oldest goes first — matches
+                // the design's "support rotation for security patching."
+                const excess = matching.length - target;
+                const sorted = [...matching].sort((a, b) => a.lastActivity.getTime() - b.lastActivity.getTime());
+                for (let i = 0; i < excess; i++) {
+                    const victim = sorted[i];
+                    debuglog("Hot-spare excess — terminating oldest", {
+                        nodeId: victim.nodeId,
+                        cfg,
+                    });
+                    void this.norsk
+                        .terminateNode(victim.nodeId)
+                        .catch((err) => this.settings.onError?.(err));
+                }
+            }
+        }
+    }
+    spareNodesMatching(cfg) {
+        const out = [];
+        for (const node of this.nodeMap.values()) {
+            if (!this.isHotSpareNode(node.nodeId))
+                continue;
+            if (node.nodeState !== "running")
+                continue;
+            if (this.poolNameForNode(node.nodeId) !== cfg.pool)
+                continue;
+            if (cfg.forBand && this.spareBandForNode(node.nodeId) !== cfg.forBand)
+                continue;
+            // A spare with jobs assigned is consumed — exclude from the count
+            // so the underflow fires and a replacement provisions.
+            const inv = this.inventoryMap.get(node.nodeId);
+            if (inv && inv.assignedJobs.length > 0)
+                continue;
+            out.push(node);
+        }
+        return out;
+    }
+    pendingSparesMatching(cfg) {
+        let n = 0;
+        for (const p of this.pendingSpareProvisions.values()) {
+            if (p.pool !== cfg.pool)
+                continue;
+            if (cfg.forBand && p.band !== cfg.forBand)
+                continue;
+            n++;
+        }
+        return n;
+    }
+    async provisionHotSpare(pool, tier, cfg) {
+        const instanceType = cfg.spec.instanceType;
+        if (!instanceType) {
+            this.settings.onError?.(new Error(`hotSpares entry for elastic pool '${cfg.pool}' needs spec.instanceType`));
+            return;
+        }
+        const nodeId = uuid.v4() + ".spare." + cfg.pool;
+        const tags = {
+            "auto-manager": "true",
+            spare: "true",
+            pool: cfg.pool,
+            tier: tier.name,
+        };
+        if (cfg.forBand)
+            tags["spareBand"] = cfg.forBand;
+        this.pendingSpareProvisions.set(nodeId, { pool: cfg.pool, band: cfg.forBand });
+        this.updateNodeMap(nodeId, "starting", undefined, undefined);
+        this.settings.onNodeStarting?.(nodeId);
+        try {
+            if (tier.kind === "aws") {
+                await this.norsk.createAwsNode({
+                    nodeId,
+                    region: cfg.spec.region ?? tier.region ?? "",
+                    instanceType,
+                    tags,
+                    writeFiles: [],
+                });
+            }
+            else if (tier.kind === "oci") {
+                await this.norsk.createOciNode({
+                    nodeId,
+                    availabilityDomain: cfg.spec.az ?? cfg.spec.region ?? tier.region ?? "",
+                    architecture: "x86_64",
+                    shape: instanceType,
+                    subnet: "",
+                    tags,
+                    writeFiles: [],
+                });
+            }
+            // Cluster path falls through — non-elastic, no auto-provision.
+        }
+        catch (err) {
+            // Failed to even submit. Drop the pending marker so the next
+            // reconcile can try again instead of waiting forever.
+            this.pendingSpareProvisions.delete(nodeId);
+            this.settings.onError?.(err);
+        }
+    }
+    // ---- Phase F — NodeService reconciliation ------------------------
+    /**
+     * @public
+     * Current placements for each configured NodeService. Returns a
+     * snapshot map of `serviceId → (nodeId → runtimeJobId)`.
+     */
+    nodeServices() {
+        return this.settings.nodeServices ?? [];
+    }
+    /**
+     * @public
+     * Nodes on which the named NodeService is currently placed,
+     * keyed by the runtime job's jobId.
+     */
+    nodeServicePlacementsFor(serviceId) {
+        return new Map(this.nodeServicePlacements.get(serviceId) ?? []);
+    }
+    /**
+     * Drive every configured NodeService toward its lifecycle target.
+     * v1 only handles `eager` — places on every eligible node.
+     * `lazy` placements + capability-loss-driven consumer recovery (F3)
+     * are deferred.
+     */
+    reconcileNodeServices() {
+        const services = this.settings.nodeServices ?? [];
+        if (services.length === 0)
+            return;
+        for (const svc of services) {
+            if (svc.lifecycle !== "eager") {
+                // Lazy lifecycle not implemented in v1; skip.
+                continue;
+            }
+            this.reconcileEagerNodeService(svc);
+        }
+    }
+    reconcileEagerNodeService(svc) {
+        const eligible = this.eligibleNodesForService(svc);
+        const placed = this.nodeServicePlacements.get(svc.serviceId) ?? new Map();
+        // Place on every eligible node that doesn't already host the service.
+        for (const nodeId of eligible) {
+            if (placed.has(nodeId))
+                continue;
+            void this.placeNodeService(svc, nodeId).catch((err) => this.settings.onError?.(err));
+        }
+        // Tear down any placement that's no longer eligible.
+        for (const [nodeId, jobId] of placed) {
+            if (eligible.has(nodeId))
+                continue;
+            void this.teardownNodeService(svc.serviceId, nodeId, jobId).catch((err) => this.settings.onError?.(err));
+        }
+    }
+    /**
+     * Identify nodes in the service's named pools whose current inventory
+     * satisfies its `consumes` requirements. Used both for placement and
+     * for tear-down decisions.
+     */
+    eligibleNodesForService(svc) {
+        const eligible = new Set();
+        const poolSet = new Set(svc.pools);
+        for (const node of this.nodeMap.values()) {
+            if (node.nodeState !== "running")
+                continue;
+            if (!poolSet.has(this.poolNameForNode(node.nodeId)))
+                continue;
+            const inv = this.inventoryMap.get(node.nodeId);
+            if (!inv || !inv.reachable || inv.cordoned)
+                continue;
+            if (this.cordonedNodes.has(node.nodeId))
+                continue;
+            if (this.nodeSatisfiesConsumes(inv, svc.consumes)) {
+                eligible.add(node.nodeId);
+            }
+        }
+        return eligible;
+    }
+    nodeSatisfiesConsumes(inv, consumes) {
+        for (const req of consumes) {
+            const cap = inv.capabilities.find((c) => c.name === req.name);
+            if (!cap)
+                return false;
+            const reserved = inv.reservedCapabilities.find((c) => c.name === req.name)?.count ?? 0;
+            if (cap.count - reserved < req.count)
+                return false;
+            if (req.attributeMatches) {
+                for (const [k, v] of Object.entries(req.attributeMatches)) {
+                    if (cap.attributes?.[k] !== v)
+                        return false;
+                }
+            }
+        }
+        return true;
+    }
+    async placeNodeService(svc, nodeId) {
+        const jobId = `${svc.serviceId}.${uuid.v4().slice(0, 8)}`;
+        debuglog("Placing NodeService", { serviceId: svc.serviceId, nodeId, jobId });
+        // Optimistically record the placement so a re-entrant reconcile
+        // doesn't double-place.
+        let placements = this.nodeServicePlacements.get(svc.serviceId);
+        if (!placements) {
+            placements = new Map();
+            this.nodeServicePlacements.set(svc.serviceId, placements);
+        }
+        placements.set(nodeId, jobId);
+        // Build the runtime Job: take the configured template, override
+        // the jobId, and tag it as a NodeService instance.
+        const job = {
+            ...svc.jobSpec,
+            jobId,
+            tags: {
+                ...(svc.jobSpec.tags ?? {}),
+                "auto-manager": "true",
+                nodeService: svc.serviceId,
+            },
+        };
+        let createdJob = false;
+        try {
+            await this.norsk.createJob(job);
+            createdJob = true;
+            await this.norsk.startJob(jobId, exports.NODE_SERVICE_ROLE, nodeId);
+            this.jobToNode.set(jobId, nodeId);
+        }
+        catch (err) {
+            // Roll back the optimistic placement on failure.
+            placements.delete(nodeId);
+            // R9: if createJob succeeded but startJob failed, the Manager
+            // has a dangling Job row that the next reconcile would orphan
+            // by allocating a fresh jobId. Best-effort delete.
+            if (createdJob) {
+                try {
+                    await this.norsk.deleteJob(jobId);
+                }
+                catch (cleanupErr) {
+                    debuglog("placeNodeService: cleanup deleteJob failed", {
+                        jobId,
+                        err: String(cleanupErr),
+                    });
+                }
+            }
+            this.settings.onError?.(err);
+        }
+    }
+    async teardownNodeService(serviceId, nodeId, jobId) {
+        debuglog("Tearing down NodeService instance", { serviceId, nodeId, jobId });
+        const placements = this.nodeServicePlacements.get(serviceId);
+        placements?.delete(nodeId);
+        try {
+            await this.norsk.stopJob(jobId, exports.NODE_SERVICE_ROLE, nodeId);
+        }
+        catch (err) {
+            // Job might be gone already; tolerate.
+            debuglog("NodeService stopJob failed (continuing)", { serviceId, err: String(err) });
+        }
+    }
+    tagsForJob(jobId) {
+        return this.jobMap.get(jobId)?.job.tags;
+    }
+    poolNameForNode(nodeId) {
+        const node = this.nodeMap.get(nodeId);
+        return node?.nodeMetadata?.tags?.["pool"] ?? "";
+    }
+    // Tier within the pool, from the node's `tier` tag. Undefined for
+    // pre-registered cluster nodes (no tier tag) — nodeMatchesTier then
+    // treats them as the pool's fixed tier.
+    tierNameForNode(nodeId) {
+        const node = this.nodeMap.get(nodeId);
+        return node?.nodeMetadata?.tags?.["tier"];
+    }
+    azForNode(nodeId) {
+        const node = this.nodeMap.get(nodeId);
+        return node?.nodeMetadata?.tags?.["az"];
+    }
+    // Cloud / failure-domain key for the resilience sameCloud rule: the node's
+    // tier kind (aws / oci / local-servers), resolved via its pool+tier tags. Matches
+    // how provisionDegradation and validation derive a pool's clouds.
+    cloudForNode(nodeId) {
+        const poolName = this.poolNameForNode(nodeId);
+        const tierName = this.tierNameForNode(nodeId);
+        const pool = (this.settings.placementPools ?? []).find((p) => p.name === poolName);
+        if (!pool)
+            return undefined;
+        const tier = tierName !== undefined
+            ? pool.tiers.find((t) => t.name === tierName)
+            : pool.tiers.find((t) => t.scaleOut === "fixed");
+        return tier?.kind;
+    }
+    handleBundleUpdated(raw) {
+        if (!raw.bundle)
+            return;
+        const b = (0, conversions_1.fromPbBundle)(raw.bundle);
+        this.bundleMap.set(b.bundleId, b);
+        // Phase C2: operator pushed a fresh spec — clear any prior broken
+        // state for this bundle and reset its replicas' restart counters
+        // so the new spec gets a clean start.
+        if (this.brokenBundles.delete(b.bundleId)) {
+            debuglog("Cleared broken flag on bundleUpdated", { bundleId: b.bundleId });
+        }
+        this.clearRestartHistoryForBundle(b.bundleId);
+        this.settings.onBundleUpdated?.(b);
+    }
+    handleBundleDeleted(raw) {
+        if (!raw.bundleId)
+            return;
+        const id = raw.bundleId.bundleId;
+        this.bundleMap.delete(id);
+        this.brokenBundles.delete(id);
+        this.clearRestartHistoryForBundle(id);
+        const stop = this.pendingStopTimers.get(id);
+        if (stop) {
+            stop.timer.cancel();
+            this.pendingStopTimers.delete(id);
+        }
+        this.settings.onBundleDeleted?.(id);
+    }
+    clearRestartHistoryForBundle(bundleId) {
+        for (const jobId of this.jobMap.keys()) {
+            const tags = this.jobMap.get(jobId)?.job.tags ?? {};
+            if (tags["bundleId"] === bundleId)
+                this.restartHistory.delete(jobId);
+        }
+    }
+    nodesByJob(jobId) {
+        // Uses the jobToNode index so the "is this job placed yet?" check
+        // works for multi-job nodes too — the older `tags["jobId"]` scan
+        // always returned empty for cluster workers (the tag is per-node
+        // and can only carry one value), which caused AutoManager to
+        // repeatedly re-attempt placement and the daemon to reject with
+        // `INVALID_ARGUMENT: Job already running on node`.
+        const nodes = new Set();
+        const nodeId = this.jobToNode.get(jobId);
+        if (nodeId !== undefined) {
+            const node = this.nodeMap.get(nodeId);
+            if (node)
+                nodes.add(node);
+        }
+        return nodes;
+    }
+    /** @internal */
+    constructor(norsk, settings) {
+        // All AutoManager state is a mirror of what the event stream most
+        // recently said — AutoManager owns no durable state. Maps are cleared
+        // and re-hydrated on (re)connect from the event-stream initial state.
+        this.jobMap = new Map();
+        this.nodeMap = new Map();
+        this.bundleMap = new Map();
+        this.inventoryMap = new Map();
+        // Per-node rejection history (timestamps). Pruned to the configured
+        // backoff window on read; entries trigger escalation to `unhealthyNodes`
+        // once the threshold is crossed.
+        this.rejectionsPerNode = new Map();
+        // Nodes excluded from placement entirely until the next clean
+        // `nodeInventoryUpdated` clears the entry.
+        this.unhealthyNodes = new Set();
+        // Recently-failed nodes — excluded from placement for `failureBackoffMs`
+        // after we observe their `nodeStopping`. Map value is the deadline.
+        this.recentFailures = new Map();
+        // Phase C2: per-replica restart history. Each entry is the time a
+        // recovery placement was attempted; older-than-1h entries are
+        // pruned on read. Used to trip the maxRestartsPerHour ceiling.
+        this.restartHistory = new Map();
+        // Canonical jobId → nodeId index. Set at placement time (either
+        // when AutoManager binds the job to an existing node via startJob,
+        // or when it provisions a new node and records the provisional
+        // marker); deleted on jobDeleted. Underpins `nodeForJob` as an
+        // O(1) lookup so consumers (proxy routers, operator UIs) don't pay
+        // a fleet-wide scan on every read — important when a single host
+        // can carry many jobs (cluster, or any future cloud N:1) and the
+        // `tags["jobId"]` shortcut on the node stops being expressive.
+        // Rebuilt on R7 reconnect from each job's history via
+        // `currentNodeIdForJob`. Entries for jobs whose node has since
+        // terminated remain in the map but resolve to undefined through
+        // nodeMap — harmless, and cleaned up the next time the job is
+        // re-placed or deleted.
+        this.jobToNode = new Map();
+        // Per-job placement record. Set at the same callsite that fires
+        // `onPlacementDecision` (live decisions during this AutoManager
+        // generation); rebuilt from job history on R7 reconnect — for
+        // hydrated jobs we have no live trace, so we synthesise one with
+        // empty `trace.attempts` and the decision inferred from the
+        // history event ("place" = re-binding to an existing node, "provision"
+        // = node started for the job). Consumers query this via
+        // `placements()` / `placementForJob()` so an operator UI shows the
+        // current placement immediately on (re)connect rather than waiting
+        // for the next live decision — same mirror-of-state pattern jobMap
+        // and nodeMap use. Cleared on jobDeleted; entries are kept across
+        // node restarts since the placement-of-record doesn't change just
+        // because a node bounced.
+        this.placementsByJob = new Map();
+        // Pending placement timers — one per scheduled job. Set when
+        // `handleJobChanged` sees a pre/active job whose `startDateTime`
+        // (minus the pool's `placementLeadMs`) is still in the future;
+        // fires placeAndProvision when the lead-time window opens.
+        // Cancelled on jobDeleted, on R7 reconnect (re-evaluated from
+        // hydrated state), and replaced on subsequent job-updates that
+        // change the schedule. Tracks `scheduledFor` so `pendingPlacements()`
+        // can render upcoming work in operator UIs.
+        this.pendingPlacementTimers = new Map();
+        // Pending stop timers — one per bundleId. Set when any job in the
+        // bundle carries the `__stopDateTime` tag and the instant is still
+        // in the future; on fire, calls `deleteBundle(bundleId)` and
+        // removes the entry. Daemon has no first-class concept of stop
+        // time so this state is rebuilt from job tags on R7 reconnect.
+        // Public surface: `setBundleStopTime(bundleId, newStopTime?)` for
+        // the future Edit-bundle UI; `bundleStopTimes()` for read.
+        this.pendingStopTimers = new Map();
+        // Outstanding deferred recovery placements. Each entry caps to one
+        // setTimeout-driven retry per JobId; subsequent triggers while a
+        // retry is in flight are no-ops (the scheduled fire will see the
+        // latest state). Prevents the tight-loop "provisionJob → JobFailed →
+        // provisionJob" pattern we observed when worker-side launch errors
+        // were instantly retryable. See 04-e2e-todo's B-daemon-retry-storm.
+        this.pendingPlacementRetries = new Map();
+        // Timestamp of the most recent `placeAndProvision` entry per JobId.
+        // Used by both initial and recovery branches to skip duplicate
+        // attempts while a placement is still in flight (the daemon has
+        // dispatched provisionJob but JobStarted hasn't landed yet, so
+        // `nodesByJob().size` is still 0 and handleJobChanged would
+        // otherwise re-fire). See 04-e2e-todo's
+        // B-placement-overretry-dispatch-fail.
+        this.lastPlacementAttemptAt = new Map();
+        // Phase C2: bundles that have tripped the restart ceiling. While in
+        // this set, AutoManager refuses recovery placements for any of the
+        // bundle's replicas. Cleared on bundleUpdated (operator pushed a
+        // fresh spec) or bundleDeleted.
+        this.brokenBundles = new Set();
+        // Phase C3: pending nodeStarting → nodeStarted grace timers. If a
+        // timer fires before the node reports running, AutoManager
+        // terminates the node and the existing nodeStopping path runs
+        // recovery per band rules.
+        this.startupGraceTimers = new Map();
+        // Phase C6: spares we have asked the provider to create but which
+        // haven't yet appeared in nodeMap with the spare tag. Counted
+        // alongside live spares so the reconciler doesn't fire duplicate
+        // provisions while a previous one is in flight.
+        this.pendingSpareProvisions = new Map();
+        // Phase C8: nodes the operator has cordoned client-side. Merged
+        // into NodeView.inventory.cordoned at placement time so existing
+        // placement filter excludes them. (When Manager grows a cordon
+        // API and emits `cordoned: true` on inventory updates, this local
+        // set becomes redundant — until then it's the API.)
+        this.cordonedNodes = new Set();
+        // Phase D4 — cumulative metrics counters since startedAt. Read via
+        // `metrics()`. Counters never reset; consumers compute deltas.
+        this.metricsCounters = {
+            placementsAttempted: 0,
+            placementsSucceeded: 0,
+            placementsFailed: 0,
+            placementsDispatchFailed: 0,
+            jobsRejected: 0,
+            bundlesBroken: 0,
+            jobsDropped: 0,
+        };
+        // Phase E — active migrations. "No concurrent migrations per Job"
+        // means migrationByJobId is a 1-to-1 index of migrations.
+        this.migrationStates = new Map();
+        this.migrationByJobId = new Map();
+        // Phase E deadlines — a per-migration timer fires if the current
+        // phase hasn't transitioned by the configured window. Re-armed
+        // on every phase transition; cleared on done/aborted/close().
+        this.migrationPhaseTimers = new Map();
+        // Phase F — NodeService runtime tracking. Per service, the set of
+        // (nodeId → runtime jobId) AutoManager has placed it on. The
+        // reconciler keeps this in sync with the configured eligible set.
+        this.nodeServicePlacements = new Map();
+        this.closed = false;
+        this.norsk = norsk;
+        this.settings = settings;
+        this.clock = settings.clock ?? new clock_1.RealClock();
+        if (!this.settings.onError) {
+            this.settings.onError = (err) => {
+                debuglog("Unhandled error: %o", err);
+            };
+        }
+    }
+    /** @internal */
+    async runInternal() {
+        this.startedAt = this.clock.now();
+        const [initialState, iter] = await (0, norsk_manager_sdk_1.eventStream)(this.norsk, {
+            pendingWindow: this.settings.pendingWindow,
+        });
+        // R7: Hydrate state from initial-state. AutoManager owns nothing
+        // durable; clear-then-populate so a reconnect drops anything the
+        // server has since forgotten about. Includes every transient map
+        // we maintain in-process — without this, a reconnect would carry
+        // forward unhealthy/restart/migration/spare state against a fleet
+        // view that may no longer match.
+        this.jobMap.clear();
+        this.nodeMap.clear();
+        this.bundleMap.clear();
+        this.inventoryMap.clear();
+        this.unhealthyNodes.clear();
+        this.recentFailures.clear();
+        this.rejectionsPerNode.clear();
+        this.restartHistory.clear();
+        this.brokenBundles.clear();
+        this.pendingSpareProvisions.clear();
+        this.jobToNode.clear();
+        this.placementsByJob.clear();
+        // Pending placement timers are session-local; the recovery loop
+        // below re-evaluates each hydrated pre/active job and schedules
+        // fresh timers (or places immediately) as appropriate.
+        for (const { timer } of this.pendingPlacementTimers.values())
+            timer.cancel();
+        this.pendingPlacementTimers.clear();
+        // Same for stop timers — the recovery loop calls reconcileStopTimer
+        // for each hydrated job so the daemon-persisted __stopDateTime tag
+        // re-arms the timer.
+        for (const { timer } of this.pendingStopTimers.values())
+            timer.cancel();
+        this.pendingStopTimers.clear();
+        for (const t of this.startupGraceTimers.values())
+            t.cancel();
+        this.startupGraceTimers.clear();
+        // Migrations are in-flight client-side state; on reconnect the
+        // worker-side actors are gone with the old connection, so any
+        // pending migration is effectively dead. Cancel deadlines first
+        // (they refer to timers we're about to cancel anyway), then drop
+        // the state maps.
+        for (const d of this.migrationPhaseTimers.values())
+            d.cancel();
+        this.migrationPhaseTimers.clear();
+        this.migrationStates.clear();
+        this.migrationByJobId.clear();
+        this.nodeServicePlacements.clear();
+        for (const jobWithHistory of initialState.activeJobs) {
+            this.jobMap.set(jobWithHistory.job.jobId, jobWithHistory);
+            // Rebuild jobToNode from history — on reconnect we don't get a
+            // fresh placement-decision stream for jobs that were already
+            // placed; the history is the source of truth.
+            const nodeId = currentNodeIdForJob(jobWithHistory);
+            if (nodeId !== undefined) {
+                this.jobToNode.set(jobWithHistory.job.jobId, nodeId);
+                // Same reasoning for the placement map: a job that was placed
+                // in a prior generation has no live `onPlacementDecision` event
+                // we can replay, so we synthesise a record from history. The
+                // trace is empty (we never had it) but the result + at +
+                // job-identity fields are all derivable.
+                const synth = synthesisePlacementFromHistory(jobWithHistory, nodeId);
+                if (synth)
+                    this.placementsByJob.set(jobWithHistory.job.jobId, synth);
+            }
+            // Fire onJobUpdated for each hydrated job so consumers can
+            // treat hydration identically to a stream of fresh
+            // job-updated events. Without this, a consumer subscribed
+            // through onJobUpdated wouldn't see existing jobs until the
+            // next genuine update — proxies that build routing tables
+            // from the event stream would stay empty until the operator
+            // touched something.
+            this.settings.onJobUpdated?.(jobWithHistory);
+        }
+        for (const runningNode of initialState.nodesRunning) {
+            const nodeId = runningNode.nodeMetadata.nodeId;
+            this.updateNodeMap(nodeId, "running", runningNode.nodeMetadata, runningNode.runningNodeMetadata);
+            // Surface hydrated nodes to consumers as started — symmetric with
+            // the job-hydration loop above replaying each job via onJobUpdated.
+            // A worker already up before AutoManager (re)connected is delivered
+            // as a `NodeRunning` snapshot, never a live `NodeStarted`; without
+            // this, downstream listeners (e.g. the norsk-mgr Cluster page, which
+            // only refetches on a node-started signal) never learn it joined.
+            const node = this.nodeMap.get(nodeId);
+            if (node)
+                this.settings.onNodeStarted?.(node);
+        }
+        for (const ba of initialState.bundles) {
+            if (ba.bundle) {
+                const b = (0, conversions_1.fromPbBundle)(ba.bundle);
+                this.bundleMap.set(b.bundleId, b);
+            }
+        }
+        for (const snap of initialState.inventories) {
+            if (snap.inventory && snap.nodeId) {
+                this.inventoryMap.set(snap.nodeId.nodeId, (0, conversions_1.fromPbNodeInventory)(snap.inventory));
+            }
+        }
+        // hotSpares + nodeServices reserved (Phase F).
+        // Recover any active jobs that we don't have a node for — these
+        // exist when AutoManager has just (re)connected and the server
+        // tells us about them via initial-state. Route through
+        // `placeOrDefer` so scheduled jobs whose `startDateTime - lead`
+        // is still in the future get a fresh timer; ones whose window
+        // has already arrived (or was missed while mgr was down) place
+        // immediately.
+        for (const entry of this.jobMap.values()) {
+            // Rebuild stop timers from the daemon-persisted __stopDateTime
+            // tag on every hydrated job. Dedupes per bundleId — first job
+            // wins, subsequent jobs in the same bundle are no-ops.
+            this.reconcileStopTimer(entry);
+            if (this.wantsPlacementNow(entry)) {
+                await this.placeOrDefer(entry);
+            }
+        }
+        // Periodic cleanup: terminate stopped nodes older than the threshold.
+        // R19: cadence derived from removeStoppedNodesAfter (1/4 of the
+        // threshold, capped to [10s, 5min]) so the sweep is proportionate
+        // to the retention window — tests that lower the threshold to
+        // seconds see the sweep fire quickly without rewiring the cadence.
+        const cleanupCadenceMs = Math.max(10000, Math.min(300000, Math.floor((this.settings.removeStoppedNodesAfter * 1000) / 4)));
+        this.cleanupInterval = this.clock.setInterval(async () => {
+            if (this.closed)
+                return; // R14: don't run cleanup after close()
+            // R13: snapshot entries up front so concurrent mutations during
+            // the await terminateNode don't reshuffle traversal.
+            const snapshot = Array.from(this.nodeMap.entries());
+            for (const [nodeId, node] of snapshot) {
+                if (this.closed)
+                    return;
+                if (node.nodeState === "stopped" &&
+                    this.clock.now().getTime() - node.lastActivity.getTime() >
+                        this.settings.removeStoppedNodesAfter * 1000) {
+                    await this.norsk.terminateNode(nodeId).catch((err) => {
+                        this.settings.onError?.(err);
+                    });
+                    this.nodeMap.delete(nodeId);
+                    this.settings.onNodeTerminated?.(nodeId);
+                }
+            }
+        }, cleanupCadenceMs);
+        // Phase C6: periodic hot-spare reconciliation tick. Event-driven
+        // reconciles handle most cases; this is the safety net for
+        // missed signals (e.g. a provider that quietly terminated a
+        // spare without Manager emitting nodeStopping).
+        const reconcileInterval = this.settings.hotSpareReconcileIntervalMs ??
+            DEFAULT_HOT_SPARE_RECONCILE_INTERVAL_MS;
+        this.hotSpareReconcileInterval = this.clock.setInterval(() => this.reconcileHotSpares(), reconcileInterval);
+        // Kick once at startup so a fresh AutoManager with hotSpares
+        // configured begins provisioning immediately rather than waiting
+        // for the first event or interval.
+        this.reconcileHotSpares();
+        // Phase F: same shape as hot-spare reconcile — periodic tick
+        // for NodeServices, with a startup kick.
+        const nsReconcileInterval = this.settings.nodeServiceReconcileIntervalMs ??
+            DEFAULT_NODE_SERVICE_RECONCILE_INTERVAL_MS;
+        this.nodeServiceReconcileInterval = this.clock.setInterval(() => this.reconcileNodeServices(), nsReconcileInterval);
+        this.reconcileNodeServices();
+        for await (const event of iter) {
+            try {
+                const eventType = event.event;
+                switch (eventType) {
+                    case "jobUpdated":
+                    case "jobPending":
+                        await this.handleJobChanged(event.jobWithHistory);
+                        break;
+                    case "jobDeleted":
+                        this.handleJobDeleted(event.jobId);
+                        break;
+                    case "jobOutOfWindow":
+                        this.handleJobDeleted(event.jobWithHistory.job.jobId);
+                        break;
+                    case "jobInfo":
+                        this.settings.onJobInfo?.(event);
+                        break;
+                    case "nodeStarting":
+                        this.handleNodeStarting(event.nodeMetadata);
+                        break;
+                    case "nodeStarted":
+                        this.handleNodeStarted(event.nodeMetadata, event.runningNodeMetadata);
+                        break;
+                    case "nodeStopping":
+                        await this.handleNodeStopping(event.nodeId, event.reason);
+                        break;
+                    case "nodeStopped":
+                        this.handleNodeStopped(event.nodeId);
+                        break;
+                    case "physicalNodeConnected":
+                        // Reserved.
+                        break;
+                    case "providerHealthChange":
+                        debuglog("Provider health change %o", event.health);
+                        break;
+                    case "eventStreamClosed":
+                        break;
+                    case "nodeInventoryUpdated":
+                        this.handleNodeInventoryUpdated(event.raw);
+                        break;
+                    case "jobRejected":
+                        this.handleJobRejected(event.raw);
+                        break;
+                    case "jobConfigUpdated":
+                        this.handleJobConfigUpdated(event.raw);
+                        break;
+                    case "bundleUpdated":
+                        this.handleBundleUpdated(event.raw);
+                        break;
+                    case "bundleDeleted":
+                        this.handleBundleDeleted(event.raw);
+                        break;
+                    case "nodeServiceUpdated":
+                    case "nodeServiceDeleted":
+                        // Phase F (reserved).
+                        break;
+                    case "migrationTargetReady":
+                        await this.handleMigrationTargetReady(event.raw);
+                        break;
+                    case "migrationSourceStoppedOutput":
+                        await this.handleMigrationSourceStoppedOutput(event.raw);
+                        break;
+                    case "migrationAborted":
+                        this.handleMigrationAborted(event.raw);
+                        break;
+                    case "auditAppended":
+                        this.settings.onAuditAppended?.(event.entry);
+                        break;
+                    default: {
+                        const exhaustiveCheck = eventType;
+                        throw new Error(`Unhandled case: ${exhaustiveCheck}`);
+                    }
+                }
+            }
+            catch (err) {
+                this.settings.onError?.(err);
+            }
+        }
+    }
+    // ---- job lifecycle ----
+    async handleJobChanged(jobWithHistory) {
+        const job = jobWithHistory.job;
+        this.jobMap.set(job.jobId, jobWithHistory);
+        this.settings.onJobUpdated?.(jobWithHistory);
+        // Reconcile the per-bundle stop timer from the well-known tag on
+        // this job. Done before the placement decisions so an already-past
+        // stop time triggers deleteBundle without waiting for placement.
+        this.reconcileStopTimer(jobWithHistory);
+        if (this.wantsPlacementNow(jobWithHistory)) {
+            await this.placeOrDefer(jobWithHistory);
+        }
+    }
+    /** Read the `__stopDateTime` tag off the job, and (re)schedule the
+     *  bundle's stop timer accordingly. Tag is set per-job but applies
+     *  to the whole bundle — first-job-wins for read; all jobs in a
+     *  bundle share the same value at submit. */
+    reconcileStopTimer(jh) {
+        const tags = jh.job.tags ?? {};
+        const bundleId = tags["bundleId"];
+        if (!bundleId)
+            return;
+        const stopStr = tags[conversions_1.STOP_DATE_TIME_TAG];
+        const stopAt = stopStr ? new Date(stopStr) : undefined;
+        const prev = this.pendingStopTimers.get(bundleId);
+        // If the tag is gone OR unchanged, just keep / drop the existing
+        // timer. Equality on millis is fine — daemon round-trips ISO
+        // strings, no float fuzz.
+        if (!stopAt || Number.isNaN(stopAt.getTime())) {
+            if (prev) {
+                prev.timer.cancel();
+                this.pendingStopTimers.delete(bundleId);
+            }
+            return;
+        }
+        if (prev && prev.stopAt.getTime() === stopAt.getTime())
+            return;
+        if (prev)
+            prev.timer.cancel();
+        this.scheduleStopTimer(bundleId, stopAt);
+    }
+    scheduleStopTimer(bundleId, stopAt) {
+        const delayMs = stopAt.getTime() - this.clock.now().getTime();
+        if (delayMs <= 0) {
+            // Already past — fire immediately and don't keep a timer entry.
+            debuglog("Stop time already past; deleting bundle now", { bundleId, stopAt: stopAt.toISOString() });
+            void this.deleteBundle(bundleId).catch((e) => this.settings.onError?.(e instanceof Error ? e : new Error(String(e))));
+            this.pendingStopTimers.delete(bundleId);
+            return;
+        }
+        debuglog("Scheduling bundle stop", { bundleId, stopAt: stopAt.toISOString(), inMs: delayMs });
+        const timer = this.clock.setTimeout(() => {
+            this.pendingStopTimers.delete(bundleId);
+            void this.deleteBundle(bundleId).catch((e) => this.settings.onError?.(e instanceof Error ? e : new Error(String(e))));
+        }, delayMs);
+        this.pendingStopTimers.set(bundleId, { timer, stopAt });
+    }
+    /**
+     * Update (or clear) a bundle's auto-stop time. Calls `updateBundle`
+     * so the new value persists via the daemon's tag store and any other
+     * clients see it through their event streams. Pass `undefined` to
+     * cancel a previously-set stop time.
+     *
+     * Throws if the bundle isn't known to this AutoManager (no live
+     * bundleMap entry); operators wanting to set stop on a not-yet-
+     * streamed bundle should retry after JobPending fires.
+     * @public
+     */
+    async setBundleStopTime(bundleId, stopDateTime) {
+        const bundle = this.bundleById(bundleId);
+        if (!bundle) {
+            throw new Error(`No bundle ${bundleId} in AutoManager state`);
+        }
+        const updated = {
+            ...bundle,
+            jobs: bundle.jobs.map((j) => j.launch?.kind === "productTemplate"
+                ? { ...j, launch: { ...j.launch, stopDateTime } }
+                : j),
+        };
+        await this.updateBundle(updated);
+    }
+    /**
+     * Snapshot of all currently-scheduled bundle stop timers. UIs render
+     * this alongside `pendingPlacements()` for an "upcoming events"
+     * view.
+     * @public
+     */
+    bundleStopTimes() {
+        return [...this.pendingStopTimers.entries()]
+            .map(([bundleId, { stopAt }]) => ({ bundleId, stopAt }))
+            .sort((a, b) => a.stopAt.getTime() - b.stopAt.getTime());
+    }
+    /** A job that should be running but currently has no node bound to
+     *  it: it carries placement `requirements`, isn't manual-start, is in
+     *  a placeable state (`pre` / `active`), and `nodesByJob` is empty.
+     *  This is the single source of truth for "this job wants a placement
+     *  and doesn't have one" — shared by initial hydration, the per-job
+     *  `handleJobChanged` path, and the capacity-driven reconcile sweep,
+     *  so the decision lives in exactly one place. */
+    wantsPlacementNow(jh) {
+        const job = jh.job;
+        if (!job.autoPlacement?.requirements)
+            return false;
+        if ((job.tags ?? {})["startMode"] === "manual")
+            return false;
+        // Durable terminal verdict (poison / bronze drop) persisted by a prior
+        // session: never re-place it. This is what makes "fully failed" survive a
+        // daemon restart — on reconnect the job hydrates with terminalState set.
+        if (job.terminalState)
+            return false;
+        if (job.state !== "pre" && job.state !== "active")
+            return false;
+        return this.nodesByJob(job.jobId).size === 0;
+    }
+    /**
+     * Re-run placement for every job that wants a node but hasn't got
+     * one. Called whenever a *capacity-increasing* event arrives — a node
+     * comes online (`nodeStarted`), a worker reports fresh inventory
+     * (`nodeInventoryUpdated`, which also covers an un-cordon or a node
+     * that has just freed reserved capacity), or a job finishes and
+     * releases its node (`jobDeleted`). Those are the only transitions
+     * that can turn a previously-unplaceable job placeable, so they are
+     * exactly when re-running the engine is worthwhile — there is no
+     * polling retry, the trigger is always a concrete event.
+     *
+     * Placement stays implemented once: each candidate routes through
+     * `placeOrDefer` (the same entry point hydration and `handleJobChanged`
+     * use), inheriting its scheduling, the in-flight gate, and the
+     * failure path. Jobs already waiting on a schedule timer or a recovery
+     * backoff are skipped — their own timers own them — so a capacity
+     * event never disturbs a future-scheduled or backing-off job.
+     *
+     * Cheap and idempotent (mirrors `reconcileHotSpares` /
+     * `reconcileNodeServices`); safe to call on every relevant event.
+     */
+    reconcilePendingPlacements() {
+        for (const jh of this.jobMap.values()) {
+            if (!this.wantsPlacementNow(jh))
+                continue;
+            const jobId = jh.job.jobId;
+            if (this.pendingPlacementTimers.has(jobId))
+                continue;
+            if (this.pendingPlacementRetries.has(jobId))
+                continue;
+            void this.placeOrDefer(jh).catch((err) => this.settings.onError?.(err));
+        }
+    }
+    /** Decide whether to place immediately or defer until the job's
+     *  scheduled `startDateTime` (minus pool lead time) arrives.
+     *  Re-entry from job-update events: cancels and re-evaluates so a
+     *  schedule change moves the timer. Hydrated jobs with `startTime -
+     *  lead` in the past place immediately (operator intent was to run;
+     *  schedule was a "when", not a "drop if missed"). */
+    async placeOrDefer(jh) {
+        const job = jh.job;
+        const requirements = job.autoPlacement?.requirements;
+        if (!requirements)
+            return;
+        const tags = job.tags ?? {};
+        // Any previous pending timer for this jobId is now stale — either
+        // we'll re-schedule below or we'll place immediately. Either way,
+        // drop the prior timer.
+        const prev = this.pendingPlacementTimers.get(job.jobId);
+        if (prev) {
+            prev.timer.cancel();
+            this.pendingPlacementTimers.delete(job.jobId);
+        }
+        const startDateTime = job.startDateTime;
+        if (!startDateTime) {
+            // No schedule — fire now, current behaviour.
+            await this.placeAndProvision(job.jobId, requirements, tags);
+            return;
+        }
+        const leadMs = this.placementLeadMsForJob(jh);
+        const fireAt = startDateTime.getTime() - leadMs;
+        const delayMs = fireAt - this.clock.now().getTime();
+        if (delayMs <= 0) {
+            // Schedule has arrived (or passed — hydration after mgr was
+            // down through the launch window). Operator's intent was for
+            // the job to be running; honour it.
+            await this.placeAndProvision(job.jobId, requirements, tags);
+            return;
+        }
+        debuglog("Deferring placement until scheduled startDateTime", {
+            jobId: job.jobId,
+            startDateTime: startDateTime.toISOString(),
+            leadMs,
+            fireInMs: delayMs,
+        });
+        const timer = this.clock.setTimeout(() => {
+            this.pendingPlacementTimers.delete(job.jobId);
+            // Re-check that we still want to place — the job may have been
+            // deleted or another instance placed it during the wait. Cheap
+            // re-test using nodesByJob (now jobToNode-backed and O(1)).
+            const live = this.jobMap.get(job.jobId);
+            if (!live)
+                return;
+            if (this.nodesByJob(job.jobId).size > 0)
+                return;
+            void this.placeAndProvision(job.jobId, requirements, tags);
+        }, delayMs);
+        this.pendingPlacementTimers.set(job.jobId, { timer, scheduledFor: new Date(fireAt) });
+    }
+    /** Maximum `placementLeadMs` across the bundle pool's tiers (worst case
+     *  — guarantees the job is ready by `startDateTime` regardless of which
+     *  tier the placement engine ends up picking). Falls back to the per-kind
+     *  default when a tier doesn't specify one. Returns 0 if the bundle or
+     *  pool isn't found — we'd rather race the schedule than indefinitely
+     *  delay placement. */
+    placementLeadMsForJob(jh) {
+        const bundleId = (jh.job.tags ?? {})["bundleId"];
+        if (!bundleId)
+            return 0;
+        const bundle = this.bundleMap.get(bundleId);
+        if (!bundle)
+            return 0;
+        const pools = this.settings.placementPools ?? [];
+        const pool = pools.find((p) => p.name === bundle.pool);
+        if (!pool)
+            return 0;
+        let max = 0;
+        for (const tier of pool.tiers) {
+            const lead = tier.placementLeadMs ?? (0, placement_1.defaultPlacementLeadMs)(tier.kind);
+            if (lead > max)
+                max = lead;
+        }
+        return max;
+    }
+    /**
+     * Currently-scheduled placements (jobs AutoManager has deferred
+     * because their `startDateTime - leadTime` is still in the future).
+     * Returns one entry per pending timer; the UI can render an
+     * "upcoming jobs" list and show when each will go live.
+     * @public
+     */
+    pendingPlacements() {
+        return [...this.pendingPlacementTimers.entries()]
+            .map(([jobId, { scheduledFor }]) => ({ jobId, scheduledFor }))
+            .sort((a, b) => a.scheduledFor.getTime() - b.scheduledFor.getTime());
+    }
+    handleJobDeleted(jobId) {
+        // Pure local-state cleanup. The daemon's deleteBundle saga is now
+        // responsible for stopping the worker-side compose stacks before
+        // emitting JobDeleted; node-level lifecycle (terminate / drain) is
+        // AutoManager policy (drainNode, hot-spare reconciliation) and
+        // intentionally not coupled to job deletion. Prior versions called
+        // `terminateNode` here, which was provider-incorrect for cluster
+        // workers (terminate is a no-op on long-lived operator-attached
+        // nodes; the worker's compose stack would leak as a result).
+        const existed = this.jobMap.delete(jobId);
+        this.restartHistory.delete(jobId);
+        this.lastPlacementAttemptAt.delete(jobId);
+        this.jobToNode.delete(jobId);
+        this.placementsByJob.delete(jobId);
+        const pending = this.pendingPlacementTimers.get(jobId);
+        if (pending) {
+            pending.timer.cancel();
+            this.pendingPlacementTimers.delete(jobId);
+        }
+        const pendingRetry = this.pendingPlacementRetries.get(jobId);
+        if (pendingRetry) {
+            clearTimeout(pendingRetry);
+            this.pendingPlacementRetries.delete(jobId);
+        }
+        if (existed)
+            this.settings.onJobDeleted?.(jobId);
+        // A job leaving frees the capacity it reserved, which may let a
+        // previously-unplaceable job land. The freed capacity becomes
+        // visible to the placement engine once the worker re-reports
+        // inventory (`nodeInventoryUpdated` will reconcile again then); this
+        // trigger covers the case where AutoManager's cached inventory
+        // already reflects the release.
+        this.reconcilePendingPlacements();
+    }
+    // ---- node lifecycle ----
+    handleNodeStarting(nodeMetadata) {
+        this.updateNodeMap(nodeMetadata.nodeId, "starting", nodeMetadata, undefined);
+        this.settings.onNodeStarting?.(nodeMetadata.nodeId);
+        this.armStartupGrace(nodeMetadata.nodeId);
+    }
+    /** Phase C3. Schedule the grace timer; replace any prior timer. */
+    armStartupGrace(nodeId) {
+        this.startupGraceTimers.get(nodeId)?.cancel();
+        const grace = this.cfgJobStartupGraceMs();
+        const timer = this.clock.setTimeout(() => {
+            this.startupGraceTimers.delete(nodeId);
+            debuglog("Node startup grace exceeded — terminating", { nodeId, grace });
+            void this.norsk
+                .terminateNode(nodeId)
+                .catch((err) => this.settings.onError?.(err));
+        }, grace);
+        this.startupGraceTimers.set(nodeId, timer);
+    }
+    cancelStartupGrace(nodeId) {
+        this.startupGraceTimers.get(nodeId)?.cancel();
+        this.startupGraceTimers.delete(nodeId);
+    }
+    handleNodeStarted(nodeMetadata, runningNodeMetadata) {
+        this.updateNodeMap(nodeMetadata.nodeId, "running", nodeMetadata, runningNodeMetadata);
+        this.cancelStartupGrace(nodeMetadata.nodeId);
+        // Phase C2: a successful start clears restart history for this
+        // job — the "fundamentally broken" diagnosis only fires when no
+        // amount of replacement is helping, so a job that has ever started
+        // is by definition not fundamentally broken.
+        const jobId = getJobIdForNode(nodeMetadata);
+        if (jobId)
+            this.restartHistory.delete(jobId);
+        // Phase C6: a pending spare just landed.
+        this.pendingSpareProvisions.delete(nodeMetadata.nodeId);
+        const node = this.nodeMap.get(nodeMetadata.nodeId);
+        if (node)
+            this.settings.onNodeStarted?.(node);
+        this.reconcileHotSpares();
+        this.reconcileNodeServices();
+        // A node coming online is new capacity — a cluster worker joining or
+        // a hot spare landing can host a job that previously had nowhere to
+        // go. Re-attempt pending placements.
+        this.reconcilePendingPlacements();
+        // R6: a migration target node has come up — advance from
+        // `launching` to `awaitingTargetReady`. The workflow may still
+        // need time to initialise; the awaitingTargetReady deadline now
+        // bounds that step independently from launching.
+        this.advanceMigrationsOnNodeStarted(nodeMetadata.nodeId);
+    }
+    advanceMigrationsOnNodeStarted(nodeId) {
+        for (const state of this.migrationStates.values()) {
+            if (state.targetNodeId !== nodeId)
+                continue;
+            if (state.phase !== "launching")
+                continue;
+            this.advanceMigration(state, "awaitingTargetReady");
+        }
+    }
+    async handleNodeStopping(nodeId, _reason) {
+        this.updateNodeMap(nodeId, "stopping", undefined, undefined);
+        this.cancelStartupGrace(nodeId);
+        // Surface to consumers so cluster-membership views (mgr's Cluster
+        // page) refresh promptly. Without this callback, a Ctrl-C'd
+        // cluster worker only surfaces in mgr UI when the eventual
+        // NodeStopped/NodeTerminated arrives — which can be many seconds
+        // later via the daemon's keepalive timeout + cleanup loop.
+        this.settings.onNodeStopping?.(nodeId);
+        // Mark the node as recently-failed so subsequent placements skip it
+        // for `failureBackoffMs`. Cleared when the node next reports a
+        // healthy `nodeInventoryUpdated`.
+        const window = this.cfgFailureBackoffMs();
+        this.recentFailures.set(nodeId, new Date(this.clock.now().getTime() + window));
+        // TODO future slice: thread the nodeStopping reason into
+        // recoveryContext so band-specific behaviour applies (gold→prefer
+        // hot spare, bronze drop, etc.).
+        const node = this.nodeMap.get(nodeId);
+        const jobId = node?.provisional?.jobId ?? getJobIdForNode(node?.nodeMetadata);
+        if (jobId) {
+            // R8: if the job is mid-migration, don't run recovery placement —
+            // the migration target is already a second live instance for the
+            // same jobId, and a fresh placeAndProvision would create a third.
+            // Abort the migration instead; operator can re-attempt once the
+            // source is back. The target on the new node is allowed to keep
+            // running (it's stoppable through the normal jobMap path).
+            const migration = this.migrationForJob(jobId);
+            if (migration && (migration.phase !== "done" && migration.phase !== "aborted")) {
+                this.failMigration(migration.migrationId, `source nodeStopping during phase ${migration.phase}`);
+            }
+            else {
+                const entry = this.jobMap.get(jobId);
+                const job = entry?.job;
+                const req = job?.autoPlacement?.requirements;
+                if (job && req && (job.state === "pre" || job.state === "active")) {
+                    await this.placeAndProvision(job.jobId, req, job.tags ?? {}, { excludeNodes: this.currentExcludeSet() });
+                }
+            }
+        }
+        // Phase C6: if the stopping node was a spare, reconcile to
+        // provision a replacement promptly (don't wait for the tick).
+        this.reconcileHotSpares();
+        // Phase F: lost capacity → reconcile NodeService placements.
+        // Also drop any tracked placement on this node (the runtime job
+        // is gone with the node).
+        this.dropNodeServicePlacementsOnNode(nodeId);
+        this.reconcileNodeServices();
+    }
+    handleNodeStopped(nodeId) {
+        this.updateNodeMap(nodeId, "stopped", undefined, undefined);
+        // Phase C6: a spare going away should trigger a replacement.
+        this.pendingSpareProvisions.delete(nodeId);
+        this.settings.onNodeStopped?.(nodeId);
+        this.reconcileHotSpares();
+        // Phase F: clear NodeService placements on stopped nodes.
+        this.dropNodeServicePlacementsOnNode(nodeId);
+        this.reconcileNodeServices();
+    }
+    dropNodeServicePlacementsOnNode(nodeId) {
+        for (const placements of this.nodeServicePlacements.values()) {
+            placements.delete(nodeId);
+        }
+    }
+    // ---- maps ----
+    updateNodeMap(nodeId, nodeState, nodeMetadata, runningNodeMetadata, provisional) {
+        const existing = this.nodeMap.get(nodeId);
+        this.nodeMap.set(nodeId, {
+            nodeId,
+            nodeState,
+            lastActivity: this.clock.now(),
+            nodeMetadata: nodeMetadata ?? existing?.nodeMetadata,
+            runningNodeMetadata: runningNodeMetadata ?? existing?.runningNodeMetadata,
+            provisional: provisional ?? existing?.provisional,
+        });
+    }
+}
+exports.AutoManager = AutoManager;
+function summariseTrace(trace) {
+    // One-line per attempted pool, listing how nodes were filtered
+    // (grouped by reason) and what happened. Compact enough that it
+    // doesn't drown a debuglog stream when many placements happen in a
+    // burst, but informative enough that "why did my job land there?"
+    // is immediately answerable.
+    const lines = trace.attempts.map((a) => {
+        const groups = {};
+        for (const f of a.filtered)
+            groups[f.reason] = (groups[f.reason] ?? 0) + 1;
+        const filterSummary = Object.entries(groups)
+            .map(([reason, count]) => `${reason}=${count}`)
+            .join(", ");
+        const winner = a.outcome.kind === "placed"
+            ? ` → placed on ${a.outcome.nodeId}${a.outcome.gpuIndex !== undefined ? ` (gpu ${a.outcome.gpuIndex})` : ""}`
+            : a.outcome.kind === "provisioned"
+                ? ` → provisioned ${a.outcome.instanceType}`
+                : a.outcome.kind === "exhausted"
+                    ? " → fixed-pool exhausted, fall through"
+                    : a.outcome.kind === "no-instance-type"
+                        ? " → no allowed instance type fits, fall through"
+                        : " → pool not configured";
+        const matchSummary = `${a.scored.length}/${a.initialCandidates} matched`;
+        const filterPart = filterSummary ? `, filtered: ${filterSummary}` : "";
+        return `[${a.pool}] ${matchSummary}${filterPart}${winner}`;
+    });
+    return `${trace.decision}: ${lines.join(" || ")}`;
+}
+function roleFromReplicaIndex(replicaIndex) {
+    // Preserve the existing primary/backup convention for replicas 0 and 1
+    // so consumers that grep on role keep working. Higher replica counts
+    // get a numeric form.
+    if (replicaIndex === 0)
+        return "primary";
+    if (replicaIndex === 1)
+        return "backup";
+    return `replica-${replicaIndex}`;
+}
+function debuglog(msg, ...param) {
+    util.debuglog("norsk-automanager")(util.formatWithOptions({ maxArrayLength: null, depth: null, colors: true }, "[" + new Date().toISOString() + "] " + msg, ...param));
+}
+function getJobIdForNode(nodeMetadata) {
+    return nodeMetadata?.tags["jobId"];
+}
+/**
+ * Walk a job's history backward and return the nodeId where it is
+ * currently placed (or was most recently placed). Prefers `running`
+ * entries; falls back to `provisioned` for jobs still booting their
+ * node. Returns undefined for jobs that have never been placed or
+ * whose history is empty (defensive — runtime data always has at
+ * least a `created` entry, which carries no nodeMetadata).
+ *
+ * This is the canonical "where does this job live" lookup and
+ * underpins `AutoManager.nodeForJob`. It works regardless of whether
+ * a node hosts one job (cloud, today) or many (cluster, or future
+ * cloud N:1) because the nodeId is recorded against the JOB rather
+ * than scanned out of NODE tags.
+ */
+function currentNodeIdForJob(jh) {
+    if (!jh)
+        return undefined;
+    // Two-pass: prefer the most recent `running` over the most recent
+    // `provisioned`. A job that has been provisioned, run, stopped,
+    // re-provisioned, but not yet run again should report the newest
+    // provisioned node — but if it's running we always want that.
+    let mostRecentProvisioned;
+    for (let i = jh.history.length - 1; i >= 0; i--) {
+        const e = jh.history[i];
+        if (e.event === "running")
+            return e.nodeMetadata.nodeId;
+        if (e.event === "provisioned" && mostRecentProvisioned === undefined) {
+            mostRecentProvisioned = e.nodeMetadata.nodeId;
+        }
+    }
+    return mostRecentProvisioned;
+}
+/**
+ * Build a `PlacementDecisionInfo` from a hydrated job's history. Used
+ * on R7 reconnect to seed the placementsByJob map for jobs that were
+ * placed in a previous generation — we have no live trace, so this
+ * synthesises a minimal record (empty `trace.attempts`) that's still
+ * shaped like a real decision so consumers can render it uniformly.
+ *
+ * `nodeId` is passed in by the caller (already resolved via
+ * `currentNodeIdForJob`), to avoid walking the history twice.
+ *
+ * Returns undefined if the job's tags are too sparse to derive
+ * `bundleId` / `replicaIndex` / `jobName` (un-managed jobs the
+ * AutoManager-driven UI couldn't usefully display anyway).
+ */
+function synthesisePlacementFromHistory(jh, nodeId) {
+    const tags = jh.job.tags ?? {};
+    const bundleId = tags["bundleId"];
+    const replicaIndexStr = tags["replicaIndex"];
+    const jobName = tags["jobName"];
+    if (!bundleId || replicaIndexStr === undefined || !jobName)
+        return undefined;
+    const replicaIndex = Number(replicaIndexStr);
+    if (!Number.isFinite(replicaIndex))
+        return undefined;
+    // Find the most recent placement event for `at` + decision kind.
+    // running > provisioned (same precedence as currentNodeIdForJob).
+    // If the only event we found is `provisioned`, decision="provision"
+    // (the node was started for this job); `running` could be either —
+    // we treat it as "place" because by the time we hydrate the node is
+    // already running, so functionally the job was bound to an existing
+    // running node. The trace stays empty either way.
+    let at;
+    let decision = "place";
+    for (let i = jh.history.length - 1; i >= 0; i--) {
+        const e = jh.history[i];
+        if (e.event === "running") {
+            at = e.timestamp instanceof Date ? e.timestamp : new Date(e.timestamp);
+            decision = "place";
+            break;
+        }
+        if (e.event === "provisioned" && at === undefined) {
+            at = e.timestamp instanceof Date ? e.timestamp : new Date(e.timestamp);
+            decision = "provision";
+        }
+    }
+    if (!at)
+        return undefined;
+    // `pool`, `tier` and `instanceType` aren't on the history entries —
+    // leave them as best-guess placeholders. Consumers that care about
+    // accurate pool/tier attribution should walk the node's tags via
+    // nodeMap.get(nodeId).nodeMetadata.tags.{pool,tier}.
+    const result = decision === "place"
+        ? { kind: "place", nodeId, pool: "", tier: "" }
+        : { kind: "provision", pool: "", tier: "", instanceType: "", launchMode: "on-demand" };
+    const trace = { attempts: [], decision };
+    return { jobId: jh.job.jobId, bundleId, replicaIndex, jobName, result, trace, at };
+}
+//# sourceMappingURL=automanager.js.map