@norskvideo/norsk-auto-manager 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +344 -0
- package/lib/src/automanager.d.ts +1070 -0
- package/lib/src/automanager.js +2746 -0
- package/lib/src/clock.d.ts +14 -0
- package/lib/src/clock.js +21 -0
- package/lib/src/conversions.d.ts +59 -0
- package/lib/src/conversions.js +416 -0
- package/lib/src/index.d.ts +9 -0
- package/lib/src/index.js +37 -0
- package/lib/src/inventory.d.ts +34 -0
- package/lib/src/inventory.js +64 -0
- package/lib/src/knownCapabilities.d.ts +18 -0
- package/lib/src/knownCapabilities.js +22 -0
- package/lib/src/placement.d.ts +268 -0
- package/lib/src/placement.js +468 -0
- package/lib/src/settingsValidation.d.ts +34 -0
- package/lib/src/settingsValidation.js +232 -0
- package/lib/src/shared/utils.d.ts +17 -0
- package/lib/src/shared/utils.js +220 -0
- package/lib/src/types.d.ts +295 -0
- package/lib/src/types.js +20 -0
- package/lib/src/validation.d.ts +68 -0
- package/lib/src/validation.js +198 -0
- package/package.json +66 -0
- package/src/automanager.ts +3570 -0
- package/src/clock.ts +29 -0
- package/src/conversions.ts +458 -0
- package/src/index.ts +27 -0
- package/src/inventory.ts +64 -0
- package/src/knownCapabilities.ts +23 -0
- package/src/placement.ts +804 -0
- package/src/settingsValidation.ts +386 -0
- package/src/types.ts +346 -0
- package/src/validation.ts +297 -0
- package/tsconfig.json +25 -0
|
@@ -0,0 +1,3570 @@
|
|
|
1
|
+
import * as util from "util";
|
|
2
|
+
import * as uuid from "uuid";
|
|
3
|
+
import * as ManagerPB from "@norskvideo/norsk-api/lib/manager_pb";
|
|
4
|
+
import { CancellableTimer, Clock, RealClock } from "./clock";
|
|
5
|
+
import {
|
|
6
|
+
Bundle as TypedBundle,
|
|
7
|
+
BundleId,
|
|
8
|
+
Capability,
|
|
9
|
+
CapabilityRequirement,
|
|
10
|
+
JobRequirements,
|
|
11
|
+
NodeInventory as TypedNodeInventory,
|
|
12
|
+
NodeService,
|
|
13
|
+
PriorityBand,
|
|
14
|
+
replicaCount,
|
|
15
|
+
ServiceId,
|
|
16
|
+
} from "./types";
|
|
17
|
+
import {
|
|
18
|
+
NodeProfile,
|
|
19
|
+
PoolDescriptor,
|
|
20
|
+
ValidationError,
|
|
21
|
+
ValidationResult,
|
|
22
|
+
validateBundle as validateBundleFn,
|
|
23
|
+
} from "./validation";
|
|
24
|
+
import {
|
|
25
|
+
AutoSettingsValidationError,
|
|
26
|
+
validateAutoSettings,
|
|
27
|
+
} from "./settingsValidation";
|
|
28
|
+
import {
|
|
29
|
+
fromPbBundle,
|
|
30
|
+
fromPbNodeInventory,
|
|
31
|
+
STOP_DATE_TIME_TAG,
|
|
32
|
+
toPbBundle,
|
|
33
|
+
} from "./conversions";
|
|
34
|
+
import {
|
|
35
|
+
defaultPlacementLeadMs,
|
|
36
|
+
nodeMatchesTier,
|
|
37
|
+
NodeView,
|
|
38
|
+
PlacementFailureReason,
|
|
39
|
+
PlacementPool,
|
|
40
|
+
PlacementResult,
|
|
41
|
+
PlacementTier,
|
|
42
|
+
PlacementTrace,
|
|
43
|
+
placeWithTrace,
|
|
44
|
+
ResilienceFlag,
|
|
45
|
+
RunningJob,
|
|
46
|
+
tierReliability,
|
|
47
|
+
} from "./placement";
|
|
48
|
+
import {
|
|
49
|
+
NorskManager,
|
|
50
|
+
eventStream,
|
|
51
|
+
JobId,
|
|
52
|
+
JobInfo,
|
|
53
|
+
JobWithHistory,
|
|
54
|
+
NodeId,
|
|
55
|
+
NodeMetadata,
|
|
56
|
+
NodeStopping,
|
|
57
|
+
Role,
|
|
58
|
+
RunningNodeMetadata,
|
|
59
|
+
} from "@norskvideo/norsk-manager-sdk";
|
|
60
|
+
import type * as ManagerSdk from "@norskvideo/norsk-manager-sdk";
|
|
61
|
+
|
|
62
|
+
////////////////////////////////////////////////////////////////////////////////
|
|
63
|
+
// AutoManager — bundle-driven scheduler. The legacy AutoJob / region+
|
|
64
|
+
// instanceType API is gone; everything goes through createBundle and the
|
|
65
|
+
// placement engine. See .ai/steve/tasks/37-auto-manager-design.md.
|
|
66
|
+
|
|
67
|
+
export type NodeState =
|
|
68
|
+
| "starting"
|
|
69
|
+
| "running"
|
|
70
|
+
| "stopping"
|
|
71
|
+
| "stopped"
|
|
72
|
+
| "terminating"
|
|
73
|
+
| "terminated";
|
|
74
|
+
|
|
75
|
+
export type NodeSummary = {
|
|
76
|
+
nodeId: NodeId;
|
|
77
|
+
nodeMetadata?: NodeMetadata;
|
|
78
|
+
runningNodeMetadata?: RunningNodeMetadata;
|
|
79
|
+
nodeState: NodeState;
|
|
80
|
+
lastActivity: Date;
|
|
81
|
+
// Captured when AutoManager kicks off node creation. Lets us recover
|
|
82
|
+
// jobId / role for nodeStopping events that fire before the node has
|
|
83
|
+
// reported back nodeMetadata (and so before tags["jobId"] / tags["role"]
|
|
84
|
+
// are available).
|
|
85
|
+
provisional?: { jobId: JobId; role: Role };
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
export interface AutoSettings {
|
|
89
|
+
url?: string;
|
|
90
|
+
/**
|
|
91
|
+
* Abort the connection / run. Aborting before `run()` connects rejects it
|
|
92
|
+
* (so a connect-with-timeout retry loop can give up cleanly without
|
|
93
|
+
* leaving an abandoned SDK connection alive); aborting a running
|
|
94
|
+
* AutoManager tears down its Manager connection, ending the run loop.
|
|
95
|
+
*/
|
|
96
|
+
signal?: AbortSignal;
|
|
97
|
+
/** Seconds-from-now window for jobs to be considered pending in the eventStream. */
|
|
98
|
+
pendingWindow: number;
|
|
99
|
+
/** Seconds stopped nodes persist before being terminated. */
|
|
100
|
+
removeStoppedNodesAfter: number;
|
|
101
|
+
/** Configured pools; a bundle names one, placement walks its tiers. */
|
|
102
|
+
placementPools: PlacementPool[];
|
|
103
|
+
/**
|
|
104
|
+
* Window (ms) for counting jobRejected events per-node. Default 5_000.
|
|
105
|
+
* Once `rejectionEscalationCount` rejections fall inside this window the
|
|
106
|
+
* node is marked "unhealthy" and excluded from placement until the next
|
|
107
|
+
* `nodeInventoryUpdated` event clears the mark.
|
|
108
|
+
*/
|
|
109
|
+
rejectionBackoffMs?: number;
|
|
110
|
+
/** Default 3. */
|
|
111
|
+
rejectionEscalationCount?: number;
|
|
112
|
+
/**
|
|
113
|
+
* After a node fails (`nodeStopping`), how long to keep it excluded from
|
|
114
|
+
* placement. Default 30_000 ms. Cleared by a subsequent
|
|
115
|
+
* `nodeInventoryUpdated` event reporting the node as reachable again.
|
|
116
|
+
*/
|
|
117
|
+
failureBackoffMs?: number;
|
|
118
|
+
/**
|
|
119
|
+
* Phase C2: per-replica restart-rate ceiling on the recovery path.
|
|
120
|
+
* When a replica's restart count in the trailing hour exceeds this,
|
|
121
|
+
* AutoManager stops re-placing it (and every other replica of the
|
|
122
|
+
* same bundle), marks the bundle broken, and fires `onBundleBroken`.
|
|
123
|
+
* Cleared when the bundle is updated or deleted, or when a fresh
|
|
124
|
+
* `jobUpdated` arrives for the broken job. Default 6 (per design).
|
|
125
|
+
*/
|
|
126
|
+
maxRestartsPerHour?: number;
|
|
127
|
+
/**
|
|
128
|
+
* Phase C3: node startup grace. If a node stays in `nodeStarting`
|
|
129
|
+
* for longer than this, AutoManager terminates it — Manager emits a
|
|
130
|
+
* `nodeStopping` and the existing band-aware recovery path replaces
|
|
131
|
+
* the job. Default 120_000 ms.
|
|
132
|
+
*
|
|
133
|
+
* (In v1, "job in starting" and "node in starting" collapse onto
|
|
134
|
+
* the same signal because `running → ready` is auto-marked. The
|
|
135
|
+
* design's distinction matters when explicit readiness lands.)
|
|
136
|
+
*/
|
|
137
|
+
jobStartupGraceMs?: number;
|
|
138
|
+
/**
|
|
139
|
+
* Phase C4 — per-band recovery policy. When set, takes precedence
|
|
140
|
+
* over the flat fields above. Bronze always drops (`onFailure:
|
|
141
|
+
* "drop"`) — no per-band restart options. Gold and silver get
|
|
142
|
+
* their own `maxRestartsPerHour`; gold additionally has
|
|
143
|
+
* `restartPlacement: "preferHotSpare"` baked in by the recovery
|
|
144
|
+
* path. Omitted bands fall back to the flat top-level fields, then
|
|
145
|
+
* to defaults.
|
|
146
|
+
*/
|
|
147
|
+
bands?: BandConfig;
|
|
148
|
+
/**
|
|
149
|
+
* Phase C4 — placement / recovery tunables grouped into one typed
|
|
150
|
+
* object. Same precedence as `bands`: object field beats flat field
|
|
151
|
+
* beats default. The flat fields above are preserved for one
|
|
152
|
+
* release; new code should prefer this shape.
|
|
153
|
+
*/
|
|
154
|
+
placement?: PlacementConfig;
|
|
155
|
+
/**
|
|
156
|
+
* Phase C6 — hot-spare reconciliation configuration. One entry per
|
|
157
|
+
* (pool, optional band). The reconciler runs on node lifecycle
|
|
158
|
+
* events and on a periodic tick: counts live spares per entry,
|
|
159
|
+
* provisions short, terminates excess. Empty/omitted = no spares.
|
|
160
|
+
*/
|
|
161
|
+
hotSpares?: HotSpareConfig[];
|
|
162
|
+
/**
|
|
163
|
+
* Phase F — NodeService configurations. Each entry describes a
|
|
164
|
+
* long-running service that consumes capabilities (typically
|
|
165
|
+
* exclusive hardware) and advertises new ones for sibling jobs
|
|
166
|
+
* to consume. AutoManager's reconciler places eager services on
|
|
167
|
+
* every eligible node and tears them down when a node loses
|
|
168
|
+
* eligibility.
|
|
169
|
+
*
|
|
170
|
+
* Worker-side capability advertisement is required for sibling
|
|
171
|
+
* jobs to actually see the provided capabilities — see
|
|
172
|
+
* `§Worker-side work` in the design doc.
|
|
173
|
+
*/
|
|
174
|
+
nodeServices?: NodeService[];
|
|
175
|
+
/**
|
|
176
|
+
* Phase F — how often the NodeService reconciler's periodic tick
|
|
177
|
+
* fires. Event-driven reconciles handle most cases; the tick is a
|
|
178
|
+
* safety net. Default 60_000 ms.
|
|
179
|
+
*/
|
|
180
|
+
nodeServiceReconcileIntervalMs?: number;
|
|
181
|
+
/**
|
|
182
|
+
* Phase C6 — how often the spare reconciler's periodic tick fires.
|
|
183
|
+
* Event-driven reconciles handle most cases; the tick is a safety
|
|
184
|
+
* net for missed signals. Default 60_000 ms.
|
|
185
|
+
*/
|
|
186
|
+
hotSpareReconcileIntervalMs?: number;
|
|
187
|
+
/**
|
|
188
|
+
* Phase E — per-phase deadline configuration for in-flight
|
|
189
|
+
* migrations. If a migration sits in a phase longer than the
|
|
190
|
+
* configured window, AutoManager auto-aborts with reason
|
|
191
|
+
* `phase timeout: <phase>`. Operators can still abort manually via
|
|
192
|
+
* `abortMigration` before the deadline fires.
|
|
193
|
+
*/
|
|
194
|
+
migration?: MigrationConfig;
|
|
195
|
+
onError?: (err: unknown) => void;
|
|
196
|
+
onNodeStarting?: (node: NodeId) => void;
|
|
197
|
+
onNodeStarted?: (node: NodeSummary) => void;
|
|
198
|
+
/** Fired when the daemon emits a NodeStopping event — the node is
|
|
199
|
+
* going away but may not have fully terminated yet. For cluster
|
|
200
|
+
* workers Ctrl-C'd at the keyboard this is the signal that fires
|
|
201
|
+
* (NodeStopped/NodeTerminated come later, after the daemon's
|
|
202
|
+
* keepalive timeout or cleanup loop). Consumers wanting prompt
|
|
203
|
+
* "worker dropped" UI updates listen here. */
|
|
204
|
+
onNodeStopping?: (nodeId: NodeId) => void;
|
|
205
|
+
onNodeStopped?: (nodeId: NodeId) => void;
|
|
206
|
+
onNodeTerminated?: (nodeId: NodeId) => void;
|
|
207
|
+
onBundleUpdated?: (bundle: TypedBundle) => void;
|
|
208
|
+
onBundleDeleted?: (bundleId: BundleId) => void;
|
|
209
|
+
/**
|
|
210
|
+
* Fired when the daemon appends a new audit-log entry (any source).
|
|
211
|
+
* Pure pass-through of the SDK's `AuditLogEntry` — AutoManager doesn't
|
|
212
|
+
* own audit semantics, it just forwards the live append so consumers
|
|
213
|
+
* can tail the log without re-polling GetAuditLog.
|
|
214
|
+
*/
|
|
215
|
+
onAuditAppended?: (entry: ManagerSdk.AuditLogEntry) => void;
|
|
216
|
+
/**
|
|
217
|
+
* Fires when a node accumulates enough rejections within
|
|
218
|
+
* `rejectionBackoffMs` to be marked unhealthy. Operators may want to
|
|
219
|
+
* surface this in monitoring.
|
|
220
|
+
*/
|
|
221
|
+
onNodeUnhealthy?: (nodeId: NodeId) => void;
|
|
222
|
+
/**
|
|
223
|
+
* Fired after `jobMap` is updated in response to a `jobUpdated` or
|
|
224
|
+
* `jobPending` event. Carries the full `JobWithHistory` so consumers
|
|
225
|
+
* can render state transitions without a separate SDK round-trip.
|
|
226
|
+
*/
|
|
227
|
+
onJobUpdated?: (jobWithHistory: JobWithHistory) => void;
|
|
228
|
+
/**
|
|
229
|
+
* Fired for an informational job message from the worker — launch
|
|
230
|
+
* progress (pulling-images / starting-containers / health), container
|
|
231
|
+
* events, service lifecycle. Lets consumers surface live launch feedback
|
|
232
|
+
* tied to a job without changing its placement state.
|
|
233
|
+
*/
|
|
234
|
+
onJobInfo?: (info: JobInfo) => void;
|
|
235
|
+
/**
|
|
236
|
+
* Fired after a job is removed from `jobMap` in response to a
|
|
237
|
+
* `jobDeleted` or `jobOutOfWindow` event.
|
|
238
|
+
*/
|
|
239
|
+
onJobDeleted?: (jobId: JobId) => void;
|
|
240
|
+
/**
|
|
241
|
+
* Fired on every `jobRejected` event after AutoManager has updated
|
|
242
|
+
* its inventory cache. Note the node is not necessarily marked
|
|
243
|
+
* unhealthy by this single occurrence — see `onNodeUnhealthy` for
|
|
244
|
+
* escalation.
|
|
245
|
+
*/
|
|
246
|
+
onJobRejected?: (info: JobRejectedInfo) => void;
|
|
247
|
+
/**
|
|
248
|
+
* Fired on `jobConfigUpdated` after AutoManager has updated its
|
|
249
|
+
* mirror of the job's `managerConfiguration`.
|
|
250
|
+
*/
|
|
251
|
+
onJobConfigUpdated?: (jobId: JobId, config: string) => void;
|
|
252
|
+
/**
|
|
253
|
+
* Fires every time the placement engine runs, regardless of outcome.
|
|
254
|
+
* Carries the full trace so consumers (e.g. an operator UI) can show
|
|
255
|
+
* "why did my job land here?" without re-running placement
|
|
256
|
+
* themselves.
|
|
257
|
+
*/
|
|
258
|
+
onPlacementDecision?: (info: PlacementDecisionInfo) => void;
|
|
259
|
+
/**
|
|
260
|
+
* Fires when the placement engine returns `kind: "failure"`. Today
|
|
261
|
+
* the failure also surfaces via `onError` as a stringified message;
|
|
262
|
+
* this typed callback is the recommended path. The two are not
|
|
263
|
+
* mutually exclusive — `onError` is preserved for back-compat.
|
|
264
|
+
*/
|
|
265
|
+
onPlacementFailed?: (info: PlacementFailedInfo) => void;
|
|
266
|
+
/**
|
|
267
|
+
* Fired whenever the underlying gRPC channel changes connectivity
|
|
268
|
+
* state. Maps the five gRPC connectivity states to the strings
|
|
269
|
+
* `idle | connecting | ready | transientFailure | shutdown`.
|
|
270
|
+
* Lets a consumer surface "connected / reconnecting / disconnected"
|
|
271
|
+
* without holding a separate NorskManager handle.
|
|
272
|
+
*/
|
|
273
|
+
onConnectionStateChange?: (state: ConnectionState) => void;
|
|
274
|
+
/**
|
|
275
|
+
* Phase-C: fired when a bundle's restart rate exceeds the band's
|
|
276
|
+
* `maxRestartsPerHour` and AutoManager marks it broken. Declared
|
|
277
|
+
* now so consumers can wire the callback; not fired in pre-Phase-C
|
|
278
|
+
* code.
|
|
279
|
+
*/
|
|
280
|
+
onBundleBroken?: (bundleId: BundleId, reason: string) => void;
|
|
281
|
+
/**
|
|
282
|
+
* Phase-C: fired when a bronze-band job fails and is dropped
|
|
283
|
+
* (rather than restarted). Declared now; not fired in pre-Phase-C
|
|
284
|
+
* code.
|
|
285
|
+
*/
|
|
286
|
+
onJobDropped?: (jobId: JobId) => void;
|
|
287
|
+
/**
|
|
288
|
+
* Phase-C: fired when a hot-spare pool's live spare count is below
|
|
289
|
+
* its configured target. Declared now; not fired in pre-Phase-C
|
|
290
|
+
* code.
|
|
291
|
+
*/
|
|
292
|
+
onSpareUnderflow?: (pool: string, current: number, target: number) => void;
|
|
293
|
+
/**
|
|
294
|
+
* Fired when a bundle's backup is best-effort placed inside a failure
|
|
295
|
+
* domain its resilience policy asked to avoid (no compliant node had
|
|
296
|
+
* capacity). The backup is running, but DR posture is reduced — surface
|
|
297
|
+
* it so an operator can act / re-place when capacity frees.
|
|
298
|
+
*/
|
|
299
|
+
onResilienceDegraded?: (info: ResilienceDegradedInfo) => void;
|
|
300
|
+
/**
|
|
301
|
+
* Phase E — fired on every migration phase transition. Use for
|
|
302
|
+
* telemetry / ops UIs. The payload is the post-transition state;
|
|
303
|
+
* `phaseHistory[0]` is launch, `phaseHistory.at(-1)` is the current
|
|
304
|
+
* phase at the time of the callback.
|
|
305
|
+
*/
|
|
306
|
+
onMigrationPhase?: (info: MigrationState) => void;
|
|
307
|
+
/** Phase E — fired once when a migration reaches `done`. */
|
|
308
|
+
onMigrationCompleted?: (migrationId: MigrationId) => void;
|
|
309
|
+
/** Phase E — fired once when a migration enters `aborted`. */
|
|
310
|
+
onMigrationAborted?: (migrationId: MigrationId, reason: string) => void;
|
|
311
|
+
/** Optional clock injection. Production unset (RealClock); tests pass MockClock. */
|
|
312
|
+
clock?: Clock;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/** Fixed role for the migration-target instance. No concurrent migrations per Job. @public */
|
|
316
|
+
export const MIGRATION_TARGET_ROLE = "migrationTarget";
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Fixed role assigned to runtime jobs that exist to back a
|
|
320
|
+
* NodeService. Worker reads `WorkerJob.role === NODE_SERVICE_ROLE`
|
|
321
|
+
* to recognise the instance as a service rather than a regular
|
|
322
|
+
* workload, and adds the service's `provides[]` capabilities to its
|
|
323
|
+
* inventory once the job reaches `ready`.
|
|
324
|
+
*
|
|
325
|
+
* @public
|
|
326
|
+
*/
|
|
327
|
+
export const NODE_SERVICE_ROLE = "nodeService";
|
|
328
|
+
|
|
329
|
+
/** @public */
|
|
330
|
+
export type MigrationId = string;
|
|
331
|
+
|
|
332
|
+
/** @public */
|
|
333
|
+
export type MigrationPhase =
|
|
334
|
+
| "launching" // target provision dispatched; waiting for target node to come up
|
|
335
|
+
| "awaitingTargetReady" // target node running; waiting for workflow to signal ready
|
|
336
|
+
| "awaitingSourceStop" // told source to cut over; waiting for stop-output signal
|
|
337
|
+
| "applying" // told target the cutover timestamp
|
|
338
|
+
| "done"
|
|
339
|
+
| "aborted";
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Phase E migration state, kept entirely in AutoManager. Per the
|
|
343
|
+
* design, "no concurrent migrations per Job" — `sourceJobId` is the
|
|
344
|
+
* primary index; `migrationId` is a stable opaque handle for ops
|
|
345
|
+
* tooling.
|
|
346
|
+
*
|
|
347
|
+
* @public
|
|
348
|
+
*/
|
|
349
|
+
export interface MigrationState {
|
|
350
|
+
migrationId: MigrationId;
|
|
351
|
+
sourceJobId: JobId;
|
|
352
|
+
sourceRole: Role;
|
|
353
|
+
targetNodeId: NodeId;
|
|
354
|
+
startedAt: Date;
|
|
355
|
+
phase: MigrationPhase;
|
|
356
|
+
phaseHistory: { phase: MigrationPhase; at: Date }[];
|
|
357
|
+
/** Set when phase becomes "aborted"; undefined while in flight or on done. */
|
|
358
|
+
abortReason?: string;
|
|
359
|
+
/** Timestamp the source reported as its last-output, captured during cutover. */
|
|
360
|
+
cutoverTimestampNs?: number;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Coarse connection state surfaced by `onConnectionStateChange`. Maps
|
|
365
|
+
* to gRPC's `ChannelConnectivityState` values via NorskManager's
|
|
366
|
+
* existing watcher.
|
|
367
|
+
*
|
|
368
|
+
* @public
|
|
369
|
+
*/
|
|
370
|
+
export type ConnectionState =
|
|
371
|
+
| "idle"
|
|
372
|
+
| "connecting"
|
|
373
|
+
| "ready"
|
|
374
|
+
| "transientFailure"
|
|
375
|
+
| "shutdown";
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* Phase C4 — per-band recovery policy. Bronze always drops on
|
|
379
|
+
* failure; gold and silver restart with their own restart-rate
|
|
380
|
+
* ceilings. Gold's `restartPlacement: "preferHotSpare"` is the
|
|
381
|
+
* default behaviour of the recovery path (placement engine biases
|
|
382
|
+
* scoring toward spare-tagged nodes when the recovering job is gold).
|
|
383
|
+
*
|
|
384
|
+
* Omitted bands fall through to the flat `AutoSettings` fields, then
|
|
385
|
+
* to compiled-in defaults.
|
|
386
|
+
*
|
|
387
|
+
* @public
|
|
388
|
+
*/
|
|
389
|
+
export interface BandConfig {
|
|
390
|
+
gold?: {
|
|
391
|
+
onFailure: "restart";
|
|
392
|
+
restartPlacement: "preferHotSpare" | "any";
|
|
393
|
+
/** Default: 6 */
|
|
394
|
+
maxRestartsPerHour?: number;
|
|
395
|
+
};
|
|
396
|
+
silver?: {
|
|
397
|
+
onFailure: "restart";
|
|
398
|
+
restartPlacement: "any";
|
|
399
|
+
/** Default: 6 */
|
|
400
|
+
maxRestartsPerHour?: number;
|
|
401
|
+
};
|
|
402
|
+
bronze?: {
|
|
403
|
+
onFailure: "drop";
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Phase C4 — placement / recovery tunables. Same fields as the flat
|
|
409
|
+
* top-level settings; this typed grouping is the recommended path.
|
|
410
|
+
* Omitted fields fall through to the flat settings, then to
|
|
411
|
+
* compiled-in defaults.
|
|
412
|
+
*
|
|
413
|
+
* @public
|
|
414
|
+
*/
|
|
415
|
+
export interface PlacementConfig {
|
|
416
|
+
/** Default: 5_000 ms. */
|
|
417
|
+
rejectionBackoffMs?: number;
|
|
418
|
+
/** Default: 3. */
|
|
419
|
+
rejectionEscalationCount?: number;
|
|
420
|
+
/** Default: 30_000 ms. */
|
|
421
|
+
failureBackoffMs?: number;
|
|
422
|
+
/** Default: 120_000 ms. */
|
|
423
|
+
jobStartupGraceMs?: number;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Phase C6 — keep `targetCount` warm-running nodes ready per pool
|
|
428
|
+
* (and optional band). The reconciler provisions short, terminates
|
|
429
|
+
* excess. Cluster pools cannot auto-scale — `targetCount` there
|
|
430
|
+
* means "refuse to place onto a spare-tagged node unless this is a
|
|
431
|
+
* recovery placement"; reconciler emits `onSpareUnderflow` and leaves
|
|
432
|
+
* the rest to operator IT.
|
|
433
|
+
*
|
|
434
|
+
* @public
|
|
435
|
+
*/
|
|
436
|
+
export interface HotSpareConfig {
|
|
437
|
+
pool: string;
|
|
438
|
+
targetCount: number;
|
|
439
|
+
/** Optional: dedicate spares to a specific band. */
|
|
440
|
+
forBand?: PriorityBand;
|
|
441
|
+
spec: NodeSpec;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* Phase C6 — what kind of node the spare reconciler should
|
|
446
|
+
* provision. For elastic (aws/oci) pools, `instanceType` is
|
|
447
|
+
* required. For cluster pools, the spec is only used to constrain
|
|
448
|
+
* which existing cluster nodes are eligible to be tagged as spare
|
|
449
|
+
* (future).
|
|
450
|
+
*
|
|
451
|
+
* @public
|
|
452
|
+
*/
|
|
453
|
+
export interface NodeSpec {
|
|
454
|
+
/** AwsInstanceType | OciShape; required for elastic pools. */
|
|
455
|
+
instanceType?: string;
|
|
456
|
+
/** Cloud-specific region or AZ; pass through to provider. */
|
|
457
|
+
region?: string;
|
|
458
|
+
az?: string;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Phase E — per-phase deadlines for in-flight migrations. Set any
|
|
463
|
+
* field to override its default; omitted fields use the compiled-in
|
|
464
|
+
* default below.
|
|
465
|
+
*
|
|
466
|
+
* @public
|
|
467
|
+
*/
|
|
468
|
+
export interface MigrationConfig {
|
|
469
|
+
/** Default 120_000 ms — combined "launch + ready" window for the target. */
|
|
470
|
+
launchingDeadlineMs?: number;
|
|
471
|
+
/** Default 120_000 ms — workflow has this long after target running to signal ready. */
|
|
472
|
+
awaitingTargetReadyDeadlineMs?: number;
|
|
473
|
+
/** Default 30_000 ms — source workflow has this long to stop output after the cutover signal. */
|
|
474
|
+
awaitingSourceStopDeadlineMs?: number;
|
|
475
|
+
/** Default 10_000 ms — target workflow has this long to apply the handover. */
|
|
476
|
+
applyingDeadlineMs?: number;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/**
|
|
480
|
+
* Phase C4 — result of `validateBundleAdmission`. `ok` means the
|
|
481
|
+
* bundle passes static validation AND every (replica, jobName)
|
|
482
|
+
* placeDryRun would succeed at the current moment. The two failure
|
|
483
|
+
* modes are distinct: `validationErrors` is "the bundle spec itself
|
|
484
|
+
* is broken" (operator must fix); `noCapacity` is "current cluster
|
|
485
|
+
* state can't host this" (operator must add capacity or wait).
|
|
486
|
+
*
|
|
487
|
+
* @public
|
|
488
|
+
*/
|
|
489
|
+
export type AdmissionResult =
|
|
490
|
+
| { kind: "ok" }
|
|
491
|
+
| { kind: "validationErrors"; errors: ValidationError[] }
|
|
492
|
+
| { kind: "noCapacity"; unplaceable: AdmissionUnplaceable[] };
|
|
493
|
+
|
|
494
|
+
/** @public */
|
|
495
|
+
export interface AdmissionUnplaceable {
|
|
496
|
+
replicaIndex: number;
|
|
497
|
+
jobName: string;
|
|
498
|
+
reason: PlacementFailureReason;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/**
|
|
502
|
+
* Phase D4 — operational metrics snapshot. Cumulative counters since
|
|
503
|
+
* `startedAt`; per-pool gauges are point-in-time at call. Consumers
|
|
504
|
+
* (telemetry pipelines, ops UIs) poll `AutoManager.metrics()` on
|
|
505
|
+
* their own cadence.
|
|
506
|
+
*
|
|
507
|
+
* @public
|
|
508
|
+
*/
|
|
509
|
+
export interface AutoManagerMetrics {
|
|
510
|
+
/** AutoManager.run() entry time; undefined before runInternal starts. */
|
|
511
|
+
startedAt?: Date;
|
|
512
|
+
/** Milliseconds since startedAt, 0 if not yet started. */
|
|
513
|
+
uptimeMs: number;
|
|
514
|
+
// Cumulative counters (since startedAt):
|
|
515
|
+
placementsAttempted: number;
|
|
516
|
+
/**
|
|
517
|
+
* Placement that successfully dispatched its wire call (startJob /
|
|
518
|
+
* createAwsNode / createOciNode). Counted post-RPC, so a successful
|
|
519
|
+
* decision whose dispatch then fails does not bump this counter —
|
|
520
|
+
* see `placementsDispatchFailed`.
|
|
521
|
+
*/
|
|
522
|
+
placementsSucceeded: number;
|
|
523
|
+
/** Placement engine returned `failure` (no suitable target). */
|
|
524
|
+
placementsFailed: number;
|
|
525
|
+
/**
|
|
526
|
+
* Decision succeeded but the downstream RPC (startJob /
|
|
527
|
+
* createAwsNode / createOciNode) rejected. The job is left
|
|
528
|
+
* unplaced; AutoManager surfaces the error via onError.
|
|
529
|
+
*/
|
|
530
|
+
placementsDispatchFailed: number;
|
|
531
|
+
jobsRejected: number;
|
|
532
|
+
bundlesBroken: number;
|
|
533
|
+
/** Bronze-band drops. */
|
|
534
|
+
jobsDropped: number;
|
|
535
|
+
// Point-in-time gauges:
|
|
536
|
+
pools: PoolMetrics[];
|
|
537
|
+
/** Bundles currently marked broken by the restart-rate ceiling. */
|
|
538
|
+
brokenBundleIds: BundleId[];
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/** @public */
|
|
542
|
+
export interface PoolMetrics {
|
|
543
|
+
poolName: string;
|
|
544
|
+
nodeCount: number;
|
|
545
|
+
/** Spare-tagged, running, no jobs assigned. */
|
|
546
|
+
liveSpares: number;
|
|
547
|
+
/** Sum of `targetCount` across hotSpares entries for this pool. */
|
|
548
|
+
spareTarget: number;
|
|
549
|
+
/**
|
|
550
|
+
* Mean reservedCapacity / totalCapacity across nodes with inventory.
|
|
551
|
+
* 0 if there are no nodes with inventory.
|
|
552
|
+
*/
|
|
553
|
+
usedCapacityFraction: number;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/**
|
|
557
|
+
* Payload for `onPlacementDecision`. The `bundleId`/`replicaIndex`/
|
|
558
|
+
* `jobName` triple identifies which bundle member the decision was made
|
|
559
|
+
* for; `result` is the placement engine's verdict; `trace` is the
|
|
560
|
+
* per-pool filter/score breakdown described in §6 of the design.
|
|
561
|
+
*
|
|
562
|
+
* @public
|
|
563
|
+
*/
|
|
564
|
+
export interface PlacementDecisionInfo {
|
|
565
|
+
jobId: JobId;
|
|
566
|
+
bundleId: BundleId;
|
|
567
|
+
replicaIndex: number;
|
|
568
|
+
jobName: string;
|
|
569
|
+
result: PlacementResult;
|
|
570
|
+
trace: PlacementTrace;
|
|
571
|
+
at: Date;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Payload for `onPlacementFailed`. `triedPools` is the ordered list of
|
|
576
|
+
* tier names AutoManager attempted within the bundle's `pool` before
|
|
577
|
+
* giving up.
|
|
578
|
+
*
|
|
579
|
+
* @public
|
|
580
|
+
*/
|
|
581
|
+
export interface PlacementFailedInfo {
|
|
582
|
+
jobId: JobId;
|
|
583
|
+
bundleId: BundleId;
|
|
584
|
+
replicaIndex: number;
|
|
585
|
+
jobName: string;
|
|
586
|
+
reason: PlacementFailureReason;
|
|
587
|
+
triedPools: string[];
|
|
588
|
+
at: Date;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
/**
|
|
592
|
+
* Payload for `onResilienceDegraded`. Fired when a bundle's backup
|
|
593
|
+
* (replicaIndex 1) is placed inside a failure domain its resilience policy
|
|
594
|
+
* asked to avoid, because no compliant node had capacity. `violated` lists
|
|
595
|
+
* the anti-affinity flags that couldn't be honoured. The placement still
|
|
596
|
+
* happened — a degraded backup beats no backup — but DR posture is reduced.
|
|
597
|
+
*
|
|
598
|
+
* @public
|
|
599
|
+
*/
|
|
600
|
+
export interface ResilienceDegradedInfo {
|
|
601
|
+
jobId: JobId;
|
|
602
|
+
bundleId: BundleId;
|
|
603
|
+
replicaIndex: number;
|
|
604
|
+
jobName: string;
|
|
605
|
+
violated: ResilienceFlag[];
|
|
606
|
+
/** Set when the primary is on interruptible (spot) capacity with no
|
|
607
|
+
* durable backup — see ResilienceDegradation.interruptiblePrimary. */
|
|
608
|
+
interruptiblePrimary?: boolean;
|
|
609
|
+
at: Date;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/**
|
|
613
|
+
* Payload for `onJobRejected`. `inventory` is the typed snapshot the
|
|
614
|
+
* worker sent alongside the rejection so consumers can render "why" in
|
|
615
|
+
* the same UI tick as the rejection itself.
|
|
616
|
+
*
|
|
617
|
+
* @public
|
|
618
|
+
*/
|
|
619
|
+
export interface JobRejectedInfo {
|
|
620
|
+
jobId: JobId;
|
|
621
|
+
nodeId: NodeId;
|
|
622
|
+
reason: ManagerPB.JobRejected["reason"];
|
|
623
|
+
inventory?: TypedNodeInventory;
|
|
624
|
+
at: Date;
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
const DEFAULT_REJECTION_BACKOFF_MS = 5_000;
|
|
628
|
+
const DEFAULT_REJECTION_ESCALATION_COUNT = 3;
|
|
629
|
+
const DEFAULT_FAILURE_BACKOFF_MS = 30_000;
|
|
630
|
+
const DEFAULT_MAX_RESTARTS_PER_HOUR = 6;
|
|
631
|
+
const RESTART_WINDOW_MS = 60 * 60 * 1000;
|
|
632
|
+
// Minimum spacing between consecutive recovery placement attempts for
|
|
633
|
+
// the same JobId. Stops the "provisionJob → JobFailed → provisionJob"
|
|
634
|
+
// tight loop that JobStore state-transition events naturally drive
|
|
635
|
+
// when worker-side launches keep failing fast. 5s is short enough that
|
|
636
|
+
// transient failures still recover quickly but long enough that even a
|
|
637
|
+
// pathological failure pattern doesn't burn through `maxRestartsPerHour`
|
|
638
|
+
// in seconds. Operator-tunable later via AutoSettings.
|
|
639
|
+
const MIN_RECOVERY_INTERVAL_MS = 5_000;
|
|
640
|
+
// Minimum spacing between any `placeAndProvision` entries for the same
|
|
641
|
+
// JobId. Tighter than MIN_RECOVERY_INTERVAL_MS — the window we need is
|
|
642
|
+
// just "time for daemon's provisionJob to land + JobStarted to round-
|
|
643
|
+
// trip back into AutoManager's nodeMap". A second is comfortably more
|
|
644
|
+
// than typical local-host dispatch latency.
|
|
645
|
+
const MIN_PLACEMENT_INTERVAL_MS = 1_000;
|
|
646
|
+
const DEFAULT_JOB_STARTUP_GRACE_MS = 120_000;
|
|
647
|
+
const DEFAULT_HOT_SPARE_RECONCILE_INTERVAL_MS = 60_000;
|
|
648
|
+
const DEFAULT_NODE_SERVICE_RECONCILE_INTERVAL_MS = 60_000;
|
|
649
|
+
// Phase E phase-deadline defaults. Tuned to the design's expectation
|
|
650
|
+
// that target launch + workflow init typically completes well under
|
|
651
|
+
// two minutes, source-stop is workflow-driven and quick, and apply
|
|
652
|
+
// is a handful of wire hops.
|
|
653
|
+
const DEFAULT_MIGRATION_LAUNCHING_DEADLINE_MS = 120_000;
|
|
654
|
+
const DEFAULT_MIGRATION_TARGET_READY_DEADLINE_MS = 120_000;
|
|
655
|
+
const DEFAULT_MIGRATION_SOURCE_STOP_DEADLINE_MS = 30_000;
|
|
656
|
+
const DEFAULT_MIGRATION_APPLYING_DEADLINE_MS = 10_000;
|
|
657
|
+
|
|
658
|
+
export class AutoManager {
|
|
659
|
+
norsk: NorskManager;
|
|
660
|
+
settings: AutoSettings;
|
|
661
|
+
clock: Clock;
|
|
662
|
+
// All AutoManager state is a mirror of what the event stream most
|
|
663
|
+
// recently said — AutoManager owns no durable state. Maps are cleared
|
|
664
|
+
// and re-hydrated on (re)connect from the event-stream initial state.
|
|
665
|
+
jobMap: Map<JobId, JobWithHistory> = new Map();
|
|
666
|
+
nodeMap: Map<NodeId, NodeSummary> = new Map();
|
|
667
|
+
bundleMap: Map<BundleId, TypedBundle> = new Map();
|
|
668
|
+
inventoryMap: Map<NodeId, TypedNodeInventory> = new Map();
|
|
669
|
+
// Per-node rejection history (timestamps). Pruned to the configured
|
|
670
|
+
// backoff window on read; entries trigger escalation to `unhealthyNodes`
|
|
671
|
+
// once the threshold is crossed.
|
|
672
|
+
private rejectionsPerNode: Map<NodeId, Date[]> = new Map();
|
|
673
|
+
// Nodes excluded from placement entirely until the next clean
|
|
674
|
+
// `nodeInventoryUpdated` clears the entry.
|
|
675
|
+
private unhealthyNodes: Set<NodeId> = new Set();
|
|
676
|
+
// Recently-failed nodes — excluded from placement for `failureBackoffMs`
|
|
677
|
+
// after we observe their `nodeStopping`. Map value is the deadline.
|
|
678
|
+
private recentFailures: Map<NodeId, Date> = new Map();
|
|
679
|
+
// Phase C2: per-replica restart history. Each entry is the time a
|
|
680
|
+
// recovery placement was attempted; older-than-1h entries are
|
|
681
|
+
// pruned on read. Used to trip the maxRestartsPerHour ceiling.
|
|
682
|
+
private restartHistory: Map<JobId, Date[]> = new Map();
|
|
683
|
+
// Canonical jobId → nodeId index. Set at placement time (either
|
|
684
|
+
// when AutoManager binds the job to an existing node via startJob,
|
|
685
|
+
// or when it provisions a new node and records the provisional
|
|
686
|
+
// marker); deleted on jobDeleted. Underpins `nodeForJob` as an
|
|
687
|
+
// O(1) lookup so consumers (proxy routers, operator UIs) don't pay
|
|
688
|
+
// a fleet-wide scan on every read — important when a single host
|
|
689
|
+
// can carry many jobs (cluster, or any future cloud N:1) and the
|
|
690
|
+
// `tags["jobId"]` shortcut on the node stops being expressive.
|
|
691
|
+
// Rebuilt on R7 reconnect from each job's history via
|
|
692
|
+
// `currentNodeIdForJob`. Entries for jobs whose node has since
|
|
693
|
+
// terminated remain in the map but resolve to undefined through
|
|
694
|
+
// nodeMap — harmless, and cleaned up the next time the job is
|
|
695
|
+
// re-placed or deleted.
|
|
696
|
+
private jobToNode: Map<JobId, NodeId> = new Map();
|
|
697
|
+
// Per-job placement record. Set at the same callsite that fires
|
|
698
|
+
// `onPlacementDecision` (live decisions during this AutoManager
|
|
699
|
+
// generation); rebuilt from job history on R7 reconnect — for
|
|
700
|
+
// hydrated jobs we have no live trace, so we synthesise one with
|
|
701
|
+
// empty `trace.attempts` and the decision inferred from the
|
|
702
|
+
// history event ("place" = re-binding to an existing node, "provision"
|
|
703
|
+
// = node started for the job). Consumers query this via
|
|
704
|
+
// `placements()` / `placementForJob()` so an operator UI shows the
|
|
705
|
+
// current placement immediately on (re)connect rather than waiting
|
|
706
|
+
// for the next live decision — same mirror-of-state pattern jobMap
|
|
707
|
+
// and nodeMap use. Cleared on jobDeleted; entries are kept across
|
|
708
|
+
// node restarts since the placement-of-record doesn't change just
|
|
709
|
+
// because a node bounced.
|
|
710
|
+
private placementsByJob: Map<JobId, PlacementDecisionInfo> = new Map();
|
|
711
|
+
// Pending placement timers — one per scheduled job. Set when
|
|
712
|
+
// `handleJobChanged` sees a pre/active job whose `startDateTime`
|
|
713
|
+
// (minus the pool's `placementLeadMs`) is still in the future;
|
|
714
|
+
// fires placeAndProvision when the lead-time window opens.
|
|
715
|
+
// Cancelled on jobDeleted, on R7 reconnect (re-evaluated from
|
|
716
|
+
// hydrated state), and replaced on subsequent job-updates that
|
|
717
|
+
// change the schedule. Tracks `scheduledFor` so `pendingPlacements()`
|
|
718
|
+
// can render upcoming work in operator UIs.
|
|
719
|
+
private pendingPlacementTimers: Map<JobId, { timer: CancellableTimer; scheduledFor: Date }> = new Map();
|
|
720
|
+
// Pending stop timers — one per bundleId. Set when any job in the
|
|
721
|
+
// bundle carries the `__stopDateTime` tag and the instant is still
|
|
722
|
+
// in the future; on fire, calls `deleteBundle(bundleId)` and
|
|
723
|
+
// removes the entry. Daemon has no first-class concept of stop
|
|
724
|
+
// time so this state is rebuilt from job tags on R7 reconnect.
|
|
725
|
+
// Public surface: `setBundleStopTime(bundleId, newStopTime?)` for
|
|
726
|
+
// the future Edit-bundle UI; `bundleStopTimes()` for read.
|
|
727
|
+
private pendingStopTimers: Map<BundleId, { timer: CancellableTimer; stopAt: Date }> = new Map();
|
|
728
|
+
// Outstanding deferred recovery placements. Each entry caps to one
|
|
729
|
+
// setTimeout-driven retry per JobId; subsequent triggers while a
|
|
730
|
+
// retry is in flight are no-ops (the scheduled fire will see the
|
|
731
|
+
// latest state). Prevents the tight-loop "provisionJob → JobFailed →
|
|
732
|
+
// provisionJob" pattern we observed when worker-side launch errors
|
|
733
|
+
// were instantly retryable. See 04-e2e-todo's B-daemon-retry-storm.
|
|
734
|
+
private pendingPlacementRetries: Map<JobId, ReturnType<typeof setTimeout>> = new Map();
|
|
735
|
+
// Timestamp of the most recent `placeAndProvision` entry per JobId.
|
|
736
|
+
// Used by both initial and recovery branches to skip duplicate
|
|
737
|
+
// attempts while a placement is still in flight (the daemon has
|
|
738
|
+
// dispatched provisionJob but JobStarted hasn't landed yet, so
|
|
739
|
+
// `nodesByJob().size` is still 0 and handleJobChanged would
|
|
740
|
+
// otherwise re-fire). See 04-e2e-todo's
|
|
741
|
+
// B-placement-overretry-dispatch-fail.
|
|
742
|
+
private lastPlacementAttemptAt: Map<JobId, Date> = new Map();
|
|
743
|
+
// Phase C2: bundles that have tripped the restart ceiling. While in
|
|
744
|
+
// this set, AutoManager refuses recovery placements for any of the
|
|
745
|
+
// bundle's replicas. Cleared on bundleUpdated (operator pushed a
|
|
746
|
+
// fresh spec) or bundleDeleted.
|
|
747
|
+
private brokenBundles: Set<BundleId> = new Set();
|
|
748
|
+
// Phase C3: pending nodeStarting → nodeStarted grace timers. If a
|
|
749
|
+
// timer fires before the node reports running, AutoManager
|
|
750
|
+
// terminates the node and the existing nodeStopping path runs
|
|
751
|
+
// recovery per band rules.
|
|
752
|
+
private startupGraceTimers: Map<NodeId, CancellableTimer> = new Map();
|
|
753
|
+
// Phase C6: spares we have asked the provider to create but which
|
|
754
|
+
// haven't yet appeared in nodeMap with the spare tag. Counted
|
|
755
|
+
// alongside live spares so the reconciler doesn't fire duplicate
|
|
756
|
+
// provisions while a previous one is in flight.
|
|
757
|
+
private pendingSpareProvisions: Map<
|
|
758
|
+
NodeId,
|
|
759
|
+
{ pool: string; band?: PriorityBand }
|
|
760
|
+
> = new Map();
|
|
761
|
+
private hotSpareReconcileInterval?: CancellableTimer;
|
|
762
|
+
// Phase C8: nodes the operator has cordoned client-side. Merged
|
|
763
|
+
// into NodeView.inventory.cordoned at placement time so existing
|
|
764
|
+
// placement filter excludes them. (When Manager grows a cordon
|
|
765
|
+
// API and emits `cordoned: true` on inventory updates, this local
|
|
766
|
+
// set becomes redundant — until then it's the API.)
|
|
767
|
+
private cordonedNodes: Set<NodeId> = new Set();
|
|
768
|
+
// Phase D4 — cumulative metrics counters since startedAt. Read via
|
|
769
|
+
// `metrics()`. Counters never reset; consumers compute deltas.
|
|
770
|
+
private metricsCounters = {
|
|
771
|
+
placementsAttempted: 0,
|
|
772
|
+
placementsSucceeded: 0,
|
|
773
|
+
placementsFailed: 0,
|
|
774
|
+
placementsDispatchFailed: 0,
|
|
775
|
+
jobsRejected: 0,
|
|
776
|
+
bundlesBroken: 0,
|
|
777
|
+
jobsDropped: 0,
|
|
778
|
+
};
|
|
779
|
+
// Phase E — active migrations. "No concurrent migrations per Job"
|
|
780
|
+
// means migrationByJobId is a 1-to-1 index of migrations.
|
|
781
|
+
private migrationStates: Map<MigrationId, MigrationState> = new Map();
|
|
782
|
+
private migrationByJobId: Map<JobId, MigrationId> = new Map();
|
|
783
|
+
// Phase E deadlines — a per-migration timer fires if the current
|
|
784
|
+
// phase hasn't transitioned by the configured window. Re-armed
|
|
785
|
+
// on every phase transition; cleared on done/aborted/close().
|
|
786
|
+
private migrationPhaseTimers: Map<MigrationId, CancellableTimer> = new Map();
|
|
787
|
+
// Phase F — NodeService runtime tracking. Per service, the set of
|
|
788
|
+
// (nodeId → runtime jobId) AutoManager has placed it on. The
|
|
789
|
+
// reconciler keeps this in sync with the configured eligible set.
|
|
790
|
+
private nodeServicePlacements: Map<ServiceId, Map<NodeId, JobId>> = new Map();
|
|
791
|
+
private nodeServiceReconcileInterval?: CancellableTimer;
|
|
792
|
+
/**
|
|
793
|
+
* The moment `runInternal()` first entered (i.e. the eventStream was
|
|
794
|
+
* established). Undefined until then. Powers "AutoManager up for N
|
|
795
|
+
* minutes" affordances.
|
|
796
|
+
* @public
|
|
797
|
+
*/
|
|
798
|
+
public startedAt?: Date;
|
|
799
|
+
private cleanupInterval?: CancellableTimer;
|
|
800
|
+
private closed: boolean = false;
|
|
801
|
+
|
|
802
|
+
/** @public */
|
|
803
|
+
public static async run(
|
|
804
|
+
settings: AutoSettings
|
|
805
|
+
): Promise<[AutoManager, Promise<void>]> {
|
|
806
|
+
|
|
807
|
+
// Phase D1 — fail fast on configuration mistakes.
|
|
808
|
+
const validationErrors = validateAutoSettings(settings);
|
|
809
|
+
if (validationErrors.length > 0) {
|
|
810
|
+
throw new AutoSettingsValidationError(validationErrors);
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
debuglog("Trying to connect", { url: settings.url });
|
|
814
|
+
|
|
815
|
+
const fire = (state: ConnectionState) =>
|
|
816
|
+
settings.onConnectionStateChange?.(state);
|
|
817
|
+
|
|
818
|
+
try {
|
|
819
|
+
const norsk: NorskManager = await NorskManager.connect({
|
|
820
|
+
url: settings.url,
|
|
821
|
+
signal: settings.signal,
|
|
822
|
+
onAttemptingToConnect: () => fire("idle"),
|
|
823
|
+
onConnecting: () => fire("connecting"),
|
|
824
|
+
onReady: () => fire("ready"),
|
|
825
|
+
onFailedToConnect: () => fire("transientFailure"),
|
|
826
|
+
onShutdown: () => fire("shutdown"),
|
|
827
|
+
});
|
|
828
|
+
|
|
829
|
+
const autoManager = new AutoManager(norsk, settings);
|
|
830
|
+
const promise = autoManager.runInternal();
|
|
831
|
+
|
|
832
|
+
return [autoManager, promise];
|
|
833
|
+
} catch (err) {
|
|
834
|
+
debuglog("Exception in connection");
|
|
835
|
+
throw err;
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/** @public */
|
|
840
|
+
public async terminateNode(nodeId: NodeId): Promise<void> {
|
|
841
|
+
await this.norsk.terminateNode(nodeId);
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
/**
|
|
845
|
+
* Phase C8 — cordon a node. AutoManager projects this onto every
|
|
846
|
+
* placement decision: cordoned nodes are filtered out, so no new
|
|
847
|
+
* placements land on them. Existing jobs continue running.
|
|
848
|
+
* Idempotent.
|
|
849
|
+
* @public
|
|
850
|
+
*/
|
|
851
|
+
public cordonNode(nodeId: NodeId): void {
|
|
852
|
+
if (this.cordonedNodes.has(nodeId)) return;
|
|
853
|
+
this.cordonedNodes.add(nodeId);
|
|
854
|
+
debuglog("Node cordoned", { nodeId });
|
|
855
|
+
// Reconcile in case a cordoned node was holding a spare slot —
|
|
856
|
+
// its un-spare-ness now opens the underflow path. Same for any
|
|
857
|
+
// NodeServices placed on the now-cordoned node.
|
|
858
|
+
this.reconcileHotSpares();
|
|
859
|
+
this.reconcileNodeServices();
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
/**
|
|
863
|
+
* Phase C8 — un-cordon a node, restoring it to the placement pool.
|
|
864
|
+
* Idempotent.
|
|
865
|
+
* @public
|
|
866
|
+
*/
|
|
867
|
+
public uncordonNode(nodeId: NodeId): void {
|
|
868
|
+
if (!this.cordonedNodes.delete(nodeId)) return;
|
|
869
|
+
debuglog("Node uncordoned", { nodeId });
|
|
870
|
+
this.reconcileHotSpares();
|
|
871
|
+
this.reconcileNodeServices();
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/**
|
|
875
|
+
* Phase C8 — whether a node is currently cordoned by AutoManager.
|
|
876
|
+
* @public
|
|
877
|
+
*/
|
|
878
|
+
public isCordoned(nodeId: NodeId): boolean {
|
|
879
|
+
return this.cordonedNodes.has(nodeId);
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
/**
|
|
883
|
+
* Phase C8 — drain a node. Cordons it (so no new placements land
|
|
884
|
+
* here) and terminates the node — Manager emits `nodeStopping`
|
|
885
|
+
* for every job hosted there, and the existing band-aware recovery
|
|
886
|
+
* path re-places them onto other nodes (excluding this cordoned
|
|
887
|
+
* one). Bronze jobs are dropped; gold prefers hot spares.
|
|
888
|
+
*
|
|
889
|
+
* v1 takes the immediate-terminate path; the design's "graceful
|
|
890
|
+
* shuffle without recovery gap" needs Phase E job migration to
|
|
891
|
+
* land first.
|
|
892
|
+
* @public
|
|
893
|
+
*/
|
|
894
|
+
public async drainNode(nodeId: NodeId): Promise<void> {
|
|
895
|
+
this.cordonNode(nodeId);
|
|
896
|
+
await this.norsk.terminateNode(nodeId);
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
/**
|
|
900
|
+
* Shut down this AutoManager. Cancels periodic intervals and closes
|
|
901
|
+
* the underlying NorskManager — which causes the event-loop iterator
|
|
902
|
+
* to terminate, resolving the `Promise<void>` returned from `run()`.
|
|
903
|
+
* Idempotent.
|
|
904
|
+
* @public
|
|
905
|
+
*/
|
|
906
|
+
public close(): void {
|
|
907
|
+
if (this.closed) return;
|
|
908
|
+
this.closed = true;
|
|
909
|
+
this.cleanupInterval?.cancel();
|
|
910
|
+
this.cleanupInterval = undefined;
|
|
911
|
+
this.hotSpareReconcileInterval?.cancel();
|
|
912
|
+
this.hotSpareReconcileInterval = undefined;
|
|
913
|
+
this.nodeServiceReconcileInterval?.cancel();
|
|
914
|
+
this.nodeServiceReconcileInterval = undefined;
|
|
915
|
+
for (const t of this.startupGraceTimers.values()) t.cancel();
|
|
916
|
+
this.startupGraceTimers.clear();
|
|
917
|
+
for (const t of this.migrationPhaseTimers.values()) t.cancel();
|
|
918
|
+
this.migrationPhaseTimers.clear();
|
|
919
|
+
for (const t of this.pendingPlacementRetries.values()) clearTimeout(t);
|
|
920
|
+
this.pendingPlacementRetries.clear();
|
|
921
|
+
this.lastPlacementAttemptAt.clear();
|
|
922
|
+
this.norsk.close();
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
/**
|
|
926
|
+
* Alias for `close()`. Provided for symmetry with the
|
|
927
|
+
* `using`-style disposal conventions emerging in TypeScript.
|
|
928
|
+
* @public
|
|
929
|
+
*/
|
|
930
|
+
public dispose(): void {
|
|
931
|
+
this.close();
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* AutoManager v1 — submit a new bundle. The bundle is validated by
|
|
936
|
+
* Manager (per §4.3) and expanded into N replica jobs server-side;
|
|
937
|
+
* AutoManager mirrors the resulting state via the bundleUpdated +
|
|
938
|
+
* jobUpdated events. The returned bundleId matches the one Manager
|
|
939
|
+
* assigns (typically the same as bundle.bundleId, but Manager has
|
|
940
|
+
* the final word).
|
|
941
|
+
*
|
|
942
|
+
* @public
|
|
943
|
+
*/
|
|
944
|
+
public async createBundle(bundle: TypedBundle): Promise<BundleId> {
|
|
945
|
+
return this.norsk.createBundle(toPbBundle(bundle));
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
/**
|
|
949
|
+
* AutoManager v1 — update an existing bundle.
|
|
950
|
+
* @public
|
|
951
|
+
*/
|
|
952
|
+
public async updateBundle(bundle: TypedBundle): Promise<void> {
|
|
953
|
+
await this.norsk.updateBundle(toPbBundle(bundle));
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
/**
|
|
957
|
+
* AutoManager v1 — delete a bundle and all its replica jobs.
|
|
958
|
+
* @public
|
|
959
|
+
*/
|
|
960
|
+
public async deleteBundle(bundleId: BundleId): Promise<void> {
|
|
961
|
+
await this.norsk.deleteBundle(bundleId);
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
/**
|
|
965
|
+
* AutoManager v1 — activate a dormant (start_mode = "manual") bundle
|
|
966
|
+
* job. Manager removes the dormant tag and re-emits jobUpdated;
|
|
967
|
+
* AutoManager's placement path picks the job up via the standard
|
|
968
|
+
* flow. See §10.2 of the design.
|
|
969
|
+
* @public
|
|
970
|
+
*/
|
|
971
|
+
public async startBundleJob(
|
|
972
|
+
bundleId: BundleId,
|
|
973
|
+
replicaIndex: number,
|
|
974
|
+
jobName: string,
|
|
975
|
+
): Promise<void> {
|
|
976
|
+
await this.norsk.startBundleJob(bundleId, replicaIndex, jobName);
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
/**
|
|
980
|
+
* AutoManager v1 — deactivate a bundle job (tag-flip).
|
|
981
|
+
*
|
|
982
|
+
* Runtime termination of an in-flight workload and empty-node
|
|
983
|
+
* release with linger are pre-go-live work tracked as Phase E.5
|
|
984
|
+
* (§10.2.1). Until they land, calling stopBundleJob on an active
|
|
985
|
+
* job marks it dormant for future placement but does not stop the
|
|
986
|
+
* currently-running worker.
|
|
987
|
+
* @public
|
|
988
|
+
*/
|
|
989
|
+
public async stopBundleJob(
|
|
990
|
+
bundleId: BundleId,
|
|
991
|
+
replicaIndex: number,
|
|
992
|
+
jobName: string,
|
|
993
|
+
): Promise<void> {
|
|
994
|
+
await this.norsk.stopBundleJob(bundleId, replicaIndex, jobName);
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
/**
|
|
998
|
+
* AutoManager v1 — typed view of the jobs Manager has told us about.
|
|
999
|
+
* `JobWithHistory` carries the state-transition log for each job so
|
|
1000
|
+
* consumers can render lifecycle timelines without a separate
|
|
1001
|
+
* round-trip.
|
|
1002
|
+
* @public
|
|
1003
|
+
*/
|
|
1004
|
+
public jobs(): JobWithHistory[] {
|
|
1005
|
+
return Array.from(this.jobMap.values());
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
public nodes(): NodeSummary[] {
|
|
1009
|
+
debuglog("Getting nodes", this.nodeMap);
|
|
1010
|
+
return Array.from(this.nodeMap.values());
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/**
|
|
1014
|
+
* AutoManager v1 — typed view of the bundle definitions Manager has
|
|
1015
|
+
* told us about. Mirror only; AutoManager owns no durable state.
|
|
1016
|
+
* @public
|
|
1017
|
+
*/
|
|
1018
|
+
public bundles(): TypedBundle[] {
|
|
1019
|
+
return Array.from(this.bundleMap.values());
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
/**
|
|
1023
|
+
* Phase D4 — snapshot of AutoManager's operational metrics.
|
|
1024
|
+
* Cumulative counters since `startedAt`; per-pool gauges computed
|
|
1025
|
+
* at call time. Cheap to call repeatedly — consumers poll on their
|
|
1026
|
+
* own cadence and compute deltas if they want rates.
|
|
1027
|
+
* @public
|
|
1028
|
+
*/
|
|
1029
|
+
public metrics(): AutoManagerMetrics {
|
|
1030
|
+
const now = this.clock.now();
|
|
1031
|
+
const uptimeMs = this.startedAt
|
|
1032
|
+
? now.getTime() - this.startedAt.getTime()
|
|
1033
|
+
: 0;
|
|
1034
|
+
const poolNames = new Set<string>();
|
|
1035
|
+
for (const p of this.settings.placementPools ?? []) poolNames.add(p.name);
|
|
1036
|
+
// Include any pool name we've seen on a node, in case nodes are
|
|
1037
|
+
// tagged with a pool that isn't in placementPools (rare but worth
|
|
1038
|
+
// surfacing — operator's likely misconfigured).
|
|
1039
|
+
for (const node of this.nodeMap.values()) {
|
|
1040
|
+
const pn = node.nodeMetadata?.tags?.["pool"];
|
|
1041
|
+
if (pn) poolNames.add(pn);
|
|
1042
|
+
}
|
|
1043
|
+
const pools: PoolMetrics[] = [];
|
|
1044
|
+
for (const poolName of poolNames) {
|
|
1045
|
+
pools.push(this.computePoolMetrics(poolName));
|
|
1046
|
+
}
|
|
1047
|
+
return {
|
|
1048
|
+
startedAt: this.startedAt,
|
|
1049
|
+
uptimeMs,
|
|
1050
|
+
...this.metricsCounters,
|
|
1051
|
+
pools,
|
|
1052
|
+
brokenBundleIds: [...this.brokenBundles],
|
|
1053
|
+
};
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
private computePoolMetrics(poolName: string): PoolMetrics {
|
|
1057
|
+
let nodeCount = 0;
|
|
1058
|
+
let liveSpares = 0;
|
|
1059
|
+
let utilSum = 0;
|
|
1060
|
+
let utilN = 0;
|
|
1061
|
+
for (const node of this.nodeMap.values()) {
|
|
1062
|
+
if (this.poolNameForNode(node.nodeId) !== poolName) continue;
|
|
1063
|
+
nodeCount++;
|
|
1064
|
+
if (
|
|
1065
|
+
this.isHotSpareNode(node.nodeId) &&
|
|
1066
|
+
node.nodeState === "running" &&
|
|
1067
|
+
(this.inventoryMap.get(node.nodeId)?.assignedJobs.length ?? 0) === 0
|
|
1068
|
+
) {
|
|
1069
|
+
liveSpares++;
|
|
1070
|
+
}
|
|
1071
|
+
const inv = this.inventoryMap.get(node.nodeId);
|
|
1072
|
+
if (inv && inv.totalCapacity > 0) {
|
|
1073
|
+
utilSum += inv.reservedCapacity / inv.totalCapacity;
|
|
1074
|
+
utilN++;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
let spareTarget = 0;
|
|
1078
|
+
for (const cfg of this.settings.hotSpares ?? []) {
|
|
1079
|
+
if (cfg.pool === poolName) spareTarget += cfg.targetCount;
|
|
1080
|
+
}
|
|
1081
|
+
return {
|
|
1082
|
+
poolName,
|
|
1083
|
+
nodeCount,
|
|
1084
|
+
liveSpares,
|
|
1085
|
+
spareTarget,
|
|
1086
|
+
usedCapacityFraction: utilN > 0 ? utilSum / utilN : 0,
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
/**
|
|
1091
|
+
* Look up a single bundle by id. Convenience over `bundles().find(...)`.
|
|
1092
|
+
* @public
|
|
1093
|
+
*/
|
|
1094
|
+
public bundleById(bundleId: BundleId): TypedBundle | undefined {
|
|
1095
|
+
return this.bundleMap.get(bundleId);
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
/**
|
|
1099
|
+
* Every job (with history) that belongs to a given bundle. Derived
|
|
1100
|
+
* from `tags.bundleId` on the job entries in jobMap. Returns an empty
|
|
1101
|
+
* array if the bundle is unknown or has no jobs.
|
|
1102
|
+
* @public
|
|
1103
|
+
*/
|
|
1104
|
+
public jobsForBundle(bundleId: BundleId): JobWithHistory[] {
|
|
1105
|
+
const out: JobWithHistory[] = [];
|
|
1106
|
+
for (const entry of this.jobMap.values()) {
|
|
1107
|
+
if ((entry.job.tags ?? {})["bundleId"] === bundleId) {
|
|
1108
|
+
out.push(entry);
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
return out;
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
/**
|
|
1115
|
+
* The node the given job is currently placed on, or undefined if
|
|
1116
|
+
* the job hasn't been placed (or has been deleted). Useful for
|
|
1117
|
+
* operator UIs that render "job X is on node Y" and for proxy
|
|
1118
|
+
* routers that need to know which worker hosts each job.
|
|
1119
|
+
*
|
|
1120
|
+
* O(1) lookup against the `jobToNode` index — set at placement
|
|
1121
|
+
* time, deleted on jobDeleted, rebuilt from job history on R7
|
|
1122
|
+
* reconnect. Works regardless of whether the node hosts one job
|
|
1123
|
+
* (cloud, today) or many (cluster, or any future cloud N:1)
|
|
1124
|
+
* because the assignment is recorded against the JOB rather than
|
|
1125
|
+
* scanned out of NODE tags.
|
|
1126
|
+
* @public
|
|
1127
|
+
*/
|
|
1128
|
+
public nodeForJob(jobId: JobId): NodeSummary | undefined {
|
|
1129
|
+
const nodeId = this.jobToNode.get(jobId);
|
|
1130
|
+
return nodeId !== undefined ? this.nodeMap.get(nodeId) : undefined;
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
/**
|
|
1134
|
+
* Current placement of every job AutoManager has bound to a node,
|
|
1135
|
+
* newest-first by decision timestamp. Mirrors `nodeForJob` but
|
|
1136
|
+
* returns the full `PlacementDecisionInfo` (so consumers get the
|
|
1137
|
+
* pool, trace, replicaIndex etc., not just the node) — and exposes
|
|
1138
|
+
* it as queryable live state rather than only via the
|
|
1139
|
+
* `onPlacementDecision` event stream.
|
|
1140
|
+
*
|
|
1141
|
+
* Hydrated jobs (placed in a previous AutoManager generation, picked
|
|
1142
|
+
* up on R7 reconnect) get an entry with an empty `trace.attempts`
|
|
1143
|
+
* and a decision inferred from job history. Consumers that *need*
|
|
1144
|
+
* a real trace should subscribe to `onPlacementDecision` for live
|
|
1145
|
+
* decisions; this getter is for "where is everything right now?"
|
|
1146
|
+
* operator views.
|
|
1147
|
+
*
|
|
1148
|
+
* @public
|
|
1149
|
+
*/
|
|
1150
|
+
public placements(): PlacementDecisionInfo[] {
|
|
1151
|
+
return [...this.placementsByJob.values()].sort((a, b) => b.at.getTime() - a.at.getTime());
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
/**
|
|
1155
|
+
* O(1) lookup for the most recent placement decision binding `jobId`
|
|
1156
|
+
* to a node. Returns undefined for jobs that have never been placed
|
|
1157
|
+
* (or whose placement attempts have only failed). @public
|
|
1158
|
+
*/
|
|
1159
|
+
public placementForJob(jobId: JobId): PlacementDecisionInfo | undefined {
|
|
1160
|
+
return this.placementsByJob.get(jobId);
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
/**
|
|
1164
|
+
* Run the placement engine against AutoManager's current inventory
|
|
1165
|
+
* snapshot and configured pools, without actually placing or
|
|
1166
|
+
* provisioning anything. Powers preview affordances in operator
|
|
1167
|
+
* UIs ("if I submitted this bundle now, where would each replica
|
|
1168
|
+
* land?").
|
|
1169
|
+
*
|
|
1170
|
+
* `jobName` is optional; if omitted the *first* job in the bundle
|
|
1171
|
+
* is used. For multi-job bundles, supply the name to get a decision
|
|
1172
|
+
* specific to that job.
|
|
1173
|
+
*
|
|
1174
|
+
* Returns undefined if the bundle's named job is missing or the
|
|
1175
|
+
* replica index is out of range.
|
|
1176
|
+
* @public
|
|
1177
|
+
*/
|
|
1178
|
+
public placeDryRun(
|
|
1179
|
+
bundle: TypedBundle,
|
|
1180
|
+
replicaIndex: number,
|
|
1181
|
+
jobName?: string
|
|
1182
|
+
): { result: PlacementResult; trace: PlacementTrace } | undefined {
|
|
1183
|
+
if (replicaIndex < 0 || replicaIndex >= replicaCount(bundle)) return undefined;
|
|
1184
|
+
const job = jobName
|
|
1185
|
+
? bundle.jobs.find((j) => j.jobName === jobName)
|
|
1186
|
+
: bundle.jobs[0];
|
|
1187
|
+
if (!job) return undefined;
|
|
1188
|
+
const pools = new Map(
|
|
1189
|
+
(this.settings.placementPools ?? []).map((p) => [p.name, p])
|
|
1190
|
+
);
|
|
1191
|
+
return placeWithTrace({
|
|
1192
|
+
bundle,
|
|
1193
|
+
job,
|
|
1194
|
+
replicaIndex,
|
|
1195
|
+
inventory: this.inventoriesAsNodeViews(),
|
|
1196
|
+
pools,
|
|
1197
|
+
});
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
/**
|
|
1201
|
+
* Static validation of a bundle against AutoManager's currently
|
|
1202
|
+
* configured pools. Equivalent to calling the module-level
|
|
1203
|
+
* `validateBundle` with `PoolDescriptor[]` derived from
|
|
1204
|
+
* `AutoSettings.placementPools` — elastic pools project their
|
|
1205
|
+
* `candidateInstanceTypes` into node profiles; cluster pools project
|
|
1206
|
+
* their currently-known live inventory.
|
|
1207
|
+
* @public
|
|
1208
|
+
*/
|
|
1209
|
+
public validateBundle(bundle: TypedBundle): ValidationResult {
|
|
1210
|
+
return validateBundleFn(bundle, this.poolDescriptorsForValidation());
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
/**
|
|
1214
|
+
* Phase C4 — packaged admission check. Combines `validateBundle`
|
|
1215
|
+
* (static spec sanity) with `placeDryRun` for every (replica,
|
|
1216
|
+
* jobName) pair. Returns `ok` only when both pass for every
|
|
1217
|
+
* member; otherwise reports the structured failure. Use as a
|
|
1218
|
+
* pre-submit guard in operator-facing tooling.
|
|
1219
|
+
*
|
|
1220
|
+
* Per the design's §3.4 band table, all bands reject at admission
|
|
1221
|
+
* on "no capacity right now" — there's no band differentiation
|
|
1222
|
+
* here. (Gold/silver get *queued* internally on capacity-after-
|
|
1223
|
+
* submit failures via the rejection re-place path, but that's a
|
|
1224
|
+
* runtime concern, not admission.)
|
|
1225
|
+
*
|
|
1226
|
+
* @public
|
|
1227
|
+
*/
|
|
1228
|
+
public validateBundleAdmission(bundle: TypedBundle): AdmissionResult {
|
|
1229
|
+
const validation = this.validateBundle(bundle);
|
|
1230
|
+
if (validation.errors.length > 0) {
|
|
1231
|
+
return { kind: "validationErrors", errors: validation.errors };
|
|
1232
|
+
}
|
|
1233
|
+
const unplaceable: AdmissionUnplaceable[] = [];
|
|
1234
|
+
for (let replicaIndex = 0; replicaIndex < replicaCount(bundle); replicaIndex++) {
|
|
1235
|
+
for (const job of bundle.jobs) {
|
|
1236
|
+
const dry = this.placeDryRun(bundle, replicaIndex, job.jobName);
|
|
1237
|
+
if (!dry) {
|
|
1238
|
+
// placeDryRun returns undefined only for malformed input
|
|
1239
|
+
// (out-of-range replica, unknown jobName) — validateBundle
|
|
1240
|
+
// should have caught those already. Treat as unplaceable.
|
|
1241
|
+
unplaceable.push({
|
|
1242
|
+
replicaIndex,
|
|
1243
|
+
jobName: job.jobName,
|
|
1244
|
+
reason: { code: "noCapacityInAnyPool", triedPools: [] },
|
|
1245
|
+
});
|
|
1246
|
+
continue;
|
|
1247
|
+
}
|
|
1248
|
+
if (dry.result.kind === "failure") {
|
|
1249
|
+
unplaceable.push({
|
|
1250
|
+
replicaIndex,
|
|
1251
|
+
jobName: job.jobName,
|
|
1252
|
+
reason: dry.result.reason,
|
|
1253
|
+
});
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
if (unplaceable.length > 0) {
|
|
1258
|
+
return { kind: "noCapacity", unplaceable };
|
|
1259
|
+
}
|
|
1260
|
+
return { kind: "ok" };
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
private poolDescriptorsForValidation(): PoolDescriptor[] {
|
|
1264
|
+
const out: PoolDescriptor[] = [];
|
|
1265
|
+
for (const pool of this.settings.placementPools ?? []) {
|
|
1266
|
+
// Union the feasibility profiles across the pool's tiers: elastic
|
|
1267
|
+
// tiers contribute the shapes they can provision; fixed (cluster)
|
|
1268
|
+
// tiers contribute the live inventory projected for them.
|
|
1269
|
+
const profiles: NodeProfile[] = [];
|
|
1270
|
+
for (const tier of pool.tiers) {
|
|
1271
|
+
if (tier.scaleOut === "elastic") {
|
|
1272
|
+
for (const opt of tier.candidateInstanceTypes) {
|
|
1273
|
+
profiles.push({
|
|
1274
|
+
totalCapacity: opt.totalCapacity,
|
|
1275
|
+
totalCores: opt.totalCores,
|
|
1276
|
+
capabilities: opt.capabilities,
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1279
|
+
} else {
|
|
1280
|
+
for (const view of this.inventoriesAsNodeViews()) {
|
|
1281
|
+
if (!nodeMatchesTier(view, pool, tier)) continue;
|
|
1282
|
+
profiles.push({
|
|
1283
|
+
totalCapacity: view.inventory.totalCapacity,
|
|
1284
|
+
totalCores: view.inventory.totalCores,
|
|
1285
|
+
capabilities: view.inventory.capabilities,
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
// A pool that can provision is judged on what it can provision (a
|
|
1291
|
+
// miss is a hard error); a purely-fixed pool's misses are warnings —
|
|
1292
|
+
// a worker may join after submission (see validation.ts).
|
|
1293
|
+
const kind = pool.tiers.some((t) => t.scaleOut === "elastic")
|
|
1294
|
+
? "elastic"
|
|
1295
|
+
: "cluster";
|
|
1296
|
+
const clouds = [...new Set(pool.tiers.map((t) => t.kind))];
|
|
1297
|
+
out.push({ poolName: pool.name, kind, nodeProfiles: profiles, clouds });
|
|
1298
|
+
}
|
|
1299
|
+
return out;
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
/**
|
|
1303
|
+
* AutoManager v1 — typed inventory snapshot for a single node, or
|
|
1304
|
+
* undefined if the worker hasn't (yet) advertised one.
|
|
1305
|
+
* @public
|
|
1306
|
+
*/
|
|
1307
|
+
public inventory(nodeId: NodeId): TypedNodeInventory | undefined {
|
|
1308
|
+
return this.inventoryMap.get(nodeId);
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
/**
|
|
1312
|
+
* AutoManager v1 — typed inventory snapshots for every node we know
|
|
1313
|
+
* about. Iteration order is insertion order.
|
|
1314
|
+
* @public
|
|
1315
|
+
*/
|
|
1316
|
+
public inventories(): { nodeId: NodeId; inventory: TypedNodeInventory }[] {
|
|
1317
|
+
return Array.from(this.inventoryMap.entries()).map(([nodeId, inventory]) => ({
|
|
1318
|
+
nodeId,
|
|
1319
|
+
inventory,
|
|
1320
|
+
}));
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
// ---- AutoManager v1 event handlers ----
|
|
1324
|
+
private handleNodeInventoryUpdated(raw: ManagerPB.NodeInventoryUpdated): void {
|
|
1325
|
+
if (!raw.nodeId || !raw.inventory) return;
|
|
1326
|
+
const nodeId = raw.nodeId.nodeId;
|
|
1327
|
+
this.inventoryMap.set(nodeId, fromPbNodeInventory(raw.inventory));
|
|
1328
|
+
// A clean inventory update clears any "unhealthy" / recently-failed
|
|
1329
|
+
// mark — the worker has reported back successfully, treat as recovered.
|
|
1330
|
+
this.unhealthyNodes.delete(nodeId);
|
|
1331
|
+
this.rejectionsPerNode.delete(nodeId);
|
|
1332
|
+
this.recentFailures.delete(nodeId);
|
|
1333
|
+
// Phase C6: inventory change can flip a spare to "consumed" (jobs
|
|
1334
|
+
// assigned > 0). Reconcile so a replacement is provisioned
|
|
1335
|
+
// promptly rather than waiting for the periodic tick.
|
|
1336
|
+
this.reconcileHotSpares();
|
|
1337
|
+
// Phase F: inventory change might newly satisfy / no-longer satisfy
|
|
1338
|
+
// a NodeService's `consumes` — reconcile placements.
|
|
1339
|
+
this.reconcileNodeServices();
|
|
1340
|
+
// Fresh inventory means capacity may have appeared (a node's first
|
|
1341
|
+
// report after Hello — the startup race — a freed reservation, or an
|
|
1342
|
+
// un-cordon). Re-attempt any job that couldn't be placed before.
|
|
1343
|
+
this.reconcilePendingPlacements();
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
/**
|
|
1347
|
+
* A running job has had its config updated at runtime (operator action,
|
|
1348
|
+
* job-driven, or other side channel). Refresh our in-memory copy so a
|
|
1349
|
+
* subsequent `redeploy` (placeAndProvision triggered by node failure
|
|
1350
|
+
* etc.) hands the new config rather than the launch one.
|
|
1351
|
+
*
|
|
1352
|
+
* Manager is responsible for persisting the new value; AutoManager
|
|
1353
|
+
* just keeps its mirror up to date.
|
|
1354
|
+
*/
|
|
1355
|
+
private handleJobConfigUpdated(raw: ManagerPB.JobConfigUpdated): void {
|
|
1356
|
+
if (!raw.jobId) return;
|
|
1357
|
+
const jobId = raw.jobId.jobId;
|
|
1358
|
+
const existing = this.jobMap.get(jobId);
|
|
1359
|
+
if (!existing) return;
|
|
1360
|
+
this.jobMap.set(jobId, {
|
|
1361
|
+
...existing,
|
|
1362
|
+
job: { ...existing.job, managerConfiguration: raw.config },
|
|
1363
|
+
});
|
|
1364
|
+
this.settings.onJobConfigUpdated?.(jobId, raw.config);
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
private handleJobRejected(raw: ManagerPB.JobRejected): void {
|
|
1368
|
+
if (!raw.nodeId || !raw.jobId) return;
|
|
1369
|
+
const nodeId = raw.nodeId.nodeId;
|
|
1370
|
+
const jobId = raw.jobId.jobId;
|
|
1371
|
+
const typedInventory = raw.inventory
|
|
1372
|
+
? fromPbNodeInventory(raw.inventory)
|
|
1373
|
+
: undefined;
|
|
1374
|
+
if (typedInventory) {
|
|
1375
|
+
// Carries the worker's latest view, but does NOT clear unhealthy —
|
|
1376
|
+
// a rejection is the opposite of "I'm fine". Update the cache only.
|
|
1377
|
+
this.inventoryMap.set(nodeId, typedInventory);
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
this.metricsCounters.jobsRejected++;
|
|
1381
|
+
this.settings.onJobRejected?.({
|
|
1382
|
+
jobId,
|
|
1383
|
+
nodeId,
|
|
1384
|
+
reason: raw.reason,
|
|
1385
|
+
inventory: typedInventory,
|
|
1386
|
+
at: this.clock.now(),
|
|
1387
|
+
});
|
|
1388
|
+
|
|
1389
|
+
const escalated = this.recordRejection(nodeId);
|
|
1390
|
+
if (escalated) {
|
|
1391
|
+
debuglog("Node escalated to unhealthy", { nodeId });
|
|
1392
|
+
this.settings.onNodeUnhealthy?.(nodeId);
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
const existing = this.jobMap.get(jobId);
|
|
1396
|
+
const existingReq = existing?.job.autoPlacement?.requirements;
|
|
1397
|
+
if (existing && existingReq) {
|
|
1398
|
+
// Re-place the job, excluding the rejecting node (and any others
|
|
1399
|
+
// currently considered unfit).
|
|
1400
|
+
const exclude = this.currentExcludeSet();
|
|
1401
|
+
exclude.add(nodeId);
|
|
1402
|
+
void this.placeAndProvision(
|
|
1403
|
+
jobId,
|
|
1404
|
+
existingReq,
|
|
1405
|
+
existing.job.tags ?? {},
|
|
1406
|
+
{ excludeNodes: exclude }
|
|
1407
|
+
).catch((err) => this.settings.onError?.(err));
|
|
1408
|
+
}
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
private recordRejection(nodeId: NodeId): boolean {
|
|
1412
|
+
const now = this.clock.now();
|
|
1413
|
+
const window = this.cfgRejectionBackoffMs();
|
|
1414
|
+
const previous = this.rejectionsPerNode.get(nodeId) ?? [];
|
|
1415
|
+
const recent = previous.filter((d) => now.getTime() - d.getTime() < window);
|
|
1416
|
+
recent.push(now);
|
|
1417
|
+
this.rejectionsPerNode.set(nodeId, recent);
|
|
1418
|
+
const threshold = this.cfgRejectionEscalationCount();
|
|
1419
|
+
if (recent.length >= threshold && !this.unhealthyNodes.has(nodeId)) {
|
|
1420
|
+
this.unhealthyNodes.add(nodeId);
|
|
1421
|
+
return true;
|
|
1422
|
+
}
|
|
1423
|
+
return false;
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
private currentExcludeSet(): Set<NodeId> {
|
|
1427
|
+
const exclude = new Set(this.unhealthyNodes);
|
|
1428
|
+
const now = this.clock.now().getTime();
|
|
1429
|
+
for (const [nodeId, deadline] of this.recentFailures) {
|
|
1430
|
+
if (deadline.getTime() < now) {
|
|
1431
|
+
this.recentFailures.delete(nodeId);
|
|
1432
|
+
} else {
|
|
1433
|
+
exclude.add(nodeId);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
return exclude;
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
/**
|
|
1440
|
+
* Phase C2 — restart-rate check. Records the current attempt; if the
|
|
1441
|
+
* trailing-hour count crosses the band's `maxRestartsPerHour`, marks
|
|
1442
|
+
* the bundle broken, fires `onBundleBroken`, and returns true
|
|
1443
|
+
* (caller must short-circuit). Returns false when within budget.
|
|
1444
|
+
*/
|
|
1445
|
+
private shouldBreakOnRestart(
|
|
1446
|
+
jobId: JobId,
|
|
1447
|
+
bundleId: BundleId,
|
|
1448
|
+
band: PriorityBand
|
|
1449
|
+
): boolean {
|
|
1450
|
+
const now = this.clock.now();
|
|
1451
|
+
const previous = this.restartHistory.get(jobId) ?? [];
|
|
1452
|
+
const recent = previous.filter(
|
|
1453
|
+
(d) => now.getTime() - d.getTime() < RESTART_WINDOW_MS
|
|
1454
|
+
);
|
|
1455
|
+
recent.push(now);
|
|
1456
|
+
this.restartHistory.set(jobId, recent);
|
|
1457
|
+
const limit = this.cfgMaxRestartsPerHour(band);
|
|
1458
|
+
if (recent.length > limit) {
|
|
1459
|
+
if (!this.brokenBundles.has(bundleId)) {
|
|
1460
|
+
this.brokenBundles.add(bundleId);
|
|
1461
|
+
debuglog("Bundle marked broken — restart rate exceeded", {
|
|
1462
|
+
bundleId,
|
|
1463
|
+
jobId,
|
|
1464
|
+
attempts: recent.length,
|
|
1465
|
+
limit,
|
|
1466
|
+
});
|
|
1467
|
+
this.metricsCounters.bundlesBroken++;
|
|
1468
|
+
const reason = `Restart rate ${recent.length}/h exceeds ceiling ${limit}/h for replica ${jobId}`;
|
|
1469
|
+
this.settings.onBundleBroken?.(bundleId, reason);
|
|
1470
|
+
// Persist the verdict back to the daemon so the poison job is not
|
|
1471
|
+
// re-placed after a restart (the in-process `brokenBundles` flag is
|
|
1472
|
+
// rebuilt-from-scratch on reconnect). Fire-and-forget; the live flag
|
|
1473
|
+
// above already suppresses further placement this session.
|
|
1474
|
+
void this.norsk
|
|
1475
|
+
.markJobFailed(jobId, reason)
|
|
1476
|
+
.catch((err) => this.settings.onError?.(err));
|
|
1477
|
+
}
|
|
1478
|
+
return true;
|
|
1479
|
+
}
|
|
1480
|
+
return false;
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
/**
|
|
1484
|
+
* Whether the named bundle has tripped the restart ceiling and is
|
|
1485
|
+
* being held by AutoManager. Cleared by `bundleUpdated` (operator
|
|
1486
|
+
* pushed a fresh spec) or `bundleDeleted`.
|
|
1487
|
+
* @public
|
|
1488
|
+
*/
|
|
1489
|
+
public isBundleBroken(bundleId: BundleId): boolean {
|
|
1490
|
+
return this.brokenBundles.has(bundleId);
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
// ---- Phase E — migration state machine ---------------------------
|
|
1494
|
+
|
|
1495
|
+
/**
|
|
1496
|
+
* Initiate a migration of a running job to a target node. AutoManager
|
|
1497
|
+
* provisions a second instance of the same Job on `targetNodeId` with
|
|
1498
|
+
* role="migrationTarget" and drives the cutover handshake. Returns
|
|
1499
|
+
* a stable `migrationId` for tracking.
|
|
1500
|
+
*
|
|
1501
|
+
* Throws if a migration is already in flight for `sourceJobId` (no
|
|
1502
|
+
* concurrent migrations per Job).
|
|
1503
|
+
*
|
|
1504
|
+
* `sourceRole` defaults to whichever role the first node hosting
|
|
1505
|
+
* `sourceJobId` has — adequate for single-instance Jobs. Specify
|
|
1506
|
+
* explicitly for multi-replica bundles.
|
|
1507
|
+
*
|
|
1508
|
+
* @public
|
|
1509
|
+
*/
|
|
1510
|
+
public async migrateJob(
|
|
1511
|
+
sourceJobId: JobId,
|
|
1512
|
+
targetNodeId: NodeId,
|
|
1513
|
+
sourceRole?: Role
|
|
1514
|
+
): Promise<MigrationId> {
|
|
1515
|
+
if (this.migrationByJobId.has(sourceJobId)) {
|
|
1516
|
+
throw new Error(
|
|
1517
|
+
`migrateJob: a migration is already in flight for jobId ${sourceJobId}`
|
|
1518
|
+
);
|
|
1519
|
+
}
|
|
1520
|
+
const resolvedSourceRole =
|
|
1521
|
+
sourceRole ?? this.inferSourceRole(sourceJobId);
|
|
1522
|
+
if (!resolvedSourceRole) {
|
|
1523
|
+
throw new Error(
|
|
1524
|
+
`migrateJob: cannot locate a running instance for jobId ${sourceJobId} — pass sourceRole explicitly`
|
|
1525
|
+
);
|
|
1526
|
+
}
|
|
1527
|
+
const migrationId = uuid.v4();
|
|
1528
|
+
const now = this.clock.now();
|
|
1529
|
+
const state: MigrationState = {
|
|
1530
|
+
migrationId,
|
|
1531
|
+
sourceJobId,
|
|
1532
|
+
sourceRole: resolvedSourceRole,
|
|
1533
|
+
targetNodeId,
|
|
1534
|
+
startedAt: now,
|
|
1535
|
+
phase: "launching",
|
|
1536
|
+
phaseHistory: [{ phase: "launching", at: now }],
|
|
1537
|
+
};
|
|
1538
|
+
this.migrationStates.set(migrationId, state);
|
|
1539
|
+
this.migrationByJobId.set(sourceJobId, migrationId);
|
|
1540
|
+
this.settings.onMigrationPhase?.(state);
|
|
1541
|
+
this.armMigrationPhaseDeadline(state);
|
|
1542
|
+
debuglog("Migration started", { migrationId, sourceJobId, targetNodeId });
|
|
1543
|
+
try {
|
|
1544
|
+
await this.norsk.migrateJob(sourceJobId, targetNodeId, migrationId);
|
|
1545
|
+
} catch (err) {
|
|
1546
|
+
this.failMigration(migrationId, `Manager.migrateJob failed: ${String(err)}`);
|
|
1547
|
+
throw err;
|
|
1548
|
+
}
|
|
1549
|
+
return migrationId;
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
/**
|
|
1553
|
+
* Abort an in-flight migration. AutoManager sends MigrationAbort to
|
|
1554
|
+
* any live source/target instances and marks the migration aborted.
|
|
1555
|
+
* No-op (with warning log) for an unknown migration id.
|
|
1556
|
+
*
|
|
1557
|
+
* @public
|
|
1558
|
+
*/
|
|
1559
|
+
public async abortMigration(
|
|
1560
|
+
migrationId: MigrationId,
|
|
1561
|
+
reason: string
|
|
1562
|
+
): Promise<void> {
|
|
1563
|
+
const state = this.migrationStates.get(migrationId);
|
|
1564
|
+
if (!state) {
|
|
1565
|
+
debuglog("abortMigration: unknown migrationId", { migrationId });
|
|
1566
|
+
return;
|
|
1567
|
+
}
|
|
1568
|
+
if (state.phase === "done" || state.phase === "aborted") return;
|
|
1569
|
+
await this.dispatchMigrationAborts(state, reason).catch((err) =>
|
|
1570
|
+
this.settings.onError?.(err)
|
|
1571
|
+
);
|
|
1572
|
+
this.markMigrationAborted(state, reason);
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
/** All known migrations (active + done + aborted). @public */
|
|
1576
|
+
public migrations(): MigrationState[] {
|
|
1577
|
+
return Array.from(this.migrationStates.values());
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
/** Look up a single migration by id. @public */
|
|
1581
|
+
public migrationById(migrationId: MigrationId): MigrationState | undefined {
|
|
1582
|
+
return this.migrationStates.get(migrationId);
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
/** Migration active for the given source jobId, if any. @public */
|
|
1586
|
+
public migrationForJob(jobId: JobId): MigrationState | undefined {
|
|
1587
|
+
const id = this.migrationByJobId.get(jobId);
|
|
1588
|
+
return id ? this.migrationStates.get(id) : undefined;
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
/**
|
|
1592
|
+
* Find the role of the (first) node currently running this Job.
|
|
1593
|
+
* Returns undefined if no node hosts the job.
|
|
1594
|
+
*/
|
|
1595
|
+
private inferSourceRole(jobId: JobId): Role | undefined {
|
|
1596
|
+
for (const node of this.nodeMap.values()) {
|
|
1597
|
+
if (getJobIdForNode(node.nodeMetadata) === jobId) {
|
|
1598
|
+
const role =
|
|
1599
|
+
node.nodeMetadata?.tags?.["role"] ?? node.provisional?.role;
|
|
1600
|
+
if (role) return role;
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
return undefined;
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
private advanceMigration(state: MigrationState, phase: MigrationPhase): void {
|
|
1607
|
+
if (state.phase === phase) return;
|
|
1608
|
+
const now = this.clock.now();
|
|
1609
|
+
state.phase = phase;
|
|
1610
|
+
state.phaseHistory.push({ phase, at: now });
|
|
1611
|
+
this.settings.onMigrationPhase?.(state);
|
|
1612
|
+
debuglog("Migration phase", { migrationId: state.migrationId, phase });
|
|
1613
|
+
this.armMigrationPhaseDeadline(state);
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
/** Phase E deadlines — re-arm the per-phase timer for the migration. */
|
|
1617
|
+
private armMigrationPhaseDeadline(state: MigrationState): void {
|
|
1618
|
+
// Always cancel any prior timer; terminal phases don't re-arm.
|
|
1619
|
+
this.migrationPhaseTimers.get(state.migrationId)?.cancel();
|
|
1620
|
+
this.migrationPhaseTimers.delete(state.migrationId);
|
|
1621
|
+
if (state.phase === "done" || state.phase === "aborted") return;
|
|
1622
|
+
const deadlineMs = this.cfgMigrationPhaseDeadline(state.phase);
|
|
1623
|
+
if (deadlineMs === undefined) return;
|
|
1624
|
+
const id = state.migrationId;
|
|
1625
|
+
const phase = state.phase;
|
|
1626
|
+
const timer = this.clock.setTimeout(() => {
|
|
1627
|
+
this.migrationPhaseTimers.delete(id);
|
|
1628
|
+
const current = this.migrationStates.get(id);
|
|
1629
|
+
// Only fire if the migration is still in the phase we armed for.
|
|
1630
|
+
if (!current || current.phase !== phase) return;
|
|
1631
|
+
debuglog("Migration phase deadline fired — aborting", {
|
|
1632
|
+
migrationId: id,
|
|
1633
|
+
phase,
|
|
1634
|
+
deadlineMs,
|
|
1635
|
+
});
|
|
1636
|
+
const reason = `phase timeout: ${phase}`;
|
|
1637
|
+
void this.dispatchMigrationAborts(current, reason).catch((err) =>
|
|
1638
|
+
this.settings.onError?.(err)
|
|
1639
|
+
);
|
|
1640
|
+
this.markMigrationAborted(current, reason);
|
|
1641
|
+
}, deadlineMs);
|
|
1642
|
+
this.migrationPhaseTimers.set(state.migrationId, timer);
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
private cfgMigrationPhaseDeadline(phase: MigrationPhase): number | undefined {
|
|
1646
|
+
const m = this.settings.migration;
|
|
1647
|
+
switch (phase) {
|
|
1648
|
+
case "launching":
|
|
1649
|
+
return m?.launchingDeadlineMs ?? DEFAULT_MIGRATION_LAUNCHING_DEADLINE_MS;
|
|
1650
|
+
case "awaitingTargetReady":
|
|
1651
|
+
return m?.awaitingTargetReadyDeadlineMs ?? DEFAULT_MIGRATION_TARGET_READY_DEADLINE_MS;
|
|
1652
|
+
case "awaitingSourceStop":
|
|
1653
|
+
return m?.awaitingSourceStopDeadlineMs ?? DEFAULT_MIGRATION_SOURCE_STOP_DEADLINE_MS;
|
|
1654
|
+
case "applying":
|
|
1655
|
+
return m?.applyingDeadlineMs ?? DEFAULT_MIGRATION_APPLYING_DEADLINE_MS;
|
|
1656
|
+
case "done":
|
|
1657
|
+
case "aborted":
|
|
1658
|
+
return undefined;
|
|
1659
|
+
}
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
private markMigrationDone(state: MigrationState): void {
|
|
1663
|
+
this.advanceMigration(state, "done");
|
|
1664
|
+
this.migrationByJobId.delete(state.sourceJobId);
|
|
1665
|
+
this.settings.onMigrationCompleted?.(state.migrationId);
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
private markMigrationAborted(state: MigrationState, reason: string): void {
|
|
1669
|
+
state.abortReason = reason;
|
|
1670
|
+
this.advanceMigration(state, "aborted");
|
|
1671
|
+
this.migrationByJobId.delete(state.sourceJobId);
|
|
1672
|
+
this.settings.onMigrationAborted?.(state.migrationId, reason);
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
private failMigration(migrationId: MigrationId, reason: string): void {
|
|
1676
|
+
const state = this.migrationStates.get(migrationId);
|
|
1677
|
+
if (!state) return;
|
|
1678
|
+
if (state.phase === "done" || state.phase === "aborted") return;
|
|
1679
|
+
this.markMigrationAborted(state, reason);
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
private async dispatchMigrationAborts(
|
|
1683
|
+
state: MigrationState,
|
|
1684
|
+
reason: string
|
|
1685
|
+
): Promise<void> {
|
|
1686
|
+
// Manager's abortMigration takes both keys and fans out to source
|
|
1687
|
+
// + target workers; AutoManager doesn't need to send anything
|
|
1688
|
+
// else.
|
|
1689
|
+
await this.norsk
|
|
1690
|
+
.abortMigration(state.sourceJobId, state.sourceRole, MIGRATION_TARGET_ROLE, reason)
|
|
1691
|
+
.catch((err) => this.settings.onError?.(err));
|
|
1692
|
+
}
|
|
1693
|
+
|
|
1694
|
+
// ---- Phase E — event handlers ------------------------------------
|
|
1695
|
+
|
|
1696
|
+
private async handleMigrationTargetReady(
|
|
1697
|
+
raw: ManagerPB.MigrationTargetReadyEvent
|
|
1698
|
+
): Promise<void> {
|
|
1699
|
+
const jobId = raw.jobKey?.jobId?.jobId;
|
|
1700
|
+
if (!jobId) return;
|
|
1701
|
+
const state = this.migrationForJob(jobId);
|
|
1702
|
+
if (!state) {
|
|
1703
|
+
debuglog("MigrationTargetReady for unknown migration", { jobId });
|
|
1704
|
+
return;
|
|
1705
|
+
}
|
|
1706
|
+
if (state.phase !== "launching" && state.phase !== "awaitingTargetReady") {
|
|
1707
|
+
debuglog("MigrationTargetReady in unexpected phase", {
|
|
1708
|
+
migrationId: state.migrationId,
|
|
1709
|
+
phase: state.phase,
|
|
1710
|
+
});
|
|
1711
|
+
return;
|
|
1712
|
+
}
|
|
1713
|
+
this.advanceMigration(state, "awaitingSourceStop");
|
|
1714
|
+
try {
|
|
1715
|
+
await this.norsk.notifyMigrationCutoverReady({
|
|
1716
|
+
jobId: state.sourceJobId,
|
|
1717
|
+
role: state.sourceRole,
|
|
1718
|
+
});
|
|
1719
|
+
} catch (err) {
|
|
1720
|
+
this.settings.onError?.(err);
|
|
1721
|
+
this.failMigration(state.migrationId, `notifyMigrationCutoverReady failed: ${String(err)}`);
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
|
|
1725
|
+
private async handleMigrationSourceStoppedOutput(
|
|
1726
|
+
raw: ManagerPB.MigrationSourceStoppedOutputEvent
|
|
1727
|
+
): Promise<void> {
|
|
1728
|
+
const jobId = raw.jobKey?.jobId?.jobId;
|
|
1729
|
+
if (!jobId) return;
|
|
1730
|
+
const state = this.migrationForJob(jobId);
|
|
1731
|
+
if (!state) return;
|
|
1732
|
+
if (state.phase !== "awaitingSourceStop") {
|
|
1733
|
+
debuglog("MigrationSourceStoppedOutput in unexpected phase", {
|
|
1734
|
+
migrationId: state.migrationId,
|
|
1735
|
+
phase: state.phase,
|
|
1736
|
+
});
|
|
1737
|
+
return;
|
|
1738
|
+
}
|
|
1739
|
+
const ts = Number(raw.lastOutputTimestampNs);
|
|
1740
|
+
state.cutoverTimestampNs = ts;
|
|
1741
|
+
this.advanceMigration(state, "applying");
|
|
1742
|
+
try {
|
|
1743
|
+
await this.norsk.notifyMigrationCutoverApply(
|
|
1744
|
+
{ jobId: state.sourceJobId, role: MIGRATION_TARGET_ROLE },
|
|
1745
|
+
raw.lastOutputTimestampNs
|
|
1746
|
+
);
|
|
1747
|
+
} catch (err) {
|
|
1748
|
+
this.settings.onError?.(err);
|
|
1749
|
+
this.failMigration(state.migrationId, `notifyMigrationCutoverApply failed: ${String(err)}`);
|
|
1750
|
+
return;
|
|
1751
|
+
}
|
|
1752
|
+
// Source workflow has already stopped output; safe to terminate the
|
|
1753
|
+
// source instance. norsk-mgr-equivalents would do this via the
|
|
1754
|
+
// existing stopJob path, but since AutoManager owns the protocol
|
|
1755
|
+
// we drive the cleanup here.
|
|
1756
|
+
// R10: don't dispatch stopJob with nodeId="" if we can't locate
|
|
1757
|
+
// the source — that just sends a malformed RPC. If the source is
|
|
1758
|
+
// already gone, treat as success; surface other mismatches.
|
|
1759
|
+
const sourceNodeId = this.findNodeForJobKey(
|
|
1760
|
+
state.sourceJobId,
|
|
1761
|
+
state.sourceRole
|
|
1762
|
+
);
|
|
1763
|
+
if (sourceNodeId === undefined) {
|
|
1764
|
+
debuglog("Migration: source instance not located in nodeMap — skipping stopJob", {
|
|
1765
|
+
migrationId: state.migrationId,
|
|
1766
|
+
jobId: state.sourceJobId,
|
|
1767
|
+
role: state.sourceRole,
|
|
1768
|
+
});
|
|
1769
|
+
} else {
|
|
1770
|
+
try {
|
|
1771
|
+
await this.norsk.stopJob(state.sourceJobId, state.sourceRole, sourceNodeId);
|
|
1772
|
+
} catch (err) {
|
|
1773
|
+
debuglog("Migration: source stopJob failed (continuing)", {
|
|
1774
|
+
migrationId: state.migrationId,
|
|
1775
|
+
err: String(err),
|
|
1776
|
+
});
|
|
1777
|
+
this.settings.onError?.(err);
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
this.markMigrationDone(state);
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
private handleMigrationAborted(raw: ManagerPB.MigrationAbortedEvent): void {
|
|
1784
|
+
const jobId = raw.jobKey?.jobId?.jobId;
|
|
1785
|
+
if (!jobId) return;
|
|
1786
|
+
const state = this.migrationForJob(jobId);
|
|
1787
|
+
if (!state) return;
|
|
1788
|
+
debuglog("Migration aborted by worker", {
|
|
1789
|
+
migrationId: state.migrationId,
|
|
1790
|
+
reason: raw.reason,
|
|
1791
|
+
});
|
|
1792
|
+
// Dispatch an abort to the other side so it tears down its
|
|
1793
|
+
// migration state too. Don't await — fire-and-forget.
|
|
1794
|
+
void this.dispatchMigrationAborts(state, raw.reason);
|
|
1795
|
+
this.markMigrationAborted(state, raw.reason);
|
|
1796
|
+
}
|
|
1797
|
+
|
|
1798
|
+
private findNodeForJobKey(jobId: JobId, role: Role): NodeId | undefined {
|
|
1799
|
+
for (const node of this.nodeMap.values()) {
|
|
1800
|
+
const tags = node.nodeMetadata?.tags ?? {};
|
|
1801
|
+
if (tags["jobId"] === jobId && tags["role"] === role) {
|
|
1802
|
+
return node.nodeId;
|
|
1803
|
+
}
|
|
1804
|
+
const prov = node.provisional;
|
|
1805
|
+
if (prov && prov.jobId === jobId && prov.role === role) {
|
|
1806
|
+
return node.nodeId;
|
|
1807
|
+
}
|
|
1808
|
+
}
|
|
1809
|
+
return undefined;
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
// ---- Phase C4 — config resolution ---------------------------------
|
|
1813
|
+
// Precedence for every tunable: settings.placement.X > settings.X
|
|
1814
|
+
// > compiled-in default. Per-band tunables use settings.bands[band]
|
|
1815
|
+
// first, with the same flat/default fallback.
|
|
1816
|
+
|
|
1817
|
+
private cfgRejectionBackoffMs(): number {
|
|
1818
|
+
return (
|
|
1819
|
+
this.settings.placement?.rejectionBackoffMs ??
|
|
1820
|
+
this.settings.rejectionBackoffMs ??
|
|
1821
|
+
DEFAULT_REJECTION_BACKOFF_MS
|
|
1822
|
+
);
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
private cfgRejectionEscalationCount(): number {
|
|
1826
|
+
return (
|
|
1827
|
+
this.settings.placement?.rejectionEscalationCount ??
|
|
1828
|
+
this.settings.rejectionEscalationCount ??
|
|
1829
|
+
DEFAULT_REJECTION_ESCALATION_COUNT
|
|
1830
|
+
);
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
private cfgFailureBackoffMs(): number {
|
|
1834
|
+
return (
|
|
1835
|
+
this.settings.placement?.failureBackoffMs ??
|
|
1836
|
+
this.settings.failureBackoffMs ??
|
|
1837
|
+
DEFAULT_FAILURE_BACKOFF_MS
|
|
1838
|
+
);
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
private cfgJobStartupGraceMs(): number {
|
|
1842
|
+
return (
|
|
1843
|
+
this.settings.placement?.jobStartupGraceMs ??
|
|
1844
|
+
this.settings.jobStartupGraceMs ??
|
|
1845
|
+
DEFAULT_JOB_STARTUP_GRACE_MS
|
|
1846
|
+
);
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
private cfgMaxRestartsPerHour(band: PriorityBand): number {
|
|
1850
|
+
// Bronze drops on failure — restart ceiling is moot, but return a
|
|
1851
|
+
// large number so the rate-check path can be called uniformly
|
|
1852
|
+
// without crashing if anyone reaches it.
|
|
1853
|
+
if (band === "bronze") return Number.POSITIVE_INFINITY;
|
|
1854
|
+
const perBand = this.settings.bands?.[band]?.maxRestartsPerHour;
|
|
1855
|
+
return (
|
|
1856
|
+
perBand ??
|
|
1857
|
+
this.settings.maxRestartsPerHour ??
|
|
1858
|
+
DEFAULT_MAX_RESTARTS_PER_HOUR
|
|
1859
|
+
);
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
// ---- AutoManager v1 placement-driven path ----
|
|
1863
|
+
//
|
|
1864
|
+
// A "modern" job — one with `requirements` set — is a member of a
|
|
1865
|
+
// bundle that Manager has expanded into N replica jobs. The job
|
|
1866
|
+
// carries bundleId / replicaIndex / jobName tags so AutoManager can
|
|
1867
|
+
// recover its bundle context and run the placement engine. The
|
|
1868
|
+
// legacy createJob path (no requirements) is unaffected.
|
|
1869
|
+
|
|
1870
|
+
private async placeAndProvision(
|
|
1871
|
+
jobId: JobId,
|
|
1872
|
+
pbRequirements: ManagerPB.JobRequirements,
|
|
1873
|
+
tags: { [key: string]: string },
|
|
1874
|
+
recovery?: { excludeNodes: Set<NodeId> }
|
|
1875
|
+
): Promise<void> {
|
|
1876
|
+
const bundleId = tags["bundleId"];
|
|
1877
|
+
const replicaIndexStr = tags["replicaIndex"];
|
|
1878
|
+
const jobName = tags["jobName"];
|
|
1879
|
+
if (!bundleId || replicaIndexStr === undefined || !jobName) {
|
|
1880
|
+
debuglog("Job has requirements but missing bundle context tags", tags);
|
|
1881
|
+
return;
|
|
1882
|
+
}
|
|
1883
|
+
const replicaIndex = parseInt(replicaIndexStr, 10);
|
|
1884
|
+
|
|
1885
|
+
// In-flight-placement gate (initial placements only). handleJobChanged
|
|
1886
|
+
// can re-fire `placeAndProvision` for the same jobId while the
|
|
1887
|
+
// daemon's provisionJob is mid-dispatch — `nodesByJob().size` is
|
|
1888
|
+
// still 0 because JobStarted hasn't landed yet. Each re-fire ends
|
|
1889
|
+
// in `INVALID_ARGUMENT: Job already running on node` from the
|
|
1890
|
+
// daemon and a `placementsDispatchFailed++`. Suppress. Recovery
|
|
1891
|
+
// placements bypass this gate; they're driven by failure events
|
|
1892
|
+
// (JobRejected, NodeStopped) and have their own MIN_RECOVERY_INTERVAL_MS
|
|
1893
|
+
// backoff further down — and the bronze-drop / band-aware-recovery
|
|
1894
|
+
// branch needs to run even when an initial placement just dispatched.
|
|
1895
|
+
const now = this.clock.now();
|
|
1896
|
+
if (!recovery) {
|
|
1897
|
+
const lastAttempt = this.lastPlacementAttemptAt.get(jobId);
|
|
1898
|
+
if (lastAttempt && now.getTime() - lastAttempt.getTime() < MIN_PLACEMENT_INTERVAL_MS) {
|
|
1899
|
+
debuglog("Placement attempt suppressed (in-flight)", { jobId, bundleId });
|
|
1900
|
+
return;
|
|
1901
|
+
}
|
|
1902
|
+
this.lastPlacementAttemptAt.set(jobId, now);
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
const bundle = this.bundleMap.get(bundleId);
|
|
1906
|
+
if (!bundle) {
|
|
1907
|
+
debuglog("Job's bundle not in map yet", { jobId, bundleId });
|
|
1908
|
+
return;
|
|
1909
|
+
}
|
|
1910
|
+
const bundleJobSpec = bundle.jobs.find((j) => j.jobName === jobName);
|
|
1911
|
+
if (!bundleJobSpec) {
|
|
1912
|
+
debuglog("Bundle has no job named", { bundleId, jobName });
|
|
1913
|
+
return;
|
|
1914
|
+
}
|
|
1915
|
+
// Use the bundle's typed requirements (richer / canonical) rather
|
|
1916
|
+
// than re-converting from PB. Keeps placement decisions consistent
|
|
1917
|
+
// with the bundle the user submitted.
|
|
1918
|
+
void pbRequirements;
|
|
1919
|
+
|
|
1920
|
+
// Band-aware recovery (Phase C1). On a recovery path:
|
|
1921
|
+
// - bronze: drop without re-placing; fire onJobDropped.
|
|
1922
|
+
// - gold: re-place, preferring hot spares.
|
|
1923
|
+
// - silver: re-place, no spare preference.
|
|
1924
|
+
// Initial placements bypass this entirely.
|
|
1925
|
+
const band = bundleJobSpec.requirements.priorityBand;
|
|
1926
|
+
if (recovery && band === "bronze") {
|
|
1927
|
+
debuglog("Bronze job dropped (no restart on failure)", { jobId, bundleId });
|
|
1928
|
+
this.metricsCounters.jobsDropped++;
|
|
1929
|
+
this.settings.onJobDropped?.(jobId);
|
|
1930
|
+
// Bronze never restarts — this is terminal. Persist the verdict so the
|
|
1931
|
+
// job isn't re-placed after a restart.
|
|
1932
|
+
void this.norsk
|
|
1933
|
+
.markJobFailed(jobId, "bronze job dropped after failure")
|
|
1934
|
+
.catch((err) => this.settings.onError?.(err));
|
|
1935
|
+
return;
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
// Phase C2 — restart-rate ceiling + broken-bundle short-circuit.
|
|
1939
|
+
// Always run the rate check on every recovery trigger — the
|
|
1940
|
+
// ceiling tracks failure rate, which is real-time, even if we
|
|
1941
|
+
// defer the actual placement below. Without this, the
|
|
1942
|
+
// minimum-interval backoff would coalesce N rapid failures into
|
|
1943
|
+
// a single recorded attempt and the rate-broken trip would never
|
|
1944
|
+
// fire.
|
|
1945
|
+
if (recovery) {
|
|
1946
|
+
if (this.brokenBundles.has(bundleId)) {
|
|
1947
|
+
debuglog("Skipping recovery — bundle is marked broken", { jobId, bundleId });
|
|
1948
|
+
return;
|
|
1949
|
+
}
|
|
1950
|
+
if (this.shouldBreakOnRestart(jobId, bundleId, band)) {
|
|
1951
|
+
return;
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
// Minimum-interval backoff between consecutive recovery placement
|
|
1956
|
+
// dispatches for the same job. The failure rate has already been
|
|
1957
|
+
// counted above; this gate only delays the next actual placement
|
|
1958
|
+
// attempt so we don't hammer the daemon/worker at 12 attempts/sec.
|
|
1959
|
+
// Coalescing: if a deferred retry is already pending, further
|
|
1960
|
+
// triggers in the window become no-ops (the timer will see the
|
|
1961
|
+
// latest state when it fires).
|
|
1962
|
+
if (recovery) {
|
|
1963
|
+
const history = this.restartHistory.get(jobId) ?? [];
|
|
1964
|
+
// The just-pushed entry is the last one; we want the entry
|
|
1965
|
+
// before it to determine spacing.
|
|
1966
|
+
const previousAttempt = history[history.length - 2];
|
|
1967
|
+
const now = this.clock.now();
|
|
1968
|
+
if (previousAttempt && now.getTime() - previousAttempt.getTime() < MIN_RECOVERY_INTERVAL_MS) {
|
|
1969
|
+
if (this.pendingPlacementRetries.has(jobId)) {
|
|
1970
|
+
debuglog("Recovery placement already pending (coalescing)", { jobId, bundleId });
|
|
1971
|
+
return;
|
|
1972
|
+
}
|
|
1973
|
+
const delay = MIN_RECOVERY_INTERVAL_MS - (now.getTime() - previousAttempt.getTime());
|
|
1974
|
+
debuglog("Recovery placement deferred (backoff)", { jobId, bundleId, delayMs: delay });
|
|
1975
|
+
const timer = setTimeout(() => {
|
|
1976
|
+
this.pendingPlacementRetries.delete(jobId);
|
|
1977
|
+
void this.placeAndProvision(jobId, pbRequirements, tags, recovery).catch((err) =>
|
|
1978
|
+
this.settings.onError?.(err),
|
|
1979
|
+
);
|
|
1980
|
+
}, delay);
|
|
1981
|
+
this.pendingPlacementRetries.set(jobId, timer);
|
|
1982
|
+
return;
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
|
|
1986
|
+
const inventory = this.inventoriesAsNodeViews();
|
|
1987
|
+
const pools = new Map(
|
|
1988
|
+
(this.settings.placementPools ?? []).map((p) => [p.name, p])
|
|
1989
|
+
);
|
|
1990
|
+
if (pools.size === 0) {
|
|
1991
|
+
debuglog("placementPools not configured — cannot place", { jobId });
|
|
1992
|
+
this.settings.onError?.(
|
|
1993
|
+
new Error(
|
|
1994
|
+
`placementPools not configured; can't place job ${jobId}. ` +
|
|
1995
|
+
`Add to AutoSettings or this job will never start.`
|
|
1996
|
+
)
|
|
1997
|
+
);
|
|
1998
|
+
return;
|
|
1999
|
+
}
|
|
2000
|
+
|
|
2001
|
+
const { result, trace }: { result: PlacementResult; trace: PlacementTrace } =
|
|
2002
|
+
placeWithTrace({
|
|
2003
|
+
bundle,
|
|
2004
|
+
job: bundleJobSpec,
|
|
2005
|
+
replicaIndex,
|
|
2006
|
+
inventory,
|
|
2007
|
+
pools,
|
|
2008
|
+
recoveryContext: recovery
|
|
2009
|
+
? { excludeNodes: recovery.excludeNodes, preferHotSpares: band === "gold" }
|
|
2010
|
+
: undefined,
|
|
2011
|
+
});
|
|
2012
|
+
|
|
2013
|
+
debuglog("Placement decision", {
|
|
2014
|
+
jobId,
|
|
2015
|
+
bundleId,
|
|
2016
|
+
replicaIndex,
|
|
2017
|
+
decision: result,
|
|
2018
|
+
trace: summariseTrace(trace),
|
|
2019
|
+
});
|
|
2020
|
+
|
|
2021
|
+
const at = this.clock.now();
|
|
2022
|
+
this.metricsCounters.placementsAttempted++;
|
|
2023
|
+
if (result.kind === "failure") {
|
|
2024
|
+
this.metricsCounters.placementsFailed++;
|
|
2025
|
+
}
|
|
2026
|
+
// R12: placementsSucceeded counted post-dispatch; see below.
|
|
2027
|
+
const info: PlacementDecisionInfo = { jobId, bundleId, replicaIndex, jobName, result, trace, at };
|
|
2028
|
+
// Mirror into live-state map so consumers can query the current
|
|
2029
|
+
// placement without subscribing from process start. Failures don't
|
|
2030
|
+
// bind a placement, so they don't go in the map — `placementForJob`
|
|
2031
|
+
// returning undefined for "we tried and failed" is the correct
|
|
2032
|
+
// signal; failures still flow through `onPlacementFailed` for
|
|
2033
|
+
// consumers that care.
|
|
2034
|
+
if (result.kind !== "failure") {
|
|
2035
|
+
this.placementsByJob.set(jobId, info);
|
|
2036
|
+
}
|
|
2037
|
+
this.settings.onPlacementDecision?.(info);
|
|
2038
|
+
|
|
2039
|
+
if (result.kind === "failure") {
|
|
2040
|
+
// A failed placement dispatched nothing, so the in-flight gate
|
|
2041
|
+
// (lastPlacementAttemptAt + MIN_PLACEMENT_INTERVAL_MS) must not
|
|
2042
|
+
// hold this job back: a capacity-increasing event can arrive
|
|
2043
|
+
// moments later and `reconcilePendingPlacements` needs to be free
|
|
2044
|
+
// to retry. The gate exists to suppress duplicate dispatch while a
|
|
2045
|
+
// real placement is in flight — a failure is not in flight.
|
|
2046
|
+
this.lastPlacementAttemptAt.delete(jobId);
|
|
2047
|
+
this.settings.onPlacementFailed?.({
|
|
2048
|
+
jobId,
|
|
2049
|
+
bundleId,
|
|
2050
|
+
replicaIndex,
|
|
2051
|
+
jobName,
|
|
2052
|
+
reason: result.reason,
|
|
2053
|
+
triedPools: result.reason.triedPools,
|
|
2054
|
+
at,
|
|
2055
|
+
});
|
|
2056
|
+
this.settings.onError?.(
|
|
2057
|
+
new Error(
|
|
2058
|
+
`Placement failed for ${jobId}: ${JSON.stringify(result.reason)}`
|
|
2059
|
+
)
|
|
2060
|
+
);
|
|
2061
|
+
return;
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
const role = roleFromReplicaIndex(replicaIndex);
|
|
2065
|
+
|
|
2066
|
+
// A degraded backup still gets placed/provisioned (best-effort) — surface
|
|
2067
|
+
// the reduced DR posture so an operator can act / re-place later.
|
|
2068
|
+
if (result.degraded) {
|
|
2069
|
+
this.settings.onResilienceDegraded?.({
|
|
2070
|
+
jobId,
|
|
2071
|
+
bundleId,
|
|
2072
|
+
replicaIndex,
|
|
2073
|
+
jobName,
|
|
2074
|
+
violated: result.degraded.violated,
|
|
2075
|
+
...(result.degraded.interruptiblePrimary ? { interruptiblePrimary: true } : {}),
|
|
2076
|
+
at,
|
|
2077
|
+
});
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
if (result.kind === "place") {
|
|
2081
|
+
// Existing node already running on the chosen target. Manager's
|
|
2082
|
+
// StartJob RPC binds the job to that node.
|
|
2083
|
+
try {
|
|
2084
|
+
await this.norsk.startJob(jobId, role, result.nodeId);
|
|
2085
|
+
this.jobToNode.set(jobId, result.nodeId);
|
|
2086
|
+
this.metricsCounters.placementsSucceeded++;
|
|
2087
|
+
} catch (err) {
|
|
2088
|
+
this.metricsCounters.placementsDispatchFailed++;
|
|
2089
|
+
this.settings.onError?.(err);
|
|
2090
|
+
}
|
|
2091
|
+
return;
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
// Provision in the chosen pool's chosen tier.
|
|
2095
|
+
const pool = pools.get(result.pool);
|
|
2096
|
+
if (!pool) return;
|
|
2097
|
+
const tier = pool.tiers.find((t) => t.name === result.tier);
|
|
2098
|
+
if (!tier) return;
|
|
2099
|
+
// `pool` + `tier` tags are load-bearing: poolNameForNode / tierNameForNode
|
|
2100
|
+
// read them back via nodeMetadata.tags to identify which pool+tier an
|
|
2101
|
+
// existing node belongs to. We must set them here, or every placement
|
|
2102
|
+
// after the node starts will filter it out as wrong-pool/wrong-tier.
|
|
2103
|
+
const provisionTags = {
|
|
2104
|
+
"auto-manager": "true",
|
|
2105
|
+
jobId,
|
|
2106
|
+
role,
|
|
2107
|
+
bundleId,
|
|
2108
|
+
replicaIndex: replicaIndexStr,
|
|
2109
|
+
jobName,
|
|
2110
|
+
pool: pool.name,
|
|
2111
|
+
tier: tier.name,
|
|
2112
|
+
// Failure-domain metadata for the resilience engine + observability.
|
|
2113
|
+
reliability: tierReliability(tier),
|
|
2114
|
+
};
|
|
2115
|
+
const nodeId = uuid.v4() + "." + jobId + "." + role;
|
|
2116
|
+
this.updateNodeMap(nodeId, "starting", undefined, undefined, { jobId, role });
|
|
2117
|
+
this.jobToNode.set(jobId, nodeId);
|
|
2118
|
+
this.settings.onNodeStarting?.(nodeId);
|
|
2119
|
+
|
|
2120
|
+
if (tier.kind === "aws") {
|
|
2121
|
+
try {
|
|
2122
|
+
await this.norsk.createAwsNode({
|
|
2123
|
+
nodeId,
|
|
2124
|
+
region: tier.region ?? "",
|
|
2125
|
+
instanceType: result.instanceType,
|
|
2126
|
+
tags: provisionTags,
|
|
2127
|
+
writeFiles: [],
|
|
2128
|
+
launchMode: result.launchMode,
|
|
2129
|
+
});
|
|
2130
|
+
this.metricsCounters.placementsSucceeded++;
|
|
2131
|
+
} catch (err) {
|
|
2132
|
+
this.metricsCounters.placementsDispatchFailed++;
|
|
2133
|
+
this.settings.onError?.(err);
|
|
2134
|
+
}
|
|
2135
|
+
} else if (tier.kind === "oci") {
|
|
2136
|
+
try {
|
|
2137
|
+
await this.norsk.createOciNode({
|
|
2138
|
+
nodeId,
|
|
2139
|
+
availabilityDomain: tier.region ?? "",
|
|
2140
|
+
architecture: "x86_64",
|
|
2141
|
+
shape: result.instanceType,
|
|
2142
|
+
subnet: "",
|
|
2143
|
+
tags: provisionTags,
|
|
2144
|
+
writeFiles: [],
|
|
2145
|
+
});
|
|
2146
|
+
this.metricsCounters.placementsSucceeded++;
|
|
2147
|
+
} catch (err) {
|
|
2148
|
+
this.metricsCounters.placementsDispatchFailed++;
|
|
2149
|
+
this.settings.onError?.(err);
|
|
2150
|
+
}
|
|
2151
|
+
} else {
|
|
2152
|
+
// Cluster tiers never auto-scale; place-on-existing only.
|
|
2153
|
+
this.settings.onError?.(
|
|
2154
|
+
new Error(
|
|
2155
|
+
`Cluster tier '${pool.name}/${tier.name}' returned a 'provision' ` +
|
|
2156
|
+
`placement; cluster tiers should only place onto existing nodes.`
|
|
2157
|
+
)
|
|
2158
|
+
);
|
|
2159
|
+
}
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2162
|
+
private inventoriesAsNodeViews(): NodeView[] {
|
|
2163
|
+
// Reservation (reservedCapacity / cores / capabilities, assignedJobs) is
|
|
2164
|
+
// not populated upstream: the worker advertises raw hardware and the
|
|
2165
|
+
// daemon forwards it verbatim, so the wire values arrive as 0 / []. Left
|
|
2166
|
+
// as-is, every node would look fully free regardless of what's running on
|
|
2167
|
+
// it — placement would over-subscribe, and the capacity-driven reconcile
|
|
2168
|
+
// could never observe a node fill up and later free up. AutoManager is the
|
|
2169
|
+
// one component that knows both which jobs it placed where (`jobToNode`)
|
|
2170
|
+
// and each job's requirement (the bundle's typed spec), so it derives the
|
|
2171
|
+
// reservation here. A non-zero wire reservation, if a producer ever sends
|
|
2172
|
+
// one, is treated as authoritative and the derivation defers to it.
|
|
2173
|
+
const placedJobsByNode = new Map<NodeId, JobId[]>();
|
|
2174
|
+
for (const [jobId, nodeId] of this.jobToNode) {
|
|
2175
|
+
const list = placedJobsByNode.get(nodeId);
|
|
2176
|
+
if (list) list.push(jobId);
|
|
2177
|
+
else placedJobsByNode.set(nodeId, [jobId]);
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
const out: NodeView[] = [];
|
|
2181
|
+
for (const [nodeId, inv] of this.inventoryMap.entries()) {
|
|
2182
|
+
const placedHere = placedJobsByNode.get(nodeId) ?? [];
|
|
2183
|
+
|
|
2184
|
+
// Prefer the wire's assigned jobs when present; otherwise list the
|
|
2185
|
+
// jobs AutoManager itself placed on this node. Either feeds the
|
|
2186
|
+
// affinity / spread constraints in the placement engine.
|
|
2187
|
+
const runningJobs: NodeView["runningJobs"] =
|
|
2188
|
+
inv.assignedJobs.length > 0
|
|
2189
|
+
? inv.assignedJobs.flatMap((a) => this.runningJobFor(a.jobId))
|
|
2190
|
+
: placedHere.flatMap((jobId) => this.runningJobFor(jobId));
|
|
2191
|
+
|
|
2192
|
+
// Phase C8: merge the AutoManager-local cordon into the wire-reported
|
|
2193
|
+
// cordoned flag. Either side can mark a node off-limits.
|
|
2194
|
+
const cordoned = this.cordonedNodes.has(nodeId)
|
|
2195
|
+
? { ...inv, cordoned: true }
|
|
2196
|
+
: inv;
|
|
2197
|
+
// Overlay derived reservation, deferring to a non-zero wire value.
|
|
2198
|
+
const reservation = this.derivedReservation(placedHere);
|
|
2199
|
+
const effectiveInv: TypedNodeInventory = {
|
|
2200
|
+
...cordoned,
|
|
2201
|
+
reservedCapacity:
|
|
2202
|
+
cordoned.reservedCapacity > 0
|
|
2203
|
+
? cordoned.reservedCapacity
|
|
2204
|
+
: reservation.capacity,
|
|
2205
|
+
reservedCores:
|
|
2206
|
+
cordoned.reservedCores > 0 ? cordoned.reservedCores : reservation.cores,
|
|
2207
|
+
reservedCapabilities:
|
|
2208
|
+
cordoned.reservedCapabilities.length > 0
|
|
2209
|
+
? cordoned.reservedCapabilities
|
|
2210
|
+
: reservation.capabilities,
|
|
2211
|
+
};
|
|
2212
|
+
|
|
2213
|
+
// poolName / tierName are read from the node's NodeMetadata (nodeMap);
|
|
2214
|
+
// nodes we haven't seen there yet resolve to "" / undefined and never
|
|
2215
|
+
// match a pool filter.
|
|
2216
|
+
out.push({
|
|
2217
|
+
nodeId,
|
|
2218
|
+
poolName: this.poolNameForNode(nodeId),
|
|
2219
|
+
tierName: this.tierNameForNode(nodeId),
|
|
2220
|
+
cloud: this.cloudForNode(nodeId),
|
|
2221
|
+
inventory: effectiveInv,
|
|
2222
|
+
runningJobs,
|
|
2223
|
+
az: this.azForNode(nodeId),
|
|
2224
|
+
// Phase C5: spare-ness is advertised via NodeMetadata.tags["spare"].
|
|
2225
|
+
isHotSpare: this.isHotSpareNode(nodeId),
|
|
2226
|
+
});
|
|
2227
|
+
}
|
|
2228
|
+
return out;
|
|
2229
|
+
}
|
|
2230
|
+
|
|
2231
|
+
/** Map a placed jobId to the RunningJob view the placement engine uses
|
|
2232
|
+
* for affinity / spread, via the job's bundle-context tags. Empty when
|
|
2233
|
+
* the job lacks bundle tags. */
|
|
2234
|
+
private runningJobFor(jobId: JobId): RunningJob[] {
|
|
2235
|
+
const tags = this.tagsForJob(jobId);
|
|
2236
|
+
if (!tags) return [];
|
|
2237
|
+
const bundleId = tags["bundleId"];
|
|
2238
|
+
const replicaIndexStr = tags["replicaIndex"];
|
|
2239
|
+
const jobName = tags["jobName"];
|
|
2240
|
+
if (!bundleId || replicaIndexStr === undefined || !jobName) return [];
|
|
2241
|
+
return [{ bundleId, replicaIndex: parseInt(replicaIndexStr, 10), jobName }];
|
|
2242
|
+
}
|
|
2243
|
+
|
|
2244
|
+
/** Sum the resource requirements of the given placed jobs into a node
|
|
2245
|
+
* reservation. Capacity / cores / capabilities come from each job's
|
|
2246
|
+
* bundle spec — the canonical typed requirements the placement engine
|
|
2247
|
+
* scores against, looked up via the job's bundleId / jobName tags. Jobs
|
|
2248
|
+
* whose bundle or spec we don't have yet contribute nothing. */
|
|
2249
|
+
private derivedReservation(jobIds: JobId[]): {
|
|
2250
|
+
capacity: number;
|
|
2251
|
+
cores: number;
|
|
2252
|
+
capabilities: Capability[];
|
|
2253
|
+
} {
|
|
2254
|
+
let capacity = 0;
|
|
2255
|
+
let cores = 0;
|
|
2256
|
+
const caps = new Map<string, number>();
|
|
2257
|
+
for (const jobId of jobIds) {
|
|
2258
|
+
const req = this.requirementsForPlacedJob(jobId);
|
|
2259
|
+
if (!req) continue;
|
|
2260
|
+
capacity += req.requiredCapacity;
|
|
2261
|
+
cores += req.requiredCores ?? 0;
|
|
2262
|
+
for (const c of req.requiredCapabilities) {
|
|
2263
|
+
caps.set(c.name, (caps.get(c.name) ?? 0) + c.count);
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
2266
|
+
const capabilities: Capability[] = [...caps.entries()].map(
|
|
2267
|
+
([name, count]) => ({ name, count })
|
|
2268
|
+
);
|
|
2269
|
+
return { capacity, cores, capabilities };
|
|
2270
|
+
}
|
|
2271
|
+
|
|
2272
|
+
/** The canonical typed requirements for a placed job, from its bundle
|
|
2273
|
+
* spec (the same source `placeAndProvision` scores against). */
|
|
2274
|
+
private requirementsForPlacedJob(jobId: JobId): JobRequirements | undefined {
|
|
2275
|
+
const tags = this.tagsForJob(jobId);
|
|
2276
|
+
if (!tags) return undefined;
|
|
2277
|
+
const bundleId = tags["bundleId"];
|
|
2278
|
+
const jobName = tags["jobName"];
|
|
2279
|
+
if (!bundleId || !jobName) return undefined;
|
|
2280
|
+
return this.bundleMap
|
|
2281
|
+
.get(bundleId)
|
|
2282
|
+
?.jobs.find((j) => j.jobName === jobName)?.requirements;
|
|
2283
|
+
}
|
|
2284
|
+
|
|
2285
|
+
/**
|
|
2286
|
+
* Phase C5 — is the named node currently tagged as a hot spare?
|
|
2287
|
+
* Spares are warm-running nodes provisioned (by the C6 reconciler)
|
|
2288
|
+
* with `NodeMetadata.tags["spare"] = "true"`. The optional
|
|
2289
|
+
* `tags["spareBand"]` ("gold" | "silver") dedicates a spare to a
|
|
2290
|
+
* band; if absent, the spare is band-agnostic.
|
|
2291
|
+
* @public
|
|
2292
|
+
*/
|
|
2293
|
+
public isHotSpareNode(nodeId: NodeId): boolean {
|
|
2294
|
+
return this.nodeMap.get(nodeId)?.nodeMetadata?.tags?.["spare"] === "true";
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
/**
|
|
2298
|
+
* Phase C5 — band a spare is dedicated to, if any. `undefined` for a
|
|
2299
|
+
* non-spare or for a band-agnostic spare.
|
|
2300
|
+
* @public
|
|
2301
|
+
*/
|
|
2302
|
+
public spareBandForNode(nodeId: NodeId): PriorityBand | undefined {
|
|
2303
|
+
if (!this.isHotSpareNode(nodeId)) return undefined;
|
|
2304
|
+
const v = this.nodeMap.get(nodeId)?.nodeMetadata?.tags?.["spareBand"];
|
|
2305
|
+
return v === "gold" || v === "silver" || v === "bronze" ? v : undefined;
|
|
2306
|
+
}
|
|
2307
|
+
|
|
2308
|
+
// ---- Phase C6 — hot-spare reconciliation -------------------------
|
|
2309
|
+
|
|
2310
|
+
/**
|
|
2311
|
+
* Drive every configured `hotSpares` entry toward its `targetCount`.
|
|
2312
|
+
* Counts live + pending spares matching (pool, band); provisions
|
|
2313
|
+
* short, terminates excess (oldest live first). Cluster pools are
|
|
2314
|
+
* non-elastic — under-target just fires `onSpareUnderflow` and
|
|
2315
|
+
* leaves provisioning to the operator. Idempotent and cheap; safe
|
|
2316
|
+
* to call on every lifecycle event plus the periodic tick.
|
|
2317
|
+
*/
|
|
2318
|
+
private reconcileHotSpares(): void {
|
|
2319
|
+
const entries = this.settings.hotSpares ?? [];
|
|
2320
|
+
if (entries.length === 0) return;
|
|
2321
|
+
|
|
2322
|
+
for (const cfg of entries) {
|
|
2323
|
+
const pool = (this.settings.placementPools ?? []).find(
|
|
2324
|
+
(p) => p.name === cfg.pool
|
|
2325
|
+
);
|
|
2326
|
+
if (!pool) {
|
|
2327
|
+
debuglog("hotSpares entry references unknown pool", { entry: cfg });
|
|
2328
|
+
continue;
|
|
2329
|
+
}
|
|
2330
|
+
|
|
2331
|
+
const matching = this.spareNodesMatching(cfg);
|
|
2332
|
+
const pending = this.pendingSparesMatching(cfg);
|
|
2333
|
+
const effective = matching.length + pending;
|
|
2334
|
+
const target = cfg.targetCount;
|
|
2335
|
+
|
|
2336
|
+
if (effective < target) {
|
|
2337
|
+
const short = target - effective;
|
|
2338
|
+
this.settings.onSpareUnderflow?.(cfg.pool, effective, target);
|
|
2339
|
+
// Warm the cheapest provisionable tier (first elastic tier in
|
|
2340
|
+
// cost order). A pool with no elastic tier can't grow spares.
|
|
2341
|
+
const provisionTier = pool.tiers.find((t) => t.scaleOut === "elastic");
|
|
2342
|
+
if (!provisionTier) {
|
|
2343
|
+
debuglog("Pool has no elastic tier — under spare target, no provision", {
|
|
2344
|
+
cfg,
|
|
2345
|
+
effective,
|
|
2346
|
+
target,
|
|
2347
|
+
});
|
|
2348
|
+
continue;
|
|
2349
|
+
}
|
|
2350
|
+
for (let i = 0; i < short; i++) {
|
|
2351
|
+
void this.provisionHotSpare(pool, provisionTier, cfg);
|
|
2352
|
+
}
|
|
2353
|
+
} else if (matching.length > target) {
|
|
2354
|
+
// Only terminate from the truly-live (non-pending) set. Sort
|
|
2355
|
+
// by lastActivity ascending so the oldest goes first — matches
|
|
2356
|
+
// the design's "support rotation for security patching."
|
|
2357
|
+
const excess = matching.length - target;
|
|
2358
|
+
const sorted = [...matching].sort(
|
|
2359
|
+
(a, b) => a.lastActivity.getTime() - b.lastActivity.getTime()
|
|
2360
|
+
);
|
|
2361
|
+
for (let i = 0; i < excess; i++) {
|
|
2362
|
+
const victim = sorted[i];
|
|
2363
|
+
debuglog("Hot-spare excess — terminating oldest", {
|
|
2364
|
+
nodeId: victim.nodeId,
|
|
2365
|
+
cfg,
|
|
2366
|
+
});
|
|
2367
|
+
void this.norsk
|
|
2368
|
+
.terminateNode(victim.nodeId)
|
|
2369
|
+
.catch((err) => this.settings.onError?.(err));
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
}
|
|
2374
|
+
|
|
2375
|
+
private spareNodesMatching(cfg: HotSpareConfig): NodeSummary[] {
|
|
2376
|
+
const out: NodeSummary[] = [];
|
|
2377
|
+
for (const node of this.nodeMap.values()) {
|
|
2378
|
+
if (!this.isHotSpareNode(node.nodeId)) continue;
|
|
2379
|
+
if (node.nodeState !== "running") continue;
|
|
2380
|
+
if (this.poolNameForNode(node.nodeId) !== cfg.pool) continue;
|
|
2381
|
+
if (cfg.forBand && this.spareBandForNode(node.nodeId) !== cfg.forBand) continue;
|
|
2382
|
+
// A spare with jobs assigned is consumed — exclude from the count
|
|
2383
|
+
// so the underflow fires and a replacement provisions.
|
|
2384
|
+
const inv = this.inventoryMap.get(node.nodeId);
|
|
2385
|
+
if (inv && inv.assignedJobs.length > 0) continue;
|
|
2386
|
+
out.push(node);
|
|
2387
|
+
}
|
|
2388
|
+
return out;
|
|
2389
|
+
}
|
|
2390
|
+
|
|
2391
|
+
private pendingSparesMatching(cfg: HotSpareConfig): number {
|
|
2392
|
+
let n = 0;
|
|
2393
|
+
for (const p of this.pendingSpareProvisions.values()) {
|
|
2394
|
+
if (p.pool !== cfg.pool) continue;
|
|
2395
|
+
if (cfg.forBand && p.band !== cfg.forBand) continue;
|
|
2396
|
+
n++;
|
|
2397
|
+
}
|
|
2398
|
+
return n;
|
|
2399
|
+
}
|
|
2400
|
+
|
|
2401
|
+
private async provisionHotSpare(
|
|
2402
|
+
pool: PlacementPool,
|
|
2403
|
+
tier: PlacementTier,
|
|
2404
|
+
cfg: HotSpareConfig
|
|
2405
|
+
): Promise<void> {
|
|
2406
|
+
const instanceType = cfg.spec.instanceType;
|
|
2407
|
+
if (!instanceType) {
|
|
2408
|
+
this.settings.onError?.(
|
|
2409
|
+
new Error(
|
|
2410
|
+
`hotSpares entry for elastic pool '${cfg.pool}' needs spec.instanceType`
|
|
2411
|
+
)
|
|
2412
|
+
);
|
|
2413
|
+
return;
|
|
2414
|
+
}
|
|
2415
|
+
|
|
2416
|
+
const nodeId = uuid.v4() + ".spare." + cfg.pool;
|
|
2417
|
+
const tags: Record<string, string> = {
|
|
2418
|
+
"auto-manager": "true",
|
|
2419
|
+
spare: "true",
|
|
2420
|
+
pool: cfg.pool,
|
|
2421
|
+
tier: tier.name,
|
|
2422
|
+
};
|
|
2423
|
+
if (cfg.forBand) tags["spareBand"] = cfg.forBand;
|
|
2424
|
+
|
|
2425
|
+
this.pendingSpareProvisions.set(nodeId, { pool: cfg.pool, band: cfg.forBand });
|
|
2426
|
+
this.updateNodeMap(nodeId, "starting", undefined, undefined);
|
|
2427
|
+
this.settings.onNodeStarting?.(nodeId);
|
|
2428
|
+
|
|
2429
|
+
try {
|
|
2430
|
+
if (tier.kind === "aws") {
|
|
2431
|
+
await this.norsk.createAwsNode({
|
|
2432
|
+
nodeId,
|
|
2433
|
+
region: cfg.spec.region ?? tier.region ?? "",
|
|
2434
|
+
instanceType,
|
|
2435
|
+
tags,
|
|
2436
|
+
writeFiles: [],
|
|
2437
|
+
});
|
|
2438
|
+
} else if (tier.kind === "oci") {
|
|
2439
|
+
await this.norsk.createOciNode({
|
|
2440
|
+
nodeId,
|
|
2441
|
+
availabilityDomain: cfg.spec.az ?? cfg.spec.region ?? tier.region ?? "",
|
|
2442
|
+
architecture: "x86_64",
|
|
2443
|
+
shape: instanceType,
|
|
2444
|
+
subnet: "",
|
|
2445
|
+
tags,
|
|
2446
|
+
writeFiles: [],
|
|
2447
|
+
});
|
|
2448
|
+
}
|
|
2449
|
+
// Cluster path falls through — non-elastic, no auto-provision.
|
|
2450
|
+
} catch (err) {
|
|
2451
|
+
// Failed to even submit. Drop the pending marker so the next
|
|
2452
|
+
// reconcile can try again instead of waiting forever.
|
|
2453
|
+
this.pendingSpareProvisions.delete(nodeId);
|
|
2454
|
+
this.settings.onError?.(err);
|
|
2455
|
+
}
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
// ---- Phase F — NodeService reconciliation ------------------------
|
|
2459
|
+
|
|
2460
|
+
/**
|
|
2461
|
+
* @public
|
|
2462
|
+
* Current placements for each configured NodeService. Returns a
|
|
2463
|
+
* snapshot map of `serviceId → (nodeId → runtimeJobId)`.
|
|
2464
|
+
*/
|
|
2465
|
+
public nodeServices(): NodeService[] {
|
|
2466
|
+
return this.settings.nodeServices ?? [];
|
|
2467
|
+
}
|
|
2468
|
+
|
|
2469
|
+
/**
|
|
2470
|
+
* @public
|
|
2471
|
+
* Nodes on which the named NodeService is currently placed,
|
|
2472
|
+
* keyed by the runtime job's jobId.
|
|
2473
|
+
*/
|
|
2474
|
+
public nodeServicePlacementsFor(serviceId: ServiceId): Map<NodeId, JobId> {
|
|
2475
|
+
return new Map(this.nodeServicePlacements.get(serviceId) ?? []);
|
|
2476
|
+
}
|
|
2477
|
+
|
|
2478
|
+
/**
|
|
2479
|
+
* Drive every configured NodeService toward its lifecycle target.
|
|
2480
|
+
* v1 only handles `eager` — places on every eligible node.
|
|
2481
|
+
* `lazy` placements + capability-loss-driven consumer recovery (F3)
|
|
2482
|
+
* are deferred.
|
|
2483
|
+
*/
|
|
2484
|
+
private reconcileNodeServices(): void {
|
|
2485
|
+
const services = this.settings.nodeServices ?? [];
|
|
2486
|
+
if (services.length === 0) return;
|
|
2487
|
+
for (const svc of services) {
|
|
2488
|
+
if (svc.lifecycle !== "eager") {
|
|
2489
|
+
// Lazy lifecycle not implemented in v1; skip.
|
|
2490
|
+
continue;
|
|
2491
|
+
}
|
|
2492
|
+
this.reconcileEagerNodeService(svc);
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
private reconcileEagerNodeService(svc: NodeService): void {
|
|
2497
|
+
const eligible = this.eligibleNodesForService(svc);
|
|
2498
|
+
const placed = this.nodeServicePlacements.get(svc.serviceId) ?? new Map<NodeId, JobId>();
|
|
2499
|
+
|
|
2500
|
+
// Place on every eligible node that doesn't already host the service.
|
|
2501
|
+
for (const nodeId of eligible) {
|
|
2502
|
+
if (placed.has(nodeId)) continue;
|
|
2503
|
+
void this.placeNodeService(svc, nodeId).catch((err) =>
|
|
2504
|
+
this.settings.onError?.(err)
|
|
2505
|
+
);
|
|
2506
|
+
}
|
|
2507
|
+
|
|
2508
|
+
// Tear down any placement that's no longer eligible.
|
|
2509
|
+
for (const [nodeId, jobId] of placed) {
|
|
2510
|
+
if (eligible.has(nodeId)) continue;
|
|
2511
|
+
void this.teardownNodeService(svc.serviceId, nodeId, jobId).catch((err) =>
|
|
2512
|
+
this.settings.onError?.(err)
|
|
2513
|
+
);
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
2516
|
+
|
|
2517
|
+
/**
|
|
2518
|
+
* Identify nodes in the service's named pools whose current inventory
|
|
2519
|
+
* satisfies its `consumes` requirements. Used both for placement and
|
|
2520
|
+
* for tear-down decisions.
|
|
2521
|
+
*/
|
|
2522
|
+
private eligibleNodesForService(svc: NodeService): Set<NodeId> {
|
|
2523
|
+
const eligible = new Set<NodeId>();
|
|
2524
|
+
const poolSet = new Set(svc.pools);
|
|
2525
|
+
for (const node of this.nodeMap.values()) {
|
|
2526
|
+
if (node.nodeState !== "running") continue;
|
|
2527
|
+
if (!poolSet.has(this.poolNameForNode(node.nodeId))) continue;
|
|
2528
|
+
const inv = this.inventoryMap.get(node.nodeId);
|
|
2529
|
+
if (!inv || !inv.reachable || inv.cordoned) continue;
|
|
2530
|
+
if (this.cordonedNodes.has(node.nodeId)) continue;
|
|
2531
|
+
if (this.nodeSatisfiesConsumes(inv, svc.consumes)) {
|
|
2532
|
+
eligible.add(node.nodeId);
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
return eligible;
|
|
2536
|
+
}
|
|
2537
|
+
|
|
2538
|
+
private nodeSatisfiesConsumes(
|
|
2539
|
+
inv: TypedNodeInventory,
|
|
2540
|
+
consumes: CapabilityRequirement[]
|
|
2541
|
+
): boolean {
|
|
2542
|
+
for (const req of consumes) {
|
|
2543
|
+
const cap = inv.capabilities.find((c) => c.name === req.name);
|
|
2544
|
+
if (!cap) return false;
|
|
2545
|
+
const reserved =
|
|
2546
|
+
inv.reservedCapabilities.find((c) => c.name === req.name)?.count ?? 0;
|
|
2547
|
+
if (cap.count - reserved < req.count) return false;
|
|
2548
|
+
if (req.attributeMatches) {
|
|
2549
|
+
for (const [k, v] of Object.entries(req.attributeMatches)) {
|
|
2550
|
+
if (cap.attributes?.[k] !== v) return false;
|
|
2551
|
+
}
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
return true;
|
|
2555
|
+
}
|
|
2556
|
+
|
|
2557
|
+
private async placeNodeService(svc: NodeService, nodeId: NodeId): Promise<void> {
|
|
2558
|
+
const jobId = `${svc.serviceId}.${uuid.v4().slice(0, 8)}`;
|
|
2559
|
+
debuglog("Placing NodeService", { serviceId: svc.serviceId, nodeId, jobId });
|
|
2560
|
+
// Optimistically record the placement so a re-entrant reconcile
|
|
2561
|
+
// doesn't double-place.
|
|
2562
|
+
let placements = this.nodeServicePlacements.get(svc.serviceId);
|
|
2563
|
+
if (!placements) {
|
|
2564
|
+
placements = new Map();
|
|
2565
|
+
this.nodeServicePlacements.set(svc.serviceId, placements);
|
|
2566
|
+
}
|
|
2567
|
+
placements.set(nodeId, jobId);
|
|
2568
|
+
// Build the runtime Job: take the configured template, override
|
|
2569
|
+
// the jobId, and tag it as a NodeService instance.
|
|
2570
|
+
const job: ManagerSdk.Job = {
|
|
2571
|
+
...svc.jobSpec,
|
|
2572
|
+
jobId,
|
|
2573
|
+
tags: {
|
|
2574
|
+
...(svc.jobSpec.tags ?? {}),
|
|
2575
|
+
"auto-manager": "true",
|
|
2576
|
+
nodeService: svc.serviceId,
|
|
2577
|
+
},
|
|
2578
|
+
};
|
|
2579
|
+
let createdJob = false;
|
|
2580
|
+
try {
|
|
2581
|
+
await this.norsk.createJob(job);
|
|
2582
|
+
createdJob = true;
|
|
2583
|
+
await this.norsk.startJob(jobId, NODE_SERVICE_ROLE, nodeId);
|
|
2584
|
+
this.jobToNode.set(jobId, nodeId);
|
|
2585
|
+
} catch (err) {
|
|
2586
|
+
// Roll back the optimistic placement on failure.
|
|
2587
|
+
placements.delete(nodeId);
|
|
2588
|
+
// R9: if createJob succeeded but startJob failed, the Manager
|
|
2589
|
+
// has a dangling Job row that the next reconcile would orphan
|
|
2590
|
+
// by allocating a fresh jobId. Best-effort delete.
|
|
2591
|
+
if (createdJob) {
|
|
2592
|
+
try {
|
|
2593
|
+
await this.norsk.deleteJob(jobId);
|
|
2594
|
+
} catch (cleanupErr) {
|
|
2595
|
+
debuglog("placeNodeService: cleanup deleteJob failed", {
|
|
2596
|
+
jobId,
|
|
2597
|
+
err: String(cleanupErr),
|
|
2598
|
+
});
|
|
2599
|
+
}
|
|
2600
|
+
}
|
|
2601
|
+
this.settings.onError?.(err);
|
|
2602
|
+
}
|
|
2603
|
+
}
|
|
2604
|
+
|
|
2605
|
+
private async teardownNodeService(
|
|
2606
|
+
serviceId: ServiceId,
|
|
2607
|
+
nodeId: NodeId,
|
|
2608
|
+
jobId: JobId
|
|
2609
|
+
): Promise<void> {
|
|
2610
|
+
debuglog("Tearing down NodeService instance", { serviceId, nodeId, jobId });
|
|
2611
|
+
const placements = this.nodeServicePlacements.get(serviceId);
|
|
2612
|
+
placements?.delete(nodeId);
|
|
2613
|
+
try {
|
|
2614
|
+
await this.norsk.stopJob(jobId, NODE_SERVICE_ROLE, nodeId);
|
|
2615
|
+
} catch (err) {
|
|
2616
|
+
// Job might be gone already; tolerate.
|
|
2617
|
+
debuglog("NodeService stopJob failed (continuing)", { serviceId, err: String(err) });
|
|
2618
|
+
}
|
|
2619
|
+
}
|
|
2620
|
+
|
|
2621
|
+
private tagsForJob(jobId: JobId): { [k: string]: string } | undefined {
|
|
2622
|
+
return this.jobMap.get(jobId)?.job.tags;
|
|
2623
|
+
}
|
|
2624
|
+
|
|
2625
|
+
private poolNameForNode(nodeId: NodeId): string {
|
|
2626
|
+
const node = this.nodeMap.get(nodeId);
|
|
2627
|
+
return node?.nodeMetadata?.tags?.["pool"] ?? "";
|
|
2628
|
+
}
|
|
2629
|
+
|
|
2630
|
+
// Tier within the pool, from the node's `tier` tag. Undefined for
|
|
2631
|
+
// pre-registered cluster nodes (no tier tag) — nodeMatchesTier then
|
|
2632
|
+
// treats them as the pool's fixed tier.
|
|
2633
|
+
private tierNameForNode(nodeId: NodeId): string | undefined {
|
|
2634
|
+
const node = this.nodeMap.get(nodeId);
|
|
2635
|
+
return node?.nodeMetadata?.tags?.["tier"];
|
|
2636
|
+
}
|
|
2637
|
+
|
|
2638
|
+
private azForNode(nodeId: NodeId): string | undefined {
|
|
2639
|
+
const node = this.nodeMap.get(nodeId);
|
|
2640
|
+
return node?.nodeMetadata?.tags?.["az"];
|
|
2641
|
+
}
|
|
2642
|
+
|
|
2643
|
+
// Cloud / failure-domain key for the resilience sameCloud rule: the node's
|
|
2644
|
+
// tier kind (aws / oci / local-servers), resolved via its pool+tier tags. Matches
|
|
2645
|
+
// how provisionDegradation and validation derive a pool's clouds.
|
|
2646
|
+
private cloudForNode(nodeId: NodeId): string | undefined {
|
|
2647
|
+
const poolName = this.poolNameForNode(nodeId);
|
|
2648
|
+
const tierName = this.tierNameForNode(nodeId);
|
|
2649
|
+
const pool = (this.settings.placementPools ?? []).find((p) => p.name === poolName);
|
|
2650
|
+
if (!pool) return undefined;
|
|
2651
|
+
const tier =
|
|
2652
|
+
tierName !== undefined
|
|
2653
|
+
? pool.tiers.find((t) => t.name === tierName)
|
|
2654
|
+
: pool.tiers.find((t) => t.scaleOut === "fixed");
|
|
2655
|
+
return tier?.kind;
|
|
2656
|
+
}
|
|
2657
|
+
|
|
2658
|
+
private handleBundleUpdated(raw: ManagerPB.BundleUpdated): void {
|
|
2659
|
+
if (!raw.bundle) return;
|
|
2660
|
+
const b = fromPbBundle(raw.bundle);
|
|
2661
|
+
this.bundleMap.set(b.bundleId, b);
|
|
2662
|
+
// Phase C2: operator pushed a fresh spec — clear any prior broken
|
|
2663
|
+
// state for this bundle and reset its replicas' restart counters
|
|
2664
|
+
// so the new spec gets a clean start.
|
|
2665
|
+
if (this.brokenBundles.delete(b.bundleId)) {
|
|
2666
|
+
debuglog("Cleared broken flag on bundleUpdated", { bundleId: b.bundleId });
|
|
2667
|
+
}
|
|
2668
|
+
this.clearRestartHistoryForBundle(b.bundleId);
|
|
2669
|
+
this.settings.onBundleUpdated?.(b);
|
|
2670
|
+
}
|
|
2671
|
+
|
|
2672
|
+
private handleBundleDeleted(raw: ManagerPB.BundleDeleted): void {
|
|
2673
|
+
if (!raw.bundleId) return;
|
|
2674
|
+
const id = raw.bundleId.bundleId;
|
|
2675
|
+
this.bundleMap.delete(id);
|
|
2676
|
+
this.brokenBundles.delete(id);
|
|
2677
|
+
this.clearRestartHistoryForBundle(id);
|
|
2678
|
+
const stop = this.pendingStopTimers.get(id);
|
|
2679
|
+
if (stop) {
|
|
2680
|
+
stop.timer.cancel();
|
|
2681
|
+
this.pendingStopTimers.delete(id);
|
|
2682
|
+
}
|
|
2683
|
+
this.settings.onBundleDeleted?.(id);
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
private clearRestartHistoryForBundle(bundleId: BundleId): void {
|
|
2687
|
+
for (const jobId of this.jobMap.keys()) {
|
|
2688
|
+
const tags = this.jobMap.get(jobId)?.job.tags ?? {};
|
|
2689
|
+
if (tags["bundleId"] === bundleId) this.restartHistory.delete(jobId);
|
|
2690
|
+
}
|
|
2691
|
+
}
|
|
2692
|
+
|
|
2693
|
+
private nodesByJob(jobId: JobId): Set<NodeSummary> {
|
|
2694
|
+
// Uses the jobToNode index so the "is this job placed yet?" check
|
|
2695
|
+
// works for multi-job nodes too — the older `tags["jobId"]` scan
|
|
2696
|
+
// always returned empty for cluster workers (the tag is per-node
|
|
2697
|
+
// and can only carry one value), which caused AutoManager to
|
|
2698
|
+
// repeatedly re-attempt placement and the daemon to reject with
|
|
2699
|
+
// `INVALID_ARGUMENT: Job already running on node`.
|
|
2700
|
+
const nodes = new Set<NodeSummary>();
|
|
2701
|
+
const nodeId = this.jobToNode.get(jobId);
|
|
2702
|
+
if (nodeId !== undefined) {
|
|
2703
|
+
const node = this.nodeMap.get(nodeId);
|
|
2704
|
+
if (node) nodes.add(node);
|
|
2705
|
+
}
|
|
2706
|
+
return nodes;
|
|
2707
|
+
}
|
|
2708
|
+
|
|
2709
|
+
/** @internal */
|
|
2710
|
+
constructor(norsk: NorskManager, settings: AutoSettings) {
|
|
2711
|
+
this.norsk = norsk;
|
|
2712
|
+
this.settings = settings;
|
|
2713
|
+
this.clock = settings.clock ?? new RealClock();
|
|
2714
|
+
if (!this.settings.onError) {
|
|
2715
|
+
this.settings.onError = (err) => {
|
|
2716
|
+
debuglog("Unhandled error: %o", err);
|
|
2717
|
+
};
|
|
2718
|
+
}
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2721
|
+
/** @internal */
|
|
2722
|
+
private async runInternal(): Promise<void> {
|
|
2723
|
+
this.startedAt = this.clock.now();
|
|
2724
|
+
const [initialState, iter] = await eventStream(this.norsk, {
|
|
2725
|
+
pendingWindow: this.settings.pendingWindow,
|
|
2726
|
+
});
|
|
2727
|
+
|
|
2728
|
+
// R7: Hydrate state from initial-state. AutoManager owns nothing
|
|
2729
|
+
// durable; clear-then-populate so a reconnect drops anything the
|
|
2730
|
+
// server has since forgotten about. Includes every transient map
|
|
2731
|
+
// we maintain in-process — without this, a reconnect would carry
|
|
2732
|
+
// forward unhealthy/restart/migration/spare state against a fleet
|
|
2733
|
+
// view that may no longer match.
|
|
2734
|
+
this.jobMap.clear();
|
|
2735
|
+
this.nodeMap.clear();
|
|
2736
|
+
this.bundleMap.clear();
|
|
2737
|
+
this.inventoryMap.clear();
|
|
2738
|
+
this.unhealthyNodes.clear();
|
|
2739
|
+
this.recentFailures.clear();
|
|
2740
|
+
this.rejectionsPerNode.clear();
|
|
2741
|
+
this.restartHistory.clear();
|
|
2742
|
+
this.brokenBundles.clear();
|
|
2743
|
+
this.pendingSpareProvisions.clear();
|
|
2744
|
+
this.jobToNode.clear();
|
|
2745
|
+
this.placementsByJob.clear();
|
|
2746
|
+
// Pending placement timers are session-local; the recovery loop
|
|
2747
|
+
// below re-evaluates each hydrated pre/active job and schedules
|
|
2748
|
+
// fresh timers (or places immediately) as appropriate.
|
|
2749
|
+
for (const { timer } of this.pendingPlacementTimers.values()) timer.cancel();
|
|
2750
|
+
this.pendingPlacementTimers.clear();
|
|
2751
|
+
// Same for stop timers — the recovery loop calls reconcileStopTimer
|
|
2752
|
+
// for each hydrated job so the daemon-persisted __stopDateTime tag
|
|
2753
|
+
// re-arms the timer.
|
|
2754
|
+
for (const { timer } of this.pendingStopTimers.values()) timer.cancel();
|
|
2755
|
+
this.pendingStopTimers.clear();
|
|
2756
|
+
for (const t of this.startupGraceTimers.values()) t.cancel();
|
|
2757
|
+
this.startupGraceTimers.clear();
|
|
2758
|
+
// Migrations are in-flight client-side state; on reconnect the
|
|
2759
|
+
// worker-side actors are gone with the old connection, so any
|
|
2760
|
+
// pending migration is effectively dead. Cancel deadlines first
|
|
2761
|
+
// (they refer to timers we're about to cancel anyway), then drop
|
|
2762
|
+
// the state maps.
|
|
2763
|
+
for (const d of this.migrationPhaseTimers.values()) d.cancel();
|
|
2764
|
+
this.migrationPhaseTimers.clear();
|
|
2765
|
+
this.migrationStates.clear();
|
|
2766
|
+
this.migrationByJobId.clear();
|
|
2767
|
+
this.nodeServicePlacements.clear();
|
|
2768
|
+
|
|
2769
|
+
for (const jobWithHistory of initialState.activeJobs) {
|
|
2770
|
+
this.jobMap.set(jobWithHistory.job.jobId, jobWithHistory);
|
|
2771
|
+
// Rebuild jobToNode from history — on reconnect we don't get a
|
|
2772
|
+
// fresh placement-decision stream for jobs that were already
|
|
2773
|
+
// placed; the history is the source of truth.
|
|
2774
|
+
const nodeId = currentNodeIdForJob(jobWithHistory);
|
|
2775
|
+
if (nodeId !== undefined) {
|
|
2776
|
+
this.jobToNode.set(jobWithHistory.job.jobId, nodeId);
|
|
2777
|
+
// Same reasoning for the placement map: a job that was placed
|
|
2778
|
+
// in a prior generation has no live `onPlacementDecision` event
|
|
2779
|
+
// we can replay, so we synthesise a record from history. The
|
|
2780
|
+
// trace is empty (we never had it) but the result + at +
|
|
2781
|
+
// job-identity fields are all derivable.
|
|
2782
|
+
const synth = synthesisePlacementFromHistory(jobWithHistory, nodeId);
|
|
2783
|
+
if (synth) this.placementsByJob.set(jobWithHistory.job.jobId, synth);
|
|
2784
|
+
}
|
|
2785
|
+
// Fire onJobUpdated for each hydrated job so consumers can
|
|
2786
|
+
// treat hydration identically to a stream of fresh
|
|
2787
|
+
// job-updated events. Without this, a consumer subscribed
|
|
2788
|
+
// through onJobUpdated wouldn't see existing jobs until the
|
|
2789
|
+
// next genuine update — proxies that build routing tables
|
|
2790
|
+
// from the event stream would stay empty until the operator
|
|
2791
|
+
// touched something.
|
|
2792
|
+
this.settings.onJobUpdated?.(jobWithHistory);
|
|
2793
|
+
}
|
|
2794
|
+
|
|
2795
|
+
for (const runningNode of initialState.nodesRunning) {
|
|
2796
|
+
const nodeId = runningNode.nodeMetadata.nodeId;
|
|
2797
|
+
this.updateNodeMap(
|
|
2798
|
+
nodeId,
|
|
2799
|
+
"running",
|
|
2800
|
+
runningNode.nodeMetadata,
|
|
2801
|
+
runningNode.runningNodeMetadata
|
|
2802
|
+
);
|
|
2803
|
+
// Surface hydrated nodes to consumers as started — symmetric with
|
|
2804
|
+
// the job-hydration loop above replaying each job via onJobUpdated.
|
|
2805
|
+
// A worker already up before AutoManager (re)connected is delivered
|
|
2806
|
+
// as a `NodeRunning` snapshot, never a live `NodeStarted`; without
|
|
2807
|
+
// this, downstream listeners (e.g. the norsk-mgr Cluster page, which
|
|
2808
|
+
// only refetches on a node-started signal) never learn it joined.
|
|
2809
|
+
const node = this.nodeMap.get(nodeId);
|
|
2810
|
+
if (node) this.settings.onNodeStarted?.(node);
|
|
2811
|
+
}
|
|
2812
|
+
|
|
2813
|
+
for (const ba of initialState.bundles) {
|
|
2814
|
+
if (ba.bundle) {
|
|
2815
|
+
const b = fromPbBundle(ba.bundle);
|
|
2816
|
+
this.bundleMap.set(b.bundleId, b);
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
for (const snap of initialState.inventories) {
|
|
2820
|
+
if (snap.inventory && snap.nodeId) {
|
|
2821
|
+
this.inventoryMap.set(snap.nodeId.nodeId, fromPbNodeInventory(snap.inventory));
|
|
2822
|
+
}
|
|
2823
|
+
}
|
|
2824
|
+
// hotSpares + nodeServices reserved (Phase F).
|
|
2825
|
+
|
|
2826
|
+
// Recover any active jobs that we don't have a node for — these
|
|
2827
|
+
// exist when AutoManager has just (re)connected and the server
|
|
2828
|
+
// tells us about them via initial-state. Route through
|
|
2829
|
+
// `placeOrDefer` so scheduled jobs whose `startDateTime - lead`
|
|
2830
|
+
// is still in the future get a fresh timer; ones whose window
|
|
2831
|
+
// has already arrived (or was missed while mgr was down) place
|
|
2832
|
+
// immediately.
|
|
2833
|
+
for (const entry of this.jobMap.values()) {
|
|
2834
|
+
// Rebuild stop timers from the daemon-persisted __stopDateTime
|
|
2835
|
+
// tag on every hydrated job. Dedupes per bundleId — first job
|
|
2836
|
+
// wins, subsequent jobs in the same bundle are no-ops.
|
|
2837
|
+
this.reconcileStopTimer(entry);
|
|
2838
|
+
if (this.wantsPlacementNow(entry)) {
|
|
2839
|
+
await this.placeOrDefer(entry);
|
|
2840
|
+
}
|
|
2841
|
+
}
|
|
2842
|
+
|
|
2843
|
+
// Periodic cleanup: terminate stopped nodes older than the threshold.
|
|
2844
|
+
// R19: cadence derived from removeStoppedNodesAfter (1/4 of the
|
|
2845
|
+
// threshold, capped to [10s, 5min]) so the sweep is proportionate
|
|
2846
|
+
// to the retention window — tests that lower the threshold to
|
|
2847
|
+
// seconds see the sweep fire quickly without rewiring the cadence.
|
|
2848
|
+
const cleanupCadenceMs = Math.max(
|
|
2849
|
+
10_000,
|
|
2850
|
+
Math.min(
|
|
2851
|
+
300_000,
|
|
2852
|
+
Math.floor((this.settings.removeStoppedNodesAfter * 1000) / 4)
|
|
2853
|
+
)
|
|
2854
|
+
);
|
|
2855
|
+
this.cleanupInterval = this.clock.setInterval(async () => {
|
|
2856
|
+
if (this.closed) return; // R14: don't run cleanup after close()
|
|
2857
|
+
// R13: snapshot entries up front so concurrent mutations during
|
|
2858
|
+
// the await terminateNode don't reshuffle traversal.
|
|
2859
|
+
const snapshot = Array.from(this.nodeMap.entries());
|
|
2860
|
+
for (const [nodeId, node] of snapshot) {
|
|
2861
|
+
if (this.closed) return;
|
|
2862
|
+
if (
|
|
2863
|
+
node.nodeState === "stopped" &&
|
|
2864
|
+
this.clock.now().getTime() - node.lastActivity.getTime() >
|
|
2865
|
+
this.settings.removeStoppedNodesAfter * 1000
|
|
2866
|
+
) {
|
|
2867
|
+
await this.norsk.terminateNode(nodeId).catch((err) => {
|
|
2868
|
+
this.settings.onError?.(err);
|
|
2869
|
+
});
|
|
2870
|
+
this.nodeMap.delete(nodeId);
|
|
2871
|
+
this.settings.onNodeTerminated?.(nodeId);
|
|
2872
|
+
}
|
|
2873
|
+
}
|
|
2874
|
+
}, cleanupCadenceMs);
|
|
2875
|
+
|
|
2876
|
+
// Phase C6: periodic hot-spare reconciliation tick. Event-driven
|
|
2877
|
+
// reconciles handle most cases; this is the safety net for
|
|
2878
|
+
// missed signals (e.g. a provider that quietly terminated a
|
|
2879
|
+
// spare without Manager emitting nodeStopping).
|
|
2880
|
+
const reconcileInterval =
|
|
2881
|
+
this.settings.hotSpareReconcileIntervalMs ??
|
|
2882
|
+
DEFAULT_HOT_SPARE_RECONCILE_INTERVAL_MS;
|
|
2883
|
+
this.hotSpareReconcileInterval = this.clock.setInterval(
|
|
2884
|
+
() => this.reconcileHotSpares(),
|
|
2885
|
+
reconcileInterval
|
|
2886
|
+
);
|
|
2887
|
+
// Kick once at startup so a fresh AutoManager with hotSpares
|
|
2888
|
+
// configured begins provisioning immediately rather than waiting
|
|
2889
|
+
// for the first event or interval.
|
|
2890
|
+
this.reconcileHotSpares();
|
|
2891
|
+
|
|
2892
|
+
// Phase F: same shape as hot-spare reconcile — periodic tick
|
|
2893
|
+
// for NodeServices, with a startup kick.
|
|
2894
|
+
const nsReconcileInterval =
|
|
2895
|
+
this.settings.nodeServiceReconcileIntervalMs ??
|
|
2896
|
+
DEFAULT_NODE_SERVICE_RECONCILE_INTERVAL_MS;
|
|
2897
|
+
this.nodeServiceReconcileInterval = this.clock.setInterval(
|
|
2898
|
+
() => this.reconcileNodeServices(),
|
|
2899
|
+
nsReconcileInterval
|
|
2900
|
+
);
|
|
2901
|
+
this.reconcileNodeServices();
|
|
2902
|
+
|
|
2903
|
+
for await (const event of iter) {
|
|
2904
|
+
try {
|
|
2905
|
+
const eventType = event.event;
|
|
2906
|
+
switch (eventType) {
|
|
2907
|
+
case "jobUpdated":
|
|
2908
|
+
case "jobPending":
|
|
2909
|
+
await this.handleJobChanged(event.jobWithHistory);
|
|
2910
|
+
break;
|
|
2911
|
+
case "jobDeleted":
|
|
2912
|
+
this.handleJobDeleted(event.jobId);
|
|
2913
|
+
break;
|
|
2914
|
+
case "jobOutOfWindow":
|
|
2915
|
+
this.handleJobDeleted(event.jobWithHistory.job.jobId);
|
|
2916
|
+
break;
|
|
2917
|
+
case "jobInfo":
|
|
2918
|
+
this.settings.onJobInfo?.(event);
|
|
2919
|
+
break;
|
|
2920
|
+
case "nodeStarting":
|
|
2921
|
+
this.handleNodeStarting(event.nodeMetadata);
|
|
2922
|
+
break;
|
|
2923
|
+
case "nodeStarted":
|
|
2924
|
+
this.handleNodeStarted(event.nodeMetadata, event.runningNodeMetadata);
|
|
2925
|
+
break;
|
|
2926
|
+
case "nodeStopping":
|
|
2927
|
+
await this.handleNodeStopping(event.nodeId, event.reason);
|
|
2928
|
+
break;
|
|
2929
|
+
case "nodeStopped":
|
|
2930
|
+
this.handleNodeStopped(event.nodeId);
|
|
2931
|
+
break;
|
|
2932
|
+
case "physicalNodeConnected":
|
|
2933
|
+
// Reserved.
|
|
2934
|
+
break;
|
|
2935
|
+
case "providerHealthChange":
|
|
2936
|
+
debuglog("Provider health change %o", event.health);
|
|
2937
|
+
break;
|
|
2938
|
+
case "eventStreamClosed":
|
|
2939
|
+
break;
|
|
2940
|
+
case "nodeInventoryUpdated":
|
|
2941
|
+
this.handleNodeInventoryUpdated(event.raw);
|
|
2942
|
+
break;
|
|
2943
|
+
case "jobRejected":
|
|
2944
|
+
this.handleJobRejected(event.raw);
|
|
2945
|
+
break;
|
|
2946
|
+
case "jobConfigUpdated":
|
|
2947
|
+
this.handleJobConfigUpdated(event.raw);
|
|
2948
|
+
break;
|
|
2949
|
+
case "bundleUpdated":
|
|
2950
|
+
this.handleBundleUpdated(event.raw);
|
|
2951
|
+
break;
|
|
2952
|
+
case "bundleDeleted":
|
|
2953
|
+
this.handleBundleDeleted(event.raw);
|
|
2954
|
+
break;
|
|
2955
|
+
case "nodeServiceUpdated":
|
|
2956
|
+
case "nodeServiceDeleted":
|
|
2957
|
+
// Phase F (reserved).
|
|
2958
|
+
break;
|
|
2959
|
+
case "migrationTargetReady":
|
|
2960
|
+
await this.handleMigrationTargetReady(event.raw);
|
|
2961
|
+
break;
|
|
2962
|
+
case "migrationSourceStoppedOutput":
|
|
2963
|
+
await this.handleMigrationSourceStoppedOutput(event.raw);
|
|
2964
|
+
break;
|
|
2965
|
+
case "migrationAborted":
|
|
2966
|
+
this.handleMigrationAborted(event.raw);
|
|
2967
|
+
break;
|
|
2968
|
+
case "auditAppended":
|
|
2969
|
+
this.settings.onAuditAppended?.(event.entry);
|
|
2970
|
+
break;
|
|
2971
|
+
default: {
|
|
2972
|
+
const exhaustiveCheck: never = eventType;
|
|
2973
|
+
throw new Error(`Unhandled case: ${exhaustiveCheck}`);
|
|
2974
|
+
}
|
|
2975
|
+
}
|
|
2976
|
+
} catch (err) {
|
|
2977
|
+
this.settings.onError?.(err);
|
|
2978
|
+
}
|
|
2979
|
+
}
|
|
2980
|
+
}
|
|
2981
|
+
|
|
2982
|
+
// ---- job lifecycle ----
|
|
2983
|
+
|
|
2984
|
+
private async handleJobChanged(jobWithHistory: JobWithHistory): Promise<void> {
|
|
2985
|
+
const job = jobWithHistory.job;
|
|
2986
|
+
this.jobMap.set(job.jobId, jobWithHistory);
|
|
2987
|
+
this.settings.onJobUpdated?.(jobWithHistory);
|
|
2988
|
+
// Reconcile the per-bundle stop timer from the well-known tag on
|
|
2989
|
+
// this job. Done before the placement decisions so an already-past
|
|
2990
|
+
// stop time triggers deleteBundle without waiting for placement.
|
|
2991
|
+
this.reconcileStopTimer(jobWithHistory);
|
|
2992
|
+
if (this.wantsPlacementNow(jobWithHistory)) {
|
|
2993
|
+
await this.placeOrDefer(jobWithHistory);
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2996
|
+
|
|
2997
|
+
/** Read the `__stopDateTime` tag off the job, and (re)schedule the
|
|
2998
|
+
* bundle's stop timer accordingly. Tag is set per-job but applies
|
|
2999
|
+
* to the whole bundle — first-job-wins for read; all jobs in a
|
|
3000
|
+
* bundle share the same value at submit. */
|
|
3001
|
+
private reconcileStopTimer(jh: JobWithHistory): void {
|
|
3002
|
+
const tags = jh.job.tags ?? {};
|
|
3003
|
+
const bundleId = tags["bundleId"];
|
|
3004
|
+
if (!bundleId) return;
|
|
3005
|
+
const stopStr = tags[STOP_DATE_TIME_TAG];
|
|
3006
|
+
const stopAt = stopStr ? new Date(stopStr) : undefined;
|
|
3007
|
+
const prev = this.pendingStopTimers.get(bundleId);
|
|
3008
|
+
// If the tag is gone OR unchanged, just keep / drop the existing
|
|
3009
|
+
// timer. Equality on millis is fine — daemon round-trips ISO
|
|
3010
|
+
// strings, no float fuzz.
|
|
3011
|
+
if (!stopAt || Number.isNaN(stopAt.getTime())) {
|
|
3012
|
+
if (prev) {
|
|
3013
|
+
prev.timer.cancel();
|
|
3014
|
+
this.pendingStopTimers.delete(bundleId);
|
|
3015
|
+
}
|
|
3016
|
+
return;
|
|
3017
|
+
}
|
|
3018
|
+
if (prev && prev.stopAt.getTime() === stopAt.getTime()) return;
|
|
3019
|
+
if (prev) prev.timer.cancel();
|
|
3020
|
+
this.scheduleStopTimer(bundleId, stopAt);
|
|
3021
|
+
}
|
|
3022
|
+
|
|
3023
|
+
private scheduleStopTimer(bundleId: BundleId, stopAt: Date): void {
|
|
3024
|
+
const delayMs = stopAt.getTime() - this.clock.now().getTime();
|
|
3025
|
+
if (delayMs <= 0) {
|
|
3026
|
+
// Already past — fire immediately and don't keep a timer entry.
|
|
3027
|
+
debuglog("Stop time already past; deleting bundle now", { bundleId, stopAt: stopAt.toISOString() });
|
|
3028
|
+
void this.deleteBundle(bundleId).catch((e) =>
|
|
3029
|
+
this.settings.onError?.(e instanceof Error ? e : new Error(String(e))),
|
|
3030
|
+
);
|
|
3031
|
+
this.pendingStopTimers.delete(bundleId);
|
|
3032
|
+
return;
|
|
3033
|
+
}
|
|
3034
|
+
debuglog("Scheduling bundle stop", { bundleId, stopAt: stopAt.toISOString(), inMs: delayMs });
|
|
3035
|
+
const timer = this.clock.setTimeout(() => {
|
|
3036
|
+
this.pendingStopTimers.delete(bundleId);
|
|
3037
|
+
void this.deleteBundle(bundleId).catch((e) =>
|
|
3038
|
+
this.settings.onError?.(e instanceof Error ? e : new Error(String(e))),
|
|
3039
|
+
);
|
|
3040
|
+
}, delayMs);
|
|
3041
|
+
this.pendingStopTimers.set(bundleId, { timer, stopAt });
|
|
3042
|
+
}
|
|
3043
|
+
|
|
3044
|
+
/**
|
|
3045
|
+
* Update (or clear) a bundle's auto-stop time. Calls `updateBundle`
|
|
3046
|
+
* so the new value persists via the daemon's tag store and any other
|
|
3047
|
+
* clients see it through their event streams. Pass `undefined` to
|
|
3048
|
+
* cancel a previously-set stop time.
|
|
3049
|
+
*
|
|
3050
|
+
* Throws if the bundle isn't known to this AutoManager (no live
|
|
3051
|
+
* bundleMap entry); operators wanting to set stop on a not-yet-
|
|
3052
|
+
* streamed bundle should retry after JobPending fires.
|
|
3053
|
+
* @public
|
|
3054
|
+
*/
|
|
3055
|
+
public async setBundleStopTime(bundleId: BundleId, stopDateTime?: Date): Promise<void> {
|
|
3056
|
+
const bundle = this.bundleById(bundleId);
|
|
3057
|
+
if (!bundle) {
|
|
3058
|
+
throw new Error(`No bundle ${bundleId} in AutoManager state`);
|
|
3059
|
+
}
|
|
3060
|
+
const updated: TypedBundle = {
|
|
3061
|
+
...bundle,
|
|
3062
|
+
jobs: bundle.jobs.map((j) =>
|
|
3063
|
+
j.launch?.kind === "productTemplate"
|
|
3064
|
+
? { ...j, launch: { ...j.launch, stopDateTime } }
|
|
3065
|
+
: j,
|
|
3066
|
+
),
|
|
3067
|
+
};
|
|
3068
|
+
await this.updateBundle(updated);
|
|
3069
|
+
}
|
|
3070
|
+
|
|
3071
|
+
/**
|
|
3072
|
+
* Snapshot of all currently-scheduled bundle stop timers. UIs render
|
|
3073
|
+
* this alongside `pendingPlacements()` for an "upcoming events"
|
|
3074
|
+
* view.
|
|
3075
|
+
* @public
|
|
3076
|
+
*/
|
|
3077
|
+
public bundleStopTimes(): Array<{ bundleId: BundleId; stopAt: Date }> {
|
|
3078
|
+
return [...this.pendingStopTimers.entries()]
|
|
3079
|
+
.map(([bundleId, { stopAt }]) => ({ bundleId, stopAt }))
|
|
3080
|
+
.sort((a, b) => a.stopAt.getTime() - b.stopAt.getTime());
|
|
3081
|
+
}
|
|
3082
|
+
|
|
3083
|
+
/** A job that should be running but currently has no node bound to
|
|
3084
|
+
* it: it carries placement `requirements`, isn't manual-start, is in
|
|
3085
|
+
* a placeable state (`pre` / `active`), and `nodesByJob` is empty.
|
|
3086
|
+
* This is the single source of truth for "this job wants a placement
|
|
3087
|
+
* and doesn't have one" — shared by initial hydration, the per-job
|
|
3088
|
+
* `handleJobChanged` path, and the capacity-driven reconcile sweep,
|
|
3089
|
+
* so the decision lives in exactly one place. */
|
|
3090
|
+
private wantsPlacementNow(jh: JobWithHistory): boolean {
|
|
3091
|
+
const job = jh.job;
|
|
3092
|
+
if (!job.autoPlacement?.requirements) return false;
|
|
3093
|
+
if ((job.tags ?? {})["startMode"] === "manual") return false;
|
|
3094
|
+
// Durable terminal verdict (poison / bronze drop) persisted by a prior
|
|
3095
|
+
// session: never re-place it. This is what makes "fully failed" survive a
|
|
3096
|
+
// daemon restart — on reconnect the job hydrates with terminalState set.
|
|
3097
|
+
if (job.terminalState) return false;
|
|
3098
|
+
if (job.state !== "pre" && job.state !== "active") return false;
|
|
3099
|
+
return this.nodesByJob(job.jobId).size === 0;
|
|
3100
|
+
}
|
|
3101
|
+
|
|
3102
|
+
/**
|
|
3103
|
+
* Re-run placement for every job that wants a node but hasn't got
|
|
3104
|
+
* one. Called whenever a *capacity-increasing* event arrives — a node
|
|
3105
|
+
* comes online (`nodeStarted`), a worker reports fresh inventory
|
|
3106
|
+
* (`nodeInventoryUpdated`, which also covers an un-cordon or a node
|
|
3107
|
+
* that has just freed reserved capacity), or a job finishes and
|
|
3108
|
+
* releases its node (`jobDeleted`). Those are the only transitions
|
|
3109
|
+
* that can turn a previously-unplaceable job placeable, so they are
|
|
3110
|
+
* exactly when re-running the engine is worthwhile — there is no
|
|
3111
|
+
* polling retry, the trigger is always a concrete event.
|
|
3112
|
+
*
|
|
3113
|
+
* Placement stays implemented once: each candidate routes through
|
|
3114
|
+
* `placeOrDefer` (the same entry point hydration and `handleJobChanged`
|
|
3115
|
+
* use), inheriting its scheduling, the in-flight gate, and the
|
|
3116
|
+
* failure path. Jobs already waiting on a schedule timer or a recovery
|
|
3117
|
+
* backoff are skipped — their own timers own them — so a capacity
|
|
3118
|
+
* event never disturbs a future-scheduled or backing-off job.
|
|
3119
|
+
*
|
|
3120
|
+
* Cheap and idempotent (mirrors `reconcileHotSpares` /
|
|
3121
|
+
* `reconcileNodeServices`); safe to call on every relevant event.
|
|
3122
|
+
*/
|
|
3123
|
+
private reconcilePendingPlacements(): void {
|
|
3124
|
+
for (const jh of this.jobMap.values()) {
|
|
3125
|
+
if (!this.wantsPlacementNow(jh)) continue;
|
|
3126
|
+
const jobId = jh.job.jobId;
|
|
3127
|
+
if (this.pendingPlacementTimers.has(jobId)) continue;
|
|
3128
|
+
if (this.pendingPlacementRetries.has(jobId)) continue;
|
|
3129
|
+
void this.placeOrDefer(jh).catch((err) => this.settings.onError?.(err));
|
|
3130
|
+
}
|
|
3131
|
+
}
|
|
3132
|
+
|
|
3133
|
+
/** Decide whether to place immediately or defer until the job's
|
|
3134
|
+
* scheduled `startDateTime` (minus pool lead time) arrives.
|
|
3135
|
+
* Re-entry from job-update events: cancels and re-evaluates so a
|
|
3136
|
+
* schedule change moves the timer. Hydrated jobs with `startTime -
|
|
3137
|
+
* lead` in the past place immediately (operator intent was to run;
|
|
3138
|
+
* schedule was a "when", not a "drop if missed"). */
|
|
3139
|
+
private async placeOrDefer(jh: JobWithHistory): Promise<void> {
|
|
3140
|
+
const job = jh.job;
|
|
3141
|
+
const requirements = job.autoPlacement?.requirements;
|
|
3142
|
+
if (!requirements) return;
|
|
3143
|
+
const tags = job.tags ?? {};
|
|
3144
|
+
|
|
3145
|
+
// Any previous pending timer for this jobId is now stale — either
|
|
3146
|
+
// we'll re-schedule below or we'll place immediately. Either way,
|
|
3147
|
+
// drop the prior timer.
|
|
3148
|
+
const prev = this.pendingPlacementTimers.get(job.jobId);
|
|
3149
|
+
if (prev) {
|
|
3150
|
+
prev.timer.cancel();
|
|
3151
|
+
this.pendingPlacementTimers.delete(job.jobId);
|
|
3152
|
+
}
|
|
3153
|
+
|
|
3154
|
+
const startDateTime = job.startDateTime;
|
|
3155
|
+
if (!startDateTime) {
|
|
3156
|
+
// No schedule — fire now, current behaviour.
|
|
3157
|
+
await this.placeAndProvision(job.jobId, requirements, tags);
|
|
3158
|
+
return;
|
|
3159
|
+
}
|
|
3160
|
+
|
|
3161
|
+
const leadMs = this.placementLeadMsForJob(jh);
|
|
3162
|
+
const fireAt = startDateTime.getTime() - leadMs;
|
|
3163
|
+
const delayMs = fireAt - this.clock.now().getTime();
|
|
3164
|
+
if (delayMs <= 0) {
|
|
3165
|
+
// Schedule has arrived (or passed — hydration after mgr was
|
|
3166
|
+
// down through the launch window). Operator's intent was for
|
|
3167
|
+
// the job to be running; honour it.
|
|
3168
|
+
await this.placeAndProvision(job.jobId, requirements, tags);
|
|
3169
|
+
return;
|
|
3170
|
+
}
|
|
3171
|
+
|
|
3172
|
+
debuglog("Deferring placement until scheduled startDateTime", {
|
|
3173
|
+
jobId: job.jobId,
|
|
3174
|
+
startDateTime: startDateTime.toISOString(),
|
|
3175
|
+
leadMs,
|
|
3176
|
+
fireInMs: delayMs,
|
|
3177
|
+
});
|
|
3178
|
+
const timer = this.clock.setTimeout(() => {
|
|
3179
|
+
this.pendingPlacementTimers.delete(job.jobId);
|
|
3180
|
+
// Re-check that we still want to place — the job may have been
|
|
3181
|
+
// deleted or another instance placed it during the wait. Cheap
|
|
3182
|
+
// re-test using nodesByJob (now jobToNode-backed and O(1)).
|
|
3183
|
+
const live = this.jobMap.get(job.jobId);
|
|
3184
|
+
if (!live) return;
|
|
3185
|
+
if (this.nodesByJob(job.jobId).size > 0) return;
|
|
3186
|
+
void this.placeAndProvision(job.jobId, requirements, tags);
|
|
3187
|
+
}, delayMs);
|
|
3188
|
+
this.pendingPlacementTimers.set(job.jobId, { timer, scheduledFor: new Date(fireAt) });
|
|
3189
|
+
}
|
|
3190
|
+
|
|
3191
|
+
/** Maximum `placementLeadMs` across the bundle pool's tiers (worst case
|
|
3192
|
+
* — guarantees the job is ready by `startDateTime` regardless of which
|
|
3193
|
+
* tier the placement engine ends up picking). Falls back to the per-kind
|
|
3194
|
+
* default when a tier doesn't specify one. Returns 0 if the bundle or
|
|
3195
|
+
* pool isn't found — we'd rather race the schedule than indefinitely
|
|
3196
|
+
* delay placement. */
|
|
3197
|
+
private placementLeadMsForJob(jh: JobWithHistory): number {
|
|
3198
|
+
const bundleId = (jh.job.tags ?? {})["bundleId"];
|
|
3199
|
+
if (!bundleId) return 0;
|
|
3200
|
+
const bundle = this.bundleMap.get(bundleId);
|
|
3201
|
+
if (!bundle) return 0;
|
|
3202
|
+
const pools = this.settings.placementPools ?? [];
|
|
3203
|
+
const pool = pools.find((p) => p.name === bundle.pool);
|
|
3204
|
+
if (!pool) return 0;
|
|
3205
|
+
let max = 0;
|
|
3206
|
+
for (const tier of pool.tiers) {
|
|
3207
|
+
const lead = tier.placementLeadMs ?? defaultPlacementLeadMs(tier.kind);
|
|
3208
|
+
if (lead > max) max = lead;
|
|
3209
|
+
}
|
|
3210
|
+
return max;
|
|
3211
|
+
}
|
|
3212
|
+
|
|
3213
|
+
/**
|
|
3214
|
+
* Currently-scheduled placements (jobs AutoManager has deferred
|
|
3215
|
+
* because their `startDateTime - leadTime` is still in the future).
|
|
3216
|
+
* Returns one entry per pending timer; the UI can render an
|
|
3217
|
+
* "upcoming jobs" list and show when each will go live.
|
|
3218
|
+
* @public
|
|
3219
|
+
*/
|
|
3220
|
+
public pendingPlacements(): Array<{ jobId: JobId; scheduledFor: Date }> {
|
|
3221
|
+
return [...this.pendingPlacementTimers.entries()]
|
|
3222
|
+
.map(([jobId, { scheduledFor }]) => ({ jobId, scheduledFor }))
|
|
3223
|
+
.sort((a, b) => a.scheduledFor.getTime() - b.scheduledFor.getTime());
|
|
3224
|
+
}
|
|
3225
|
+
|
|
3226
|
+
private handleJobDeleted(jobId: JobId): void {
|
|
3227
|
+
// Pure local-state cleanup. The daemon's deleteBundle saga is now
|
|
3228
|
+
// responsible for stopping the worker-side compose stacks before
|
|
3229
|
+
// emitting JobDeleted; node-level lifecycle (terminate / drain) is
|
|
3230
|
+
// AutoManager policy (drainNode, hot-spare reconciliation) and
|
|
3231
|
+
// intentionally not coupled to job deletion. Prior versions called
|
|
3232
|
+
// `terminateNode` here, which was provider-incorrect for cluster
|
|
3233
|
+
// workers (terminate is a no-op on long-lived operator-attached
|
|
3234
|
+
// nodes; the worker's compose stack would leak as a result).
|
|
3235
|
+
const existed = this.jobMap.delete(jobId);
|
|
3236
|
+
this.restartHistory.delete(jobId);
|
|
3237
|
+
this.lastPlacementAttemptAt.delete(jobId);
|
|
3238
|
+
this.jobToNode.delete(jobId);
|
|
3239
|
+
this.placementsByJob.delete(jobId);
|
|
3240
|
+
const pending = this.pendingPlacementTimers.get(jobId);
|
|
3241
|
+
if (pending) {
|
|
3242
|
+
pending.timer.cancel();
|
|
3243
|
+
this.pendingPlacementTimers.delete(jobId);
|
|
3244
|
+
}
|
|
3245
|
+
const pendingRetry = this.pendingPlacementRetries.get(jobId);
|
|
3246
|
+
if (pendingRetry) {
|
|
3247
|
+
clearTimeout(pendingRetry);
|
|
3248
|
+
this.pendingPlacementRetries.delete(jobId);
|
|
3249
|
+
}
|
|
3250
|
+
if (existed) this.settings.onJobDeleted?.(jobId);
|
|
3251
|
+
// A job leaving frees the capacity it reserved, which may let a
|
|
3252
|
+
// previously-unplaceable job land. The freed capacity becomes
|
|
3253
|
+
// visible to the placement engine once the worker re-reports
|
|
3254
|
+
// inventory (`nodeInventoryUpdated` will reconcile again then); this
|
|
3255
|
+
// trigger covers the case where AutoManager's cached inventory
|
|
3256
|
+
// already reflects the release.
|
|
3257
|
+
this.reconcilePendingPlacements();
|
|
3258
|
+
}
|
|
3259
|
+
|
|
3260
|
+
// ---- node lifecycle ----
|
|
3261
|
+
|
|
3262
|
+
private handleNodeStarting(nodeMetadata: NodeMetadata): void {
|
|
3263
|
+
this.updateNodeMap(nodeMetadata.nodeId, "starting", nodeMetadata, undefined);
|
|
3264
|
+
this.settings.onNodeStarting?.(nodeMetadata.nodeId);
|
|
3265
|
+
this.armStartupGrace(nodeMetadata.nodeId);
|
|
3266
|
+
}
|
|
3267
|
+
|
|
3268
|
+
/** Phase C3. Schedule the grace timer; replace any prior timer. */
|
|
3269
|
+
private armStartupGrace(nodeId: NodeId): void {
|
|
3270
|
+
this.startupGraceTimers.get(nodeId)?.cancel();
|
|
3271
|
+
const grace = this.cfgJobStartupGraceMs();
|
|
3272
|
+
const timer = this.clock.setTimeout(() => {
|
|
3273
|
+
this.startupGraceTimers.delete(nodeId);
|
|
3274
|
+
debuglog("Node startup grace exceeded — terminating", { nodeId, grace });
|
|
3275
|
+
void this.norsk
|
|
3276
|
+
.terminateNode(nodeId)
|
|
3277
|
+
.catch((err) => this.settings.onError?.(err));
|
|
3278
|
+
}, grace);
|
|
3279
|
+
this.startupGraceTimers.set(nodeId, timer);
|
|
3280
|
+
}
|
|
3281
|
+
|
|
3282
|
+
private cancelStartupGrace(nodeId: NodeId): void {
|
|
3283
|
+
this.startupGraceTimers.get(nodeId)?.cancel();
|
|
3284
|
+
this.startupGraceTimers.delete(nodeId);
|
|
3285
|
+
}
|
|
3286
|
+
|
|
3287
|
+
private handleNodeStarted(
|
|
3288
|
+
nodeMetadata: NodeMetadata,
|
|
3289
|
+
runningNodeMetadata: RunningNodeMetadata
|
|
3290
|
+
): void {
|
|
3291
|
+
this.updateNodeMap(
|
|
3292
|
+
nodeMetadata.nodeId,
|
|
3293
|
+
"running",
|
|
3294
|
+
nodeMetadata,
|
|
3295
|
+
runningNodeMetadata
|
|
3296
|
+
);
|
|
3297
|
+
this.cancelStartupGrace(nodeMetadata.nodeId);
|
|
3298
|
+
// Phase C2: a successful start clears restart history for this
|
|
3299
|
+
// job — the "fundamentally broken" diagnosis only fires when no
|
|
3300
|
+
// amount of replacement is helping, so a job that has ever started
|
|
3301
|
+
// is by definition not fundamentally broken.
|
|
3302
|
+
const jobId = getJobIdForNode(nodeMetadata);
|
|
3303
|
+
if (jobId) this.restartHistory.delete(jobId);
|
|
3304
|
+
// Phase C6: a pending spare just landed.
|
|
3305
|
+
this.pendingSpareProvisions.delete(nodeMetadata.nodeId);
|
|
3306
|
+
const node = this.nodeMap.get(nodeMetadata.nodeId);
|
|
3307
|
+
if (node) this.settings.onNodeStarted?.(node);
|
|
3308
|
+
this.reconcileHotSpares();
|
|
3309
|
+
this.reconcileNodeServices();
|
|
3310
|
+
// A node coming online is new capacity — a cluster worker joining or
|
|
3311
|
+
// a hot spare landing can host a job that previously had nowhere to
|
|
3312
|
+
// go. Re-attempt pending placements.
|
|
3313
|
+
this.reconcilePendingPlacements();
|
|
3314
|
+
// R6: a migration target node has come up — advance from
|
|
3315
|
+
// `launching` to `awaitingTargetReady`. The workflow may still
|
|
3316
|
+
// need time to initialise; the awaitingTargetReady deadline now
|
|
3317
|
+
// bounds that step independently from launching.
|
|
3318
|
+
this.advanceMigrationsOnNodeStarted(nodeMetadata.nodeId);
|
|
3319
|
+
}
|
|
3320
|
+
|
|
3321
|
+
private advanceMigrationsOnNodeStarted(nodeId: NodeId): void {
|
|
3322
|
+
for (const state of this.migrationStates.values()) {
|
|
3323
|
+
if (state.targetNodeId !== nodeId) continue;
|
|
3324
|
+
if (state.phase !== "launching") continue;
|
|
3325
|
+
this.advanceMigration(state, "awaitingTargetReady");
|
|
3326
|
+
}
|
|
3327
|
+
}
|
|
3328
|
+
|
|
3329
|
+
private async handleNodeStopping(
|
|
3330
|
+
nodeId: NodeId,
|
|
3331
|
+
_reason: NodeStopping["reason"]
|
|
3332
|
+
): Promise<void> {
|
|
3333
|
+
this.updateNodeMap(nodeId, "stopping", undefined, undefined);
|
|
3334
|
+
this.cancelStartupGrace(nodeId);
|
|
3335
|
+
// Surface to consumers so cluster-membership views (mgr's Cluster
|
|
3336
|
+
// page) refresh promptly. Without this callback, a Ctrl-C'd
|
|
3337
|
+
// cluster worker only surfaces in mgr UI when the eventual
|
|
3338
|
+
// NodeStopped/NodeTerminated arrives — which can be many seconds
|
|
3339
|
+
// later via the daemon's keepalive timeout + cleanup loop.
|
|
3340
|
+
this.settings.onNodeStopping?.(nodeId);
|
|
3341
|
+
// Mark the node as recently-failed so subsequent placements skip it
|
|
3342
|
+
// for `failureBackoffMs`. Cleared when the node next reports a
|
|
3343
|
+
// healthy `nodeInventoryUpdated`.
|
|
3344
|
+
const window = this.cfgFailureBackoffMs();
|
|
3345
|
+
this.recentFailures.set(
|
|
3346
|
+
nodeId,
|
|
3347
|
+
new Date(this.clock.now().getTime() + window)
|
|
3348
|
+
);
|
|
3349
|
+
|
|
3350
|
+
// TODO future slice: thread the nodeStopping reason into
|
|
3351
|
+
// recoveryContext so band-specific behaviour applies (gold→prefer
|
|
3352
|
+
// hot spare, bronze drop, etc.).
|
|
3353
|
+
const node = this.nodeMap.get(nodeId);
|
|
3354
|
+
const jobId = node?.provisional?.jobId ?? getJobIdForNode(node?.nodeMetadata);
|
|
3355
|
+
if (jobId) {
|
|
3356
|
+
// R8: if the job is mid-migration, don't run recovery placement —
|
|
3357
|
+
// the migration target is already a second live instance for the
|
|
3358
|
+
// same jobId, and a fresh placeAndProvision would create a third.
|
|
3359
|
+
// Abort the migration instead; operator can re-attempt once the
|
|
3360
|
+
// source is back. The target on the new node is allowed to keep
|
|
3361
|
+
// running (it's stoppable through the normal jobMap path).
|
|
3362
|
+
const migration = this.migrationForJob(jobId);
|
|
3363
|
+
if (migration && (migration.phase !== "done" && migration.phase !== "aborted")) {
|
|
3364
|
+
this.failMigration(
|
|
3365
|
+
migration.migrationId,
|
|
3366
|
+
`source nodeStopping during phase ${migration.phase}`
|
|
3367
|
+
);
|
|
3368
|
+
} else {
|
|
3369
|
+
const entry = this.jobMap.get(jobId);
|
|
3370
|
+
const job = entry?.job;
|
|
3371
|
+
const req = job?.autoPlacement?.requirements;
|
|
3372
|
+
if (job && req && (job.state === "pre" || job.state === "active")) {
|
|
3373
|
+
await this.placeAndProvision(
|
|
3374
|
+
job.jobId,
|
|
3375
|
+
req,
|
|
3376
|
+
job.tags ?? {},
|
|
3377
|
+
{ excludeNodes: this.currentExcludeSet() }
|
|
3378
|
+
);
|
|
3379
|
+
}
|
|
3380
|
+
}
|
|
3381
|
+
}
|
|
3382
|
+
// Phase C6: if the stopping node was a spare, reconcile to
|
|
3383
|
+
// provision a replacement promptly (don't wait for the tick).
|
|
3384
|
+
this.reconcileHotSpares();
|
|
3385
|
+
// Phase F: lost capacity → reconcile NodeService placements.
|
|
3386
|
+
// Also drop any tracked placement on this node (the runtime job
|
|
3387
|
+
// is gone with the node).
|
|
3388
|
+
this.dropNodeServicePlacementsOnNode(nodeId);
|
|
3389
|
+
this.reconcileNodeServices();
|
|
3390
|
+
}
|
|
3391
|
+
|
|
3392
|
+
private handleNodeStopped(nodeId: NodeId): void {
|
|
3393
|
+
this.updateNodeMap(nodeId, "stopped", undefined, undefined);
|
|
3394
|
+
// Phase C6: a spare going away should trigger a replacement.
|
|
3395
|
+
this.pendingSpareProvisions.delete(nodeId);
|
|
3396
|
+
this.settings.onNodeStopped?.(nodeId);
|
|
3397
|
+
this.reconcileHotSpares();
|
|
3398
|
+
// Phase F: clear NodeService placements on stopped nodes.
|
|
3399
|
+
this.dropNodeServicePlacementsOnNode(nodeId);
|
|
3400
|
+
this.reconcileNodeServices();
|
|
3401
|
+
}
|
|
3402
|
+
|
|
3403
|
+
private dropNodeServicePlacementsOnNode(nodeId: NodeId): void {
|
|
3404
|
+
for (const placements of this.nodeServicePlacements.values()) {
|
|
3405
|
+
placements.delete(nodeId);
|
|
3406
|
+
}
|
|
3407
|
+
}
|
|
3408
|
+
|
|
3409
|
+
// ---- maps ----
|
|
3410
|
+
|
|
3411
|
+
private updateNodeMap(
|
|
3412
|
+
nodeId: NodeId,
|
|
3413
|
+
nodeState: NodeState,
|
|
3414
|
+
nodeMetadata: NodeMetadata | undefined,
|
|
3415
|
+
runningNodeMetadata: RunningNodeMetadata | undefined,
|
|
3416
|
+
provisional?: { jobId: JobId; role: Role }
|
|
3417
|
+
): void {
|
|
3418
|
+
const existing = this.nodeMap.get(nodeId);
|
|
3419
|
+
this.nodeMap.set(nodeId, {
|
|
3420
|
+
nodeId,
|
|
3421
|
+
nodeState,
|
|
3422
|
+
lastActivity: this.clock.now(),
|
|
3423
|
+
nodeMetadata: nodeMetadata ?? existing?.nodeMetadata,
|
|
3424
|
+
runningNodeMetadata: runningNodeMetadata ?? existing?.runningNodeMetadata,
|
|
3425
|
+
provisional: provisional ?? existing?.provisional,
|
|
3426
|
+
});
|
|
3427
|
+
}
|
|
3428
|
+
}
|
|
3429
|
+
|
|
3430
|
+
function summariseTrace(trace: PlacementTrace): string {
|
|
3431
|
+
// One-line per attempted pool, listing how nodes were filtered
|
|
3432
|
+
// (grouped by reason) and what happened. Compact enough that it
|
|
3433
|
+
// doesn't drown a debuglog stream when many placements happen in a
|
|
3434
|
+
// burst, but informative enough that "why did my job land there?"
|
|
3435
|
+
// is immediately answerable.
|
|
3436
|
+
const lines = trace.attempts.map((a) => {
|
|
3437
|
+
const groups: Record<string, number> = {};
|
|
3438
|
+
for (const f of a.filtered) groups[f.reason] = (groups[f.reason] ?? 0) + 1;
|
|
3439
|
+
const filterSummary = Object.entries(groups)
|
|
3440
|
+
.map(([reason, count]) => `${reason}=${count}`)
|
|
3441
|
+
.join(", ");
|
|
3442
|
+
const winner =
|
|
3443
|
+
a.outcome.kind === "placed"
|
|
3444
|
+
? ` → placed on ${a.outcome.nodeId}${a.outcome.gpuIndex !== undefined ? ` (gpu ${a.outcome.gpuIndex})` : ""}`
|
|
3445
|
+
: a.outcome.kind === "provisioned"
|
|
3446
|
+
? ` → provisioned ${a.outcome.instanceType}`
|
|
3447
|
+
: a.outcome.kind === "exhausted"
|
|
3448
|
+
? " → fixed-pool exhausted, fall through"
|
|
3449
|
+
: a.outcome.kind === "no-instance-type"
|
|
3450
|
+
? " → no allowed instance type fits, fall through"
|
|
3451
|
+
: " → pool not configured";
|
|
3452
|
+
const matchSummary = `${a.scored.length}/${a.initialCandidates} matched`;
|
|
3453
|
+
const filterPart = filterSummary ? `, filtered: ${filterSummary}` : "";
|
|
3454
|
+
return `[${a.pool}] ${matchSummary}${filterPart}${winner}`;
|
|
3455
|
+
});
|
|
3456
|
+
return `${trace.decision}: ${lines.join(" || ")}`;
|
|
3457
|
+
}
|
|
3458
|
+
|
|
3459
|
+
function roleFromReplicaIndex(replicaIndex: number): Role {
|
|
3460
|
+
// Preserve the existing primary/backup convention for replicas 0 and 1
|
|
3461
|
+
// so consumers that grep on role keep working. Higher replica counts
|
|
3462
|
+
// get a numeric form.
|
|
3463
|
+
if (replicaIndex === 0) return "primary";
|
|
3464
|
+
if (replicaIndex === 1) return "backup";
|
|
3465
|
+
return `replica-${replicaIndex}`;
|
|
3466
|
+
}
|
|
3467
|
+
|
|
3468
|
+
function debuglog(msg: string, ...param: unknown[]) {
|
|
3469
|
+
util.debuglog("norsk-automanager")(
|
|
3470
|
+
util.formatWithOptions(
|
|
3471
|
+
{ maxArrayLength: null, depth: null, colors: true },
|
|
3472
|
+
"[" + new Date().toISOString() + "] " + msg,
|
|
3473
|
+
...param
|
|
3474
|
+
)
|
|
3475
|
+
);
|
|
3476
|
+
}
|
|
3477
|
+
|
|
3478
|
+
export function getJobIdForNode(nodeMetadata: NodeMetadata | undefined): JobId | undefined {
|
|
3479
|
+
return nodeMetadata?.tags["jobId"];
|
|
3480
|
+
}
|
|
3481
|
+
|
|
3482
|
+
/**
|
|
3483
|
+
* Walk a job's history backward and return the nodeId where it is
|
|
3484
|
+
* currently placed (or was most recently placed). Prefers `running`
|
|
3485
|
+
* entries; falls back to `provisioned` for jobs still booting their
|
|
3486
|
+
* node. Returns undefined for jobs that have never been placed or
|
|
3487
|
+
* whose history is empty (defensive — runtime data always has at
|
|
3488
|
+
* least a `created` entry, which carries no nodeMetadata).
|
|
3489
|
+
*
|
|
3490
|
+
* This is the canonical "where does this job live" lookup and
|
|
3491
|
+
* underpins `AutoManager.nodeForJob`. It works regardless of whether
|
|
3492
|
+
* a node hosts one job (cloud, today) or many (cluster, or future
|
|
3493
|
+
* cloud N:1) because the nodeId is recorded against the JOB rather
|
|
3494
|
+
* than scanned out of NODE tags.
|
|
3495
|
+
*/
|
|
3496
|
+
export function currentNodeIdForJob(jh: JobWithHistory | undefined): NodeId | undefined {
|
|
3497
|
+
if (!jh) return undefined;
|
|
3498
|
+
// Two-pass: prefer the most recent `running` over the most recent
|
|
3499
|
+
// `provisioned`. A job that has been provisioned, run, stopped,
|
|
3500
|
+
// re-provisioned, but not yet run again should report the newest
|
|
3501
|
+
// provisioned node — but if it's running we always want that.
|
|
3502
|
+
let mostRecentProvisioned: NodeId | undefined;
|
|
3503
|
+
for (let i = jh.history.length - 1; i >= 0; i--) {
|
|
3504
|
+
const e = jh.history[i];
|
|
3505
|
+
if (e.event === "running") return e.nodeMetadata.nodeId;
|
|
3506
|
+
if (e.event === "provisioned" && mostRecentProvisioned === undefined) {
|
|
3507
|
+
mostRecentProvisioned = e.nodeMetadata.nodeId;
|
|
3508
|
+
}
|
|
3509
|
+
}
|
|
3510
|
+
return mostRecentProvisioned;
|
|
3511
|
+
}
|
|
3512
|
+
|
|
3513
|
+
/**
|
|
3514
|
+
* Build a `PlacementDecisionInfo` from a hydrated job's history. Used
|
|
3515
|
+
* on R7 reconnect to seed the placementsByJob map for jobs that were
|
|
3516
|
+
* placed in a previous generation — we have no live trace, so this
|
|
3517
|
+
* synthesises a minimal record (empty `trace.attempts`) that's still
|
|
3518
|
+
* shaped like a real decision so consumers can render it uniformly.
|
|
3519
|
+
*
|
|
3520
|
+
* `nodeId` is passed in by the caller (already resolved via
|
|
3521
|
+
* `currentNodeIdForJob`), to avoid walking the history twice.
|
|
3522
|
+
*
|
|
3523
|
+
* Returns undefined if the job's tags are too sparse to derive
|
|
3524
|
+
* `bundleId` / `replicaIndex` / `jobName` (un-managed jobs the
|
|
3525
|
+
* AutoManager-driven UI couldn't usefully display anyway).
|
|
3526
|
+
*/
|
|
3527
|
+
function synthesisePlacementFromHistory(jh: JobWithHistory, nodeId: NodeId): PlacementDecisionInfo | undefined {
|
|
3528
|
+
const tags = jh.job.tags ?? {};
|
|
3529
|
+
const bundleId = tags["bundleId"];
|
|
3530
|
+
const replicaIndexStr = tags["replicaIndex"];
|
|
3531
|
+
const jobName = tags["jobName"];
|
|
3532
|
+
if (!bundleId || replicaIndexStr === undefined || !jobName) return undefined;
|
|
3533
|
+
const replicaIndex = Number(replicaIndexStr);
|
|
3534
|
+
if (!Number.isFinite(replicaIndex)) return undefined;
|
|
3535
|
+
|
|
3536
|
+
// Find the most recent placement event for `at` + decision kind.
|
|
3537
|
+
// running > provisioned (same precedence as currentNodeIdForJob).
|
|
3538
|
+
// If the only event we found is `provisioned`, decision="provision"
|
|
3539
|
+
// (the node was started for this job); `running` could be either —
|
|
3540
|
+
// we treat it as "place" because by the time we hydrate the node is
|
|
3541
|
+
// already running, so functionally the job was bound to an existing
|
|
3542
|
+
// running node. The trace stays empty either way.
|
|
3543
|
+
let at: Date | undefined;
|
|
3544
|
+
let decision: "place" | "provision" = "place";
|
|
3545
|
+
for (let i = jh.history.length - 1; i >= 0; i--) {
|
|
3546
|
+
const e = jh.history[i];
|
|
3547
|
+
if (e.event === "running") {
|
|
3548
|
+
at = e.timestamp instanceof Date ? e.timestamp : new Date(e.timestamp);
|
|
3549
|
+
decision = "place";
|
|
3550
|
+
break;
|
|
3551
|
+
}
|
|
3552
|
+
if (e.event === "provisioned" && at === undefined) {
|
|
3553
|
+
at = e.timestamp instanceof Date ? e.timestamp : new Date(e.timestamp);
|
|
3554
|
+
decision = "provision";
|
|
3555
|
+
}
|
|
3556
|
+
}
|
|
3557
|
+
if (!at) return undefined;
|
|
3558
|
+
|
|
3559
|
+
// `pool`, `tier` and `instanceType` aren't on the history entries —
|
|
3560
|
+
// leave them as best-guess placeholders. Consumers that care about
|
|
3561
|
+
// accurate pool/tier attribution should walk the node's tags via
|
|
3562
|
+
// nodeMap.get(nodeId).nodeMetadata.tags.{pool,tier}.
|
|
3563
|
+
const result: PlacementResult =
|
|
3564
|
+
decision === "place"
|
|
3565
|
+
? { kind: "place", nodeId, pool: "", tier: "" }
|
|
3566
|
+
: { kind: "provision", pool: "", tier: "", instanceType: "", launchMode: "on-demand" };
|
|
3567
|
+
const trace: PlacementTrace = { attempts: [], decision };
|
|
3568
|
+
return { jobId: jh.job.jobId, bundleId, replicaIndex, jobName, result, trace, at };
|
|
3569
|
+
}
|
|
3570
|
+
|