@animastor/orchestration 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +75 -0
- package/package.json +51 -0
- package/src/host/host-bindings.js +138 -0
- package/src/index.js +99 -0
- package/src/orchestration/index.js +40 -0
- package/src/orchestration/orchestrator.js +630 -0
- package/src/orchestration/runtime-result-consumer.js +54 -0
- package/src/orchestration/scene-callbacks.js +542 -0
- package/src/orchestration/scene-orchestrator.js +601 -0
- package/src/orchestration/scene-restoration.js +129 -0
- package/src/orchestration/scene-utils.js +36 -0
- package/src/runtime/active-scenes-index.js +164 -0
- package/src/runtime/audio-fsm-port.js +187 -0
- package/src/runtime/circuit-breaker.js +588 -0
- package/src/runtime/counter-reconciliation.js +308 -0
- package/src/runtime/dispatch-engine.js +1768 -0
- package/src/runtime/failure-taxonomy.js +125 -0
- package/src/runtime/hub-cancel-port.js +122 -0
- package/src/runtime/layer-config-port.js +136 -0
- package/src/runtime/lease-manager.js +607 -0
- package/src/runtime/persistence-port.js +162 -0
- package/src/runtime/placeholder-audio-port.js +117 -0
- package/src/runtime/progress-events-port.js +121 -0
- package/src/runtime/reconciliation-engine.js +2405 -0
- package/src/runtime/retry-budget-manager.js +229 -0
- package/src/runtime/runtime-metrics.js +427 -0
- package/src/runtime/runtime-result-emitter.js +123 -0
- package/src/runtime/runtime-scheduler.js +629 -0
- package/src/runtime/scene-data-port.js +114 -0
- package/src/runtime/scene-window.js +827 -0
- package/src/runtime/video-fsm-port.js +189 -0
- package/src/runtime/worker-health.js +270 -0
- package/test/counter-reconciliation.test.js +164 -0
- package/test/mocks/redis-mock.js +221 -0
- package/test/test-bindings.js +60 -0
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
// ======================================================
|
|
2
|
+
// O-8 PORT: VideoFsmPort — runtime/orchestration
|
|
3
|
+
// video-FSM contract
|
|
4
|
+
// ======================================================
|
|
5
|
+
// The runtime and orchestration tiers must not know the video-orchestrator
|
|
6
|
+
// host service. The Redis key grammar (`animastor:video-orch:*`), the JSON
|
|
7
|
+
// state envelope, the valid-transition map, the group-file validation
|
|
8
|
+
// threshold and the merge/source-cap pipeline details stay HOST-side
|
|
9
|
+
// (backend/src/services/video-orchestrator.js); the tiers drive the video
|
|
10
|
+
// scene FSM ONLY through this port, wired by the composition root
|
|
11
|
+
// (backend.cjs) at startup.
|
|
12
|
+
//
|
|
13
|
+
// runtime/** + orchestration/** (tier consumers)
|
|
14
|
+
// ↓ consume video-FSM ops through
|
|
15
|
+
// runtime/video-fsm-port.js ← THIS PORT (tier-owned contract)
|
|
16
|
+
// ↑ wired by the host
|
|
17
|
+
// storage/video-fsm-adapter (host FSM adapter)
|
|
18
|
+
//
|
|
19
|
+
// This is the O-2/O-3/O-4/O-5/O-7 port convention (runtime/persistence-
|
|
20
|
+
// port.js, runtime/scene-data-port.js, runtime/placeholder-audio-port.js,
|
|
21
|
+
// runtime/progress-events-port.js, runtime/audio-fsm-port.js): a zero-
|
|
22
|
+
// require contract module, a fail-fast call-time resolver, one set/wired/
|
|
23
|
+
// reset surface. No Redis key strings, no JSON state encoding, no
|
|
24
|
+
// transition-map internals and no "future ops" cross the boundary — the
|
|
25
|
+
// contract below is EXACTLY the set of operations the two tiers use today
|
|
26
|
+
// (measured at O-8 reconnaissance, §32.25; the O-6 audit had measured
|
|
27
|
+
// "14 ops" for the combined MediaFsmPort video half — the tiers' actual
|
|
28
|
+
// usage is 13 ops + the phase constants; setMerging/allGroupsDone are
|
|
29
|
+
// host-only internals the tiers never touch):
|
|
30
|
+
//
|
|
31
|
+
// getState — read the video-FSM phase record (dispatch
|
|
32
|
+
// cache-hit snapshot, invariant checks)
|
|
33
|
+
// initState — create/replace the per-dispatch group state
|
|
34
|
+
// (scene-orchestrator executeVideoDispatch)
|
|
35
|
+
// setWaitingChunks — GENERATING → WAITING_CHUNKS before jobs are sent
|
|
36
|
+
// setDone — → DONE recovery leg (MERGING with merged file)
|
|
37
|
+
// setFailed — → FAILED (orchestrator failStage facade sync,
|
|
38
|
+
// recovery legs)
|
|
39
|
+
// deleteState — drop a stale/unknown-phase record (recovery
|
|
40
|
+
// unknown-phase sweep)
|
|
41
|
+
// markGroupDone — mark a cache-hit group done in state
|
|
42
|
+
// groupSuffixes — list group suffixes from a state record
|
|
43
|
+
// groupFilePath — resolve a group file path (stale-file unlink
|
|
44
|
+
// before re-dispatch)
|
|
45
|
+
// isGroupFileValid — validate a group file on disk (cache-hit check,
|
|
46
|
+
// recovery completeness)
|
|
47
|
+
// completeGroup — drive group-completeness + merge from a group
|
|
48
|
+
// result / recovery (host-owned merge logic)
|
|
49
|
+
// failWaitingScene — WAITING_CHUNKS → FAILED with hub-dedup cleanup
|
|
50
|
+
// (stall watchdog, recovery)
|
|
51
|
+
// scanAllStates — enumerate every non-terminal record (stall
|
|
52
|
+
// watchdog, startup recovery)
|
|
53
|
+
// PHASES — the phase-name constants the tiers compare
|
|
54
|
+
// orchState.phase against (NEW, GENERATING,
|
|
55
|
+
// WAITING_CHUNKS, MERGING, DONE, FAILED). Phase
|
|
56
|
+
// NAMES are the FSM's public vocabulary; everything
|
|
57
|
+
// about their STORAGE (key grammar, envelope,
|
|
58
|
+
// transitions) stays host-side in the adapter.
|
|
59
|
+
//
|
|
60
|
+
// Optional-load semantics are the ADAPTER's concern, not the port's: the
|
|
61
|
+
// reconciliation optional-load legs (try/catch → return 0 when the host
|
|
62
|
+
// service is absent) keep their exact pre-O-8 behavior by wrapping the
|
|
63
|
+
// ADAPTER's lazy require, not by relaxing this fail-fast contract.
|
|
64
|
+
//
|
|
65
|
+
// Deliberately NOT in the port: the audio FSM (runtime/audio-fsm-port —
|
|
66
|
+
// the O-7 seam; two independent FSMs never merge, the O-6 MediaFsmPort
|
|
67
|
+
// rejection stands), key()/PREFIX/createState/transitionState/setMerging/
|
|
68
|
+
// allGroupsDone internals (Redis/storage knowledge), any combined
|
|
69
|
+
// MediaFsmPort surface and any CRUD "for later".
|
|
70
|
+
//
|
|
71
|
+
// Docs: docs/architecture/generation-module-extraction-reconnaissance.md §32.25 (O-8 DONE)
|
|
72
|
+
// ======================================================
|
|
73
|
+
|
|
74
|
+
const OPS = [
|
|
75
|
+
'getState',
|
|
76
|
+
'initState',
|
|
77
|
+
'setWaitingChunks',
|
|
78
|
+
'setDone',
|
|
79
|
+
'setFailed',
|
|
80
|
+
'deleteState',
|
|
81
|
+
'markGroupDone',
|
|
82
|
+
'groupSuffixes',
|
|
83
|
+
'groupFilePath',
|
|
84
|
+
'isGroupFileValid',
|
|
85
|
+
'completeGroup',
|
|
86
|
+
'failWaitingScene',
|
|
87
|
+
'scanAllStates',
|
|
88
|
+
];
|
|
89
|
+
|
|
90
|
+
// Phase names the tier consumers compare against — the FSM's public
|
|
91
|
+
// vocabulary (a frozen set: adding one is a contract change, not a key
|
|
92
|
+
// grammar change — the adapter owns the storage behind these names).
|
|
93
|
+
const PHASE_CONSTANTS = [
|
|
94
|
+
'NEW',
|
|
95
|
+
'GENERATING',
|
|
96
|
+
'WAITING_CHUNKS',
|
|
97
|
+
'MERGING',
|
|
98
|
+
'DONE',
|
|
99
|
+
'FAILED',
|
|
100
|
+
];
|
|
101
|
+
|
|
102
|
+
let impl = null;
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Wire the host video-FSM adapter. Called once by the composition root
|
|
106
|
+
* (backend.cjs) BEFORE any runtime/orchestration video-FSM consumer runs.
|
|
107
|
+
* The adapter must expose every op in OPS as a function; the PHASES
|
|
108
|
+
* constants resolve lazily at call time (videoFsmPhases()) so wiring never
|
|
109
|
+
* forces a host-service load — the O-2..O-7 lazy-resolution discipline.
|
|
110
|
+
* @param {object} adapter — flat map with every op in OPS as a function
|
|
111
|
+
* plus a PHASES constant object (validated on first phase read, not here).
|
|
112
|
+
*/
|
|
113
|
+
function setVideoFsmPort(adapter) {
|
|
114
|
+
if (!adapter) {
|
|
115
|
+
throw new Error('video-fsm-port: adapter is required');
|
|
116
|
+
}
|
|
117
|
+
for (const op of OPS) {
|
|
118
|
+
if (typeof adapter[op] !== 'function') {
|
|
119
|
+
throw new Error(`video-fsm-port: adapter must implement ${op}()`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (!('PHASES' in adapter)) {
|
|
123
|
+
throw new Error('video-fsm-port: adapter must carry the PHASES constants');
|
|
124
|
+
}
|
|
125
|
+
impl = adapter;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Resolve the port at CALL time — fail-fast when unwired. */
|
|
129
|
+
function videoFsm() {
|
|
130
|
+
if (!impl) {
|
|
131
|
+
throw new Error(
|
|
132
|
+
'video-fsm-port: not wired — the composition root must call ' +
|
|
133
|
+
'setVideoFsmPort(adapter) (storage/video-fsm-adapter) ' +
|
|
134
|
+
'before runtime/orchestration video-FSM consumers run (O-8)'
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
return impl;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Shorthand op resolver: `videoFsmOp('getState')(...)`.
|
|
142
|
+
* Fails fast when the port or the single op is missing — a missing phase
|
|
143
|
+
* writer silently degrading into a no-op would corrupt the video FSM
|
|
144
|
+
* (ghost WAITING_CHUNKS scenes, lost recovery legs).
|
|
145
|
+
* @param {string} op — op name from OPS
|
|
146
|
+
*/
|
|
147
|
+
function videoFsmOp(op) {
|
|
148
|
+
const fn = impl?.[op];
|
|
149
|
+
if (typeof fn !== 'function') {
|
|
150
|
+
throw new Error(`video-fsm-port: op '${op}' is not wired (O-8)`);
|
|
151
|
+
}
|
|
152
|
+
return fn;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Phase constants through the port (fail-fast at CALL time — never loaded
|
|
156
|
+
* during composition-root wiring, preserving the lazy-resolution parity
|
|
157
|
+
* with test stubs). */
|
|
158
|
+
function videoFsmPhases() {
|
|
159
|
+
const phases = videoFsm().PHASES;
|
|
160
|
+
if (!phases || typeof phases !== 'object') {
|
|
161
|
+
throw new Error('video-fsm-port: PHASES is not wired (O-8)');
|
|
162
|
+
}
|
|
163
|
+
for (const phase of PHASE_CONSTANTS) {
|
|
164
|
+
if (!(phase in phases)) {
|
|
165
|
+
throw new Error(`video-fsm-port: PHASES is missing the ${phase} constant (O-8)`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return phases;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function isVideoFsmPortWired() {
|
|
172
|
+
return impl !== null;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Test hygiene: drop the wiring (mocha hooks). */
|
|
176
|
+
function _resetVideoFsmPort() {
|
|
177
|
+
impl = null;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
module.exports = {
|
|
181
|
+
OPS,
|
|
182
|
+
PHASE_CONSTANTS,
|
|
183
|
+
setVideoFsmPort,
|
|
184
|
+
videoFsm,
|
|
185
|
+
videoFsmOp,
|
|
186
|
+
videoFsmPhases,
|
|
187
|
+
isVideoFsmPortWired,
|
|
188
|
+
_resetVideoFsmPort,
|
|
189
|
+
};
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
// ======================================================
|
|
2
|
+
// Worker Health - v3.0.0 (fail-closed visibility)
|
|
3
|
+
// ======================================================
|
|
4
|
+
// Two independent characteristics per worker:
|
|
5
|
+
// 1. liveness — ONLINE/OFFLINE (fresh heartbeat key);
|
|
6
|
+
// 2. access scope — SYSTEM / SHARE / PRIVATE (heartbeat payload).
|
|
7
|
+
//
|
|
8
|
+
// RULE: a heartbeat NEVER means "available to the current caller" by itself.
|
|
9
|
+
// Availability is always liveness ∧ scope.
|
|
10
|
+
//
|
|
11
|
+
// Heartbeat keys `animastor:worker:heartbeat:<type>:<worker_id>` carry a JSON
|
|
12
|
+
// payload authored by the GPU hub (PW-4: only after credential auth):
|
|
13
|
+
// { type, worker_id, ts, current_job_id,
|
|
14
|
+
// workspace_id: <uuid>|null, mode: 'private'|'share'|'system', ... }
|
|
15
|
+
//
|
|
16
|
+
// Scope classification — FAIL CLOSED:
|
|
17
|
+
// mode 'system' → SYSTEM pool (Animastor-operated);
|
|
18
|
+
// mode 'share' → SHARE pool (community capacity);
|
|
19
|
+
// both count as global capacity;
|
|
20
|
+
// mode 'private' + workspace_id set → PRIVATE worker — visible/countable
|
|
21
|
+
// ONLY for its owning workspace;
|
|
22
|
+
// mode 'private' + active public share policy (SH-1, D3)
|
|
23
|
+
// → PRIVATE **and** global capacity
|
|
24
|
+
// (double count is the normative D3
|
|
25
|
+
// decision — see below);
|
|
26
|
+
// anything else (mode missing/unknown, private without workspace)
|
|
27
|
+
// → UNAUTHORIZED — counted NOWHERE.
|
|
28
|
+
//
|
|
29
|
+
// D3 (worker-sharing-model-design.md §7.4): `system.*` is a CAPACITY
|
|
30
|
+
// indicator — "workers currently able to serve the system pool" — NOT a
|
|
31
|
+
// physical inventory. A private worker with an active public share policy
|
|
32
|
+
// genuinely is able to serve the system pool, so it counts in BOTH the
|
|
33
|
+
// owner's private_* bucket and the global pool. Restated invariant:
|
|
34
|
+
// "a private worker WITHOUT an active public policy is never in the global
|
|
35
|
+
// count". Policy-LESS heartbeats are classified exactly as before — the
|
|
36
|
+
// share_policy field is optional (forward/backward compatibility).
|
|
37
|
+
//
|
|
38
|
+
// The marker must come from a hub that ran with the kill-switch ON: the hub
|
|
39
|
+
// only ever writes `share_policy` into heartbeats when SHARE_FEATURES_ENABLED
|
|
40
|
+
// is on and the policy is active — so a disabled switch degrades counts to
|
|
41
|
+
// today's behavior automatically.
|
|
42
|
+
//
|
|
43
|
+
// Expiry is re-checked on read (expires_at carried in the marker): a stale
|
|
44
|
+
// heartbeat cannot extend a policy past its expires_at.
|
|
45
|
+
// A missing/invalid credential never becomes SYSTEM or SHARE: the hub no
|
|
46
|
+
// longer writes scope-less heartbeats, and this module refuses to count
|
|
47
|
+
// them as defense-in-depth.
|
|
48
|
+
|
|
49
|
+
const { lazyHostBinding } = require('../host/host-bindings');
|
|
50
|
+
// §32.30: former host requires resolve lazily through the host-bindings seam.
|
|
51
|
+
const config = lazyHostBinding('config');
|
|
52
|
+
|
|
53
|
+
const logPrefix = '[WORKER]';
|
|
54
|
+
|
|
55
|
+
function log(msg) {
|
|
56
|
+
console.log(`${logPrefix} ${msg}`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Report a worker heartbeat: mark a worker as alive.
|
|
61
|
+
* Kept for completeness — in production the GPU hub writes heartbeats itself
|
|
62
|
+
* (beacon/claim/result/error) and is the author of the scope fields.
|
|
63
|
+
* `scope` = { workspaceId, mode }; absent → SYSTEM worker.
|
|
64
|
+
*/
|
|
65
|
+
async function reportHeartbeat(redis, type, workerId, currentJobId = null, scope = {}) {
|
|
66
|
+
const key = config.WORKER_HEARTBEAT_KEY(type, workerId);
|
|
67
|
+
const payload = JSON.stringify({
|
|
68
|
+
type,
|
|
69
|
+
worker_id: workerId,
|
|
70
|
+
ts: Date.now(),
|
|
71
|
+
current_job_id: currentJobId || null,
|
|
72
|
+
workspace_id: scope.workspaceId || null,
|
|
73
|
+
mode: scope.mode || null
|
|
74
|
+
});
|
|
75
|
+
await redis.set(key, payload, 'EX', config.WORKER_HEARTBEAT_TTL);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Parse a heartbeat payload; null on missing/corrupt/shapeless data. */
|
|
79
|
+
function parseHeartbeat(raw) {
|
|
80
|
+
if (!raw) return null;
|
|
81
|
+
try {
|
|
82
|
+
const data = JSON.parse(raw);
|
|
83
|
+
if (!data || typeof data !== 'object' || typeof data.ts !== 'number') return null;
|
|
84
|
+
return data;
|
|
85
|
+
} catch (_) {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* All FRESH heartbeat entries of a type (liveness-filtered; scope fields kept).
|
|
92
|
+
* B7: SCAN вместо keys() — не блокируем Redis.
|
|
93
|
+
*/
|
|
94
|
+
async function scanFreshHeartbeats(redis, type) {
|
|
95
|
+
const pattern = config.WORKER_HEARTBEAT_TYPE_PATTERN(type);
|
|
96
|
+
const keys = [];
|
|
97
|
+
let cursor = '0';
|
|
98
|
+
do {
|
|
99
|
+
const [nextCursor, batch] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 200);
|
|
100
|
+
cursor = nextCursor;
|
|
101
|
+
keys.push(...batch);
|
|
102
|
+
} while (cursor !== '0');
|
|
103
|
+
|
|
104
|
+
const now = Date.now();
|
|
105
|
+
const maxAge = config.WORKER_HEARTBEAT_TTL * 1000;
|
|
106
|
+
const entries = [];
|
|
107
|
+
for (const key of keys) {
|
|
108
|
+
const data = parseHeartbeat(await redis.get(key));
|
|
109
|
+
if (!data) continue;
|
|
110
|
+
if (now - data.ts >= maxAge) continue;
|
|
111
|
+
entries.push(data);
|
|
112
|
+
}
|
|
113
|
+
return entries;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* SH-1 (D3): is the heartbeat's optional share_policy marker ACTIVE at
|
|
118
|
+
* read time? Fail closed: malformed marker, non-public scope, an
|
|
119
|
+
* expires_at in the past (relative to the read — a stale marker can never
|
|
120
|
+
* extend a policy) or the kill-switch OFF (bit-for-bit today's counting,
|
|
121
|
+
* §8.7 — stale marker heartbeats left over from a flag flip are ignored)
|
|
122
|
+
* → not active.
|
|
123
|
+
*/
|
|
124
|
+
function hasActiveSharePolicyMarker(entry, now = Date.now()) {
|
|
125
|
+
if (config.shareFeaturesEnabled() !== true) return false; // kill-switch
|
|
126
|
+
const p = entry && entry.share_policy;
|
|
127
|
+
if (!p || typeof p !== 'object') return false;
|
|
128
|
+
if (p.scope_kind !== 'public') return false; // V1: public only
|
|
129
|
+
if (p.expires_at == null) return true; // NULL = "until stopped"
|
|
130
|
+
return Number(p.expires_at) > now; // expiry re-checked on read
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Global capacity scope: Animastor-operated SYSTEM workers and volunteered
|
|
135
|
+
* SHARE workers. FAIL CLOSED: a heartbeat without an explicit mode is NEVER
|
|
136
|
+
* global capacity — it is UNAUTHORIZED and counted nowhere.
|
|
137
|
+
* D3: a policy-active PRIVATE worker is able to serve the system pool and
|
|
138
|
+
* therefore counts as global capacity too (double count with private_*).
|
|
139
|
+
*/
|
|
140
|
+
function isSystemScope(entry, now = Date.now()) {
|
|
141
|
+
if (entry.mode === 'system' || entry.mode === 'share') return true;
|
|
142
|
+
if (entry.mode === 'private' && hasActiveSharePolicyMarker(entry, now)) return true;
|
|
143
|
+
return false;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** PRIVATE scope of exactly one workspace (share/system are never private). */
|
|
147
|
+
function isPrivateScopeOf(entry, workspaceId) {
|
|
148
|
+
return !!workspaceId && entry.mode === 'private' && entry.workspace_id === workspaceId;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function countWhere(entries, pred, busyOnly = false) {
|
|
152
|
+
let n = 0;
|
|
153
|
+
for (const e of entries) {
|
|
154
|
+
if (!pred(e)) continue;
|
|
155
|
+
if (busyOnly && !e.current_job_id) continue;
|
|
156
|
+
n++;
|
|
157
|
+
}
|
|
158
|
+
return n;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Global/system capacity: alive workers serving the common pool.
|
|
163
|
+
* PRIVATE workers without an active public policy are never counted here —
|
|
164
|
+
* ONLINE ≠ available to everyone. Policy-active private workers count (D3).
|
|
165
|
+
*/
|
|
166
|
+
async function getAliveCount(redis, type) {
|
|
167
|
+
const entries = await scanFreshHeartbeats(redis, type);
|
|
168
|
+
return countWhere(entries, (e) => isSystemScope(e, Date.now()));
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Alive PRIVATE workers of one workspace (never visible to other workspaces). */
|
|
172
|
+
async function getPrivateAliveCount(redis, type, workspaceId) {
|
|
173
|
+
if (!workspaceId) return 0;
|
|
174
|
+
const entries = await scanFreshHeartbeats(redis, type);
|
|
175
|
+
return countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId));
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Busy (current_job_id set) workers of the system/shared pool. */
|
|
179
|
+
async function getBusyCount(redis, type) {
|
|
180
|
+
const entries = await scanFreshHeartbeats(redis, type);
|
|
181
|
+
return countWhere(entries, (e) => isSystemScope(e, Date.now()), true);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Busy PRIVATE workers of one workspace. */
|
|
185
|
+
async function getPrivateBusyCount(redis, type, workspaceId) {
|
|
186
|
+
if (!workspaceId) return 0;
|
|
187
|
+
const entries = await scanFreshHeartbeats(redis, type);
|
|
188
|
+
return countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId), true);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* One-pass availability snapshot for a caller (single SCAN per type):
|
|
193
|
+
* system.* — the global/shared pool (what every caller may use); per D3
|
|
194
|
+
* this includes policy-active private workers of ANY owner;
|
|
195
|
+
* private.* — the caller's OWN private workers (null workspace → zeros).
|
|
196
|
+
* available.* — PHYSICAL union visible to this caller: system pool ∪ own
|
|
197
|
+
* private workers, each PHYSICAL worker counted ONCE
|
|
198
|
+
* (deduplicated by worker_id). This is the "how many workers
|
|
199
|
+
* can I use" metric for UI counters — it deliberately does
|
|
200
|
+
* NOT sum the overlapping D3 buckets (a policy-active private
|
|
201
|
+
* worker is one physical unit, not two). Scheduler/capacity
|
|
202
|
+
* consumers keep using system.* / private.* unchanged.
|
|
203
|
+
* Foreign private workers appear in the system buckets ONLY when they carry
|
|
204
|
+
* an active public share policy (D3) — never in this caller's private.*.
|
|
205
|
+
*/
|
|
206
|
+
async function getAvailability(redis, workspaceId = null) {
|
|
207
|
+
const out = {
|
|
208
|
+
system: {}, system_busy: {},
|
|
209
|
+
private: {}, private_busy: {},
|
|
210
|
+
available: {}, available_busy: {},
|
|
211
|
+
};
|
|
212
|
+
const now = Date.now();
|
|
213
|
+
for (const type of config.WORKER_HEARTBEAT_TYPES) {
|
|
214
|
+
const entries = await scanFreshHeartbeats(redis, type);
|
|
215
|
+
out.system[type] = countWhere(entries, (e) => isSystemScope(e, now));
|
|
216
|
+
out.system_busy[type] = countWhere(entries, (e) => isSystemScope(e, now), true);
|
|
217
|
+
out.private[type] = countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId));
|
|
218
|
+
out.private_busy[type] = countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId), true);
|
|
219
|
+
// Physical union: dedupe by worker_id — a heartbeat key IS one physical
|
|
220
|
+
// worker; sharing changes access, never the number of units.
|
|
221
|
+
const usable = new Set();
|
|
222
|
+
const usableBusy = new Set();
|
|
223
|
+
for (const e of entries) {
|
|
224
|
+
if (!e || !e.worker_id) continue;
|
|
225
|
+
const mine = isPrivateScopeOf(e, workspaceId);
|
|
226
|
+
if (!mine && !isSystemScope(e, now)) continue;
|
|
227
|
+
usable.add(e.worker_id);
|
|
228
|
+
if (e.current_job_id) usableBusy.add(e.worker_id);
|
|
229
|
+
}
|
|
230
|
+
out.available[type] = usable.size;
|
|
231
|
+
out.available_busy[type] = usableBusy.size;
|
|
232
|
+
}
|
|
233
|
+
return out;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Global/system status of all worker types: { audio, image, video }.
|
|
238
|
+
* System/shared pool ONLY — private workers never inflate global capacity.
|
|
239
|
+
*/
|
|
240
|
+
async function getStatus(redis) {
|
|
241
|
+
const status = {};
|
|
242
|
+
for (const type of config.WORKER_HEARTBEAT_TYPES) {
|
|
243
|
+
status[type] = await getAliveCount(redis, type);
|
|
244
|
+
}
|
|
245
|
+
return status;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Availability for a caller: at least one system worker alive, OR (when the
|
|
250
|
+
* caller's workspace is known) at least one of the workspace's OWN private
|
|
251
|
+
* workers alive. Foreign private workers never contribute.
|
|
252
|
+
*/
|
|
253
|
+
async function isAvailable(redis, type, workspaceId = null) {
|
|
254
|
+
const entries = await scanFreshHeartbeats(redis, type);
|
|
255
|
+
const now = Date.now();
|
|
256
|
+
if (countWhere(entries, (e) => isSystemScope(e, now)) > 0) return true;
|
|
257
|
+
if (!workspaceId) return false;
|
|
258
|
+
return countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId)) > 0;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
module.exports = {
|
|
262
|
+
reportHeartbeat,
|
|
263
|
+
getAliveCount,
|
|
264
|
+
getPrivateAliveCount,
|
|
265
|
+
getBusyCount,
|
|
266
|
+
getPrivateBusyCount,
|
|
267
|
+
getAvailability,
|
|
268
|
+
getStatus,
|
|
269
|
+
isAvailable
|
|
270
|
+
};
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// ======================================================
|
|
2
|
+
// Counter Reconciliation Tests (B10 verification)
|
|
3
|
+
// ======================================================
|
|
4
|
+
|
|
5
|
+
const { expect } = require('chai');
|
|
6
|
+
const { createMockRedis } = require('./mocks/redis-mock');
|
|
7
|
+
const counterReconciliation = require('../src/runtime/counter-reconciliation');
|
|
8
|
+
|
|
9
|
+
describe('Counter Reconciliation', () => {
|
|
10
|
+
let redis;
|
|
11
|
+
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
redis = createMockRedis();
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
// ── correctCounterWithLua ─────────────────────────
|
|
17
|
+
describe('correctCounterWithLua (B10 regression)', () => {
|
|
18
|
+
it('corrects a drifted counter value', async () => {
|
|
19
|
+
// Simulate drift: counter says 5, but only 3 leases exist
|
|
20
|
+
await redis.set('animastor:runtime:active-audio', '5');
|
|
21
|
+
|
|
22
|
+
const result = await counterReconciliation.correctCounterWithLua(redis, 'audio', 3);
|
|
23
|
+
expect(result.success).to.equal(true);
|
|
24
|
+
expect(result.action).to.equal('corrected');
|
|
25
|
+
expect(result.old).to.equal('5');
|
|
26
|
+
expect(result.new).to.equal('3');
|
|
27
|
+
|
|
28
|
+
// Verify the counter was actually corrected
|
|
29
|
+
const val = await redis.get('animastor:runtime:active-audio');
|
|
30
|
+
expect(val).to.equal('3');
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it('corrects when counter is zero but leases exist', async () => {
|
|
34
|
+
// Drift: counter is 0, but lease scan finds 2
|
|
35
|
+
// Actually the correct behavior is: counter is too LOW
|
|
36
|
+
await redis.set('animastor:runtime:active-image', '0');
|
|
37
|
+
|
|
38
|
+
const result = await counterReconciliation.correctCounterWithLua(redis, 'image', 2);
|
|
39
|
+
expect(result.success).to.equal(true);
|
|
40
|
+
expect(result.old).to.equal('0');
|
|
41
|
+
expect(result.new).to.equal('2');
|
|
42
|
+
|
|
43
|
+
const val = await redis.get('animastor:runtime:active-image');
|
|
44
|
+
expect(val).to.equal('2');
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it('corrects when counter key does not exist', async () => {
|
|
48
|
+
const result = await counterReconciliation.correctCounterWithLua(redis, 'video', 1);
|
|
49
|
+
expect(result.success).to.equal(true);
|
|
50
|
+
expect(result.old).to.equal(null);
|
|
51
|
+
|
|
52
|
+
const val = await redis.get('animastor:runtime:active-video');
|
|
53
|
+
expect(val).to.equal('1');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it('B10 bug: ioredis null → "" no longer causes silent no-op', async () => {
|
|
57
|
+
// This test verifies that the fix for B10 works:
|
|
58
|
+
// Previously, correctCounterWithLua passed `null` as an "expected" guard arg,
|
|
59
|
+
// which ioredis serialized to "" (truthy in Lua), causing the guard to
|
|
60
|
+
// always fire and skip the SET. The guard was removed.
|
|
61
|
+
await redis.set('animastor:runtime:active-audio', '10');
|
|
62
|
+
|
|
63
|
+
const result = await counterReconciliation.correctCounterWithLua(redis, 'audio', 7);
|
|
64
|
+
expect(result.success).to.equal(true);
|
|
65
|
+
expect(result.old).to.equal('10');
|
|
66
|
+
expect(result.new).to.equal('7');
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
// ── getCounterWithDriftCheck ──────────────────────
|
|
71
|
+
describe('getCounterWithDriftCheck', () => {
|
|
72
|
+
it('reports no drift when counter matches lease count', async () => {
|
|
73
|
+
const result = await counterReconciliation.getCounterWithDriftCheck(redis, 'audio');
|
|
74
|
+
expect(result.stage).to.equal('audio');
|
|
75
|
+
expect(result.leaseCount).to.equal(0);
|
|
76
|
+
expect(result.counterValue).to.equal(0);
|
|
77
|
+
expect(result.drift).to.equal(0);
|
|
78
|
+
expect(result.correct).to.equal(true);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('detects positive drift (counter > actual leases)', async () => {
|
|
82
|
+
// Set counter higher with no leases
|
|
83
|
+
await redis.set('animastor:runtime:active-audio', '3');
|
|
84
|
+
|
|
85
|
+
const result = await counterReconciliation.getCounterWithDriftCheck(redis, 'audio');
|
|
86
|
+
expect(result.drift).to.equal(3);
|
|
87
|
+
expect(result.correct).to.equal(false);
|
|
88
|
+
expect(result.counterValue).to.equal(3);
|
|
89
|
+
expect(result.leaseCount).to.equal(0);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
// ── reconcileCounters ─────────────────────────────
|
|
94
|
+
describe('reconcileCounters', () => {
|
|
95
|
+
it('reports no corrections when counters are balanced', async () => {
|
|
96
|
+
const report = await counterReconciliation.reconcileCounters(redis);
|
|
97
|
+
|
|
98
|
+
expect(report.summary.correctedCount).to.equal(0);
|
|
99
|
+
expect(report.summary.totalDrift).to.equal(0);
|
|
100
|
+
for (const stage of ['audio', 'image', 'video']) {
|
|
101
|
+
expect(report.stages[stage].corrected).to.equal(false);
|
|
102
|
+
expect(report.stages[stage].drift).to.equal(0);
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it('corrects drifted counters across all stages', async () => {
|
|
107
|
+
// Create drift: audio=2, image=3, video=1
|
|
108
|
+
await redis.set('animastor:runtime:active-audio', '2');
|
|
109
|
+
await redis.set('animastor:runtime:active-image', '3');
|
|
110
|
+
await redis.set('animastor:runtime:active-video', '1');
|
|
111
|
+
|
|
112
|
+
const report = await counterReconciliation.reconcileCounters(redis);
|
|
113
|
+
|
|
114
|
+
// All 3 should be corrected to 0 (no leases in mock)
|
|
115
|
+
expect(report.summary.correctedCount).to.equal(3);
|
|
116
|
+
|
|
117
|
+
// Verify counters are corrected
|
|
118
|
+
expect(await redis.get('animastor:runtime:active-audio')).to.equal('0');
|
|
119
|
+
expect(await redis.get('animastor:runtime:active-image')).to.equal('0');
|
|
120
|
+
expect(await redis.get('animastor:runtime:active-video')).to.equal('0');
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('corrects counter even with non-numeric initial value', async () => {
|
|
124
|
+
await redis.set('animastor:runtime:active-audio', 'not-a-number');
|
|
125
|
+
|
|
126
|
+
const report = await counterReconciliation.reconcileCounters(redis);
|
|
127
|
+
// Counter should be corrected to 0 (no leases in mock)
|
|
128
|
+
expect(report.stages.audio.corrected).to.equal(true);
|
|
129
|
+
expect(await redis.get('animastor:runtime:active-audio')).to.equal('0');
|
|
130
|
+
});
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
// ── countActiveLeasesByStage ──────────────────────
|
|
134
|
+
describe('countActiveLeasesByStage', () => {
|
|
135
|
+
it('returns 0 when no leases exist', async () => {
|
|
136
|
+
const count = await counterReconciliation.countActiveLeasesByStage(redis, 'audio');
|
|
137
|
+
expect(count).to.equal(0);
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
// ── getCurrentCounter ────────────────────────────
|
|
142
|
+
describe('getCurrentCounter', () => {
|
|
143
|
+
it('returns 0 for missing key', async () => {
|
|
144
|
+
const val = await counterReconciliation.getCurrentCounter(redis, 'audio');
|
|
145
|
+
expect(val).to.equal(0);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it('returns stored integer value', async () => {
|
|
149
|
+
await redis.set('animastor:runtime:active-audio', '42');
|
|
150
|
+
const val = await counterReconciliation.getCurrentCounter(redis, 'audio');
|
|
151
|
+
expect(val).to.equal(42);
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
// ── manualCounterCorrection ───────────────────────
|
|
156
|
+
describe('manualCounterCorrection', () => {
|
|
157
|
+
it('sets counter to exact value', async () => {
|
|
158
|
+
await redis.set('animastor:runtime:active-audio', '99');
|
|
159
|
+
const result = await counterReconciliation.manualCounterCorrection(redis, 'audio', 5);
|
|
160
|
+
expect(result.success).to.equal(true);
|
|
161
|
+
expect(await redis.get('animastor:runtime:active-audio')).to.equal('5');
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
});
|