@animastor/orchestration 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +75 -0
  3. package/package.json +51 -0
  4. package/src/host/host-bindings.js +138 -0
  5. package/src/index.js +99 -0
  6. package/src/orchestration/index.js +40 -0
  7. package/src/orchestration/orchestrator.js +630 -0
  8. package/src/orchestration/runtime-result-consumer.js +54 -0
  9. package/src/orchestration/scene-callbacks.js +542 -0
  10. package/src/orchestration/scene-orchestrator.js +601 -0
  11. package/src/orchestration/scene-restoration.js +129 -0
  12. package/src/orchestration/scene-utils.js +36 -0
  13. package/src/runtime/active-scenes-index.js +164 -0
  14. package/src/runtime/audio-fsm-port.js +187 -0
  15. package/src/runtime/circuit-breaker.js +588 -0
  16. package/src/runtime/counter-reconciliation.js +308 -0
  17. package/src/runtime/dispatch-engine.js +1768 -0
  18. package/src/runtime/failure-taxonomy.js +125 -0
  19. package/src/runtime/hub-cancel-port.js +122 -0
  20. package/src/runtime/layer-config-port.js +136 -0
  21. package/src/runtime/lease-manager.js +607 -0
  22. package/src/runtime/persistence-port.js +162 -0
  23. package/src/runtime/placeholder-audio-port.js +117 -0
  24. package/src/runtime/progress-events-port.js +121 -0
  25. package/src/runtime/reconciliation-engine.js +2405 -0
  26. package/src/runtime/retry-budget-manager.js +229 -0
  27. package/src/runtime/runtime-metrics.js +427 -0
  28. package/src/runtime/runtime-result-emitter.js +123 -0
  29. package/src/runtime/runtime-scheduler.js +629 -0
  30. package/src/runtime/scene-data-port.js +114 -0
  31. package/src/runtime/scene-window.js +827 -0
  32. package/src/runtime/video-fsm-port.js +189 -0
  33. package/src/runtime/worker-health.js +270 -0
  34. package/test/counter-reconciliation.test.js +164 -0
  35. package/test/mocks/redis-mock.js +221 -0
  36. package/test/test-bindings.js +60 -0
@@ -0,0 +1,189 @@
1
+ // ======================================================
2
+ // O-8 PORT: VideoFsmPort — runtime/orchestration
3
+ // video-FSM contract
4
+ // ======================================================
5
+ // The runtime and orchestration tiers must not know the video-orchestrator
6
+ // host service. The Redis key grammar (`animastor:video-orch:*`), the JSON
7
+ // state envelope, the valid-transition map, the group-file validation
8
+ // threshold and the merge/source-cap pipeline details stay HOST-side
9
+ // (backend/src/services/video-orchestrator.js); the tiers drive the video
10
+ // scene FSM ONLY through this port, wired by the composition root
11
+ // (backend.cjs) at startup.
12
+ //
13
+ // runtime/** + orchestration/** (tier consumers)
14
+ // ↓ consume video-FSM ops through
15
+ // runtime/video-fsm-port.js ← THIS PORT (tier-owned contract)
16
+ // ↑ wired by the host
17
+ // storage/video-fsm-adapter (host FSM adapter)
18
+ //
19
+ // This is the O-2/O-3/O-4/O-5/O-7 port convention (runtime/persistence-
20
+ // port.js, runtime/scene-data-port.js, runtime/placeholder-audio-port.js,
21
+ // runtime/progress-events-port.js, runtime/audio-fsm-port.js): a zero-
22
+ // require contract module, a fail-fast call-time resolver, one set/wired/
23
+ // reset surface. No Redis key strings, no JSON state encoding, no
24
+ // transition-map internals and no "future ops" cross the boundary — the
25
+ // contract below is EXACTLY the set of operations the two tiers use today
26
+ // (measured at O-8 reconnaissance, §32.25; the O-6 audit had measured
27
+ // "14 ops" for the combined MediaFsmPort video half — the tiers' actual
28
+ // usage is 13 ops + the phase constants; setMerging/allGroupsDone are
29
+ // host-only internals the tiers never touch):
30
+ //
31
+ // getState — read the video-FSM phase record (dispatch
32
+ // cache-hit snapshot, invariant checks)
33
+ // initState — create/replace the per-dispatch group state
34
+ // (scene-orchestrator executeVideoDispatch)
35
+ // setWaitingChunks — GENERATING → WAITING_CHUNKS before jobs are sent
36
+ // setDone — → DONE recovery leg (MERGING with merged file)
37
+ // setFailed — → FAILED (orchestrator failStage facade sync,
38
+ // recovery legs)
39
+ // deleteState — drop a stale/unknown-phase record (recovery
40
+ // unknown-phase sweep)
41
+ // markGroupDone — mark a cache-hit group done in state
42
+ // groupSuffixes — list group suffixes from a state record
43
+ // groupFilePath — resolve a group file path (stale-file unlink
44
+ // before re-dispatch)
45
+ // isGroupFileValid — validate a group file on disk (cache-hit check,
46
+ // recovery completeness)
47
+ // completeGroup — drive group-completeness + merge from a group
48
+ // result / recovery (host-owned merge logic)
49
+ // failWaitingScene — WAITING_CHUNKS → FAILED with hub-dedup cleanup
50
+ // (stall watchdog, recovery)
51
+ // scanAllStates — enumerate every non-terminal record (stall
52
+ // watchdog, startup recovery)
53
+ // PHASES — the phase-name constants the tiers compare
54
+ // orchState.phase against (NEW, GENERATING,
55
+ // WAITING_CHUNKS, MERGING, DONE, FAILED). Phase
56
+ // NAMES are the FSM's public vocabulary; everything
57
+ // about their STORAGE (key grammar, envelope,
58
+ // transitions) stays host-side in the adapter.
59
+ //
60
+ // Optional-load semantics are the ADAPTER's concern, not the port's: the
61
+ // reconciliation optional-load legs (try/catch → return 0 when the host
62
+ // service is absent) keep their exact pre-O-8 behavior by wrapping the
63
+ // ADAPTER's lazy require, not by relaxing this fail-fast contract.
64
+ //
65
+ // Deliberately NOT in the port: the audio FSM (runtime/audio-fsm-port —
66
+ // the O-7 seam; two independent FSMs never merge, the O-6 MediaFsmPort
67
+ // rejection stands), key()/PREFIX/createState/transitionState/setMerging/
68
+ // allGroupsDone internals (Redis/storage knowledge), any combined
69
+ // MediaFsmPort surface and any CRUD "for later".
70
+ //
71
+ // Docs: docs/architecture/generation-module-extraction-reconnaissance.md §32.25 (O-8 DONE)
72
+ // ======================================================
73
+
74
+ const OPS = [
75
+ 'getState',
76
+ 'initState',
77
+ 'setWaitingChunks',
78
+ 'setDone',
79
+ 'setFailed',
80
+ 'deleteState',
81
+ 'markGroupDone',
82
+ 'groupSuffixes',
83
+ 'groupFilePath',
84
+ 'isGroupFileValid',
85
+ 'completeGroup',
86
+ 'failWaitingScene',
87
+ 'scanAllStates',
88
+ ];
89
+
90
+ // Phase names the tier consumers compare against — the FSM's public
91
+ // vocabulary (a frozen set: adding one is a contract change, not a key
92
+ // grammar change — the adapter owns the storage behind these names).
93
+ const PHASE_CONSTANTS = [
94
+ 'NEW',
95
+ 'GENERATING',
96
+ 'WAITING_CHUNKS',
97
+ 'MERGING',
98
+ 'DONE',
99
+ 'FAILED',
100
+ ];
101
+
102
+ let impl = null;
103
+
104
+ /**
105
+ * Wire the host video-FSM adapter. Called once by the composition root
106
+ * (backend.cjs) BEFORE any runtime/orchestration video-FSM consumer runs.
107
+ * The adapter must expose every op in OPS as a function; the PHASES
108
+ * constants resolve lazily at call time (videoFsmPhases()) so wiring never
109
+ * forces a host-service load — the O-2..O-7 lazy-resolution discipline.
110
+ * @param {object} adapter — flat map with every op in OPS as a function
111
+ * plus a PHASES constant object (validated on first phase read, not here).
112
+ */
113
+ function setVideoFsmPort(adapter) {
114
+ if (!adapter) {
115
+ throw new Error('video-fsm-port: adapter is required');
116
+ }
117
+ for (const op of OPS) {
118
+ if (typeof adapter[op] !== 'function') {
119
+ throw new Error(`video-fsm-port: adapter must implement ${op}()`);
120
+ }
121
+ }
122
+ if (!('PHASES' in adapter)) {
123
+ throw new Error('video-fsm-port: adapter must carry the PHASES constants');
124
+ }
125
+ impl = adapter;
126
+ }
127
+
128
+ /** Resolve the port at CALL time — fail-fast when unwired. */
129
+ function videoFsm() {
130
+ if (!impl) {
131
+ throw new Error(
132
+ 'video-fsm-port: not wired — the composition root must call ' +
133
+ 'setVideoFsmPort(adapter) (storage/video-fsm-adapter) ' +
134
+ 'before runtime/orchestration video-FSM consumers run (O-8)'
135
+ );
136
+ }
137
+ return impl;
138
+ }
139
+
140
+ /**
141
+ * Shorthand op resolver: `videoFsmOp('getState')(...)`.
142
+ * Fails fast when the port or the single op is missing — a missing phase
143
+ * writer silently degrading into a no-op would corrupt the video FSM
144
+ * (ghost WAITING_CHUNKS scenes, lost recovery legs).
145
+ * @param {string} op — op name from OPS
146
+ */
147
+ function videoFsmOp(op) {
148
+ const fn = impl?.[op];
149
+ if (typeof fn !== 'function') {
150
+ throw new Error(`video-fsm-port: op '${op}' is not wired (O-8)`);
151
+ }
152
+ return fn;
153
+ }
154
+
155
+ /** Phase constants through the port (fail-fast at CALL time — never loaded
156
+ * during composition-root wiring, preserving the lazy-resolution parity
157
+ * with test stubs). */
158
+ function videoFsmPhases() {
159
+ const phases = videoFsm().PHASES;
160
+ if (!phases || typeof phases !== 'object') {
161
+ throw new Error('video-fsm-port: PHASES is not wired (O-8)');
162
+ }
163
+ for (const phase of PHASE_CONSTANTS) {
164
+ if (!(phase in phases)) {
165
+ throw new Error(`video-fsm-port: PHASES is missing the ${phase} constant (O-8)`);
166
+ }
167
+ }
168
+ return phases;
169
+ }
170
+
171
+ function isVideoFsmPortWired() {
172
+ return impl !== null;
173
+ }
174
+
175
+ /** Test hygiene: drop the wiring (mocha hooks). */
176
+ function _resetVideoFsmPort() {
177
+ impl = null;
178
+ }
179
+
180
+ module.exports = {
181
+ OPS,
182
+ PHASE_CONSTANTS,
183
+ setVideoFsmPort,
184
+ videoFsm,
185
+ videoFsmOp,
186
+ videoFsmPhases,
187
+ isVideoFsmPortWired,
188
+ _resetVideoFsmPort,
189
+ };
@@ -0,0 +1,270 @@
1
+ // ======================================================
2
+ // Worker Health - v3.0.0 (fail-closed visibility)
3
+ // ======================================================
4
+ // Two independent characteristics per worker:
5
+ // 1. liveness — ONLINE/OFFLINE (fresh heartbeat key);
6
+ // 2. access scope — SYSTEM / SHARE / PRIVATE (heartbeat payload).
7
+ //
8
+ // RULE: a heartbeat NEVER means "available to the current caller" by itself.
9
+ // Availability is always liveness ∧ scope.
10
+ //
11
+ // Heartbeat keys `animastor:worker:heartbeat:<type>:<worker_id>` carry a JSON
12
+ // payload authored by the GPU hub (PW-4: only after credential auth):
13
+ // { type, worker_id, ts, current_job_id,
14
+ // workspace_id: <uuid>|null, mode: 'private'|'share'|'system', ... }
15
+ //
16
+ // Scope classification — FAIL CLOSED:
17
+ // mode 'system' → SYSTEM pool (Animastor-operated);
18
+ // mode 'share' → SHARE pool (community capacity);
19
+ // both count as global capacity;
20
+ // mode 'private' + workspace_id set → PRIVATE worker — visible/countable
21
+ // ONLY for its owning workspace;
22
+ // mode 'private' + active public share policy (SH-1, D3)
23
+ // → PRIVATE **and** global capacity
24
+ // (double count is the normative D3
25
+ // decision — see below);
26
+ // anything else (mode missing/unknown, private without workspace)
27
+ // → UNAUTHORIZED — counted NOWHERE.
28
+ //
29
+ // D3 (worker-sharing-model-design.md §7.4): `system.*` is a CAPACITY
30
+ // indicator — "workers currently able to serve the system pool" — NOT a
31
+ // physical inventory. A private worker with an active public share policy
32
+ // genuinely is able to serve the system pool, so it counts in BOTH the
33
+ // owner's private_* bucket and the global pool. Restated invariant:
34
+ // "a private worker WITHOUT an active public policy is never in the global
35
+ // count". Policy-LESS heartbeats are classified exactly as before — the
36
+ // share_policy field is optional (forward/backward compatibility).
37
+ //
38
+ // The marker must come from a hub that ran with the kill-switch ON: the hub
39
+ // only ever writes `share_policy` into heartbeats when SHARE_FEATURES_ENABLED
40
+ // is on and the policy is active — so a disabled switch degrades counts to
41
+ // today's behavior automatically.
42
+ //
43
+ // Expiry is re-checked on read (expires_at carried in the marker): a stale
44
+ // heartbeat cannot extend a policy past its expires_at.
45
+ // A missing/invalid credential never becomes SYSTEM or SHARE: the hub no
46
+ // longer writes scope-less heartbeats, and this module refuses to count
47
+ // them as defense-in-depth.
48
+
49
+ const { lazyHostBinding } = require('../host/host-bindings');
50
+ // §32.30: former host requires resolve lazily through the host-bindings seam.
51
+ const config = lazyHostBinding('config');
52
+
53
+ const logPrefix = '[WORKER]';
54
+
55
+ function log(msg) {
56
+ console.log(`${logPrefix} ${msg}`);
57
+ }
58
+
59
+ /**
60
+ * Report a worker heartbeat: mark a worker as alive.
61
+ * Kept for completeness — in production the GPU hub writes heartbeats itself
62
+ * (beacon/claim/result/error) and is the author of the scope fields.
63
+ * `scope` = { workspaceId, mode }; absent → SYSTEM worker.
64
+ */
65
+ async function reportHeartbeat(redis, type, workerId, currentJobId = null, scope = {}) {
66
+ const key = config.WORKER_HEARTBEAT_KEY(type, workerId);
67
+ const payload = JSON.stringify({
68
+ type,
69
+ worker_id: workerId,
70
+ ts: Date.now(),
71
+ current_job_id: currentJobId || null,
72
+ workspace_id: scope.workspaceId || null,
73
+ mode: scope.mode || null
74
+ });
75
+ await redis.set(key, payload, 'EX', config.WORKER_HEARTBEAT_TTL);
76
+ }
77
+
78
+ /** Parse a heartbeat payload; null on missing/corrupt/shapeless data. */
79
+ function parseHeartbeat(raw) {
80
+ if (!raw) return null;
81
+ try {
82
+ const data = JSON.parse(raw);
83
+ if (!data || typeof data !== 'object' || typeof data.ts !== 'number') return null;
84
+ return data;
85
+ } catch (_) {
86
+ return null;
87
+ }
88
+ }
89
+
90
+ /**
91
+ * All FRESH heartbeat entries of a type (liveness-filtered; scope fields kept).
92
+ * B7: SCAN вместо keys() — не блокируем Redis.
93
+ */
94
+ async function scanFreshHeartbeats(redis, type) {
95
+ const pattern = config.WORKER_HEARTBEAT_TYPE_PATTERN(type);
96
+ const keys = [];
97
+ let cursor = '0';
98
+ do {
99
+ const [nextCursor, batch] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 200);
100
+ cursor = nextCursor;
101
+ keys.push(...batch);
102
+ } while (cursor !== '0');
103
+
104
+ const now = Date.now();
105
+ const maxAge = config.WORKER_HEARTBEAT_TTL * 1000;
106
+ const entries = [];
107
+ for (const key of keys) {
108
+ const data = parseHeartbeat(await redis.get(key));
109
+ if (!data) continue;
110
+ if (now - data.ts >= maxAge) continue;
111
+ entries.push(data);
112
+ }
113
+ return entries;
114
+ }
115
+
116
+ /**
117
+ * SH-1 (D3): is the heartbeat's optional share_policy marker ACTIVE at
118
+ * read time? Fail closed: malformed marker, non-public scope, an
119
+ * expires_at in the past (relative to the read — a stale marker can never
120
+ * extend a policy) or the kill-switch OFF (bit-for-bit today's counting,
121
+ * §8.7 — stale marker heartbeats left over from a flag flip are ignored)
122
+ * → not active.
123
+ */
124
+ function hasActiveSharePolicyMarker(entry, now = Date.now()) {
125
+ if (config.shareFeaturesEnabled() !== true) return false; // kill-switch
126
+ const p = entry && entry.share_policy;
127
+ if (!p || typeof p !== 'object') return false;
128
+ if (p.scope_kind !== 'public') return false; // V1: public only
129
+ if (p.expires_at == null) return true; // NULL = "until stopped"
130
+ return Number(p.expires_at) > now; // expiry re-checked on read
131
+ }
132
+
133
+ /**
134
+ * Global capacity scope: Animastor-operated SYSTEM workers and volunteered
135
+ * SHARE workers. FAIL CLOSED: a heartbeat without an explicit mode is NEVER
136
+ * global capacity — it is UNAUTHORIZED and counted nowhere.
137
+ * D3: a policy-active PRIVATE worker is able to serve the system pool and
138
+ * therefore counts as global capacity too (double count with private_*).
139
+ */
140
+ function isSystemScope(entry, now = Date.now()) {
141
+ if (entry.mode === 'system' || entry.mode === 'share') return true;
142
+ if (entry.mode === 'private' && hasActiveSharePolicyMarker(entry, now)) return true;
143
+ return false;
144
+ }
145
+
146
+ /** PRIVATE scope of exactly one workspace (share/system are never private). */
147
+ function isPrivateScopeOf(entry, workspaceId) {
148
+ return !!workspaceId && entry.mode === 'private' && entry.workspace_id === workspaceId;
149
+ }
150
+
151
+ function countWhere(entries, pred, busyOnly = false) {
152
+ let n = 0;
153
+ for (const e of entries) {
154
+ if (!pred(e)) continue;
155
+ if (busyOnly && !e.current_job_id) continue;
156
+ n++;
157
+ }
158
+ return n;
159
+ }
160
+
161
+ /**
162
+ * Global/system capacity: alive workers serving the common pool.
163
+ * PRIVATE workers without an active public policy are never counted here —
164
+ * ONLINE ≠ available to everyone. Policy-active private workers count (D3).
165
+ */
166
+ async function getAliveCount(redis, type) {
167
+ const entries = await scanFreshHeartbeats(redis, type);
168
+ return countWhere(entries, (e) => isSystemScope(e, Date.now()));
169
+ }
170
+
171
+ /** Alive PRIVATE workers of one workspace (never visible to other workspaces). */
172
+ async function getPrivateAliveCount(redis, type, workspaceId) {
173
+ if (!workspaceId) return 0;
174
+ const entries = await scanFreshHeartbeats(redis, type);
175
+ return countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId));
176
+ }
177
+
178
+ /** Busy (current_job_id set) workers of the system/shared pool. */
179
+ async function getBusyCount(redis, type) {
180
+ const entries = await scanFreshHeartbeats(redis, type);
181
+ return countWhere(entries, (e) => isSystemScope(e, Date.now()), true);
182
+ }
183
+
184
+ /** Busy PRIVATE workers of one workspace. */
185
+ async function getPrivateBusyCount(redis, type, workspaceId) {
186
+ if (!workspaceId) return 0;
187
+ const entries = await scanFreshHeartbeats(redis, type);
188
+ return countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId), true);
189
+ }
190
+
191
+ /**
192
+ * One-pass availability snapshot for a caller (single SCAN per type):
193
+ * system.* — the global/shared pool (what every caller may use); per D3
194
+ * this includes policy-active private workers of ANY owner;
195
+ * private.* — the caller's OWN private workers (null workspace → zeros).
196
+ * available.* — PHYSICAL union visible to this caller: system pool ∪ own
197
+ * private workers, each PHYSICAL worker counted ONCE
198
+ * (deduplicated by worker_id). This is the "how many workers
199
+ * can I use" metric for UI counters — it deliberately does
200
+ * NOT sum the overlapping D3 buckets (a policy-active private
201
+ * worker is one physical unit, not two). Scheduler/capacity
202
+ * consumers keep using system.* / private.* unchanged.
203
+ * Foreign private workers appear in the system buckets ONLY when they carry
204
+ * an active public share policy (D3) — never in this caller's private.*.
205
+ */
206
+ async function getAvailability(redis, workspaceId = null) {
207
+ const out = {
208
+ system: {}, system_busy: {},
209
+ private: {}, private_busy: {},
210
+ available: {}, available_busy: {},
211
+ };
212
+ const now = Date.now();
213
+ for (const type of config.WORKER_HEARTBEAT_TYPES) {
214
+ const entries = await scanFreshHeartbeats(redis, type);
215
+ out.system[type] = countWhere(entries, (e) => isSystemScope(e, now));
216
+ out.system_busy[type] = countWhere(entries, (e) => isSystemScope(e, now), true);
217
+ out.private[type] = countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId));
218
+ out.private_busy[type] = countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId), true);
219
+ // Physical union: dedupe by worker_id — a heartbeat key IS one physical
220
+ // worker; sharing changes access, never the number of units.
221
+ const usable = new Set();
222
+ const usableBusy = new Set();
223
+ for (const e of entries) {
224
+ if (!e || !e.worker_id) continue;
225
+ const mine = isPrivateScopeOf(e, workspaceId);
226
+ if (!mine && !isSystemScope(e, now)) continue;
227
+ usable.add(e.worker_id);
228
+ if (e.current_job_id) usableBusy.add(e.worker_id);
229
+ }
230
+ out.available[type] = usable.size;
231
+ out.available_busy[type] = usableBusy.size;
232
+ }
233
+ return out;
234
+ }
235
+
236
+ /**
237
+ * Global/system status of all worker types: { audio, image, video }.
238
+ * System/shared pool ONLY — private workers never inflate global capacity.
239
+ */
240
+ async function getStatus(redis) {
241
+ const status = {};
242
+ for (const type of config.WORKER_HEARTBEAT_TYPES) {
243
+ status[type] = await getAliveCount(redis, type);
244
+ }
245
+ return status;
246
+ }
247
+
248
+ /**
249
+ * Availability for a caller: at least one system worker alive, OR (when the
250
+ * caller's workspace is known) at least one of the workspace's OWN private
251
+ * workers alive. Foreign private workers never contribute.
252
+ */
253
+ async function isAvailable(redis, type, workspaceId = null) {
254
+ const entries = await scanFreshHeartbeats(redis, type);
255
+ const now = Date.now();
256
+ if (countWhere(entries, (e) => isSystemScope(e, now)) > 0) return true;
257
+ if (!workspaceId) return false;
258
+ return countWhere(entries, (e) => isPrivateScopeOf(e, workspaceId)) > 0;
259
+ }
260
+
261
+ module.exports = {
262
+ reportHeartbeat,
263
+ getAliveCount,
264
+ getPrivateAliveCount,
265
+ getBusyCount,
266
+ getPrivateBusyCount,
267
+ getAvailability,
268
+ getStatus,
269
+ isAvailable
270
+ };
@@ -0,0 +1,164 @@
1
+ // ======================================================
2
+ // Counter Reconciliation Tests (B10 verification)
3
+ // ======================================================
4
+
5
+ const { expect } = require('chai');
6
+ const { createMockRedis } = require('./mocks/redis-mock');
7
+ const counterReconciliation = require('../src/runtime/counter-reconciliation');
8
+
9
+ describe('Counter Reconciliation', () => {
10
+ let redis;
11
+
12
+ beforeEach(() => {
13
+ redis = createMockRedis();
14
+ });
15
+
16
+ // ── correctCounterWithLua ─────────────────────────
17
+ describe('correctCounterWithLua (B10 regression)', () => {
18
+ it('corrects a drifted counter value', async () => {
19
+ // Simulate drift: counter says 5, but only 3 leases exist
20
+ await redis.set('animastor:runtime:active-audio', '5');
21
+
22
+ const result = await counterReconciliation.correctCounterWithLua(redis, 'audio', 3);
23
+ expect(result.success).to.equal(true);
24
+ expect(result.action).to.equal('corrected');
25
+ expect(result.old).to.equal('5');
26
+ expect(result.new).to.equal('3');
27
+
28
+ // Verify the counter was actually corrected
29
+ const val = await redis.get('animastor:runtime:active-audio');
30
+ expect(val).to.equal('3');
31
+ });
32
+
33
+ it('corrects when counter is zero but leases exist', async () => {
34
+ // Drift: counter is 0, but lease scan finds 2
35
+ // Actually the correct behavior is: counter is too LOW
36
+ await redis.set('animastor:runtime:active-image', '0');
37
+
38
+ const result = await counterReconciliation.correctCounterWithLua(redis, 'image', 2);
39
+ expect(result.success).to.equal(true);
40
+ expect(result.old).to.equal('0');
41
+ expect(result.new).to.equal('2');
42
+
43
+ const val = await redis.get('animastor:runtime:active-image');
44
+ expect(val).to.equal('2');
45
+ });
46
+
47
+ it('corrects when counter key does not exist', async () => {
48
+ const result = await counterReconciliation.correctCounterWithLua(redis, 'video', 1);
49
+ expect(result.success).to.equal(true);
50
+ expect(result.old).to.equal(null);
51
+
52
+ const val = await redis.get('animastor:runtime:active-video');
53
+ expect(val).to.equal('1');
54
+ });
55
+
56
+ it('B10 bug: ioredis null → "" no longer causes silent no-op', async () => {
57
+ // This test verifies that the fix for B10 works:
58
+ // Previously, correctCounterWithLua passed `null` as an "expected" guard arg,
59
+ // which ioredis serialized to "" (truthy in Lua), causing the guard to
60
+ // always fire and skip the SET. The guard was removed.
61
+ await redis.set('animastor:runtime:active-audio', '10');
62
+
63
+ const result = await counterReconciliation.correctCounterWithLua(redis, 'audio', 7);
64
+ expect(result.success).to.equal(true);
65
+ expect(result.old).to.equal('10');
66
+ expect(result.new).to.equal('7');
67
+ });
68
+ });
69
+
70
+ // ── getCounterWithDriftCheck ──────────────────────
71
+ describe('getCounterWithDriftCheck', () => {
72
+ it('reports no drift when counter matches lease count', async () => {
73
+ const result = await counterReconciliation.getCounterWithDriftCheck(redis, 'audio');
74
+ expect(result.stage).to.equal('audio');
75
+ expect(result.leaseCount).to.equal(0);
76
+ expect(result.counterValue).to.equal(0);
77
+ expect(result.drift).to.equal(0);
78
+ expect(result.correct).to.equal(true);
79
+ });
80
+
81
+ it('detects positive drift (counter > actual leases)', async () => {
82
+ // Set counter higher with no leases
83
+ await redis.set('animastor:runtime:active-audio', '3');
84
+
85
+ const result = await counterReconciliation.getCounterWithDriftCheck(redis, 'audio');
86
+ expect(result.drift).to.equal(3);
87
+ expect(result.correct).to.equal(false);
88
+ expect(result.counterValue).to.equal(3);
89
+ expect(result.leaseCount).to.equal(0);
90
+ });
91
+ });
92
+
93
+ // ── reconcileCounters ─────────────────────────────
94
+ describe('reconcileCounters', () => {
95
+ it('reports no corrections when counters are balanced', async () => {
96
+ const report = await counterReconciliation.reconcileCounters(redis);
97
+
98
+ expect(report.summary.correctedCount).to.equal(0);
99
+ expect(report.summary.totalDrift).to.equal(0);
100
+ for (const stage of ['audio', 'image', 'video']) {
101
+ expect(report.stages[stage].corrected).to.equal(false);
102
+ expect(report.stages[stage].drift).to.equal(0);
103
+ }
104
+ });
105
+
106
+ it('corrects drifted counters across all stages', async () => {
107
+ // Create drift: audio=2, image=3, video=1
108
+ await redis.set('animastor:runtime:active-audio', '2');
109
+ await redis.set('animastor:runtime:active-image', '3');
110
+ await redis.set('animastor:runtime:active-video', '1');
111
+
112
+ const report = await counterReconciliation.reconcileCounters(redis);
113
+
114
+ // All 3 should be corrected to 0 (no leases in mock)
115
+ expect(report.summary.correctedCount).to.equal(3);
116
+
117
+ // Verify counters are corrected
118
+ expect(await redis.get('animastor:runtime:active-audio')).to.equal('0');
119
+ expect(await redis.get('animastor:runtime:active-image')).to.equal('0');
120
+ expect(await redis.get('animastor:runtime:active-video')).to.equal('0');
121
+ });
122
+
123
+ it('corrects counter even with non-numeric initial value', async () => {
124
+ await redis.set('animastor:runtime:active-audio', 'not-a-number');
125
+
126
+ const report = await counterReconciliation.reconcileCounters(redis);
127
+ // Counter should be corrected to 0 (no leases in mock)
128
+ expect(report.stages.audio.corrected).to.equal(true);
129
+ expect(await redis.get('animastor:runtime:active-audio')).to.equal('0');
130
+ });
131
+ });
132
+
133
+ // ── countActiveLeasesByStage ──────────────────────
134
+ describe('countActiveLeasesByStage', () => {
135
+ it('returns 0 when no leases exist', async () => {
136
+ const count = await counterReconciliation.countActiveLeasesByStage(redis, 'audio');
137
+ expect(count).to.equal(0);
138
+ });
139
+ });
140
+
141
+ // ── getCurrentCounter ────────────────────────────
142
+ describe('getCurrentCounter', () => {
143
+ it('returns 0 for missing key', async () => {
144
+ const val = await counterReconciliation.getCurrentCounter(redis, 'audio');
145
+ expect(val).to.equal(0);
146
+ });
147
+
148
+ it('returns stored integer value', async () => {
149
+ await redis.set('animastor:runtime:active-audio', '42');
150
+ const val = await counterReconciliation.getCurrentCounter(redis, 'audio');
151
+ expect(val).to.equal(42);
152
+ });
153
+ });
154
+
155
+ // ── manualCounterCorrection ───────────────────────
156
+ describe('manualCounterCorrection', () => {
157
+ it('sets counter to exact value', async () => {
158
+ await redis.set('animastor:runtime:active-audio', '99');
159
+ const result = await counterReconciliation.manualCounterCorrection(redis, 'audio', 5);
160
+ expect(result.success).to.equal(true);
161
+ expect(await redis.get('animastor:runtime:active-audio')).to.equal('5');
162
+ });
163
+ });
164
+ });