@animastor/orchestration 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +75 -0
  3. package/package.json +51 -0
  4. package/src/host/host-bindings.js +138 -0
  5. package/src/index.js +99 -0
  6. package/src/orchestration/index.js +40 -0
  7. package/src/orchestration/orchestrator.js +630 -0
  8. package/src/orchestration/runtime-result-consumer.js +54 -0
  9. package/src/orchestration/scene-callbacks.js +542 -0
  10. package/src/orchestration/scene-orchestrator.js +601 -0
  11. package/src/orchestration/scene-restoration.js +129 -0
  12. package/src/orchestration/scene-utils.js +36 -0
  13. package/src/runtime/active-scenes-index.js +164 -0
  14. package/src/runtime/audio-fsm-port.js +187 -0
  15. package/src/runtime/circuit-breaker.js +588 -0
  16. package/src/runtime/counter-reconciliation.js +308 -0
  17. package/src/runtime/dispatch-engine.js +1768 -0
  18. package/src/runtime/failure-taxonomy.js +125 -0
  19. package/src/runtime/hub-cancel-port.js +122 -0
  20. package/src/runtime/layer-config-port.js +136 -0
  21. package/src/runtime/lease-manager.js +607 -0
  22. package/src/runtime/persistence-port.js +162 -0
  23. package/src/runtime/placeholder-audio-port.js +117 -0
  24. package/src/runtime/progress-events-port.js +121 -0
  25. package/src/runtime/reconciliation-engine.js +2405 -0
  26. package/src/runtime/retry-budget-manager.js +229 -0
  27. package/src/runtime/runtime-metrics.js +427 -0
  28. package/src/runtime/runtime-result-emitter.js +123 -0
  29. package/src/runtime/runtime-scheduler.js +629 -0
  30. package/src/runtime/scene-data-port.js +114 -0
  31. package/src/runtime/scene-window.js +827 -0
  32. package/src/runtime/video-fsm-port.js +189 -0
  33. package/src/runtime/worker-health.js +270 -0
  34. package/test/counter-reconciliation.test.js +164 -0
  35. package/test/mocks/redis-mock.js +221 -0
  36. package/test/test-bindings.js +60 -0
@@ -0,0 +1,2405 @@
1
+ // ======================================================
2
+ // Reconciliation Engine - v1.0.0
3
+ // ======================================================
4
+ // Self-healing engine that detects and fixes inconsistencies
5
+ // between state machine and actual assets on disk/registry.
6
+ //
7
+ // T6: Единый reconciliation-контур (R4/К4). Включает:
8
+ // — startup-recovery (version staleness, audio-orch, chunk recovery)
9
+ // — audio-recovery (scan animastor:result:* keys)
10
+ // — cleanup-service (expired audio scene locks)
11
+ // — reconciliation-engine (orphan states, stale leases, drift)
12
+ // Все фазы — через единый цикл reconcileCycle() с распределённым локом.
13
+
14
+ const fs = require('fs').promises;
15
+ const syncFs = require('fs');
16
+ const syncPath = require('path');
17
+
18
+ const { lazyHostBinding } = require('../host/host-bindings');
19
+ // §32.30: former host requires resolve lazily through the host-bindings seam.
20
+ const state = lazyHostBinding('state');
21
+ // S-4: filename grammar composed from the canonical owner (bytes unchanged)
22
+ const artifactNaming = require('@animastor/generation').artifactNaming;
23
+ // S-2: registered stage list resolved from media registry
24
+ const mediaRegistry = require('@animastor/generation').mediaRegistry;
25
+ // O-2: persistence arrives ONLY through the PersistencePort (host adapter:
26
+ // storage/runtime-persistence-adapter) — the storage barrel, PG repositories
27
+ // and the database handle left this file. The Redis asset-registry and the
28
+ // audio-path composition are consumed as persistence OPERATIONS through the
29
+ // same port (their implementations stay host-side).
30
+ const persist = require('./persistence-port').persist;
31
+ const config = lazyHostBinding('config');
32
+ // S-5: the event journal is an append-only observability sink (zero requires,
33
+ // no orchestration policy) — the ONLY runtime→Generation dependency allowed
34
+ // without a seam (s5-runtime-orchestration-cycle.test.js S5-A).
35
+ const journal = lazyHostBinding('journal');
36
+ const runtimeScheduler = require('./runtime-scheduler');
37
+ const counterReconciliation = require('./counter-reconciliation');
38
+ const dispatchEngine = require('./dispatch-engine');
39
+ const leaseManager = require('./lease-manager');
40
+ // S-5: orchestration-owned behavior (FSM-safe facade writers, rollbacks)
41
+ // reaches runtime ONLY through the composition-root-injected seam registry.
42
+ // No orchestration module is imported from this file (S-5).
43
+ const orchestrationSeams = lazyHostBinding('seams');
44
+
45
+ const logPrefix = '[RECONCILE]';
46
+
47
+ function log(msg) {
48
+ console.log(`${logPrefix} ${msg}`);
49
+ }
50
+
51
+ function warn(msg) {
52
+ console.warn(`${logPrefix} ⚠️ ${msg}`);
53
+ }
54
+
55
+ function error(msg) {
56
+ console.error(`${logPrefix} ❌ ${msg}`);
57
+ }
58
+
59
+ // ======================================================
60
+ // RECONCILIATION REPORT
61
+ // ======================================================
62
+
63
+ class ReconciliationReport {
64
+ constructor() {
65
+ this.orphanStates = [];
66
+ this.orphanAssets = [];
67
+ this.partialBuilds = [];
68
+ this.staleLocks = [];
69
+ this.inconsistentScenes = [];
70
+ }
71
+
72
+ toSummary() {
73
+ return {
74
+ totalOrphanStates: this.orphanStates.length,
75
+ totalOrphanAssets: this.orphanAssets.length,
76
+ totalPartialBuilds: this.partialBuilds.length,
77
+ totalStaleLocks: this.staleLocks.length,
78
+ totalInconsistent: this.inconsistentScenes.length,
79
+ orphanStates: this.orphanStates,
80
+ orphanAssets: this.orphanAssets,
81
+ partialBuilds: this.partialBuilds,
82
+ staleLocks: this.staleLocks,
83
+ inconsistentScenes: this.inconsistentScenes
84
+ };
85
+ }
86
+ }
87
+
88
+ // ======================================================
89
+ // ORPHAN STATE CHECKS
90
+ // ======================================================
91
+
92
+ /**
93
+ * Canonical build identity of a book (audit d9d67a3): the SAME source the
94
+ * rest of the runtime uses (rebuildWorkList, generate-next, dispatch) —
95
+ * book.json manifest.build_id. The former hardcoded 'default' produced
96
+ * false orphan detections for every book with a real build directory.
97
+ */
98
+ function resolveBookBuildId(bookId) {
99
+ try {
100
+ // O-3: scene content via the SceneDataPort (host adapter:
101
+ // storage/scene-data-adapter) — the Book Model facade require left
102
+ // the reconciliation engine.
103
+ const loadedBook = require('./scene-data-port').sceneDataOp('loadBook')(bookId);
104
+ return loadedBook?.manifest?.build_id || 'default';
105
+ } catch (_) {
106
+ return 'default';
107
+ }
108
+ }
109
+
110
+ /**
111
+ * Check for VIDEO_READY state but no video file.
112
+ * T8: использует per-asset state (video=ready) вместо линейного.
113
+ */
114
+ async function checkOrphanVideoState(redis, bookId, chapterId, sceneId) {
115
+ const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
116
+ const isVideoReady = assetStates && assetStates.video &&
117
+ (assetStates.video === state.AssetState.READY || assetStates.video === state.AssetState.PLACEHOLDER);
118
+
119
+ if (!isVideoReady) {
120
+ return null;
121
+ }
122
+
123
+ const buildId = resolveBookBuildId(bookId);
124
+ const videoPath = `/data/output/${buildId}/${bookId}_${chapterId}_${sceneId}.mp4`;
125
+
126
+ try {
127
+ await fs.access(videoPath);
128
+ return null; // File exists, state is valid
129
+ } catch (err) {
130
+ return {
131
+ type: 'orphan_video_state',
132
+ scene: { bookId, chapterId, sceneId },
133
+ state: 'video_ready',
134
+ missingFile: videoPath,
135
+ recommendation: 'regenerate_video'
136
+ };
137
+ }
138
+ }
139
+
140
+ /**
141
+ * Check for IMAGE_READY state but no image file.
142
+ * T8: использует per-asset state (image=ready) вместо линейного.
143
+ */
144
+ async function checkOrphanImageState(redis, bookId, chapterId, sceneId) {
145
+ const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
146
+ const isImageReady = assetStates && assetStates.image &&
147
+ (assetStates.image === state.AssetState.READY || assetStates.image === state.AssetState.PLACEHOLDER);
148
+
149
+ if (!isImageReady) {
150
+ return null;
151
+ }
152
+
153
+ // Note: build_id is read from manifest, not from scene-state (which no longer exists after T8).
154
+ // Orphan-check uses the canonical book build identity (audit d9d67a3).
155
+ const buildId = resolveBookBuildId(bookId);
156
+ const imageModule = lazyHostBinding('media.image');
157
+ const imageInfo = imageModule.resolveCanonicalSceneImage(
158
+ '/data/output',
159
+ buildId,
160
+ bookId,
161
+ chapterId,
162
+ sceneId,
163
+ );
164
+
165
+ if (!imageInfo) {
166
+ return {
167
+ type: 'orphan_image_state',
168
+ scene: { bookId, chapterId, sceneId },
169
+ state: 'image_ready',
170
+ missingFile: 'image_not_found',
171
+ recommendation: 'regenerate_image'
172
+ };
173
+ }
174
+
175
+ return null;
176
+ }
177
+
178
+ /**
179
+ * Check for AUDIO_READY state but no audio file.
180
+ * T8: использует per-asset state (audio=ready) вместо линейного.
181
+ */
182
+ async function checkOrphanAudioState(redis, bookId, chapterId, sceneId) {
183
+ const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
184
+ const isAudioReady = assetStates && assetStates.audio &&
185
+ (assetStates.audio === state.AssetState.READY || assetStates.audio === state.AssetState.PLACEHOLDER);
186
+
187
+ if (!isAudioReady) {
188
+ return null;
189
+ }
190
+
191
+ // Canonical book build identity (audit d9d67a3) — the former hardcoded
192
+ // 'default' false-flagged every book with a real build directory.
193
+ const buildId = resolveBookBuildId(bookId);
194
+
195
+ // O-2: canonical audio path via the PersistencePort (filesystem op —
196
+ // the host adapter delegates to storage/filesystem-store unchanged).
197
+ const audioPath = persist('filesystem.getSceneAudioPath')(
198
+ config.OUTPUT_DIR,
199
+ buildId,
200
+ bookId,
201
+ chapterId,
202
+ sceneId
203
+ );
204
+
205
+ try {
206
+ await fs.access(audioPath);
207
+ return null; // File exists, state is valid
208
+ } catch (err) {
209
+ return {
210
+ type: 'orphan_audio_state',
211
+ scene: { bookId, chapterId, sceneId },
212
+ state: 'audio_ready',
213
+ missingFile: audioPath,
214
+ recommendation: 'regenerate_audio'
215
+ };
216
+ }
217
+ }
218
+
219
+ // ======================================================
220
+ // ORPHAN ASSET CHECKS
221
+ // ======================================================
222
+
223
+ /**
224
+ * Check for assets in registry but no files.
225
+ */
226
+ async function checkOrphanAssets(redis, bookId, chapterId, sceneId) {
227
+ const assets = await persist('registry.getSceneAssetsRedis')(redis, bookId, chapterId, sceneId);
228
+
229
+ if (!assets) {
230
+ return null;
231
+ }
232
+
233
+ const orphanAssets = [];
234
+
235
+ if (assets.audio?.canonical && assets.audio.ready) {
236
+ try {
237
+ await fs.access(assets.audio.canonical);
238
+ } catch (err) {
239
+ orphanAssets.push({
240
+ type: 'missing_audio_file',
241
+ scene: { bookId, chapterId, sceneId },
242
+ path: assets.audio.canonical
243
+ });
244
+ }
245
+ }
246
+
247
+ if (assets.image?.path && assets.image.ready) {
248
+ try {
249
+ await fs.access(assets.image.path);
250
+ } catch (err) {
251
+ orphanAssets.push({
252
+ type: 'missing_image_file',
253
+ scene: { bookId, chapterId, sceneId },
254
+ path: assets.image.path
255
+ });
256
+ }
257
+ }
258
+
259
+ if (assets.video?.path && assets.video.ready) {
260
+ try {
261
+ await fs.access(assets.video.path);
262
+ } catch (err) {
263
+ orphanAssets.push({
264
+ type: 'missing_video_file',
265
+ scene: { bookId, chapterId, sceneId },
266
+ path: assets.video.path
267
+ });
268
+ }
269
+ }
270
+
271
+ return orphanAssets.length > 0 ? orphanAssets : null;
272
+ }
273
+
274
+ // ======================================================
275
+ // ORPHAN GENERATING CHECK (audit d9d67a3)
276
+ // ======================================================
277
+
278
+ /**
279
+ * Cancellation precedence helper: true if the user explicitly cancelled
280
+ * generation for this book (Redis cancel flag and/or PG tombstone).
281
+ * Reconciliation must never resurrect a cancelled book.
282
+ */
283
+ async function isBookGenerationCancelled(redis, bookId) {
284
+ try {
285
+ const sceneWindow = require('./scene-window');
286
+ if (await sceneWindow.isCancelled(redis, bookId)) return true;
287
+ } catch (_) { /* flag unreadable — fall through to tombstone */ }
288
+ try {
289
+ return await persist('cancel.isCancelled')(bookId);
290
+ } catch (_) {
291
+ return false;
292
+ }
293
+ }
294
+
295
+ /**
296
+ * Check for orphan GENERATING states: a per-asset state stuck in GENERATING
297
+ * with NO evidence of a live generation — no dispatch lease, no dispatch
298
+ * metadata, no iu-in-flight marker (image), no active PG generation task.
299
+ *
300
+ * A live dispatch always holds lease+metadata from dispatch to finalization
301
+ * (dispatch order: acquireLease → setDispatchMetadata → markGenerating), so
302
+ * the absence of ALL evidence proves the generation is dead (crash/restart/
303
+ * TTL expiry with the rollback swallowed). A live job is NEVER repaired —
304
+ * age alone is not a reason (long video jobs keep their lease renewed).
305
+ *
306
+ * Fail-safe: if PG is unavailable the active-task check is treated as
307
+ * "task exists" and the state is NOT repaired.
308
+ */
309
+ async function checkOrphanGeneratingState(redis, bookId, chapterId, sceneId) {
310
+ const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
311
+ const orphans = [];
312
+
313
+ for (const stage of mediaRegistry.listMediaTypes()) {
314
+ if (assetStates[stage] !== state.AssetState.GENERATING) continue;
315
+
316
+ const evidence = await dispatchEngine.getDispatchEvidence(redis, bookId, chapterId, sceneId, stage);
317
+ if (evidence.alive) continue;
318
+
319
+ let hasActiveTask = false;
320
+ try {
321
+ // Pre-O-2 semantics preserved: a PG error is swallowed by the
322
+ // caller-side catch and treated as "task exists" (fail-safe).
323
+ hasActiveTask = await persist('tasks.hasActiveTaskForScene')(bookId, sceneId, stage);
324
+ } catch (_) {
325
+ hasActiveTask = true; // PG unavailable — fail safe, do NOT repair
326
+ }
327
+ if (hasActiveTask) continue;
328
+
329
+ orphans.push({
330
+ type: 'orphan_generating',
331
+ stage,
332
+ scene: { bookId, chapterId, sceneId },
333
+ state: 'generating',
334
+ recommendation: 'rollback_to_pending'
335
+ });
336
+ }
337
+
338
+ return orphans.length > 0 ? orphans : null;
339
+ }
340
+
341
+ // ======================================================
342
+ // PARTIAL BUILD CHECKS
343
+ // ======================================================
344
+
345
+ /**
346
+ * Check for partially built scenes.
347
+ * T8: использует per-asset state вместо линейного.
348
+ */
349
+ async function checkPartialBuilds(redis, bookId, chapterId, sceneId) {
350
+ const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
351
+ const assets = await persist('registry.getSceneAssetsRedis')(redis, bookId, chapterId, sceneId);
352
+
353
+ // Audio READY/PLACEHOLDER but no image asset
354
+ const audioReady = assetStates && assetStates.audio &&
355
+ (assetStates.audio === state.AssetState.READY || assetStates.audio === state.AssetState.PLACEHOLDER);
356
+ if (audioReady && !assets?.image) {
357
+ return {
358
+ type: 'partial_audio_only',
359
+ scene: { bookId, chapterId, sceneId },
360
+ currentStage: 'audio_ready',
361
+ missingStage: 'image',
362
+ recommendation: 'progress_to_image'
363
+ };
364
+ }
365
+
366
+ // Image READY but no video asset
367
+ const imageReady = assetStates && assetStates.image &&
368
+ (assetStates.image === state.AssetState.READY || assetStates.image === state.AssetState.PLACEHOLDER);
369
+ if (imageReady && !assets?.video) {
370
+ return {
371
+ type: 'partial_image_video_missing',
372
+ scene: { bookId, chapterId, sceneId },
373
+ currentStage: 'image_ready',
374
+ missingStage: 'video',
375
+ recommendation: 'progress_to_video'
376
+ };
377
+ }
378
+
379
+ return null;
380
+ }
381
+
382
+ // ======================================================
383
+ // B1: STALLED AUDIO CHUNKS WATCHDOG
384
+ // ======================================================
385
+ // Сканирует audio-orch state машины в фазе WAITING_CHUNKS.
386
+ // Если last_chunk_at + AUDIO_CHUNK_STALL_MS < Date.now() —
387
+ // сцена зависла: вызываем failWaitingScene(), которая чистит
388
+ // hub-dedup, сбрасывает metadata и публикует failStage.
389
+ //
390
+ // Важно: watchdog НЕ конкурирует с gpu-hub timeout — они ловят
391
+ // разные сценарии (стоп прогресса vs мёртвый воркер).
392
+
393
+ async function checkStalledAudioScenes(redis, deps) {
394
+ // O-7: the audio FSM is consumed through the AudioFsmPort (host adapter:
395
+ // storage/audio-fsm-adapter). The pre-O-7 optional-load semantics are
396
+ // preserved EXACTLY: when the audio-orchestrator host service cannot be
397
+ // required, this watchdog leg returns 0 (missing module ≠ stalled scene).
398
+ // The try/catch therefore wraps the ADAPTER's lazy require — the port
399
+ // itself stays fail-fast.
400
+ let audioOrch;
401
+ let PHASES;
402
+ try {
403
+ const audioFsmPort = require('./audio-fsm-port');
404
+ audioOrch = audioFsmPort.audioFsm();
405
+ PHASES = audioFsmPort.audioFsmPhases();
406
+ } catch (_) {
407
+ return 0;
408
+ }
409
+
410
+ const { AUDIO_CHUNK_STALL_MS } = config.TIMEOUTS;
411
+ const allStates = await audioOrch.scanAllStates(redis);
412
+ if (allStates.length === 0) return 0;
413
+
414
+ let stalled = 0;
415
+ for (const entry of allStates) {
416
+ const { bookId, chapterId, sceneId, state: orchState } = entry;
417
+ if (orchState.phase !== PHASES.WAITING_CHUNKS) continue;
418
+
419
+ const buildId = orchState.build_id || 'default';
420
+
421
+ // ── LAST_CHUNK_AT || STARTED_AT ──
422
+ // Если completeChunk ни разу не был вызван (HTTP 500 на всех чанках),
423
+ // last_chunk_at = null, но started_at есть. Используем started_at как
424
+ // точку отсчёта застоя. Если и started_at нет — проверяем чанки на диске.
425
+ const lastChunkAt = orchState.last_chunk_at;
426
+ const startedAt = orchState.started_at;
427
+ const threshold = lastChunkAt || startedAt;
428
+
429
+ if (threshold) {
430
+ const age = Date.now() - threshold;
431
+ if (age < AUDIO_CHUNK_STALL_MS) continue;
432
+
433
+ const reason = `chunk_stall_timeout:${Math.round(age / 1000)}s`;
434
+ warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${reason} (last_chunk_at=${!!lastChunkAt} started_at=${!!startedAt})`);
435
+
436
+ try {
437
+ await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId, reason, {
438
+ orchestrator: deps.orchestrator
439
+ });
440
+ stalled++;
441
+ } catch (err) {
442
+ warn(`[STALLED-AUDIO] failWaitingScene error: ${err.message}`);
443
+ }
444
+ } else {
445
+ // ── НЕТ НИ LAST_CHUNK_AT, НИ STARTED_AT ──
446
+ // Проверяем чанки на диске. Если все на месте — доигрываем merge.
447
+ // Если ни одного чанка нет — это fresh state без dispatch, не трогаем.
448
+ const expectedCount = parseInt(orchState.expected_count || '1', 10);
449
+ const pad = (n) => String(n).padStart(4, '0');
450
+ const OUTPUT_DIR = config.OUTPUT_DIR;
451
+ const buildDir = syncPath.join(OUTPUT_DIR, buildId);
452
+ let presentCount = 0;
453
+ for (let i = 1; i <= expectedCount; i++) {
454
+ const chunkPath = syncPath.join(buildDir, artifactNaming.sceneChunkAudioName(bookId, chapterId, sceneId, i));
455
+ if (syncFs.existsSync(chunkPath)) presentCount++;
456
+ }
457
+
458
+ if (presentCount === expectedCount) {
459
+ log(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ALL ${presentCount} chunks on disk, no last_chunk_at. Calling completeChunk to trigger merge.`);
460
+ try {
461
+ await audioOrch.completeChunk(redis, bookId, chapterId, sceneId, 'recovery', buildId, {
462
+ audio: deps.audio || lazyHostBinding('media.audio'),
463
+ orchestrator: deps.orchestrator,
464
+ dispatchId: 'recovery-reconcile',
465
+ });
466
+ stalled++;
467
+ } catch (err) {
468
+ warn(`[STALLED-AUDIO] completeChunk recovery failed: ${err.message}`);
469
+ try {
470
+ await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
471
+ `recovery_fallback:${err.message}`, { orchestrator: deps.orchestrator });
472
+ stalled++;
473
+ } catch (fsErr) {
474
+ warn(`[STALLED-AUDIO] failWaitingScene fallback failed: ${fsErr.message}`);
475
+ }
476
+ }
477
+ } else if (presentCount > 0) {
478
+ warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${presentCount}/${expectedCount} on disk, no last_chunk_at. Partial chunks, failing scene.`);
479
+ try {
480
+ await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
481
+ `partial_chunks:${presentCount}/${expectedCount}_no_timestamps`, { orchestrator: deps.orchestrator });
482
+ stalled++;
483
+ } catch (err) {
484
+ warn(`[STALLED-AUDIO] failWaitingScene failed: ${err.message}`);
485
+ }
486
+ }
487
+ // Если presentCount === 0 — это fresh state, ничего не делаем
488
+ }
489
+ }
490
+
491
+ if (stalled > 0) log(`[STALLED-AUDIO] ${stalled} stalled audio scenes recovered`);
492
+ return stalled;
493
+ }
494
+
495
+ // ======================================================
496
+ // B2: STALLED VIDEO GROUPS WATCHDOG
497
+ // ======================================================
498
+ // Сканирует video-orch state машины в фазе WAITING_CHUNKS.
499
+ // Если last_group_at/started_at + VIDEO_CHUNK_STALL_MS < Date.now() —
500
+ // сцена зависла: вызываем failWaitingScene(), которая чистит hub-dedup
501
+ // недостающих групп и публикует failStage (→ re-dispatch).
502
+
503
+ async function checkStalledVideoScenes(redis, deps) {
504
+ // O-8: the video FSM is consumed through the VideoFsmPort (host adapter:
505
+ // storage/video-fsm-adapter). The pre-O-8 optional-load semantics are
506
+ // preserved EXACTLY: when the video-orchestrator host service cannot be
507
+ // required, this watchdog leg returns 0 (missing module ≠ stalled scene).
508
+ // The try/catch therefore wraps the ADAPTER's lazy require — the port
509
+ // itself stays fail-fast.
510
+ let videoOrch;
511
+ let PHASES;
512
+ try {
513
+ const videoFsmPort = require('./video-fsm-port');
514
+ videoOrch = videoFsmPort.videoFsm();
515
+ PHASES = videoFsmPort.videoFsmPhases();
516
+ } catch (_) {
517
+ return 0;
518
+ }
519
+
520
+ const { VIDEO_CHUNK_STALL_MS } = config.TIMEOUTS;
521
+ const allStates = await videoOrch.scanAllStates(redis);
522
+ if (allStates.length === 0) return 0;
523
+
524
+ let stalled = 0;
525
+ for (const entry of allStates) {
526
+ const { bookId, chapterId, sceneId, state: orchState } = entry;
527
+ if (orchState.phase !== PHASES.WAITING_CHUNKS) continue;
528
+
529
+ // ── ПЕРЕСМОТР ПОРОГА ЗАСТОЯ ──
530
+ // Видео-генерация долгая по природе (LTX 5-10 мин на группу, на слабом
531
+ // GPU — 20-30+ мин). Жёсткий константный порог мог бы убить нормальную
532
+ // долгую генерацию. Порог застоя СТРОГО БОЛЬШЕ per-job timeout из
533
+ // layer-config (video_timeout_minutes): hub затаймливает воркера раньше,
534
+ // watchdog срабатывает только если hub/worker НЕ объявили ни результат,
535
+ // ни ошибку за весь интервал.
536
+ let stallThreshold = VIDEO_CHUNK_STALL_MS;
537
+ try {
538
+ const layerRaw = await redis.get(`animastor:layer-config:${bookId}`);
539
+ if (layerRaw) {
540
+ const lc = JSON.parse(layerRaw);
541
+ const perJobMs = (Number(lc.video_timeout_minutes) || 60) * 60 * 1000;
542
+ stallThreshold = Math.max(stallThreshold, perJobMs + 5 * 60 * 1000);
543
+ }
544
+ } catch (_) {}
545
+
546
+ const buildId = orchState.build_id || 'default';
547
+ const lastGroupAt = orchState.last_group_at;
548
+ const startedAt = orchState.started_at;
549
+ const threshold = lastGroupAt || startedAt;
550
+
551
+ if (threshold) {
552
+ const age = Date.now() - threshold;
553
+ if (age < stallThreshold) continue;
554
+
555
+ const reason = `video_group_stall_timeout:${Math.round(age / 1000)}s`;
556
+ warn(`[STALLED-VIDEO] ${bookId}/${chapterId}/${sceneId} — ${reason} (threshold=${Math.round(stallThreshold / 60000)}min, last_group_at=${!!lastGroupAt} started_at=${!!startedAt})`);
557
+ try {
558
+ await videoOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId, reason, {
559
+ orchestrator: deps.orchestrator,
560
+ });
561
+ stalled++;
562
+ } catch (err) {
563
+ warn(`[STALLED-VIDEO] failWaitingScene error: ${err.message}`);
564
+ }
565
+ } else {
566
+ // Нет ни last_group_at, ни started_at — свежий state без dispatch.
567
+ // Если все группы на диске — доигрываем merge.
568
+ const suffixes = videoOrch.groupSuffixes(orchState);
569
+ const presentCount = suffixes.filter(s => videoOrch.isGroupFileValid(buildId, bookId, chapterId, sceneId, s)).length;
570
+ if (presentCount === suffixes.length && suffixes.length > 0) {
571
+ log(`[STALLED-VIDEO] ${bookId}/${chapterId}/${sceneId} — ALL ${presentCount} groups on disk, no last_group_at. Calling completeGroup to trigger merge.`);
572
+ try {
573
+ await videoOrch.completeGroup(redis, bookId, chapterId, sceneId, '', buildId, {
574
+ orchestrator: deps.orchestrator,
575
+ dispatchId: 'recovery-reconcile',
576
+ });
577
+ stalled++;
578
+ } catch (err) {
579
+ warn(`[STALLED-VIDEO] completeGroup recovery failed: ${err.message}`);
580
+ try {
581
+ await videoOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
582
+ `recovery_fallback:${err.message}`, { orchestrator: deps.orchestrator });
583
+ stalled++;
584
+ } catch (fsErr) {
585
+ warn(`[STALLED-VIDEO] failWaitingScene fallback failed: ${fsErr.message}`);
586
+ }
587
+ }
588
+ } else if (presentCount > 0) {
589
+ warn(`[STALLED-VIDEO] ${bookId}/${chapterId}/${sceneId} — ${presentCount}/${suffixes.length} on disk, no last_group_at. Partial groups, failing scene.`);
590
+ try {
591
+ await videoOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
592
+ `partial_groups:${presentCount}/${suffixes.length}_no_timestamps`, { orchestrator: deps.orchestrator });
593
+ stalled++;
594
+ } catch (err) {
595
+ warn(`[STALLED-VIDEO] failWaitingScene failed: ${err.message}`);
596
+ }
597
+ }
598
+ }
599
+ }
600
+
601
+ if (stalled > 0) log(`[STALLED-VIDEO] ${stalled} stalled video scenes recovered`);
602
+ return stalled;
603
+ }
604
+
605
+ // ======================================================
606
+ // T7.6: AUDIO-ORCH INVARIANT CHECKS
607
+ // ======================================================
608
+ // Проверяет соответствие между audio-orch phase и asset state:
609
+ // phase=DONE ⇔ asset.audio = READY
610
+ // phase=FAILED ⇒ asset.audio = FAILED (или PENDING — после re-dispatch)
611
+ // промежуточные ⇒ asset.audio ∈ {PENDING, GENERATING, DIRTY}
612
+
613
+ async function checkAudioOrchInvariants(redis, bookId, chapterId, sceneId) {
614
+ // O-7: through the AudioFsmPort (host adapter: storage/audio-fsm-adapter)
615
+ // — the audio-orchestrator host service left this file. Mandatory load
616
+ // (pre-O-7 behavior: a broken require here throws, no optional-load).
617
+ const audioOrch = require('./audio-fsm-port').audioFsm();
618
+ const PHASES = require('./audio-fsm-port').audioFsmPhases();
619
+ const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
620
+ const orchState = await audioOrch.getState(redis, bookId, chapterId, sceneId);
621
+
622
+ if (!orchState || !assetStates) return null;
623
+
624
+ const phase = orchState.phase;
625
+ const audioState = assetStates.audio;
626
+ const violations = [];
627
+
628
+ if (phase === PHASES.DONE) {
629
+ if (audioState !== state.AssetState.READY && audioState !== state.AssetState.PLACEHOLDER) {
630
+ violations.push({
631
+ type: 'audio_orch_invariant_done',
632
+ scene: { bookId, chapterId, sceneId },
633
+ phase,
634
+ audioState,
635
+ expected: 'READY or PLACEHOLDER',
636
+ recommendation: 'run_completeStage'
637
+ });
638
+ }
639
+ } else if (phase === PHASES.FAILED) {
640
+ if (audioState !== state.AssetState.FAILED && audioState !== state.AssetState.PENDING) {
641
+ violations.push({
642
+ type: 'audio_orch_invariant_failed',
643
+ scene: { bookId, chapterId, sceneId },
644
+ phase,
645
+ audioState,
646
+ expected: 'FAILED or PENDING',
647
+ recommendation: 'mark_dirty'
648
+ });
649
+ }
650
+ } else if (phase !== PHASES.NEW && phase !== PHASES.PLACEHOLDER_READY) {
651
+ // Intermediate phases: GENERATING, WAITING_CHUNKS, MERGING
652
+ if (audioState === state.AssetState.READY) {
653
+ violations.push({
654
+ type: 'audio_orch_invariant_intermediate',
655
+ scene: { bookId, chapterId, sceneId },
656
+ phase,
657
+ audioState,
658
+ expected: 'PENDING|GENERATING|DIRTY',
659
+ recommendation: 'mark_dirty_or_set_done'
660
+ });
661
+ }
662
+ }
663
+
664
+ return violations.length > 0 ? violations : null;
665
+ }
666
+
667
+ // ======================================================
668
+ // VIDEO-ORCH INVARIANT CHECKS (зеркало checkAudioOrchInvariants)
669
+ // ======================================================
670
+ // phase=DONE ⇒ asset.video = READY (или PLACEHOLDER)
671
+ // phase=FAILED ⇒ asset.video = FAILED (или PENDING — после re-dispatch)
672
+ // промежуточные ⇒ asset.video = PENDING|GENERATING|DIRTY
673
+
674
+ async function checkVideoOrchInvariants(redis, bookId, chapterId, sceneId) {
675
+ // O-8: through the VideoFsmPort (host adapter: storage/video-fsm-adapter)
676
+ // — the video-orchestrator host service left this file. Mandatory load
677
+ // (pre-O-8 behavior: a broken require here throws, no optional-load).
678
+ const videoOrch = require('./video-fsm-port').videoFsm();
679
+ const PHASES = require('./video-fsm-port').videoFsmPhases();
680
+ const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
681
+ const orchState = await videoOrch.getState(redis, bookId, chapterId, sceneId);
682
+
683
+ if (!orchState || !assetStates) return null;
684
+
685
+ const phase = orchState.phase;
686
+ const videoState = assetStates.video;
687
+ const violations = [];
688
+
689
+ if (phase === PHASES.DONE) {
690
+ if (videoState !== state.AssetState.READY && videoState !== state.AssetState.PLACEHOLDER) {
691
+ violations.push({
692
+ type: 'video_orch_invariant_done',
693
+ scene: { bookId, chapterId, sceneId },
694
+ phase,
695
+ videoState,
696
+ expected: 'READY or PLACEHOLDER',
697
+ recommendation: 'run_completeStage'
698
+ });
699
+ }
700
+ } else if (phase === PHASES.FAILED) {
701
+ if (videoState !== state.AssetState.FAILED && videoState !== state.AssetState.PENDING) {
702
+ violations.push({
703
+ type: 'video_orch_invariant_failed',
704
+ scene: { bookId, chapterId, sceneId },
705
+ phase,
706
+ videoState,
707
+ expected: 'FAILED or PENDING',
708
+ recommendation: 'mark_dirty'
709
+ });
710
+ }
711
+ } else if (phase !== PHASES.NEW && phase !== PHASES.GENERATING) {
712
+ // WAITING_CHUNKS, MERGING — ассет не должен быть READY
713
+ if (videoState === state.AssetState.READY) {
714
+ violations.push({
715
+ type: 'video_orch_invariant_intermediate',
716
+ scene: { bookId, chapterId, sceneId },
717
+ phase,
718
+ videoState,
719
+ expected: 'PENDING|GENERATING|DIRTY',
720
+ recommendation: 'mark_dirty_or_set_done'
721
+ });
722
+ }
723
+ }
724
+
725
+ return violations.length > 0 ? violations : null;
726
+ }
727
+
728
+ // ======================================================
729
+ // STALE LOCK CHECKS
730
+ // ======================================================
731
+
732
+ const LOCK_KEYS = [
733
+ 'animastor:audio-scene-lock',
734
+ 'animastor:video-lock',
735
+ 'animastor:audio-merge-lock',
736
+ 'animastor:scene-transition-lock'
737
+ ];
738
+
739
+ /**
740
+ * Check for stale locks (lock exists but no heartbeat).
741
+ */
742
+ async function checkStaleLocks(redis, bookId, chapterId, sceneId) {
743
+ const staleLocks = [];
744
+
745
+ for (const lockPrefix of LOCK_KEYS) {
746
+ const lockKey = `${lockPrefix}:${bookId}:${chapterId}:${sceneId}`;
747
+ const lockData = await redis.get(lockKey);
748
+
749
+ if (lockData) {
750
+ // Check heartbeat
751
+ const heartbeatKey = `animastor:scene-heartbeat:${bookId}:${chapterId}:${sceneId}`;
752
+ const heartbeatTime = await redis.get(heartbeatKey);
753
+
754
+ if (!heartbeatTime) {
755
+ staleLocks.push({
756
+ type: 'lock_without_heartbeat',
757
+ lockKey,
758
+ scene: { bookId, chapterId, sceneId },
759
+ lockData: JSON.parse(lockData),
760
+ recommendation: 'release_lock'
761
+ });
762
+ } else {
763
+ // Check if heartbeat is stale (> 5 minutes)
764
+ const now = Date.now();
765
+ const lastHeartbeat = parseInt(heartbeatTime, 10);
766
+ if (now - lastHeartbeat > 5 * 60 * 1000) {
767
+ staleLocks.push({
768
+ type: 'stale_heartbeat',
769
+ lockKey,
770
+ scene: { bookId, chapterId, sceneId },
771
+ lastHeartbeat,
772
+ recommendation: 'clear_heartbeat'
773
+ });
774
+ }
775
+ }
776
+ }
777
+ }
778
+
779
+ return staleLocks.length > 0 ? staleLocks : null;
780
+ }
781
+
782
+ // ======================================================
783
+ // DISPATCH LEASE CHECKS
784
+ // ======================================================
785
+
786
+ /**
787
+ * Check for stale dispatch leases.
788
+ *
789
+ * Liveness is decided by the lease key's REMAINING TTL — the canonical
790
+ * signal that renewal is still running — NOT by metadata.started_at.
791
+ * renewLeaseIfOwner re-pins the TTL to the renewal target every 30 s while
792
+ * the owner is alive, so a healthy long-running (even 60-min video) dispatch
793
+ * never goes stale, while a lease whose owner died (crash/restart) decays and
794
+ * is flagged once its TTL drops below target - STALE_LEASE_GRACE_S.
795
+ * (audit c8b79f6 / incident 2026-08-28: started_at-based age falsely killed
796
+ * live renewed video dispatches at 27 min → duplicate re-dispatch loop.)
797
+ */
798
+ async function checkStaleDispatchLeases(redis, bookId, chapterId, sceneId) {
799
+ const staleLeases = [];
800
+
801
+ // Check all stages for lease existence
802
+ const stages = mediaRegistry.listMediaTypes();
803
+
804
+ for (const stage of stages) {
805
+ const leaseKey = dispatchEngine.getLeaseKey(bookId, chapterId, sceneId, stage);
806
+ const token = await redis.get(leaseKey);
807
+
808
+ if (token) {
809
+ // Check dispatch metadata for age
810
+ const metaKey = dispatchEngine.getDispatchMetaKey(bookId, chapterId, sceneId, stage);
811
+ const metadata = await redis.get(metaKey);
812
+
813
+ if (metadata) {
814
+ // Remaining TTL is the source of truth for liveness. A lease
815
+ // being actively renewed keeps its TTL pinned near the renewal
816
+ // target regardless of how old started_at is.
817
+ const leaseTtlS = await redis.ttl(leaseKey);
818
+
819
+ if (leaseManager.isLeaseStale(leaseTtlS, stage)) {
820
+ staleLeases.push({
821
+ type: 'stale_dispatch_lease',
822
+ stage,
823
+ leaseKey,
824
+ scene: { bookId, chapterId, sceneId },
825
+ leaseTtlS,
826
+ renewalTargetS: leaseManager.getRenewalTargetTtlS(stage),
827
+ token,
828
+ recommendation: 'release_lease_and_move_to_pending'
829
+ });
830
+ }
831
+ } else {
832
+ // Lease exists but no metadata - might be orphaned
833
+ staleLeases.push({
834
+ type: 'orphan_dispatch_lease',
835
+ stage,
836
+ leaseKey,
837
+ scene: { bookId, chapterId, sceneId },
838
+ token,
839
+ recommendation: 'release_lease_and_move_to_pending'
840
+ });
841
+ }
842
+ }
843
+ }
844
+
845
+ return staleLeases.length > 0 ? staleLeases : null;
846
+ }
847
+
848
+ // ======================================================
849
+ // SCENE RECONCILIATION
850
+ // ======================================================
851
+
852
+ /**
853
+ * Reconcile a single scene.
854
+ */
855
+ async function reconcileScene(redis, bookId, chapterId, sceneId) {
856
+ const report = new ReconciliationReport();
857
+
858
+ try {
859
+ // Check orphan states
860
+ const orphanVideo = await checkOrphanVideoState(redis, bookId, chapterId, sceneId);
861
+ if (orphanVideo) {
862
+ report.orphanStates.push(orphanVideo);
863
+ report.inconsistentScenes.push({
864
+ scene: { bookId, chapterId, sceneId },
865
+ issue: orphanVideo.type
866
+ });
867
+ }
868
+
869
+ const orphanImage = await checkOrphanImageState(redis, bookId, chapterId, sceneId);
870
+ if (orphanImage) {
871
+ report.orphanStates.push(orphanImage);
872
+ if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
873
+ report.inconsistentScenes.push({
874
+ scene: { bookId, chapterId, sceneId },
875
+ issue: orphanImage.type
876
+ });
877
+ }
878
+ }
879
+
880
+ const orphanAudio = await checkOrphanAudioState(redis, bookId, chapterId, sceneId);
881
+ if (orphanAudio) {
882
+ report.orphanStates.push(orphanAudio);
883
+ if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
884
+ report.inconsistentScenes.push({
885
+ scene: { bookId, chapterId, sceneId },
886
+ issue: orphanAudio.type
887
+ });
888
+ }
889
+ }
890
+
891
+ // Check orphan assets
892
+ const orphanAssets = await checkOrphanAssets(redis, bookId, chapterId, sceneId);
893
+ if (orphanAssets) {
894
+ report.orphanAssets.push(...orphanAssets);
895
+ if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
896
+ report.inconsistentScenes.push({
897
+ scene: { bookId, chapterId, sceneId },
898
+ issue: 'orphan_assets'
899
+ });
900
+ }
901
+ }
902
+
903
+ // Check orphan GENERATING states (audit d9d67a3). Pushed BEFORE the
904
+ // partial-build checks: those de-duplicate by sceneId, and the orphan
905
+ // repair must win over a PROGRESS_TO_* resurrection attempt.
906
+ const orphanGenerating = await checkOrphanGeneratingState(redis, bookId, chapterId, sceneId);
907
+ if (orphanGenerating) {
908
+ report.orphanStates.push(...orphanGenerating);
909
+ for (const o of orphanGenerating) {
910
+ report.inconsistentScenes.push({
911
+ scene: { bookId, chapterId, sceneId },
912
+ stage: o.stage,
913
+ issue: 'orphan_generating'
914
+ });
915
+ }
916
+ }
917
+
918
+ // Check partial builds
919
+ const partial = await checkPartialBuilds(redis, bookId, chapterId, sceneId);
920
+ if (partial) {
921
+ report.partialBuilds.push(partial);
922
+ if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
923
+ report.inconsistentScenes.push({
924
+ scene: { bookId, chapterId, sceneId },
925
+ issue: partial.type
926
+ });
927
+ }
928
+ }
929
+
930
+ // Check stale locks
931
+ const staleLocks = await checkStaleLocks(redis, bookId, chapterId, sceneId);
932
+ if (staleLocks) {
933
+ report.staleLocks.push(...staleLocks);
934
+ if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
935
+ report.inconsistentScenes.push({
936
+ scene: { bookId, chapterId, sceneId },
937
+ issue: 'stale_locks'
938
+ });
939
+ }
940
+ }
941
+
942
+ // Check stale dispatch leases
943
+ const staleLeases = await checkStaleDispatchLeases(redis, bookId, chapterId, sceneId);
944
+ if (staleLeases) {
945
+ report.staleLocks.push(...staleLeases);
946
+ for (const lease of staleLeases) {
947
+ if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId && s.issue === 'stale_dispatch_lease')) {
948
+ report.inconsistentScenes.push({
949
+ scene: { bookId, chapterId, sceneId },
950
+ issue: 'stale_dispatch_lease'
951
+ });
952
+ }
953
+ }
954
+ }
955
+
956
+ // T7.6: Check audio-orch invariants
957
+ const audioOrchViolations = await checkAudioOrchInvariants(redis, bookId, chapterId, sceneId);
958
+ if (audioOrchViolations) {
959
+ for (const v of audioOrchViolations) {
960
+ report.inconsistentScenes.push({
961
+ scene: v.scene,
962
+ issue: v.type
963
+ });
964
+ }
965
+ log(`[INVARIANT] ${bookId}/${chapterId}/${sceneId}: ${audioOrchViolations.length} audio-orch violations`);
966
+ }
967
+
968
+ // Video-orch invariants (зеркало аудио)
969
+ const videoOrchViolations = await checkVideoOrchInvariants(redis, bookId, chapterId, sceneId);
970
+ if (videoOrchViolations) {
971
+ for (const v of videoOrchViolations) {
972
+ report.inconsistentScenes.push({
973
+ scene: v.scene,
974
+ issue: v.type
975
+ });
976
+ }
977
+ log(`[INVARIANT] ${bookId}/${chapterId}/${sceneId}: ${videoOrchViolations.length} video-orch violations`);
978
+ }
979
+
980
+ return report;
981
+ } catch (err) {
982
+ error(`Reconciliation error for ${bookId}/${chapterId}/${sceneId}: ${err.message}`);
983
+ return report;
984
+ }
985
+ }
986
+
987
+ // ======================================================
988
+ // RECOMMENDATIONS & AUTO-FIXES
989
+ // ======================================================
990
+
991
+ /**
992
+ * Get recommendations for fixing inconsistencies.
993
+ */
994
+ function getFixRecommendations(inconsistentScenes) {
995
+ return inconsistentScenes.map(item => {
996
+ const { scene, issue } = item;
997
+
998
+ switch (issue) {
999
+ case 'orphan_video_state':
1000
+ return {
1001
+ scene,
1002
+ action: 'REGENERATE_MISSING_ASSET',
1003
+ reason: 'VIDEO_READY but no video file',
1004
+ safeToExecute: true
1005
+ };
1006
+ case 'orphan_image_state':
1007
+ return {
1008
+ scene,
1009
+ action: 'REGENERATE_MISSING_ASSET',
1010
+ reason: 'IMAGE_READY but no image file',
1011
+ safeToExecute: true
1012
+ };
1013
+ case 'orphan_audio_state':
1014
+ return {
1015
+ scene,
1016
+ action: 'REGENERATE_MISSING_ASSET',
1017
+ reason: 'AUDIO_READY but no audio file',
1018
+ safeToExecute: true
1019
+ };
1020
+ case 'partial_audio_only':
1021
+ return {
1022
+ scene,
1023
+ action: 'PROGRESS_TO_IMAGE',
1024
+ reason: 'Audio ready but image not started',
1025
+ safeToExecute: true
1026
+ };
1027
+ case 'partial_image_video_missing':
1028
+ return {
1029
+ scene,
1030
+ action: 'PROGRESS_TO_VIDEO',
1031
+ reason: 'Image ready but video not started',
1032
+ safeToExecute: true
1033
+ };
1034
+ case 'stale_locks':
1035
+ return {
1036
+ scene,
1037
+ action: 'RELEASE_STALE_LOCKS',
1038
+ reason: 'Locks exist without heartbeat',
1039
+ safeToExecute: true
1040
+ };
1041
+ case 'orphan_assets':
1042
+ return {
1043
+ scene,
1044
+ action: 'RECOVER_ORPHAN_ASSETS',
1045
+ reason: 'Registry has assets but files are missing',
1046
+ safeToExecute: true
1047
+ };
1048
+ case 'stuck_state':
1049
+ return {
1050
+ scene,
1051
+ action: 'MOVE_TO_PENDING',
1052
+ reason: `Stuck in ${item.state} for ${item.ageMinutes} minutes`,
1053
+ safeToExecute: true
1054
+ };
1055
+ case 'orphan_generating':
1056
+ // Audit d9d67a3: GENERATING with no lease/dispatch-meta/
1057
+ // in-flight marker/active PG task — the generation is proven
1058
+ // dead. Stage-scoped MOVE_TO_PENDING performs the strict
1059
+ // FSM-safe repair GENERATING → DIRTY → PENDING.
1060
+ return {
1061
+ scene,
1062
+ stage: item.stage,
1063
+ action: 'MOVE_TO_PENDING',
1064
+ reason: `Orphan GENERATING (${item.stage}): no lease/dispatch/in-flight/PG task`,
1065
+ safeToExecute: true
1066
+ };
1067
+ case 'counter_drift':
1068
+ return {
1069
+ scene,
1070
+ stage: item.stage,
1071
+ action: 'RECONCILE_COUNTER_DRIFT',
1072
+ reason: `Counter drift: ${item.drift} (leases=${item.leaseCount}, counter=${item.counterValue})`,
1073
+ safeToExecute: true
1074
+ };
1075
+ case 'stale_dispatch_lease':
1076
+ return {
1077
+ scene,
1078
+ action: 'RELEASE_STALE_LEASE',
1079
+ reason: `Stale dispatch lease detected`,
1080
+ safeToExecute: true
1081
+ };
1082
+ default:
1083
+ return {
1084
+ scene,
1085
+ action: 'REVIEW_MANUALLY',
1086
+ reason: issue,
1087
+ safeToExecute: false
1088
+ };
1089
+ }
1090
+ });
1091
+ }
1092
+
1093
+ /**
1094
+ * Fix actions that resurrect work (re-add a scene to the active index /
1095
+ * re-mark stages for generation). Audit d9d67a3: they must NEVER run for a
1096
+ * book the user explicitly cancelled — the cancellation tombstone / cancel
1097
+ * flag takes precedence over any reconciliation recommendation.
1098
+ */
1099
+ const RESURRECTION_ACTIONS = new Set([
1100
+ 'REGENERATE_MISSING_ASSET',
1101
+ 'PROGRESS_TO_IMAGE',
1102
+ 'PROGRESS_TO_VIDEO',
1103
+ 'RECOVER_ORPHAN_ASSETS',
1104
+ 'RELEASE_STALE_LEASE',
1105
+ ]);
1106
+
1107
+ /**
1108
+ * Apply a fix recommendation.
1109
+ * Returns { success: boolean, action, scene, details }
1110
+ */
1111
+ async function applyFix(redis, fix) {
1112
+ const { scene, action } = fix;
1113
+
1114
+ try {
1115
+ // Cancellation precedence (audit d9d67a3): a fresh tombstone/cancel
1116
+ // flag blocks every resurrection path. State-hygiene repairs that do
1117
+ // NOT re-add the scene to the active index (stage-scoped
1118
+ // MOVE_TO_PENDING) are still allowed — they remove a ghost, not
1119
+ // resurrect work.
1120
+ if (RESURRECTION_ACTIONS.has(action) &&
1121
+ await isBookGenerationCancelled(redis, scene.bookId)) {
1122
+ return {
1123
+ success: false,
1124
+ action,
1125
+ scene,
1126
+ details: 'skipped: book generation cancelled (tombstone/cancel flag precedence)'
1127
+ };
1128
+ }
1129
+
1130
+ switch (action) {
1131
+ case 'MOVE_TO_PENDING': {
1132
+ // S-5: FSM-safe rollback via the injected orchestration seam
1133
+ const rollbackStageToPending = orchestrationSeams.getOrchestrationSeam('rollbackStageToPending');
1134
+
1135
+ // ── Stage-scoped orphan GENERATING repair (audit d9d67a3) ──
1136
+ if (fix.stage) {
1137
+ // Strict re-verification right before the write (TOCTOU):
1138
+ // a concurrent dispatch may have acquired the stage
1139
+ // between detection and fix. A live stage is NEVER
1140
+ // rolled back.
1141
+ const evidence = await dispatchEngine.getDispatchEvidence(
1142
+ redis, scene.bookId, scene.chapterId, scene.sceneId, fix.stage);
1143
+ if (evidence.alive) {
1144
+ return {
1145
+ success: false, action, scene,
1146
+ details: `repair aborted: generation alive (${evidence.reason})`
1147
+ };
1148
+ }
1149
+ const currentStates = await state.getAssetStates(
1150
+ redis, scene.bookId, scene.chapterId, scene.sceneId);
1151
+ if (currentStates[fix.stage] !== state.AssetState.GENERATING) {
1152
+ return {
1153
+ success: false, action, scene,
1154
+ details: `repair aborted: state now ${currentStates[fix.stage]}`
1155
+ };
1156
+ }
1157
+
1158
+ // FSM-safe path GENERATING → DIRTY → PENDING (a direct
1159
+ // GENERATING → PENDING is forbidden). Errors are NOT
1160
+ // swallowed (rollbackStageToPending logs + journals +
1161
+ // counts failures itself).
1162
+ const rollback = await rollbackStageToPending(
1163
+ redis, scene.bookId, scene.chapterId, scene.sceneId,
1164
+ fix.stage, null, 'orphan_generating_repair');
1165
+ if (!rollback.changed) {
1166
+ return { success: false, action, scene, details: rollback.reason || 'rollback_failed' };
1167
+ }
1168
+
1169
+ await journal.appendSceneEvent(redis, scene.bookId, scene.chapterId, scene.sceneId,
1170
+ 'AUTO_RECOVER', 'PENDING',
1171
+ { stage: fix.stage, fromState: 'generating', recoveredBy: 'reconciliation-engine' }
1172
+ ).catch(() => {});
1173
+
1174
+ // Make the repaired stage dispatchable again — but never
1175
+ // for a cancelled book (tombstone/cancel flag precedence:
1176
+ // only an explicit new generation clears them).
1177
+ if (!(await isBookGenerationCancelled(redis, scene.bookId))) {
1178
+ await runtimeScheduler.addSceneToActiveIndex(
1179
+ redis, scene.bookId, scene.chapterId, scene.sceneId);
1180
+ }
1181
+
1182
+ return {
1183
+ success: true, action, scene,
1184
+ details: `${fix.stage}: GENERATING → DIRTY → PENDING (orphan repair)`
1185
+ };
1186
+ }
1187
+
1188
+ // ── Legacy stuck_state path: whole-scene DIRTY ──
1189
+ // Mark all per-asset states as DIRTY for redispatch
1190
+ // M5: Route through orchestrator.markDirtyScene instead of direct state.setAssetState
1191
+ // S-5: asset-state writer via the injected orchestration seam
1192
+ await orchestrationSeams.getOrchestrationSeam('markDirtyScene')(redis, scene.bookId, scene.chapterId, scene.sceneId);
1193
+
1194
+ // Remove from active index to avoid immediate re-scheduling
1195
+ await runtimeScheduler.removeSceneFromActiveIndex(redis, scene.bookId, scene.chapterId, scene.sceneId);
1196
+
1197
+ // Extract previous state from fix reason (format: "Stuck in <state> for <n> minutes")
1198
+ const fromState = fix.reason?.match(/Stuck in (\w+)/)?.[1] || 'unknown';
1199
+
1200
+ // Log to journal
1201
+ await journal.appendSceneEvent(redis, scene.bookId, scene.chapterId, scene.sceneId,
1202
+ 'AUTO_RECOVER', 'DIRTY',
1203
+ { fromState, recoveredBy: 'reconciliation-engine' }
1204
+ );
1205
+
1206
+ return { success: true, action, scene, details: 'moved to DIRTY + assets DIRTY' };
1207
+ }
1208
+
1209
+ case 'RELEASE_STALE_LOCKS': {
1210
+ let removedCount = 0;
1211
+ for (const lockPrefix of LOCK_KEYS) {
1212
+ const lockKey = `${lockPrefix}:${scene.bookId}:${scene.chapterId}:${scene.sceneId}`;
1213
+ const removed = await redis.del(lockKey);
1214
+ removedCount += removed;
1215
+ }
1216
+
1217
+ return { success: removedCount > 0, action, scene, details: `removed ${removedCount} locks` };
1218
+ }
1219
+
1220
+ case 'RELEASE_STALE_LEASE': {
1221
+ const dispatchEngine = require('./dispatch-engine');
1222
+ let removedCount = 0;
1223
+ const clearedDispatchIds = [];
1224
+
1225
+ // Release all dispatch leases for this scene through the
1226
+ // canonical cancellation path (lease + owned quota + renewal
1227
+ // timer + completion marker). A bare `del` would leak the
1228
+ // quota slot and the in-memory renewal timer, and could race a
1229
+ // concurrent re-acquire (no token check).
1230
+ for (const stage of mediaRegistry.listMediaTypes()) {
1231
+ const result = await dispatchEngine.cancelActiveDispatch(
1232
+ redis, scene.bookId, scene.chapterId, scene.sceneId, stage,
1233
+ 'stale_lease_recovery'
1234
+ );
1235
+ if (result.cancelled) {
1236
+ removedCount++;
1237
+ if (result.dispatchId) clearedDispatchIds.push(result.dispatchId);
1238
+ }
1239
+ }
1240
+
1241
+ // Defensive de-duplication (audit c8b79f6 §2): purge the hub
1242
+ // queue/running copies owned by the cancelled dispatch(es) so a
1243
+ // recovered scene cannot accumulate duplicate job copies that a
1244
+ // worker would later drain. Best-effort — hub unavailability
1245
+ // must not block state recovery. O-9: the hub call goes
1246
+ // through the HubCancelPort (hub HTTP stays host-side; the
1247
+ // cancelActiveDispatch loop above stays a direct
1248
+ // dispatch-engine call — Redis domain operation).
1249
+ if (clearedDispatchIds.length > 0) {
1250
+ try {
1251
+ await require('./hub-cancel-port').hubCancelOp('clearHubDispatches')(clearedDispatchIds, { context: 'STALE_LEASE_RECOVERY' });
1252
+ } catch (hubErr) {
1253
+ warn(`RELEASE_STALE_LEASE hub cleanup failed (non-fatal): ${hubErr.message}`);
1254
+ }
1255
+ }
1256
+
1257
+ // Mark per-asset states as DIRTY for redispatch
1258
+ // M5: Route through orchestrator.markDirtyScene instead of direct state.setAssetState
1259
+ // M5 Шаг 3: syncLinearState уже внутри markDirtyScene
1260
+ // S-5: asset-state writer via the injected orchestration seam
1261
+ const markDirtyScene = orchestrationSeams.getOrchestrationSeam('markDirtyScene');
1262
+ await markDirtyScene(redis, scene.bookId, scene.chapterId, scene.sceneId);
1263
+
1264
+ // Add back to active index
1265
+ await runtimeScheduler.addSceneToActiveIndex(redis, scene.bookId, scene.chapterId, scene.sceneId);
1266
+
1267
+ return { success: removedCount > 0, action, scene, details: `released ${removedCount} stale leases` };
1268
+ }
1269
+
1270
+ case 'REGENERATE_MISSING_ASSET': {
1271
+ // Derive asset type from action reason or issue field
1272
+ const reasons = (fix.reason + ' ' + (fix.issue || '')).toLowerCase();
1273
+ const pendingState =
1274
+ reasons.includes('audio') || reasons.includes('tts')
1275
+ ? 'audio_pending'
1276
+ : reasons.includes('video')
1277
+ ? 'video_pending'
1278
+ : reasons.includes('image') || reasons.includes('iu')
1279
+ ? 'image_pending'
1280
+ : null;
1281
+
1282
+ if (!pendingState) {
1283
+ return { success: false, action, scene, details: `unknown asset type (reason: ${fix.reason})` };
1284
+ }
1285
+
1286
+ // M5 Шаг 4: Route through orchestrator.setScenePending (which includes syncLinearState)
1287
+ // S-5: asset-state writer via the injected orchestration seam
1288
+ const setScenePending = orchestrationSeams.getOrchestrationSeam('setScenePending');
1289
+ const assetType = pendingState === 'audio_pending' ? 'audio'
1290
+ : pendingState === 'image_pending' ? 'image'
1291
+ : pendingState === 'video_pending' ? 'video'
1292
+ : null;
1293
+ if (assetType) {
1294
+ await setScenePending(redis, scene.bookId, scene.chapterId, scene.sceneId, assetType);
1295
+ }
1296
+
1297
+ await runtimeScheduler.addSceneToActiveIndex(
1298
+ redis,
1299
+ scene.bookId,
1300
+ scene.chapterId,
1301
+ scene.sceneId
1302
+ );
1303
+
1304
+ return { success: true, action, scene, details: `marked as ${pendingState}` };
1305
+ }
1306
+
1307
+ case 'PROGRESS_TO_IMAGE': {
1308
+ // M5 Шаг 4: Route through orchestrator.setScenePending (includes syncLinearState)
1309
+ // S-5: asset-state writer via the injected orchestration seam
1310
+ await orchestrationSeams.getOrchestrationSeam('setScenePending')(redis, scene.bookId, scene.chapterId, scene.sceneId, 'image');
1311
+
1312
+ await runtimeScheduler.addSceneToActiveIndex(
1313
+ redis,
1314
+ scene.bookId,
1315
+ scene.chapterId,
1316
+ scene.sceneId
1317
+ );
1318
+
1319
+ return { success: true, action, scene, details: 'progressed to IMAGE_PENDING' };
1320
+ }
1321
+
1322
+ case 'RECONCILE_COUNTER_DRIFT': {
1323
+ // Get the current lease count and fix counter
1324
+ const stage = fix.stage || 'audio';
1325
+ const leaseCount = await counterReconciliation.countActiveLeasesByStage(redis, stage);
1326
+
1327
+ await counterReconciliation.correctCounterWithLua(redis, stage, leaseCount);
1328
+
1329
+ return { success: true, action, scene, details: `counter corrected to ${leaseCount}` };
1330
+ }
1331
+
1332
+ case 'PROGRESS_TO_VIDEO': {
1333
+ // M5 Шаг 4: Route through orchestrator.setScenePending (includes syncLinearState)
1334
+ // S-5: asset-state writer via the injected orchestration seam
1335
+ await orchestrationSeams.getOrchestrationSeam('setScenePending')(redis, scene.bookId, scene.chapterId, scene.sceneId, 'video');
1336
+
1337
+ await runtimeScheduler.addSceneToActiveIndex(
1338
+ redis,
1339
+ scene.bookId,
1340
+ scene.chapterId,
1341
+ scene.sceneId
1342
+ );
1343
+
1344
+ return { success: true, action, scene, details: 'progressed to VIDEO_PENDING' };
1345
+ }
1346
+
1347
+ case 'RECOVER_ORPHAN_ASSETS': {
1348
+ // Clear registry entries for missing assets
1349
+ const assets = await persist('registry.getSceneAssetsRedis')(redis, scene.bookId, scene.chapterId, scene.sceneId);
1350
+ if (assets) {
1351
+ if (assets.audio && assets.audio.canonical) {
1352
+ await persist('registry.registerSceneAudioRedis')(redis, scene.bookId, scene.chapterId, scene.sceneId, {
1353
+ canonicalPath: assets.audio.canonical,
1354
+ ready: false
1355
+ });
1356
+ }
1357
+ if (assets.image && assets.image.path) {
1358
+ await persist('registry.registerSceneImageRedis')(redis, scene.bookId, scene.chapterId, scene.sceneId, {
1359
+ path: assets.image.path,
1360
+ ready: false
1361
+ });
1362
+ }
1363
+ if (assets.video && assets.video.path) {
1364
+ await persist('registry.registerSceneVideoRedis')(redis, scene.bookId, scene.chapterId, scene.sceneId, {
1365
+ path: assets.video.path,
1366
+ ready: false
1367
+ });
1368
+ }
1369
+ }
1370
+
1371
+ // M5 Шаг 4: Route through orchestrator.setScenePending (includes syncLinearState)
1372
+ // S-5: asset-state writer via the injected orchestration seam
1373
+ await orchestrationSeams.getOrchestrationSeam('setScenePending')(redis, scene.bookId, scene.chapterId, scene.sceneId, 'audio');
1374
+
1375
+ return { success: true, action, scene, details: 'registry marked for recovery' };
1376
+ }
1377
+
1378
+ default:
1379
+ return { success: false, action, scene, details: 'unknown_action' };
1380
+ }
1381
+ } catch (err) {
1382
+ error(`Fix execution error for ${scene.bookId}/${scene.chapterId}/${scene.sceneId}: ${err.message}`);
1383
+ return { success: false, action, scene, details: err.message };
1384
+ }
1385
+ }
1386
+
1387
+ // ======================================================
1388
+ // BULK RECONCILIATION
1389
+ // ======================================================
1390
+
1391
+ /**
1392
+ * Reconcile all scenes for a specific book.
1393
+ */
1394
+ async function reconcileBook(redis, bookId) {
1395
+ log(`Starting book reconciliation: ${bookId}`);
1396
+
1397
+ const report = new ReconciliationReport();
1398
+ let scanned = 0;
1399
+
1400
+ // Scan event journals for scenes
1401
+ const journalPattern = `animastor:event-journal:${bookId}:*`;
1402
+ let cursor = 0;
1403
+
1404
+ do {
1405
+ const result = await redis.scan(cursor, 'MATCH', journalPattern, 'COUNT', 200);
1406
+ cursor = parseInt(result[0], 10);
1407
+ const keys = result[1];
1408
+
1409
+ for (const key of keys) {
1410
+ const parts = key.split(':');
1411
+ if (parts.length >= 5) {
1412
+ const chapterId = parts[3];
1413
+ const sceneId = parts[4];
1414
+
1415
+ const sceneReport = await reconcileScene(redis, bookId, chapterId, sceneId);
1416
+
1417
+ report.orphanStates.push(...sceneReport.orphanStates);
1418
+ report.orphanAssets.push(...sceneReport.orphanAssets);
1419
+ report.partialBuilds.push(...sceneReport.partialBuilds);
1420
+ report.staleLocks.push(...sceneReport.staleLocks);
1421
+ report.inconsistentScenes.push(...sceneReport.inconsistentScenes);
1422
+
1423
+ scanned++;
1424
+ }
1425
+ }
1426
+ } while (cursor !== 0);
1427
+
1428
+ log(`Reconciliation complete: ${scanned} scenes scanned, ${report.inconsistentScenes.length} inconsistencies found`);
1429
+
1430
+ return report;
1431
+ }
1432
+
1433
+ /**
1434
+ * Reconcile all scenes across all books.
1435
+ */
1436
+ async function reconcileAll(redis) {
1437
+ log('Starting full system reconciliation');
1438
+
1439
+ const report = new ReconciliationReport();
1440
+ let scanned = 0;
1441
+
1442
+ // Find all scenes via event journal
1443
+ const journalPattern = 'animastor:event-journal:*';
1444
+ let cursor = 0;
1445
+
1446
+ do {
1447
+ const result = await redis.scan(cursor, 'MATCH', journalPattern, 'COUNT', 500);
1448
+ cursor = parseInt(result[0], 10);
1449
+ const keys = result[1];
1450
+
1451
+ for (const key of keys) {
1452
+ const parts = key.split(':');
1453
+ if (parts.length >= 5) {
1454
+ const bookId = parts[2];
1455
+ const chapterId = parts[3];
1456
+ const sceneId = parts[4];
1457
+
1458
+ const sceneReport = await reconcileScene(redis, bookId, chapterId, sceneId);
1459
+
1460
+ report.orphanStates.push(...sceneReport.orphanStates);
1461
+ report.orphanAssets.push(...sceneReport.orphanAssets);
1462
+ report.partialBuilds.push(...sceneReport.partialBuilds);
1463
+ report.staleLocks.push(...sceneReport.staleLocks);
1464
+ report.inconsistentScenes.push(...sceneReport.inconsistentScenes);
1465
+
1466
+ scanned++;
1467
+ }
1468
+ }
1469
+ } while (cursor !== 0);
1470
+
1471
+ // Check counter drift once globally (leases vs counters are system-wide, not per-scene)
1472
+ for (const stage of mediaRegistry.listMediaTypes()) {
1473
+ const driftCheck = await counterReconciliation.getCounterWithDriftCheck(redis, stage);
1474
+ if (driftCheck && !driftCheck.correct) {
1475
+ report.inconsistentScenes.push({
1476
+ scene: { bookId: '*', chapterId: '*', sceneId: '*' },
1477
+ stage,
1478
+ drift: driftCheck.drift,
1479
+ leaseCount: driftCheck.leaseCount,
1480
+ counterValue: driftCheck.counterValue,
1481
+ issue: 'counter_drift'
1482
+ });
1483
+ }
1484
+ }
1485
+
1486
+ log(`Full reconciliation complete: ${scanned} scenes scanned, ${report.inconsistentScenes.length} inconsistencies found`);
1487
+
1488
+ return report;
1489
+ }
1490
+
1491
+ /**
1492
+ * Get current metrics from reconciliation engine.
1493
+ */
1494
+ async function getMetrics(redis) {
1495
+ return {
1496
+ activeChecks: 'all_scenes_via_journal_scan'
1497
+ };
1498
+ }
1499
+
1500
+ // ======================================================
1501
+ // T6: ЕДИНЫЙ RECONCILIATION-ЦИКЛ
1502
+ // ======================================================
1503
+ // Заменяет 4 несогласованных механизма восстановления:
1504
+ // startup-recovery, audio-recovery, cleanup-service, reconcileAll
1505
+ //
1506
+ // Фазы:
1507
+ // A. Result/error key recovery (из audio-recovery.cjs)
1508
+ // B. Cleanup expired locks (из cleanup-service.cjs)
1509
+ // C. Startup-specific (из startup-recovery.js) — только при startup:true
1510
+ // D. Full scene reconciliation (из reconciliation-engine)
1511
+ //
1512
+ // Каждый прогон пишет RECOVERY_STARTED/RECOVERY_COMPLETED в event-journal.
1513
+ // Один распределённый CLEANUP_LOCK на цикл — прогоны не пересекаются.
1514
+
1515
+ /**
1516
+ * Единый reconciliation-цикл. Выполняет все фазы восстановления.
1517
+ *
1518
+ * @param {Object} redis
1519
+ * @param {Object} deps — зависимости { state, config, postgres, orchestrator, taskHandler, ... }
1520
+ * @param {Object} [options]
1521
+ * @param {boolean} [options.startup=false] — первый прогон при старте (доп. шаги C)
1522
+ * @param {string} [options.scope] — если указан, обработать только одну книгу/сцену
1523
+ * Формат: "bookId" (книга), "bookId:chapterId:sceneId" (одна сцена)
1524
+ * @returns {Promise<{ok:boolean, phases:string[], summary:object}>}
1525
+ */
1526
+ async function reconcileCycle(redis, deps = {}, options = {}) {
1527
+ const { startup = false, scope = null } = options;
1528
+ const startTime = Date.now();
1529
+ const phases = [];
1530
+ const summary = { totalScanned: 0, itemsRecovered: 0, staleLocks: 0, errors: [] };
1531
+
1532
+ // ── Acquire distributed CLEANUP_LOCK ──
1533
+ const lockKey = config.REDIS.CLEANUP_LOCK || 'animastor:cleanup-lock';
1534
+ const lockToken = `reconcile-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
1535
+ const lockAcquired = await redis.set(lockKey, lockToken, 'NX', 'EX', 120);
1536
+ if (!lockAcquired) {
1537
+ log('Cycle skipped — CLEANUP_LOCK held by another instance');
1538
+ return { ok: false, reason: 'lock_held', phases: ['lock_skipped'], summary };
1539
+ }
1540
+
1541
+ const releaseLock = async () => {
1542
+ try {
1543
+ const current = await redis.get(lockKey);
1544
+ if (current === lockToken) await redis.del(lockKey);
1545
+ } catch (_) {}
1546
+ };
1547
+
1548
+ try {
1549
+ // ── Journal: RECOVERY_STARTED ──
1550
+ try {
1551
+ await journal.appendSceneEvent(redis, '_system', '_recovery', '_cycle',
1552
+ journal.EventType.RECOVERY_STARTED, 'INITIATED',
1553
+ { startup, scope, phases: [] }
1554
+ );
1555
+ } catch (_) {}
1556
+
1557
+ // ══════════════════════════════════════════════
1558
+ // PHASE A: Result/error key recovery (audio-recovery)
1559
+ // ══════════════════════════════════════════════
1560
+ if (deps.taskHandler) {
1561
+ try {
1562
+ const aCount = await recoverResultKeys(redis, deps, scope);
1563
+ summary.itemsRecovered += aCount;
1564
+ phases.push(`result_recovery:${aCount}`);
1565
+ } catch (err) {
1566
+ warn(`Phase A failed: ${err.message}`);
1567
+ summary.errors.push(`result_recovery: ${err.message}`);
1568
+ }
1569
+ }
1570
+
1571
+ // ══════════════════════════════════════════════
1572
+ // PHASE B1: Stalled audio chunks watchdog
1573
+ // ══════════════════════════════════════════════
1574
+ try {
1575
+ const b1Count = await checkStalledAudioScenes(redis, deps);
1576
+ if (b1Count > 0) phases.push(`stalled_audio:${b1Count}`);
1577
+ } catch (err) {
1578
+ warn(`Phase B1 failed: ${err.message}`);
1579
+ summary.errors.push(`stalled_audio: ${err.message}`);
1580
+ }
1581
+
1582
+ // ══════════════════════════════════════════════
1583
+ // PHASE B2: Stalled video groups watchdog
1584
+ // ══════════════════════════════════════════════
1585
+ try {
1586
+ const b2Count = await checkStalledVideoScenes(redis, deps);
1587
+ if (b2Count > 0) phases.push(`stalled_video:${b2Count}`);
1588
+ } catch (err) {
1589
+ warn(`Phase B2 failed: ${err.message}`);
1590
+ summary.errors.push(`stalled_video: ${err.message}`);
1591
+ }
1592
+
1593
+ // ══════════════════════════════════════════════
1594
+ // PHASE C: Startup-specific (startup-recovery)
1595
+ // ══════════════════════════════════════════════
1596
+ if (startup) {
1597
+ // C0: Recover Redis chunks from disk
1598
+ if (typeof deps.recoverAllBooksFromDisk === 'function') {
1599
+ try {
1600
+ await deps.recoverAllBooksFromDisk();
1601
+ phases.push('chunk_recovery:ok');
1602
+ } catch (err) {
1603
+ warn(`Phase C0 failed: ${err.message}`);
1604
+ summary.errors.push(`chunk_recovery: ${err.message}`);
1605
+ }
1606
+ }
1607
+
1608
+ // C1: Audio-orch state recovery
1609
+ try {
1610
+ const c1Count = await recoverAudioOrchStates(redis, deps);
1611
+ summary.itemsRecovered += c1Count;
1612
+ phases.push(`audio_orch:${c1Count}`);
1613
+ } catch (err) {
1614
+ warn(`Phase C1 failed: ${err.message}`);
1615
+ summary.errors.push(`audio_orch: ${err.message}`);
1616
+ }
1617
+
1618
+ // C1b: Video-orch state recovery
1619
+ try {
1620
+ const c1bCount = await recoverVideoOrchStates(redis, deps);
1621
+ summary.itemsRecovered += c1bCount;
1622
+ phases.push(`video_orch:${c1bCount}`);
1623
+ } catch (err) {
1624
+ warn(`Phase C1b failed: ${err.message}`);
1625
+ summary.errors.push(`video_orch: ${err.message}`);
1626
+ }
1627
+
1628
+ // C2: Version staleness check
1629
+ try {
1630
+ const c2Count = await checkVersionStaleness(redis, deps);
1631
+ if (c2Count > 0) phases.push(`version_stale:${c2Count}`);
1632
+ } catch (err) {
1633
+ warn(`Phase C2 failed: ${err.message}`);
1634
+ summary.errors.push(`version_stale: ${err.message}`);
1635
+ }
1636
+
1637
+ // C3 removed (S1.4): was log-only PNG scan, no state mutation.
1638
+ // recoverIuImagesFromDisk() too expensive for every 60s cycle without value.
1639
+
1640
+ // C4: Reconcile missing scene counters from PG
1641
+ try {
1642
+ const c4Count = await reconcileMissingSceneState(redis, deps);
1643
+ if (c4Count > 0) phases.push(`counter_reconcile:${c4Count}`);
1644
+ } catch (err) {
1645
+ warn(`Phase C4 failed: ${err.message}`);
1646
+ summary.errors.push(`counter_reconcile: ${err.message}`);
1647
+ }
1648
+
1649
+ // C5: Resume incomplete sessions (optional — requires runBackgroundWindowGeneration)
1650
+ if (typeof deps.resumeIncompleteSessions === 'function' && typeof deps.runBackgroundWindowGeneration === 'function') {
1651
+ try {
1652
+ await deps.resumeIncompleteSessions(log, deps.runBackgroundWindowGeneration);
1653
+ phases.push('session_resume');
1654
+ } catch (err) {
1655
+ warn(`Phase C5 failed: ${err.message}`);
1656
+ summary.errors.push(`session_resume: ${err.message}`);
1657
+ }
1658
+ }
1659
+ }
1660
+
1661
+ // ══════════════════════════════════════════════
1662
+ // PHASE C6: Restore durable layer-config from book.json (Кирпич №2)
1663
+ // ══════════════════════════════════════════════
1664
+ // Сервис layer-config владеет форматом: заполняет ТОЛЬКО отсутствующие
1665
+ // Redis-ключи из durable-копии (нормализуя значения). В норме — no-op.
1666
+ // Запускается каждый цикл (не только startup), чтобы heal удалений
1667
+ // ключа cleanBookRedisKeys на cache-clear / import / recovery, пока
1668
+ // книга существует — конфиг не пропадает из Redis до рестарта.
1669
+
1670
+ if (!scope) {
1671
+ // O-10: through the LayerConfigPort (host adapter:
1672
+ // storage/layer-config-adapter). The pre-O-10 optional-load
1673
+ // semantics are preserved EXACTLY: when the host service
1674
+ // cannot be loaded, this phase is skipped with a warn (a
1675
+ // missing module ≠ failed recovery). The outer try/catch
1676
+ // wraps the PORT call — the adapter's lazy call-time
1677
+ // resolver throws at the same point the pre-O-10 inner
1678
+ // require did, and the caller's catch handles it
1679
+ // identically (the pre-O-10 inner try set layerConfig to
1680
+ // null and skipped silently; the C6 error path below is
1681
+ // reached only when the CALL fails, same as before).
1682
+ try {
1683
+ const c6Count = await require('./layer-config-port').layerConfigOp('restoreFromBooks')(redis);
1684
+ if (c6Count > 0) phases.push(`layer_config_restore:${c6Count}`);
1685
+ } catch (err) {
1686
+ warn(`Phase C6 failed: ${err.message}`);
1687
+ summary.errors.push(`layer_config_restore: ${err.message}`);
1688
+ }
1689
+ }
1690
+
1691
+ // ══════════════════════════════════════════════
1692
+ // PHASE C7: Retry pending entity purges (deleted scenes/units)
1693
+ // ══════════════════════════════════════════════
1694
+ // The entity-delete cleanup records a marker in the pending-purge set
1695
+ // whenever a purge is only partially complete. Retry here keeps the
1696
+ // "delete = gone everywhere" guarantee without hiding failures.
1697
+ if (deps.entityCleanup && typeof deps.entityCleanup.retryPendingPurges === 'function') {
1698
+ try {
1699
+ const pCount = await deps.entityCleanup.retryPendingPurges();
1700
+ if (pCount > 0) phases.push(`pending_purge:${pCount}`);
1701
+ } catch (err) {
1702
+ warn(`Phase C7 failed: ${err.message}`);
1703
+ summary.errors.push(`pending_purge: ${err.message}`);
1704
+ }
1705
+ }
1706
+
1707
+ // ══════════════════════════════════════════════
1708
+ // PHASE C8: WORK_TO_DO rebuild (Option E) — startup-only
1709
+ // ══════════════════════════════════════════════
1710
+ // Авто-возобновление прерванной генерации после потери Redis. Только
1711
+ // первый цикл (startup:true) — периодический цикл работу НЕ возрождает
1712
+ // (политика Recon #4 §10.3: без user action генерация не стартует).
1713
+ if (startup && !scope) {
1714
+ try {
1715
+ const c7Count = await rebuildWorkList(redis);
1716
+ if (c7Count > 0) phases.push(`worklist_rebuild:${c7Count}`);
1717
+ } catch (err) {
1718
+ warn(`Phase C8 failed: ${err.message}`);
1719
+ summary.errors.push(`worklist_rebuild: ${err.message}`);
1720
+ }
1721
+ }
1722
+
1723
+ // ══════════════════════════════════════════════
1724
+ // PHASE D: Full scene reconciliation
1725
+ // ══════════════════════════════════════════════
1726
+ if (!scope || scope.includes(':')) {
1727
+ try {
1728
+ const dReport = await reconcileAll(redis);
1729
+ summary.totalScanned = dReport.orphanStates.length + dReport.orphanAssets.length;
1730
+ phases.push(`reconcile:${dReport.inconsistentScenes.length}_issues`);
1731
+ // Auto-fix safe issues
1732
+ const fixes = getFixRecommendations(dReport.inconsistentScenes);
1733
+ for (const fix of fixes.filter(f => f.safeToExecute)) {
1734
+ try {
1735
+ await applyFix(redis, fix);
1736
+ } catch (fixErr) {
1737
+ warn(`Auto-fix failed for ${fix.scene.sceneId}: ${fixErr.message}`);
1738
+ }
1739
+ }
1740
+ } catch (err) {
1741
+ warn(`Phase D failed: ${err.message}`);
1742
+ summary.errors.push(`reconcile: ${err.message}`);
1743
+ }
1744
+ }
1745
+
1746
+ // ── Journal: RECOVERY_COMPLETED ──
1747
+ try {
1748
+ await journal.appendSceneEvent(redis, '_system', '_recovery', '_cycle',
1749
+ journal.EventType.RECOVERY_COMPLETED, 'DONE',
1750
+ { startup, scope, phases, summary, elapsedMs: Date.now() - startTime }
1751
+ );
1752
+ } catch (_) {}
1753
+
1754
+ log(`Cycle complete [${phases.join(', ')}] in ${Date.now() - startTime}ms`);
1755
+ return { ok: true, phases, summary };
1756
+
1757
+ } finally {
1758
+ await releaseLock();
1759
+ }
1760
+ }
1761
+
1762
+ // ── PHASE A: Recover result/error keys ─────────────────
1763
+ // Из audio-recovery.cjs: сканирует animastor:result:* и animastor:error:*,
1764
+ // доигрывает потерянные результаты через taskHandler.handleTaskResult.
1765
+ // Для error-ключей вызывает orchestrator.failStage.
1766
+ async function recoverResultKeys(redis, deps, scope) {
1767
+ const taskHandler = deps.taskHandler;
1768
+ if (!taskHandler || typeof taskHandler.handleTaskResult !== 'function') return 0;
1769
+
1770
+ let recovered = 0;
1771
+
1772
+ // Отклонения dispatch-identity — терминальные для результата: dispatch
1773
+ // уже финализирован/отменён, файл писать нельзя. Ключ удаляем, чтобы
1774
+ // recovery не ретраил его каждые 60 сек до истечения 1-часового TTL
1775
+ // (бывший бесконечный цикл по _g2.._g5 после no_active_dispatch).
1776
+ const TERMINAL_REJECTIONS = new Set([
1777
+ 'no_active_dispatch',
1778
+ 'stale_dispatch',
1779
+ 'missing_dispatch_id',
1780
+ 'metadata_missing_dispatch_id',
1781
+ 'dispatch_metadata_changed',
1782
+ 'lease_token_mismatch',
1783
+ ]);
1784
+
1785
+ // Scan animastor:result:*
1786
+ let cursor = '0';
1787
+ do {
1788
+ const [nextCursor, keys] = await redis.scan(cursor, 'MATCH', 'animastor:result:*', 'COUNT', 200);
1789
+ cursor = nextCursor;
1790
+ for (const key of keys) {
1791
+ try {
1792
+ const raw = await redis.get(key);
1793
+ if (!raw) continue;
1794
+
1795
+ let job_id, result_base64, buildId, dispatchId;
1796
+
1797
+ // Try JSON (new format: { job_id, result_base64, build_id })
1798
+ if (raw.startsWith('{')) {
1799
+ try {
1800
+ const data = JSON.parse(raw);
1801
+ job_id = data.job_id;
1802
+ result_base64 = data.result_base64;
1803
+ buildId = data.build_id;
1804
+ dispatchId = data.dispatch_id;
1805
+ } catch (_) {
1806
+ await redis.del(key);
1807
+ continue;
1808
+ }
1809
+ } else {
1810
+ // Data URL fallback (old format)
1811
+ result_base64 = raw;
1812
+ const keyParts = key.split(':');
1813
+ const buildPart = keyParts[2] || '';
1814
+ const combinedId = `${keyParts[3] || ''}_${keyParts[4] || ''}_${keyParts[5] || ''}`;
1815
+ job_id = `${combinedId}:${keyParts[6] || 'audio'}`;
1816
+ buildId = buildPart;
1817
+ }
1818
+
1819
+ if (!job_id || !result_base64) continue;
1820
+
1821
+ // Scope filter
1822
+ if (scope && !job_id.startsWith(scope)) continue;
1823
+
1824
+ if (!buildId || !dispatchId) {
1825
+ warn(`Skipping legacy result without build/dispatch identity: ${job_id}`);
1826
+ continue;
1827
+ }
1828
+ try {
1829
+ await taskHandler.handleTaskResult(job_id, result_base64, buildId, dispatchId);
1830
+ } catch (rejectErr) {
1831
+ if (rejectErr && TERMINAL_REJECTIONS.has(rejectErr.code)) {
1832
+ warn(`Result ${job_id} terminally rejected (${rejectErr.code}) — dropping result key, no infinite retry`);
1833
+ await redis.del(key);
1834
+ } else {
1835
+ throw rejectErr;
1836
+ }
1837
+ continue;
1838
+ }
1839
+ await redis.del(key);
1840
+ recovered++;
1841
+ } catch (itemErr) {
1842
+ warn(`Result recovery item failed: ${itemErr.message}`);
1843
+ }
1844
+ }
1845
+ } while (cursor !== '0');
1846
+
1847
+ // Scan animastor:error:* (T3: форвард ошибок, которые не дошли до backend)
1848
+ cursor = '0';
1849
+ do {
1850
+ const [nextCursor, keys] = await redis.scan(cursor, 'MATCH', 'animastor:error:*', 'COUNT', 200);
1851
+ cursor = nextCursor;
1852
+ for (const key of keys) {
1853
+ try {
1854
+ const raw = await redis.get(key);
1855
+ if (!raw) continue;
1856
+
1857
+ let job_id, buildId, reason, dispatchId;
1858
+ try {
1859
+ const data = JSON.parse(raw);
1860
+ job_id = data.job_id;
1861
+ buildId = data.build_id;
1862
+ reason = data.reason;
1863
+ dispatchId = data.dispatch_id;
1864
+ } catch (_) {
1865
+ await redis.del(key);
1866
+ continue;
1867
+ }
1868
+
1869
+ if (!job_id) continue;
1870
+ if (scope && !job_id.startsWith(scope)) continue;
1871
+
1872
+ // Parse job_id to get stage and scene info
1873
+ const jobSchema = require('@animastor/contracts').jobProtocolV2;
1874
+ const parsed = jobSchema.parseJobId(job_id);
1875
+ if (parsed) {
1876
+ const stageMap = { audio_chunk: 'audio', iu_image: 'image', scene_image: 'image', scene_video: 'video' };
1877
+ const stage = stageMap[parsed.kind];
1878
+ if (stage && deps.orchestrator) {
1879
+ await deps.orchestrator.failStage(redis, parsed.bookId, parsed.chapterId, parsed.sceneId,
1880
+ stage, buildId, reason || 'recovered_error', { dispatchId });
1881
+ recovered++;
1882
+ }
1883
+ }
1884
+ await redis.del(key);
1885
+ } catch (itemErr) {
1886
+ warn(`Error recovery item failed: ${itemErr.message}`);
1887
+ }
1888
+ }
1889
+ } while (cursor !== '0');
1890
+
1891
+ return recovered;
1892
+ }
1893
+
1894
+ // ── PHASE C1: Recover audio-orch states ────────────────
1895
+ // Из startup-recovery.js: не-терминальные фазы → FAILED,
1896
+ // MERGING → DONE если файл есть, иначе FAILED.
1897
+ async function recoverAudioOrchStates(redis, deps) {
1898
+ // O-7: through the AudioFsmPort (host adapter: storage/audio-fsm-adapter).
1899
+ // The pre-O-7 optional-load semantics are preserved EXACTLY: when the
1900
+ // audio-orchestrator host service cannot be required, this recovery leg
1901
+ // returns 0 (missing module ≠ recoverable state). The try/catch wraps
1902
+ // the ADAPTER's lazy require — the port itself stays fail-fast.
1903
+ let audioOrch;
1904
+ try {
1905
+ audioOrch = require('./audio-fsm-port').audioFsm();
1906
+ } catch (_) {
1907
+ return 0;
1908
+ }
1909
+
1910
+ const allStates = await audioOrch.scanAllStates(redis);
1911
+ if (allStates.length === 0) return 0;
1912
+
1913
+ let recovered = 0;
1914
+ const OUTPUT_DIR = config.OUTPUT_DIR;
1915
+
1916
+ for (const entry of allStates) {
1917
+ const { bookId, chapterId, sceneId, state: orchState } = entry;
1918
+ const phase = orchState.phase;
1919
+ const buildId = orchState.build_id || 'default';
1920
+ const mergedPath = syncPath.join(OUTPUT_DIR, buildId, artifactNaming.sceneAudioName(bookId, chapterId, sceneId));
1921
+
1922
+ switch (phase) {
1923
+ case 'GENERATING':
1924
+ case 'WAITING_CHUNKS':
1925
+ // ── RECOVERY: сначала проверяем, все ли чанки на диске ──
1926
+ // Если все чанки на месте — доигрываем merge, не FAIL'им.
1927
+ // Иначе FAILED → re-dispatch.
1928
+ const expectedChunkCount = parseInt(orchState.expected_count || '0', 10);
1929
+ const chunks = lazyHostBinding('media.audio.chunks');
1930
+ const presentChunks = chunks.findExistingSceneChunks(bookId, chapterId, sceneId, buildId, expectedChunkCount > 0 ? expectedChunkCount : null);
1931
+ if (presentChunks.length > 0) {
1932
+ const actualCount = expectedChunkCount > 0 ? expectedChunkCount : presentChunks.length;
1933
+ log(`[AUDIO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: ${phase} — ${presentChunks.length}/${actualCount} chunks on disk, driving merge`);
1934
+ // Восстанавливаем expected_count если он был 0
1935
+ if (expectedChunkCount === 0 && presentChunks.length > 0) {
1936
+ orchState.expected_count = presentChunks.length;
1937
+ await audioOrch.setState(redis, bookId, chapterId, sceneId, orchState);
1938
+ }
1939
+ try {
1940
+ await audioOrch.completeChunk(redis, bookId, chapterId, sceneId, 'recovery', buildId, {
1941
+ audio: deps.audio || lazyHostBinding('media.audio'),
1942
+ orchestrator: deps.orchestrator,
1943
+ dispatchId: 'startup-recovery',
1944
+ });
1945
+ recovered++;
1946
+ break;
1947
+ } catch (mergeErr) {
1948
+ warn(`[AUDIO-ORCH] Recover merge failed for ${bookId}/${chapterId}/${sceneId}: ${mergeErr.message} — falling through to FAILED`);
1949
+ }
1950
+ }
1951
+ log(`[AUDIO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: ${phase} → FAILED (${presentChunks.length}/${expectedChunkCount || '?'} chunks on disk)`);
1952
+ await audioOrch.setFailed(redis, bookId, chapterId, sceneId, 'restart_recovery');
1953
+ // F1: sync asset.audio → FAILED after audioOrch.setFailed (R3 follow-up)
1954
+ await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'audio', state.AssetState.FAILED);
1955
+ // T6: Также сбрасываем asset state, чтобы scheduler передиспатчил
1956
+ if (deps.orchestrator) {
1957
+ await deps.orchestrator.markDirtyScene(redis, bookId, chapterId, sceneId, ['audio']);
1958
+ }
1959
+ recovered++;
1960
+ break;
1961
+ case 'MERGING':
1962
+ if (syncFs.existsSync(mergedPath)) {
1963
+ log(`[AUDIO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: MERGING → DONE`);
1964
+ await audioOrch.setDone(redis, bookId, chapterId, sceneId);
1965
+ // R3: sync asset state after audioOrch.setDone — asset.audio → READY
1966
+ await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'audio', state.AssetState.READY);
1967
+ } else {
1968
+ log(`[AUDIO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: MERGING → FAILED`);
1969
+ await audioOrch.setFailed(redis, bookId, chapterId, sceneId, 'restart_merge_missing');
1970
+ // F1: sync asset.audio → FAILED after audioOrch.setFailed (R3 follow-up)
1971
+ await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'audio', state.AssetState.FAILED);
1972
+ if (deps.orchestrator) {
1973
+ await deps.orchestrator.markDirtyScene(redis, bookId, chapterId, sceneId, ['audio']);
1974
+ }
1975
+ }
1976
+ recovered++;
1977
+ break;
1978
+ case 'PLACEHOLDER_READY':
1979
+ // Leave as is, scheduler will dispatch
1980
+ break;
1981
+ case 'DONE':
1982
+ case 'FAILED':
1983
+ // Terminal, leave as is
1984
+ break;
1985
+ default:
1986
+ await audioOrch.deleteState(redis, bookId, chapterId, sceneId);
1987
+ break;
1988
+ }
1989
+ }
1990
+
1991
+ if (recovered > 0) log(`[AUDIO-ORCH] ${recovered} non-terminal states recovered`);
1992
+ return recovered;
1993
+ }
1994
+
1995
+ // ── PHASE C1b: Recover video-orch states ───────────────
1996
+ // Зеркало recoverAudioOrchStates: не-терминальные фазы → FAILED (re-dispatch),
1997
+ // MERGING/WAITING_CHUNKS с полным набором групп → доигрываем merge.
1998
+ async function recoverVideoOrchStates(redis, deps) {
1999
+ // O-8: through the VideoFsmPort (host adapter: storage/video-fsm-adapter).
2000
+ // The pre-O-8 optional-load semantics are preserved EXACTLY: when the
2001
+ // video-orchestrator host service cannot be required, this recovery leg
2002
+ // returns 0 (missing module ≠ recoverable state). The try/catch wraps
2003
+ // the ADAPTER's lazy require — the port itself stays fail-fast.
2004
+ let videoOrch;
2005
+ try {
2006
+ videoOrch = require('./video-fsm-port').videoFsm();
2007
+ } catch (_) {
2008
+ return 0;
2009
+ }
2010
+
2011
+ const allStates = await videoOrch.scanAllStates(redis);
2012
+ if (allStates.length === 0) return 0;
2013
+
2014
+ let recovered = 0;
2015
+ const OUTPUT_DIR = config.OUTPUT_DIR;
2016
+
2017
+ for (const entry of allStates) {
2018
+ const { bookId, chapterId, sceneId, state: orchState } = entry;
2019
+ const phase = orchState.phase;
2020
+ const buildId = orchState.build_id || 'default';
2021
+ const suffixes = videoOrch.groupSuffixes(orchState);
2022
+ const mergedPath = syncPath.join(OUTPUT_DIR, buildId, artifactNaming.sceneVideoName(bookId, chapterId, sceneId));
2023
+
2024
+ switch (phase) {
2025
+ case 'GENERATING':
2026
+ case 'WAITING_CHUNKS':
2027
+ // ── RECOVERY: сначала проверяем, все ли группы на диске ──
2028
+ const presentCount = suffixes.filter(s => videoOrch.isGroupFileValid(buildId, bookId, chapterId, sceneId, s)).length;
2029
+ if (presentCount === suffixes.length && suffixes.length > 0) {
2030
+ log(`[VIDEO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: ${phase} — ${presentCount}/${suffixes.length} groups on disk, driving merge`);
2031
+ try {
2032
+ await videoOrch.completeGroup(redis, bookId, chapterId, sceneId, '', buildId, {
2033
+ orchestrator: deps.orchestrator,
2034
+ dispatchId: 'startup-recovery',
2035
+ });
2036
+ recovered++;
2037
+ break;
2038
+ } catch (mergeErr) {
2039
+ warn(`[VIDEO-ORCH] Recover merge failed for ${bookId}/${chapterId}/${sceneId}: ${mergeErr.message} — falling through to FAILED`);
2040
+ }
2041
+ }
2042
+ log(`[VIDEO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: ${phase} → FAILED (${presentCount}/${suffixes.length} groups on disk)`);
2043
+ await videoOrch.setFailed(redis, bookId, chapterId, sceneId, 'restart_recovery');
2044
+ // F1: sync asset.video → FAILED после videoOrch.setFailed
2045
+ await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'video', state.AssetState.FAILED);
2046
+ if (deps.orchestrator) {
2047
+ await deps.orchestrator.markDirtyScene(redis, bookId, chapterId, sceneId, ['video']);
2048
+ }
2049
+ recovered++;
2050
+ break;
2051
+ case 'MERGING':
2052
+ if (syncFs.existsSync(mergedPath)) {
2053
+ log(`[VIDEO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: MERGING → DONE`);
2054
+ await videoOrch.setDone(redis, bookId, chapterId, sceneId);
2055
+ await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'video', state.AssetState.READY);
2056
+ } else {
2057
+ log(`[VIDEO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: MERGING → FAILED`);
2058
+ await videoOrch.setFailed(redis, bookId, chapterId, sceneId, 'restart_merge_missing');
2059
+ await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'video', state.AssetState.FAILED);
2060
+ if (deps.orchestrator) {
2061
+ await deps.orchestrator.markDirtyScene(redis, bookId, chapterId, sceneId, ['video']);
2062
+ }
2063
+ }
2064
+ recovered++;
2065
+ break;
2066
+ case 'DONE':
2067
+ case 'FAILED':
2068
+ // Terminal — leave as is
2069
+ break;
2070
+ default:
2071
+ await videoOrch.deleteState(redis, bookId, chapterId, sceneId);
2072
+ break;
2073
+ }
2074
+ }
2075
+
2076
+ if (recovered > 0) log(`[VIDEO-ORCH] ${recovered} non-terminal states recovered`);
2077
+ return recovered;
2078
+ }
2079
+
2080
+ // ── PHASE C2: Version staleness check ───────────────────
2081
+ // Из startup-recovery.js: для stale-ассетов → markDirtyScene
2082
+ // O-2: the raw deps.postgres handle is GONE — the book-wide version scan is
2083
+ // the sceneVersions.getVersionStalenessRows() port op (SQL lives in the host
2084
+ // adapter verbatim). Pre-O-2 a missing deps.postgres skipped the phase; the
2085
+ // unwired-port throw hits the same catch → warn + return 0.
2086
+ async function checkVersionStaleness(redis, deps) {
2087
+ const { orchestrator } = deps;
2088
+ if (!orchestrator) return 0;
2089
+
2090
+ try {
2091
+ const result = { rows: await persist('sceneVersions.getVersionStalenessRows')() };
2092
+
2093
+ const sceneMap = new Map();
2094
+ for (const row of result.rows) {
2095
+ const key = `${row.book_id}|${row.chapter_id}|${row.scene_id}`;
2096
+ if (!sceneMap.has(key)) {
2097
+ sceneMap.set(key, { bookId: row.book_id, chapterId: row.chapter_id, sceneId: row.scene_id, stale: false });
2098
+ }
2099
+ const entry = sceneMap.get(key);
2100
+ if ((row.scene_content_version != null && row.content_version != null && row.scene_content_version < row.content_version) ||
2101
+ (row.scene_audio_config_version != null && row.audio_config_version != null && row.scene_audio_config_version < row.audio_config_version)) {
2102
+ entry.stale = true;
2103
+ }
2104
+ }
2105
+
2106
+ let outdated = 0;
2107
+ for (const entry of sceneMap.values()) {
2108
+ if (entry.stale) {
2109
+ log(`[VERSION-STALE] ${entry.bookId}/${entry.chapterId}/${entry.sceneId}`);
2110
+ await orchestrator.markDirtyScene(redis, entry.bookId, entry.chapterId, entry.sceneId);
2111
+ outdated++;
2112
+ }
2113
+ }
2114
+ return outdated;
2115
+ } catch (err) {
2116
+ warn(`Version staleness check failed: ${err.message}`);
2117
+ return 0;
2118
+ }
2119
+ }
2120
+
2121
+ // ── PHASE C3: REMOVED in S1.4 (2026-07-19) ─────────────
2122
+ // Был log-only PNG scan, сканировал всю OUTPUT_DIR по 60s. Не мутировал состояние,
2123
+ // только логировал счётчик сцен с IU images. Полная walks FS без пользы для reconcile.
2124
+ // Аналогичная функция recoverIuImagesFromDisk осталась в services/startup-recovery.js
2125
+ // (вызывается явно, не на каждый reconcile-цикл).
2126
+
2127
+ // ── PHASE C4: Reconcile missing scene counters from PG ─
2128
+ // Из startup-recovery.js: логирует книги с PG записями без Redis-счётчиков.
2129
+ // O-2: the raw deps.postgres handle is GONE — the DISTINCT book scan is the
2130
+ // same listBooks() port op the worklist rebuild uses (identical SQL, host
2131
+ // adapter). Unwired port → throw → catch → return 0 (pre-O-2: no
2132
+ // deps.postgres → return 0).
2133
+ async function reconcileMissingSceneState(redis, deps) {
2134
+ try {
2135
+ const bookResult = { rows: await persist('listBooks')() };
2136
+ let count = 0;
2137
+ for (const row of bookResult.rows) {
2138
+ const totalKey = `animastor:book-scenes:${row.book_id}:total`;
2139
+ const totalRaw = await redis.get(totalKey);
2140
+ if (!totalRaw || parseInt(totalRaw, 10) === 0) {
2141
+ log(`[COUNTER-LOG-ONLY] Book ${row.book_id} has PG records but no Redis counters`);
2142
+ count++;
2143
+ }
2144
+ }
2145
+ return count;
2146
+ } catch (err) {
2147
+ warn(`Missing scene state check failed: ${err.message}`);
2148
+ return 0;
2149
+ }
2150
+ }
2151
+
2152
+ // ── PHASE C7: WORK_TO_DO rebuild (Option E) ──────────
2153
+ // Cathedral Option E — см. docs/architecture/work-list-rebuild-design.md
2154
+ // (Recon #4 design proof). После полной потери Redis пересчитывает WORK_TO_DO
2155
+ // из PG + FS и МАТЕРИАЛИЗУЕТ runtime-состояние: per-asset Redis states + SADD
2156
+ // в active index. Генерацию НЕ запускает — это делает обычный scheduler tick
2157
+ // (единственный владелец execution, SCHEDULER_TICK_LOCK).
2158
+ //
2159
+ // Доказанные в Recon #4 инварианты:
2160
+ // 1. Предикат НЕ читает Redis asset state (C0 over-marks READY/READY/READY
2161
+ // для любой сцены с .mp3) — только PG + FS-проба.
2162
+ // 2. States пишутся ДО SADD: scheduler решает «что диспатчить» по Redis
2163
+ // states (shouldScheduleAssets), пустой ключ = NEW = «диспатчить всё».
2164
+ // 3. build_id пинится из manifest.book_id — та же формула, что в attemptDispatch.
2165
+ // 4. Книги с cancellation tombstone (Operation #1) пропускаются fail-closed.
2166
+ // 5. Фаза последняя среди state-write C-фаз и идёт ПОСЛЕ C6 (layer-config
2167
+ // восстановлен — нужен для enabled-слоёв).
2168
+ // 6. Ошибки PG — fail-closed: без PG предикат невычислим, фаза прерывается
2169
+ // (нельзя воскресить отменённую работу или потерять dirty-маркеры).
2170
+ async function rebuildWorkList(redis) {
2171
+ // Все зависимости — lazy require на момент вызова, чтобы фаза была
2172
+ // самодостаточной (не зависит от того, с какими зависимостями был
2173
+ // загружен reconciliation-engine) и чтобы unit-тесты не могли подменить
2174
+ // storage/state через require.cache.
2175
+
2176
+ // §32.30: former host requires resolve lazily through the host-bindings seam.
2177
+ const state = lazyHostBinding('state');
2178
+ // O-3: scene content via the SceneDataPort (host adapter:
2179
+ // storage/scene-data-adapter) — the Book Model facade require left this
2180
+ // function. loadBook/collectScenes consumed as `sceneData(op)(...)`.
2181
+ const sceneData = require('./scene-data-port').sceneDataOp;
2182
+ // O-2: PG persistence (scene-assets + cancel tombstone + the two raw
2183
+ // barrel queries) via the PersistencePort; the storage barrel require
2184
+ // left this function.
2185
+ const persist = require('./persistence-port').persist;
2186
+ // O-10: per-book layer config arrives ONLY through the LayerConfigPort
2187
+ // (host adapter: storage/layer-config-adapter) — the layer-config host
2188
+ // service (../services/layer-config) left this function.
2189
+ const layerConfigOp = require('./layer-config-port').layerConfigOp;
2190
+ // O-4: placeholder audio arrives ONLY through the PlaceholderAudioPort
2191
+ // (host adapter: storage/placeholder-audio-adapter) — the ffmpeg/fs
2192
+ // host service left this function.
2193
+ const placeholderAudioOp = require('./placeholder-audio-port').placeholderAudioOp;
2194
+ const sceneWindow = require('./scene-window');
2195
+ const runtimeScheduler = require('./runtime-scheduler');
2196
+ // S-5: asset-state writers via the injected orchestration seams
2197
+ const markDirtyScene = orchestrationSeams.getOrchestrationSeam('markDirtyScene');
2198
+ const setScenePending = orchestrationSeams.getOrchestrationSeam('setScenePending');
2199
+
2200
+ const AssetState = state.AssetState;
2201
+ const STAGES = mediaRegistry.listMediaTypes();
2202
+
2203
+ // 0. Fail-closed: книги с cancellation tombstone (Operation #1)
2204
+ const cancelled = new Set();
2205
+ const cancelRows = await persist('cancel.getAllCancelled')();
2206
+ for (const r of cancelRows) cancelled.add(r.book_id);
2207
+
2208
+ // 1. Книги с историей генерации — из PG (у книги без PG-строк нет работы)
2209
+ // O-2: the raw barrel query moved verbatim to the host adapter
2210
+ // (storage/runtime-persistence-adapter.listBooks).
2211
+ const books = (await persist('listBooks')()).map(r => r.book_id);
2212
+
2213
+ let scenesAdded = 0;
2214
+ for (const bookId of books) {
2215
+ if (cancelled.has(bookId)) {
2216
+ log(`[WORKLIST] ${bookId}: skipped (cancellation tombstone)`);
2217
+ continue;
2218
+ }
2219
+
2220
+ try {
2221
+ const loadedBook = sceneData('loadBook')(bookId);
2222
+ if (!loadedBook) {
2223
+ log(`[WORKLIST] ${bookId}: no book.json on disk — skipped`);
2224
+ continue;
2225
+ }
2226
+
2227
+ const buildId = loadedBook?.manifest?.build_id || 'default';
2228
+ const buildDir = syncPath.join(config.OUTPUT_DIR, buildId);
2229
+ const layerCfg = await layerConfigOp('get')(redis, bookId);
2230
+ const enabled = Object.fromEntries(
2231
+ mediaRegistry.listMediaTypes().map(t => [t, layerCfg[t + '_enabled'] !== false])
2232
+ );
2233
+
2234
+ // 2. Dirty-маркеры из PG (audit gap 2): каждый marker РЕАЛЬНО влияет
2235
+ // на WORK_TO_DO. Раньше dirtyScenes собирался, но не использовался —
2236
+ // dirty_unit_ids и per-asset статусы (stale/failed/pending) теряли
2237
+ // regeneration intent. Теперь:
2238
+ // - staleAssets: ready-строка с version mismatch → PENDING (как раньше)
2239
+ // - dirtyStages: scene_assets.status IN (stale,failed,pending) → PENDING
2240
+ // даже при валидном файле на диске (leftover от прошлого билда)
2241
+ // - unitDirtyScenes: dirty_unit_ids → image PENDING (force-regen юнитов)
2242
+ const staleAssets = {}; // "ch:sc:asset" → ready, но version mismatch
2243
+ const dirtyStages = {}; // "ch:sc:asset" → status IN (stale,failed,pending)
2244
+ const unitDirtyScenes = new Set(); // "ch:sc" → dirty_unit_ids (image force-regen)
2245
+ const dirtyRows = await persist('sceneAssets.getDirtyScenesByVersion')(bookId);
2246
+ for (const row of dirtyRows) {
2247
+ const contentStale = row.scene_content_version != null && row.content_version != null &&
2248
+ row.scene_content_version < row.content_version;
2249
+ const audioCfgStale = row.scene_audio_config_version != null && row.audio_config_version != null &&
2250
+ row.scene_audio_config_version < row.audio_config_version;
2251
+ if (row.status === 'ready' &&
2252
+ (contentStale || (row.asset_type === 'audio' && audioCfgStale))) {
2253
+ staleAssets[`${row.chapter_id}:${row.scene_id}:${row.asset_type}`] = true;
2254
+ }
2255
+ }
2256
+
2257
+ // 2b. dirty_unit_ids (granular force-regen) → image; per-asset статусы
2258
+ // (stale/failed/pending) → PENDING. asset_type обязателен, чтобы
2259
+ // знать, КАКАЯ стадия грязная (раньше терялся).
2260
+ // O-2: dirty-marker reads via the PersistencePort. Pre-O-2 these
2261
+ // were two raw barrel queries — the SQL moved verbatim into the
2262
+ // host adapter op `sceneAssets.getDirtyMarkers(bookId)` which
2263
+ // returns { unitScenes, assetStages } in the same row shapes.
2264
+ const markers = await persist('sceneAssets.getDirtyMarkers')(bookId);
2265
+ for (const row of markers.unitScenes) unitDirtyScenes.add(`${row.chapter_id}:${row.scene_id}`);
2266
+ for (const row of markers.assetStages) dirtyStages[`${row.chapter_id}:${row.scene_id}:${row.asset_type}`] = true;
2267
+
2268
+ // 3. Пройти все сцены книги: FS-проба + предикат §4 (Recon #3)
2269
+ const scenes = sceneData('collectScenes')(loadedBook);
2270
+ for (const s of scenes) {
2271
+ const chapterId = s.chapter_id;
2272
+ const sceneId = s.scene_id;
2273
+ const key = `${chapterId}:${sceneId}`;
2274
+
2275
+ const fsStatus = await sceneWindow.getSceneFilesStatus(buildDir, bookId, chapterId, sceneId);
2276
+ let audioReal = false;
2277
+ if (fsStatus.audio.exists) {
2278
+ try { audioReal = await placeholderAudioOp('hasRealAudio')(bookId, chapterId, sceneId, buildId); }
2279
+ catch (err) { warn(`Rebuild: hasRealAudio probe failed for ${bookId}/${chapterId}/${sceneId}: ${err.message}`); }
2280
+ }
2281
+
2282
+ const stateMap = {};
2283
+ let needsWork = false;
2284
+ for (const stage of STAGES) {
2285
+ if (!enabled[stage]) continue; // disabled: PENDING не пишем (isInFlight-override в shouldScheduleAssets)
2286
+ const fileValid = fsStatus[stage].exists && (stage !== 'audio' || audioReal);
2287
+ const stale = !!staleAssets[`${key}:${stage}`];
2288
+ const dirty = !!dirtyStages[`${key}:${stage}`];
2289
+ const unitDirty = stage === 'image' && unitDirtyScenes.has(key);
2290
+ if (fileValid && !stale && !dirty && !unitDirty) {
2291
+ stateMap[stage] = AssetState.READY;
2292
+ } else {
2293
+ stateMap[stage] = AssetState.PENDING;
2294
+ needsWork = true;
2295
+ }
2296
+ }
2297
+
2298
+ if (!needsWork) continue;
2299
+
2300
+ // 4. Материализовать states ДО SADD — через SAFE facade (audit gap 1).
2301
+ // Нельзя downgrade-ить выполняющуюся генерацию:
2302
+ // GENERATING/PENDING/PLACEHOLDER → НЕ трогаем (in-flight/очередь
2303
+ // владеют стадией; recovery не имеет права их откатывать).
2304
+ // PENDING-таргеты → orchestrator.setScenePending (валидация переходов:
2305
+ // GENERATING→PENDING rejected самой таблицей переходов).
2306
+ // READY-таргеты → guarded restore-fact write (S2.4): только из
2307
+ // NEW/DIRTY/FAILED/READY, никогда поверх GENERATING/PENDING.
2308
+ // C0 over-mark (READY без файла) → READY→DIRTY (markDirtyScene) →
2309
+ // →PENDING (setScenePending) — self-heal P1 через валидные переходы.
2310
+ const currentStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
2311
+ for (const [stage, target] of Object.entries(stateMap)) {
2312
+ const current = currentStates[stage] || state.AssetState.NEW;
2313
+ // GENERATING/PENDING — выполняющаяся/очередная генерация:
2314
+ // recovery не имеет права downgrade-ить её (audit gap 1).
2315
+ if (current === state.AssetState.GENERATING ||
2316
+ current === state.AssetState.PENDING) {
2317
+ log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: ${current} — untouched by recovery (in-flight/queued)`);
2318
+ continue;
2319
+ }
2320
+ if (target === state.AssetState.PENDING) {
2321
+ if (current === state.AssetState.READY || current === state.AssetState.PLACEHOLDER) {
2322
+ // C0 over-mark (P1, READY без файла) / placeholder-аудио:
2323
+ // прямой READY→PENDING / PLACEHOLDER→PENDING невалиден;
2324
+ // READY→DIRTY / PLACEHOLDER→DIRTY валиден → затем DIRTY→PENDING.
2325
+ // TOCTOU-guard (audit follow-up): scheduler tick работает
2326
+ // конкурентно (runtime-loop стартует ДО startup-цикла),
2327
+ // markVersionStaleDirty → dispatch может успеть выставить
2328
+ // GENERATING между нашим guard-read и markDirtyScene (который
2329
+ // пишет DIRTY через unsafe без re-validation). Re-read прямо
2330
+ // перед записью: если состояние изменилось — стадией уже
2331
+ // владеет кто-то другой, recovery отступает. setScenePending
2332
+ // ниже сам re-reads + validates (вторая линия обороны).
2333
+ const fresh = (await state.getAssetStates(redis, bookId, chapterId, sceneId))[stage] || state.AssetState.NEW;
2334
+ if (fresh !== current) {
2335
+ log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: state changed ${current}→${fresh} during rebuild — untouched (owner present)`);
2336
+ continue;
2337
+ }
2338
+ log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: ${current} → DIRTY → PENDING (valid transitions)`);
2339
+ await markDirtyScene(redis, bookId, chapterId, sceneId, [stage]);
2340
+ await setScenePending(redis, bookId, chapterId, sceneId, stage);
2341
+ } else {
2342
+ // NEW/DIRTY/FAILED → PENDING — валидные переходы.
2343
+ await setScenePending(redis, bookId, chapterId, sceneId, stage);
2344
+ }
2345
+ } else {
2346
+ // target === READY — restore-fact (файл валиден на диске):
2347
+ // guarded unsafe write, S2.4 (restore path). Пропускаются
2348
+ // GENERATING/PENDING (см. guard выше); из остальных состояний
2349
+ // (NEW/DIRTY/FAILED/PLACEHOLDER/READY) восстановить READY
2350
+ // безопасно — файл доказан FS-пробой.
2351
+ if (current !== state.AssetState.READY) {
2352
+ await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, stage, state.AssetState.READY);
2353
+ }
2354
+ }
2355
+ }
2356
+ await runtimeScheduler.addSceneToActiveIndex(redis, bookId, chapterId, sceneId);
2357
+ scenesAdded++;
2358
+ }
2359
+ } catch (err) {
2360
+ warn(`Rebuild failed for book ${bookId}: ${err.message}`);
2361
+ }
2362
+ }
2363
+
2364
+ if (scenesAdded > 0) log(`[WORKLIST] ${scenesAdded} scene(s) re-added for generation resume`);
2365
+ return scenesAdded;
2366
+ }
2367
+
2368
+ // ======================================================
2369
+ // EXPORTS
2370
+ // ======================================================
2371
+
2372
+ module.exports = {
2373
+ reconcileScene,
2374
+ reconcileBook,
2375
+ reconcileAll,
2376
+ reconcileCycle,
2377
+ recoverResultKeys,
2378
+ recoverAudioOrchStates,
2379
+ recoverVideoOrchStates,
2380
+ checkVersionStaleness,
2381
+ reconcileMissingSceneState,
2382
+ rebuildWorkList,
2383
+ checkStalledAudioScenes,
2384
+ checkStalledVideoScenes,
2385
+
2386
+ checkAudioOrchInvariants,
2387
+ checkVideoOrchInvariants,
2388
+
2389
+ checkOrphanVideoState,
2390
+ checkOrphanImageState,
2391
+ checkOrphanAudioState,
2392
+ checkOrphanAssets,
2393
+ checkOrphanGeneratingState,
2394
+ checkPartialBuilds,
2395
+ checkStaleLocks,
2396
+ checkStaleDispatchLeases,
2397
+
2398
+ isBookGenerationCancelled,
2399
+ resolveBookBuildId,
2400
+
2401
+ getFixRecommendations,
2402
+ applyFix,
2403
+ getMetrics,
2404
+ ReconciliationReport
2405
+ };