@animastor/orchestration 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +75 -0
- package/package.json +51 -0
- package/src/host/host-bindings.js +138 -0
- package/src/index.js +99 -0
- package/src/orchestration/index.js +40 -0
- package/src/orchestration/orchestrator.js +630 -0
- package/src/orchestration/runtime-result-consumer.js +54 -0
- package/src/orchestration/scene-callbacks.js +542 -0
- package/src/orchestration/scene-orchestrator.js +601 -0
- package/src/orchestration/scene-restoration.js +129 -0
- package/src/orchestration/scene-utils.js +36 -0
- package/src/runtime/active-scenes-index.js +164 -0
- package/src/runtime/audio-fsm-port.js +187 -0
- package/src/runtime/circuit-breaker.js +588 -0
- package/src/runtime/counter-reconciliation.js +308 -0
- package/src/runtime/dispatch-engine.js +1768 -0
- package/src/runtime/failure-taxonomy.js +125 -0
- package/src/runtime/hub-cancel-port.js +122 -0
- package/src/runtime/layer-config-port.js +136 -0
- package/src/runtime/lease-manager.js +607 -0
- package/src/runtime/persistence-port.js +162 -0
- package/src/runtime/placeholder-audio-port.js +117 -0
- package/src/runtime/progress-events-port.js +121 -0
- package/src/runtime/reconciliation-engine.js +2405 -0
- package/src/runtime/retry-budget-manager.js +229 -0
- package/src/runtime/runtime-metrics.js +427 -0
- package/src/runtime/runtime-result-emitter.js +123 -0
- package/src/runtime/runtime-scheduler.js +629 -0
- package/src/runtime/scene-data-port.js +114 -0
- package/src/runtime/scene-window.js +827 -0
- package/src/runtime/video-fsm-port.js +189 -0
- package/src/runtime/worker-health.js +270 -0
- package/test/counter-reconciliation.test.js +164 -0
- package/test/mocks/redis-mock.js +221 -0
- package/test/test-bindings.js +60 -0
|
@@ -0,0 +1,2405 @@
|
|
|
1
|
+
// ======================================================
|
|
2
|
+
// Reconciliation Engine - v1.0.0
|
|
3
|
+
// ======================================================
|
|
4
|
+
// Self-healing engine that detects and fixes inconsistencies
|
|
5
|
+
// between state machine and actual assets on disk/registry.
|
|
6
|
+
//
|
|
7
|
+
// T6: Единый reconciliation-контур (R4/К4). Включает:
|
|
8
|
+
// — startup-recovery (version staleness, audio-orch, chunk recovery)
|
|
9
|
+
// — audio-recovery (scan animastor:result:* keys)
|
|
10
|
+
// — cleanup-service (expired audio scene locks)
|
|
11
|
+
// — reconciliation-engine (orphan states, stale leases, drift)
|
|
12
|
+
// Все фазы — через единый цикл reconcileCycle() с распределённым локом.
|
|
13
|
+
|
|
14
|
+
const fs = require('fs').promises;
|
|
15
|
+
const syncFs = require('fs');
|
|
16
|
+
const syncPath = require('path');
|
|
17
|
+
|
|
18
|
+
const { lazyHostBinding } = require('../host/host-bindings');
|
|
19
|
+
// §32.30: former host requires resolve lazily through the host-bindings seam.
|
|
20
|
+
const state = lazyHostBinding('state');
|
|
21
|
+
// S-4: filename grammar composed from the canonical owner (bytes unchanged)
|
|
22
|
+
const artifactNaming = require('@animastor/generation').artifactNaming;
|
|
23
|
+
// S-2: registered stage list resolved from media registry
|
|
24
|
+
const mediaRegistry = require('@animastor/generation').mediaRegistry;
|
|
25
|
+
// O-2: persistence arrives ONLY through the PersistencePort (host adapter:
|
|
26
|
+
// storage/runtime-persistence-adapter) — the storage barrel, PG repositories
|
|
27
|
+
// and the database handle left this file. The Redis asset-registry and the
|
|
28
|
+
// audio-path composition are consumed as persistence OPERATIONS through the
|
|
29
|
+
// same port (their implementations stay host-side).
|
|
30
|
+
const persist = require('./persistence-port').persist;
|
|
31
|
+
const config = lazyHostBinding('config');
|
|
32
|
+
// S-5: the event journal is an append-only observability sink (zero requires,
|
|
33
|
+
// no orchestration policy) — the ONLY runtime→Generation dependency allowed
|
|
34
|
+
// without a seam (s5-runtime-orchestration-cycle.test.js S5-A).
|
|
35
|
+
const journal = lazyHostBinding('journal');
|
|
36
|
+
const runtimeScheduler = require('./runtime-scheduler');
|
|
37
|
+
const counterReconciliation = require('./counter-reconciliation');
|
|
38
|
+
const dispatchEngine = require('./dispatch-engine');
|
|
39
|
+
const leaseManager = require('./lease-manager');
|
|
40
|
+
// S-5: orchestration-owned behavior (FSM-safe facade writers, rollbacks)
|
|
41
|
+
// reaches runtime ONLY through the composition-root-injected seam registry.
|
|
42
|
+
// No orchestration module is imported from this file (S-5).
|
|
43
|
+
const orchestrationSeams = lazyHostBinding('seams');
|
|
44
|
+
|
|
45
|
+
const logPrefix = '[RECONCILE]';
|
|
46
|
+
|
|
47
|
+
function log(msg) {
|
|
48
|
+
console.log(`${logPrefix} ${msg}`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function warn(msg) {
|
|
52
|
+
console.warn(`${logPrefix} ⚠️ ${msg}`);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function error(msg) {
|
|
56
|
+
console.error(`${logPrefix} ❌ ${msg}`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// ======================================================
|
|
60
|
+
// RECONCILIATION REPORT
|
|
61
|
+
// ======================================================
|
|
62
|
+
|
|
63
|
+
class ReconciliationReport {
|
|
64
|
+
constructor() {
|
|
65
|
+
this.orphanStates = [];
|
|
66
|
+
this.orphanAssets = [];
|
|
67
|
+
this.partialBuilds = [];
|
|
68
|
+
this.staleLocks = [];
|
|
69
|
+
this.inconsistentScenes = [];
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
toSummary() {
|
|
73
|
+
return {
|
|
74
|
+
totalOrphanStates: this.orphanStates.length,
|
|
75
|
+
totalOrphanAssets: this.orphanAssets.length,
|
|
76
|
+
totalPartialBuilds: this.partialBuilds.length,
|
|
77
|
+
totalStaleLocks: this.staleLocks.length,
|
|
78
|
+
totalInconsistent: this.inconsistentScenes.length,
|
|
79
|
+
orphanStates: this.orphanStates,
|
|
80
|
+
orphanAssets: this.orphanAssets,
|
|
81
|
+
partialBuilds: this.partialBuilds,
|
|
82
|
+
staleLocks: this.staleLocks,
|
|
83
|
+
inconsistentScenes: this.inconsistentScenes
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// ======================================================
|
|
89
|
+
// ORPHAN STATE CHECKS
|
|
90
|
+
// ======================================================
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Canonical build identity of a book (audit d9d67a3): the SAME source the
|
|
94
|
+
* rest of the runtime uses (rebuildWorkList, generate-next, dispatch) —
|
|
95
|
+
* book.json manifest.build_id. The former hardcoded 'default' produced
|
|
96
|
+
* false orphan detections for every book with a real build directory.
|
|
97
|
+
*/
|
|
98
|
+
function resolveBookBuildId(bookId) {
|
|
99
|
+
try {
|
|
100
|
+
// O-3: scene content via the SceneDataPort (host adapter:
|
|
101
|
+
// storage/scene-data-adapter) — the Book Model facade require left
|
|
102
|
+
// the reconciliation engine.
|
|
103
|
+
const loadedBook = require('./scene-data-port').sceneDataOp('loadBook')(bookId);
|
|
104
|
+
return loadedBook?.manifest?.build_id || 'default';
|
|
105
|
+
} catch (_) {
|
|
106
|
+
return 'default';
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Check for VIDEO_READY state but no video file.
|
|
112
|
+
* T8: использует per-asset state (video=ready) вместо линейного.
|
|
113
|
+
*/
|
|
114
|
+
async function checkOrphanVideoState(redis, bookId, chapterId, sceneId) {
|
|
115
|
+
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
116
|
+
const isVideoReady = assetStates && assetStates.video &&
|
|
117
|
+
(assetStates.video === state.AssetState.READY || assetStates.video === state.AssetState.PLACEHOLDER);
|
|
118
|
+
|
|
119
|
+
if (!isVideoReady) {
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const buildId = resolveBookBuildId(bookId);
|
|
124
|
+
const videoPath = `/data/output/${buildId}/${bookId}_${chapterId}_${sceneId}.mp4`;
|
|
125
|
+
|
|
126
|
+
try {
|
|
127
|
+
await fs.access(videoPath);
|
|
128
|
+
return null; // File exists, state is valid
|
|
129
|
+
} catch (err) {
|
|
130
|
+
return {
|
|
131
|
+
type: 'orphan_video_state',
|
|
132
|
+
scene: { bookId, chapterId, sceneId },
|
|
133
|
+
state: 'video_ready',
|
|
134
|
+
missingFile: videoPath,
|
|
135
|
+
recommendation: 'regenerate_video'
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Check for IMAGE_READY state but no image file.
|
|
142
|
+
* T8: использует per-asset state (image=ready) вместо линейного.
|
|
143
|
+
*/
|
|
144
|
+
async function checkOrphanImageState(redis, bookId, chapterId, sceneId) {
|
|
145
|
+
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
146
|
+
const isImageReady = assetStates && assetStates.image &&
|
|
147
|
+
(assetStates.image === state.AssetState.READY || assetStates.image === state.AssetState.PLACEHOLDER);
|
|
148
|
+
|
|
149
|
+
if (!isImageReady) {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Note: build_id is read from manifest, not from scene-state (which no longer exists after T8).
|
|
154
|
+
// Orphan-check uses the canonical book build identity (audit d9d67a3).
|
|
155
|
+
const buildId = resolveBookBuildId(bookId);
|
|
156
|
+
const imageModule = lazyHostBinding('media.image');
|
|
157
|
+
const imageInfo = imageModule.resolveCanonicalSceneImage(
|
|
158
|
+
'/data/output',
|
|
159
|
+
buildId,
|
|
160
|
+
bookId,
|
|
161
|
+
chapterId,
|
|
162
|
+
sceneId,
|
|
163
|
+
);
|
|
164
|
+
|
|
165
|
+
if (!imageInfo) {
|
|
166
|
+
return {
|
|
167
|
+
type: 'orphan_image_state',
|
|
168
|
+
scene: { bookId, chapterId, sceneId },
|
|
169
|
+
state: 'image_ready',
|
|
170
|
+
missingFile: 'image_not_found',
|
|
171
|
+
recommendation: 'regenerate_image'
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Check for AUDIO_READY state but no audio file.
|
|
180
|
+
* T8: использует per-asset state (audio=ready) вместо линейного.
|
|
181
|
+
*/
|
|
182
|
+
async function checkOrphanAudioState(redis, bookId, chapterId, sceneId) {
|
|
183
|
+
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
184
|
+
const isAudioReady = assetStates && assetStates.audio &&
|
|
185
|
+
(assetStates.audio === state.AssetState.READY || assetStates.audio === state.AssetState.PLACEHOLDER);
|
|
186
|
+
|
|
187
|
+
if (!isAudioReady) {
|
|
188
|
+
return null;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Canonical book build identity (audit d9d67a3) — the former hardcoded
|
|
192
|
+
// 'default' false-flagged every book with a real build directory.
|
|
193
|
+
const buildId = resolveBookBuildId(bookId);
|
|
194
|
+
|
|
195
|
+
// O-2: canonical audio path via the PersistencePort (filesystem op —
|
|
196
|
+
// the host adapter delegates to storage/filesystem-store unchanged).
|
|
197
|
+
const audioPath = persist('filesystem.getSceneAudioPath')(
|
|
198
|
+
config.OUTPUT_DIR,
|
|
199
|
+
buildId,
|
|
200
|
+
bookId,
|
|
201
|
+
chapterId,
|
|
202
|
+
sceneId
|
|
203
|
+
);
|
|
204
|
+
|
|
205
|
+
try {
|
|
206
|
+
await fs.access(audioPath);
|
|
207
|
+
return null; // File exists, state is valid
|
|
208
|
+
} catch (err) {
|
|
209
|
+
return {
|
|
210
|
+
type: 'orphan_audio_state',
|
|
211
|
+
scene: { bookId, chapterId, sceneId },
|
|
212
|
+
state: 'audio_ready',
|
|
213
|
+
missingFile: audioPath,
|
|
214
|
+
recommendation: 'regenerate_audio'
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// ======================================================
|
|
220
|
+
// ORPHAN ASSET CHECKS
|
|
221
|
+
// ======================================================
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Check for assets in registry but no files.
|
|
225
|
+
*/
|
|
226
|
+
async function checkOrphanAssets(redis, bookId, chapterId, sceneId) {
|
|
227
|
+
const assets = await persist('registry.getSceneAssetsRedis')(redis, bookId, chapterId, sceneId);
|
|
228
|
+
|
|
229
|
+
if (!assets) {
|
|
230
|
+
return null;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const orphanAssets = [];
|
|
234
|
+
|
|
235
|
+
if (assets.audio?.canonical && assets.audio.ready) {
|
|
236
|
+
try {
|
|
237
|
+
await fs.access(assets.audio.canonical);
|
|
238
|
+
} catch (err) {
|
|
239
|
+
orphanAssets.push({
|
|
240
|
+
type: 'missing_audio_file',
|
|
241
|
+
scene: { bookId, chapterId, sceneId },
|
|
242
|
+
path: assets.audio.canonical
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
if (assets.image?.path && assets.image.ready) {
|
|
248
|
+
try {
|
|
249
|
+
await fs.access(assets.image.path);
|
|
250
|
+
} catch (err) {
|
|
251
|
+
orphanAssets.push({
|
|
252
|
+
type: 'missing_image_file',
|
|
253
|
+
scene: { bookId, chapterId, sceneId },
|
|
254
|
+
path: assets.image.path
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
if (assets.video?.path && assets.video.ready) {
|
|
260
|
+
try {
|
|
261
|
+
await fs.access(assets.video.path);
|
|
262
|
+
} catch (err) {
|
|
263
|
+
orphanAssets.push({
|
|
264
|
+
type: 'missing_video_file',
|
|
265
|
+
scene: { bookId, chapterId, sceneId },
|
|
266
|
+
path: assets.video.path
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
return orphanAssets.length > 0 ? orphanAssets : null;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// ======================================================
|
|
275
|
+
// ORPHAN GENERATING CHECK (audit d9d67a3)
|
|
276
|
+
// ======================================================
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Cancellation precedence helper: true if the user explicitly cancelled
|
|
280
|
+
* generation for this book (Redis cancel flag and/or PG tombstone).
|
|
281
|
+
* Reconciliation must never resurrect a cancelled book.
|
|
282
|
+
*/
|
|
283
|
+
async function isBookGenerationCancelled(redis, bookId) {
|
|
284
|
+
try {
|
|
285
|
+
const sceneWindow = require('./scene-window');
|
|
286
|
+
if (await sceneWindow.isCancelled(redis, bookId)) return true;
|
|
287
|
+
} catch (_) { /* flag unreadable — fall through to tombstone */ }
|
|
288
|
+
try {
|
|
289
|
+
return await persist('cancel.isCancelled')(bookId);
|
|
290
|
+
} catch (_) {
|
|
291
|
+
return false;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* Check for orphan GENERATING states: a per-asset state stuck in GENERATING
|
|
297
|
+
* with NO evidence of a live generation — no dispatch lease, no dispatch
|
|
298
|
+
* metadata, no iu-in-flight marker (image), no active PG generation task.
|
|
299
|
+
*
|
|
300
|
+
* A live dispatch always holds lease+metadata from dispatch to finalization
|
|
301
|
+
* (dispatch order: acquireLease → setDispatchMetadata → markGenerating), so
|
|
302
|
+
* the absence of ALL evidence proves the generation is dead (crash/restart/
|
|
303
|
+
* TTL expiry with the rollback swallowed). A live job is NEVER repaired —
|
|
304
|
+
* age alone is not a reason (long video jobs keep their lease renewed).
|
|
305
|
+
*
|
|
306
|
+
* Fail-safe: if PG is unavailable the active-task check is treated as
|
|
307
|
+
* "task exists" and the state is NOT repaired.
|
|
308
|
+
*/
|
|
309
|
+
async function checkOrphanGeneratingState(redis, bookId, chapterId, sceneId) {
|
|
310
|
+
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
311
|
+
const orphans = [];
|
|
312
|
+
|
|
313
|
+
for (const stage of mediaRegistry.listMediaTypes()) {
|
|
314
|
+
if (assetStates[stage] !== state.AssetState.GENERATING) continue;
|
|
315
|
+
|
|
316
|
+
const evidence = await dispatchEngine.getDispatchEvidence(redis, bookId, chapterId, sceneId, stage);
|
|
317
|
+
if (evidence.alive) continue;
|
|
318
|
+
|
|
319
|
+
let hasActiveTask = false;
|
|
320
|
+
try {
|
|
321
|
+
// Pre-O-2 semantics preserved: a PG error is swallowed by the
|
|
322
|
+
// caller-side catch and treated as "task exists" (fail-safe).
|
|
323
|
+
hasActiveTask = await persist('tasks.hasActiveTaskForScene')(bookId, sceneId, stage);
|
|
324
|
+
} catch (_) {
|
|
325
|
+
hasActiveTask = true; // PG unavailable — fail safe, do NOT repair
|
|
326
|
+
}
|
|
327
|
+
if (hasActiveTask) continue;
|
|
328
|
+
|
|
329
|
+
orphans.push({
|
|
330
|
+
type: 'orphan_generating',
|
|
331
|
+
stage,
|
|
332
|
+
scene: { bookId, chapterId, sceneId },
|
|
333
|
+
state: 'generating',
|
|
334
|
+
recommendation: 'rollback_to_pending'
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
return orphans.length > 0 ? orphans : null;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// ======================================================
|
|
342
|
+
// PARTIAL BUILD CHECKS
|
|
343
|
+
// ======================================================
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* Check for partially built scenes.
|
|
347
|
+
* T8: использует per-asset state вместо линейного.
|
|
348
|
+
*/
|
|
349
|
+
async function checkPartialBuilds(redis, bookId, chapterId, sceneId) {
|
|
350
|
+
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
351
|
+
const assets = await persist('registry.getSceneAssetsRedis')(redis, bookId, chapterId, sceneId);
|
|
352
|
+
|
|
353
|
+
// Audio READY/PLACEHOLDER but no image asset
|
|
354
|
+
const audioReady = assetStates && assetStates.audio &&
|
|
355
|
+
(assetStates.audio === state.AssetState.READY || assetStates.audio === state.AssetState.PLACEHOLDER);
|
|
356
|
+
if (audioReady && !assets?.image) {
|
|
357
|
+
return {
|
|
358
|
+
type: 'partial_audio_only',
|
|
359
|
+
scene: { bookId, chapterId, sceneId },
|
|
360
|
+
currentStage: 'audio_ready',
|
|
361
|
+
missingStage: 'image',
|
|
362
|
+
recommendation: 'progress_to_image'
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// Image READY but no video asset
|
|
367
|
+
const imageReady = assetStates && assetStates.image &&
|
|
368
|
+
(assetStates.image === state.AssetState.READY || assetStates.image === state.AssetState.PLACEHOLDER);
|
|
369
|
+
if (imageReady && !assets?.video) {
|
|
370
|
+
return {
|
|
371
|
+
type: 'partial_image_video_missing',
|
|
372
|
+
scene: { bookId, chapterId, sceneId },
|
|
373
|
+
currentStage: 'image_ready',
|
|
374
|
+
missingStage: 'video',
|
|
375
|
+
recommendation: 'progress_to_video'
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
return null;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// ======================================================
|
|
383
|
+
// B1: STALLED AUDIO CHUNKS WATCHDOG
|
|
384
|
+
// ======================================================
|
|
385
|
+
// Сканирует audio-orch state машины в фазе WAITING_CHUNKS.
|
|
386
|
+
// Если last_chunk_at + AUDIO_CHUNK_STALL_MS < Date.now() —
|
|
387
|
+
// сцена зависла: вызываем failWaitingScene(), которая чистит
|
|
388
|
+
// hub-dedup, сбрасывает metadata и публикует failStage.
|
|
389
|
+
//
|
|
390
|
+
// Важно: watchdog НЕ конкурирует с gpu-hub timeout — они ловят
|
|
391
|
+
// разные сценарии (стоп прогресса vs мёртвый воркер).
|
|
392
|
+
|
|
393
|
+
async function checkStalledAudioScenes(redis, deps) {
|
|
394
|
+
// O-7: the audio FSM is consumed through the AudioFsmPort (host adapter:
|
|
395
|
+
// storage/audio-fsm-adapter). The pre-O-7 optional-load semantics are
|
|
396
|
+
// preserved EXACTLY: when the audio-orchestrator host service cannot be
|
|
397
|
+
// required, this watchdog leg returns 0 (missing module ≠ stalled scene).
|
|
398
|
+
// The try/catch therefore wraps the ADAPTER's lazy require — the port
|
|
399
|
+
// itself stays fail-fast.
|
|
400
|
+
let audioOrch;
|
|
401
|
+
let PHASES;
|
|
402
|
+
try {
|
|
403
|
+
const audioFsmPort = require('./audio-fsm-port');
|
|
404
|
+
audioOrch = audioFsmPort.audioFsm();
|
|
405
|
+
PHASES = audioFsmPort.audioFsmPhases();
|
|
406
|
+
} catch (_) {
|
|
407
|
+
return 0;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
const { AUDIO_CHUNK_STALL_MS } = config.TIMEOUTS;
|
|
411
|
+
const allStates = await audioOrch.scanAllStates(redis);
|
|
412
|
+
if (allStates.length === 0) return 0;
|
|
413
|
+
|
|
414
|
+
let stalled = 0;
|
|
415
|
+
for (const entry of allStates) {
|
|
416
|
+
const { bookId, chapterId, sceneId, state: orchState } = entry;
|
|
417
|
+
if (orchState.phase !== PHASES.WAITING_CHUNKS) continue;
|
|
418
|
+
|
|
419
|
+
const buildId = orchState.build_id || 'default';
|
|
420
|
+
|
|
421
|
+
// ── LAST_CHUNK_AT || STARTED_AT ──
|
|
422
|
+
// Если completeChunk ни разу не был вызван (HTTP 500 на всех чанках),
|
|
423
|
+
// last_chunk_at = null, но started_at есть. Используем started_at как
|
|
424
|
+
// точку отсчёта застоя. Если и started_at нет — проверяем чанки на диске.
|
|
425
|
+
const lastChunkAt = orchState.last_chunk_at;
|
|
426
|
+
const startedAt = orchState.started_at;
|
|
427
|
+
const threshold = lastChunkAt || startedAt;
|
|
428
|
+
|
|
429
|
+
if (threshold) {
|
|
430
|
+
const age = Date.now() - threshold;
|
|
431
|
+
if (age < AUDIO_CHUNK_STALL_MS) continue;
|
|
432
|
+
|
|
433
|
+
const reason = `chunk_stall_timeout:${Math.round(age / 1000)}s`;
|
|
434
|
+
warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${reason} (last_chunk_at=${!!lastChunkAt} started_at=${!!startedAt})`);
|
|
435
|
+
|
|
436
|
+
try {
|
|
437
|
+
await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId, reason, {
|
|
438
|
+
orchestrator: deps.orchestrator
|
|
439
|
+
});
|
|
440
|
+
stalled++;
|
|
441
|
+
} catch (err) {
|
|
442
|
+
warn(`[STALLED-AUDIO] failWaitingScene error: ${err.message}`);
|
|
443
|
+
}
|
|
444
|
+
} else {
|
|
445
|
+
// ── НЕТ НИ LAST_CHUNK_AT, НИ STARTED_AT ──
|
|
446
|
+
// Проверяем чанки на диске. Если все на месте — доигрываем merge.
|
|
447
|
+
// Если ни одного чанка нет — это fresh state без dispatch, не трогаем.
|
|
448
|
+
const expectedCount = parseInt(orchState.expected_count || '1', 10);
|
|
449
|
+
const pad = (n) => String(n).padStart(4, '0');
|
|
450
|
+
const OUTPUT_DIR = config.OUTPUT_DIR;
|
|
451
|
+
const buildDir = syncPath.join(OUTPUT_DIR, buildId);
|
|
452
|
+
let presentCount = 0;
|
|
453
|
+
for (let i = 1; i <= expectedCount; i++) {
|
|
454
|
+
const chunkPath = syncPath.join(buildDir, artifactNaming.sceneChunkAudioName(bookId, chapterId, sceneId, i));
|
|
455
|
+
if (syncFs.existsSync(chunkPath)) presentCount++;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
if (presentCount === expectedCount) {
|
|
459
|
+
log(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ALL ${presentCount} chunks on disk, no last_chunk_at. Calling completeChunk to trigger merge.`);
|
|
460
|
+
try {
|
|
461
|
+
await audioOrch.completeChunk(redis, bookId, chapterId, sceneId, 'recovery', buildId, {
|
|
462
|
+
audio: deps.audio || lazyHostBinding('media.audio'),
|
|
463
|
+
orchestrator: deps.orchestrator,
|
|
464
|
+
dispatchId: 'recovery-reconcile',
|
|
465
|
+
});
|
|
466
|
+
stalled++;
|
|
467
|
+
} catch (err) {
|
|
468
|
+
warn(`[STALLED-AUDIO] completeChunk recovery failed: ${err.message}`);
|
|
469
|
+
try {
|
|
470
|
+
await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
|
|
471
|
+
`recovery_fallback:${err.message}`, { orchestrator: deps.orchestrator });
|
|
472
|
+
stalled++;
|
|
473
|
+
} catch (fsErr) {
|
|
474
|
+
warn(`[STALLED-AUDIO] failWaitingScene fallback failed: ${fsErr.message}`);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
} else if (presentCount > 0) {
|
|
478
|
+
warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${presentCount}/${expectedCount} on disk, no last_chunk_at. Partial chunks, failing scene.`);
|
|
479
|
+
try {
|
|
480
|
+
await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
|
|
481
|
+
`partial_chunks:${presentCount}/${expectedCount}_no_timestamps`, { orchestrator: deps.orchestrator });
|
|
482
|
+
stalled++;
|
|
483
|
+
} catch (err) {
|
|
484
|
+
warn(`[STALLED-AUDIO] failWaitingScene failed: ${err.message}`);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
// Если presentCount === 0 — это fresh state, ничего не делаем
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
if (stalled > 0) log(`[STALLED-AUDIO] ${stalled} stalled audio scenes recovered`);
|
|
492
|
+
return stalled;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// ======================================================
|
|
496
|
+
// B2: STALLED VIDEO GROUPS WATCHDOG
|
|
497
|
+
// ======================================================
|
|
498
|
+
// Сканирует video-orch state машины в фазе WAITING_CHUNKS.
|
|
499
|
+
// Если last_group_at/started_at + VIDEO_CHUNK_STALL_MS < Date.now() —
|
|
500
|
+
// сцена зависла: вызываем failWaitingScene(), которая чистит hub-dedup
|
|
501
|
+
// недостающих групп и публикует failStage (→ re-dispatch).
|
|
502
|
+
|
|
503
|
+
async function checkStalledVideoScenes(redis, deps) {
|
|
504
|
+
// O-8: the video FSM is consumed through the VideoFsmPort (host adapter:
|
|
505
|
+
// storage/video-fsm-adapter). The pre-O-8 optional-load semantics are
|
|
506
|
+
// preserved EXACTLY: when the video-orchestrator host service cannot be
|
|
507
|
+
// required, this watchdog leg returns 0 (missing module ≠ stalled scene).
|
|
508
|
+
// The try/catch therefore wraps the ADAPTER's lazy require — the port
|
|
509
|
+
// itself stays fail-fast.
|
|
510
|
+
let videoOrch;
|
|
511
|
+
let PHASES;
|
|
512
|
+
try {
|
|
513
|
+
const videoFsmPort = require('./video-fsm-port');
|
|
514
|
+
videoOrch = videoFsmPort.videoFsm();
|
|
515
|
+
PHASES = videoFsmPort.videoFsmPhases();
|
|
516
|
+
} catch (_) {
|
|
517
|
+
return 0;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
const { VIDEO_CHUNK_STALL_MS } = config.TIMEOUTS;
|
|
521
|
+
const allStates = await videoOrch.scanAllStates(redis);
|
|
522
|
+
if (allStates.length === 0) return 0;
|
|
523
|
+
|
|
524
|
+
let stalled = 0;
|
|
525
|
+
for (const entry of allStates) {
|
|
526
|
+
const { bookId, chapterId, sceneId, state: orchState } = entry;
|
|
527
|
+
if (orchState.phase !== PHASES.WAITING_CHUNKS) continue;
|
|
528
|
+
|
|
529
|
+
// ── ПЕРЕСМОТР ПОРОГА ЗАСТОЯ ──
|
|
530
|
+
// Видео-генерация долгая по природе (LTX 5-10 мин на группу, на слабом
|
|
531
|
+
// GPU — 20-30+ мин). Жёсткий константный порог мог бы убить нормальную
|
|
532
|
+
// долгую генерацию. Порог застоя СТРОГО БОЛЬШЕ per-job timeout из
|
|
533
|
+
// layer-config (video_timeout_minutes): hub затаймливает воркера раньше,
|
|
534
|
+
// watchdog срабатывает только если hub/worker НЕ объявили ни результат,
|
|
535
|
+
// ни ошибку за весь интервал.
|
|
536
|
+
let stallThreshold = VIDEO_CHUNK_STALL_MS;
|
|
537
|
+
try {
|
|
538
|
+
const layerRaw = await redis.get(`animastor:layer-config:${bookId}`);
|
|
539
|
+
if (layerRaw) {
|
|
540
|
+
const lc = JSON.parse(layerRaw);
|
|
541
|
+
const perJobMs = (Number(lc.video_timeout_minutes) || 60) * 60 * 1000;
|
|
542
|
+
stallThreshold = Math.max(stallThreshold, perJobMs + 5 * 60 * 1000);
|
|
543
|
+
}
|
|
544
|
+
} catch (_) {}
|
|
545
|
+
|
|
546
|
+
const buildId = orchState.build_id || 'default';
|
|
547
|
+
const lastGroupAt = orchState.last_group_at;
|
|
548
|
+
const startedAt = orchState.started_at;
|
|
549
|
+
const threshold = lastGroupAt || startedAt;
|
|
550
|
+
|
|
551
|
+
if (threshold) {
|
|
552
|
+
const age = Date.now() - threshold;
|
|
553
|
+
if (age < stallThreshold) continue;
|
|
554
|
+
|
|
555
|
+
const reason = `video_group_stall_timeout:${Math.round(age / 1000)}s`;
|
|
556
|
+
warn(`[STALLED-VIDEO] ${bookId}/${chapterId}/${sceneId} — ${reason} (threshold=${Math.round(stallThreshold / 60000)}min, last_group_at=${!!lastGroupAt} started_at=${!!startedAt})`);
|
|
557
|
+
try {
|
|
558
|
+
await videoOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId, reason, {
|
|
559
|
+
orchestrator: deps.orchestrator,
|
|
560
|
+
});
|
|
561
|
+
stalled++;
|
|
562
|
+
} catch (err) {
|
|
563
|
+
warn(`[STALLED-VIDEO] failWaitingScene error: ${err.message}`);
|
|
564
|
+
}
|
|
565
|
+
} else {
|
|
566
|
+
// Нет ни last_group_at, ни started_at — свежий state без dispatch.
|
|
567
|
+
// Если все группы на диске — доигрываем merge.
|
|
568
|
+
const suffixes = videoOrch.groupSuffixes(orchState);
|
|
569
|
+
const presentCount = suffixes.filter(s => videoOrch.isGroupFileValid(buildId, bookId, chapterId, sceneId, s)).length;
|
|
570
|
+
if (presentCount === suffixes.length && suffixes.length > 0) {
|
|
571
|
+
log(`[STALLED-VIDEO] ${bookId}/${chapterId}/${sceneId} — ALL ${presentCount} groups on disk, no last_group_at. Calling completeGroup to trigger merge.`);
|
|
572
|
+
try {
|
|
573
|
+
await videoOrch.completeGroup(redis, bookId, chapterId, sceneId, '', buildId, {
|
|
574
|
+
orchestrator: deps.orchestrator,
|
|
575
|
+
dispatchId: 'recovery-reconcile',
|
|
576
|
+
});
|
|
577
|
+
stalled++;
|
|
578
|
+
} catch (err) {
|
|
579
|
+
warn(`[STALLED-VIDEO] completeGroup recovery failed: ${err.message}`);
|
|
580
|
+
try {
|
|
581
|
+
await videoOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
|
|
582
|
+
`recovery_fallback:${err.message}`, { orchestrator: deps.orchestrator });
|
|
583
|
+
stalled++;
|
|
584
|
+
} catch (fsErr) {
|
|
585
|
+
warn(`[STALLED-VIDEO] failWaitingScene fallback failed: ${fsErr.message}`);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
} else if (presentCount > 0) {
|
|
589
|
+
warn(`[STALLED-VIDEO] ${bookId}/${chapterId}/${sceneId} — ${presentCount}/${suffixes.length} on disk, no last_group_at. Partial groups, failing scene.`);
|
|
590
|
+
try {
|
|
591
|
+
await videoOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
|
|
592
|
+
`partial_groups:${presentCount}/${suffixes.length}_no_timestamps`, { orchestrator: deps.orchestrator });
|
|
593
|
+
stalled++;
|
|
594
|
+
} catch (err) {
|
|
595
|
+
warn(`[STALLED-VIDEO] failWaitingScene failed: ${err.message}`);
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
if (stalled > 0) log(`[STALLED-VIDEO] ${stalled} stalled video scenes recovered`);
|
|
602
|
+
return stalled;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// ======================================================
|
|
606
|
+
// T7.6: AUDIO-ORCH INVARIANT CHECKS
|
|
607
|
+
// ======================================================
|
|
608
|
+
// Проверяет соответствие между audio-orch phase и asset state:
|
|
609
|
+
// phase=DONE ⇔ asset.audio = READY
|
|
610
|
+
// phase=FAILED ⇒ asset.audio = FAILED (или PENDING — после re-dispatch)
|
|
611
|
+
// промежуточные ⇒ asset.audio ∈ {PENDING, GENERATING, DIRTY}
|
|
612
|
+
|
|
613
|
+
async function checkAudioOrchInvariants(redis, bookId, chapterId, sceneId) {
|
|
614
|
+
// O-7: through the AudioFsmPort (host adapter: storage/audio-fsm-adapter)
|
|
615
|
+
// — the audio-orchestrator host service left this file. Mandatory load
|
|
616
|
+
// (pre-O-7 behavior: a broken require here throws, no optional-load).
|
|
617
|
+
const audioOrch = require('./audio-fsm-port').audioFsm();
|
|
618
|
+
const PHASES = require('./audio-fsm-port').audioFsmPhases();
|
|
619
|
+
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
620
|
+
const orchState = await audioOrch.getState(redis, bookId, chapterId, sceneId);
|
|
621
|
+
|
|
622
|
+
if (!orchState || !assetStates) return null;
|
|
623
|
+
|
|
624
|
+
const phase = orchState.phase;
|
|
625
|
+
const audioState = assetStates.audio;
|
|
626
|
+
const violations = [];
|
|
627
|
+
|
|
628
|
+
if (phase === PHASES.DONE) {
|
|
629
|
+
if (audioState !== state.AssetState.READY && audioState !== state.AssetState.PLACEHOLDER) {
|
|
630
|
+
violations.push({
|
|
631
|
+
type: 'audio_orch_invariant_done',
|
|
632
|
+
scene: { bookId, chapterId, sceneId },
|
|
633
|
+
phase,
|
|
634
|
+
audioState,
|
|
635
|
+
expected: 'READY or PLACEHOLDER',
|
|
636
|
+
recommendation: 'run_completeStage'
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
} else if (phase === PHASES.FAILED) {
|
|
640
|
+
if (audioState !== state.AssetState.FAILED && audioState !== state.AssetState.PENDING) {
|
|
641
|
+
violations.push({
|
|
642
|
+
type: 'audio_orch_invariant_failed',
|
|
643
|
+
scene: { bookId, chapterId, sceneId },
|
|
644
|
+
phase,
|
|
645
|
+
audioState,
|
|
646
|
+
expected: 'FAILED or PENDING',
|
|
647
|
+
recommendation: 'mark_dirty'
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
} else if (phase !== PHASES.NEW && phase !== PHASES.PLACEHOLDER_READY) {
|
|
651
|
+
// Intermediate phases: GENERATING, WAITING_CHUNKS, MERGING
|
|
652
|
+
if (audioState === state.AssetState.READY) {
|
|
653
|
+
violations.push({
|
|
654
|
+
type: 'audio_orch_invariant_intermediate',
|
|
655
|
+
scene: { bookId, chapterId, sceneId },
|
|
656
|
+
phase,
|
|
657
|
+
audioState,
|
|
658
|
+
expected: 'PENDING|GENERATING|DIRTY',
|
|
659
|
+
recommendation: 'mark_dirty_or_set_done'
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
return violations.length > 0 ? violations : null;
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
// ======================================================
|
|
668
|
+
// VIDEO-ORCH INVARIANT CHECKS (зеркало checkAudioOrchInvariants)
|
|
669
|
+
// ======================================================
|
|
670
|
+
// phase=DONE ⇒ asset.video = READY (или PLACEHOLDER)
|
|
671
|
+
// phase=FAILED ⇒ asset.video = FAILED (или PENDING — после re-dispatch)
|
|
672
|
+
// промежуточные ⇒ asset.video = PENDING|GENERATING|DIRTY
|
|
673
|
+
|
|
674
|
+
async function checkVideoOrchInvariants(redis, bookId, chapterId, sceneId) {
|
|
675
|
+
// O-8: through the VideoFsmPort (host adapter: storage/video-fsm-adapter)
|
|
676
|
+
// — the video-orchestrator host service left this file. Mandatory load
|
|
677
|
+
// (pre-O-8 behavior: a broken require here throws, no optional-load).
|
|
678
|
+
const videoOrch = require('./video-fsm-port').videoFsm();
|
|
679
|
+
const PHASES = require('./video-fsm-port').videoFsmPhases();
|
|
680
|
+
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
681
|
+
const orchState = await videoOrch.getState(redis, bookId, chapterId, sceneId);
|
|
682
|
+
|
|
683
|
+
if (!orchState || !assetStates) return null;
|
|
684
|
+
|
|
685
|
+
const phase = orchState.phase;
|
|
686
|
+
const videoState = assetStates.video;
|
|
687
|
+
const violations = [];
|
|
688
|
+
|
|
689
|
+
if (phase === PHASES.DONE) {
|
|
690
|
+
if (videoState !== state.AssetState.READY && videoState !== state.AssetState.PLACEHOLDER) {
|
|
691
|
+
violations.push({
|
|
692
|
+
type: 'video_orch_invariant_done',
|
|
693
|
+
scene: { bookId, chapterId, sceneId },
|
|
694
|
+
phase,
|
|
695
|
+
videoState,
|
|
696
|
+
expected: 'READY or PLACEHOLDER',
|
|
697
|
+
recommendation: 'run_completeStage'
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
} else if (phase === PHASES.FAILED) {
|
|
701
|
+
if (videoState !== state.AssetState.FAILED && videoState !== state.AssetState.PENDING) {
|
|
702
|
+
violations.push({
|
|
703
|
+
type: 'video_orch_invariant_failed',
|
|
704
|
+
scene: { bookId, chapterId, sceneId },
|
|
705
|
+
phase,
|
|
706
|
+
videoState,
|
|
707
|
+
expected: 'FAILED or PENDING',
|
|
708
|
+
recommendation: 'mark_dirty'
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
} else if (phase !== PHASES.NEW && phase !== PHASES.GENERATING) {
|
|
712
|
+
// WAITING_CHUNKS, MERGING — ассет не должен быть READY
|
|
713
|
+
if (videoState === state.AssetState.READY) {
|
|
714
|
+
violations.push({
|
|
715
|
+
type: 'video_orch_invariant_intermediate',
|
|
716
|
+
scene: { bookId, chapterId, sceneId },
|
|
717
|
+
phase,
|
|
718
|
+
videoState,
|
|
719
|
+
expected: 'PENDING|GENERATING|DIRTY',
|
|
720
|
+
recommendation: 'mark_dirty_or_set_done'
|
|
721
|
+
});
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
return violations.length > 0 ? violations : null;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
// ======================================================
|
|
729
|
+
// STALE LOCK CHECKS
|
|
730
|
+
// ======================================================
|
|
731
|
+
|
|
732
|
+
const LOCK_KEYS = [
|
|
733
|
+
'animastor:audio-scene-lock',
|
|
734
|
+
'animastor:video-lock',
|
|
735
|
+
'animastor:audio-merge-lock',
|
|
736
|
+
'animastor:scene-transition-lock'
|
|
737
|
+
];
|
|
738
|
+
|
|
739
|
+
/**
|
|
740
|
+
* Check for stale locks (lock exists but no heartbeat).
|
|
741
|
+
*/
|
|
742
|
+
async function checkStaleLocks(redis, bookId, chapterId, sceneId) {
|
|
743
|
+
const staleLocks = [];
|
|
744
|
+
|
|
745
|
+
for (const lockPrefix of LOCK_KEYS) {
|
|
746
|
+
const lockKey = `${lockPrefix}:${bookId}:${chapterId}:${sceneId}`;
|
|
747
|
+
const lockData = await redis.get(lockKey);
|
|
748
|
+
|
|
749
|
+
if (lockData) {
|
|
750
|
+
// Check heartbeat
|
|
751
|
+
const heartbeatKey = `animastor:scene-heartbeat:${bookId}:${chapterId}:${sceneId}`;
|
|
752
|
+
const heartbeatTime = await redis.get(heartbeatKey);
|
|
753
|
+
|
|
754
|
+
if (!heartbeatTime) {
|
|
755
|
+
staleLocks.push({
|
|
756
|
+
type: 'lock_without_heartbeat',
|
|
757
|
+
lockKey,
|
|
758
|
+
scene: { bookId, chapterId, sceneId },
|
|
759
|
+
lockData: JSON.parse(lockData),
|
|
760
|
+
recommendation: 'release_lock'
|
|
761
|
+
});
|
|
762
|
+
} else {
|
|
763
|
+
// Check if heartbeat is stale (> 5 minutes)
|
|
764
|
+
const now = Date.now();
|
|
765
|
+
const lastHeartbeat = parseInt(heartbeatTime, 10);
|
|
766
|
+
if (now - lastHeartbeat > 5 * 60 * 1000) {
|
|
767
|
+
staleLocks.push({
|
|
768
|
+
type: 'stale_heartbeat',
|
|
769
|
+
lockKey,
|
|
770
|
+
scene: { bookId, chapterId, sceneId },
|
|
771
|
+
lastHeartbeat,
|
|
772
|
+
recommendation: 'clear_heartbeat'
|
|
773
|
+
});
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
return staleLocks.length > 0 ? staleLocks : null;
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
// ======================================================
|
|
783
|
+
// DISPATCH LEASE CHECKS
|
|
784
|
+
// ======================================================
|
|
785
|
+
|
|
786
|
+
/**
|
|
787
|
+
* Check for stale dispatch leases.
|
|
788
|
+
*
|
|
789
|
+
* Liveness is decided by the lease key's REMAINING TTL — the canonical
|
|
790
|
+
* signal that renewal is still running — NOT by metadata.started_at.
|
|
791
|
+
* renewLeaseIfOwner re-pins the TTL to the renewal target every 30 s while
|
|
792
|
+
* the owner is alive, so a healthy long-running (even 60-min video) dispatch
|
|
793
|
+
* never goes stale, while a lease whose owner died (crash/restart) decays and
|
|
794
|
+
* is flagged once its TTL drops below target - STALE_LEASE_GRACE_S.
|
|
795
|
+
* (audit c8b79f6 / incident 2026-08-28: started_at-based age falsely killed
|
|
796
|
+
* live renewed video dispatches at 27 min → duplicate re-dispatch loop.)
|
|
797
|
+
*/
|
|
798
|
+
async function checkStaleDispatchLeases(redis, bookId, chapterId, sceneId) {
|
|
799
|
+
const staleLeases = [];
|
|
800
|
+
|
|
801
|
+
// Check all stages for lease existence
|
|
802
|
+
const stages = mediaRegistry.listMediaTypes();
|
|
803
|
+
|
|
804
|
+
for (const stage of stages) {
|
|
805
|
+
const leaseKey = dispatchEngine.getLeaseKey(bookId, chapterId, sceneId, stage);
|
|
806
|
+
const token = await redis.get(leaseKey);
|
|
807
|
+
|
|
808
|
+
if (token) {
|
|
809
|
+
// Check dispatch metadata for age
|
|
810
|
+
const metaKey = dispatchEngine.getDispatchMetaKey(bookId, chapterId, sceneId, stage);
|
|
811
|
+
const metadata = await redis.get(metaKey);
|
|
812
|
+
|
|
813
|
+
if (metadata) {
|
|
814
|
+
// Remaining TTL is the source of truth for liveness. A lease
|
|
815
|
+
// being actively renewed keeps its TTL pinned near the renewal
|
|
816
|
+
// target regardless of how old started_at is.
|
|
817
|
+
const leaseTtlS = await redis.ttl(leaseKey);
|
|
818
|
+
|
|
819
|
+
if (leaseManager.isLeaseStale(leaseTtlS, stage)) {
|
|
820
|
+
staleLeases.push({
|
|
821
|
+
type: 'stale_dispatch_lease',
|
|
822
|
+
stage,
|
|
823
|
+
leaseKey,
|
|
824
|
+
scene: { bookId, chapterId, sceneId },
|
|
825
|
+
leaseTtlS,
|
|
826
|
+
renewalTargetS: leaseManager.getRenewalTargetTtlS(stage),
|
|
827
|
+
token,
|
|
828
|
+
recommendation: 'release_lease_and_move_to_pending'
|
|
829
|
+
});
|
|
830
|
+
}
|
|
831
|
+
} else {
|
|
832
|
+
// Lease exists but no metadata - might be orphaned
|
|
833
|
+
staleLeases.push({
|
|
834
|
+
type: 'orphan_dispatch_lease',
|
|
835
|
+
stage,
|
|
836
|
+
leaseKey,
|
|
837
|
+
scene: { bookId, chapterId, sceneId },
|
|
838
|
+
token,
|
|
839
|
+
recommendation: 'release_lease_and_move_to_pending'
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
return staleLeases.length > 0 ? staleLeases : null;
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
// ======================================================
|
|
849
|
+
// SCENE RECONCILIATION
|
|
850
|
+
// ======================================================
|
|
851
|
+
|
|
852
|
+
/**
|
|
853
|
+
* Reconcile a single scene.
|
|
854
|
+
*/
|
|
855
|
+
async function reconcileScene(redis, bookId, chapterId, sceneId) {
|
|
856
|
+
const report = new ReconciliationReport();
|
|
857
|
+
|
|
858
|
+
try {
|
|
859
|
+
// Check orphan states
|
|
860
|
+
const orphanVideo = await checkOrphanVideoState(redis, bookId, chapterId, sceneId);
|
|
861
|
+
if (orphanVideo) {
|
|
862
|
+
report.orphanStates.push(orphanVideo);
|
|
863
|
+
report.inconsistentScenes.push({
|
|
864
|
+
scene: { bookId, chapterId, sceneId },
|
|
865
|
+
issue: orphanVideo.type
|
|
866
|
+
});
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
const orphanImage = await checkOrphanImageState(redis, bookId, chapterId, sceneId);
|
|
870
|
+
if (orphanImage) {
|
|
871
|
+
report.orphanStates.push(orphanImage);
|
|
872
|
+
if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
|
|
873
|
+
report.inconsistentScenes.push({
|
|
874
|
+
scene: { bookId, chapterId, sceneId },
|
|
875
|
+
issue: orphanImage.type
|
|
876
|
+
});
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
const orphanAudio = await checkOrphanAudioState(redis, bookId, chapterId, sceneId);
|
|
881
|
+
if (orphanAudio) {
|
|
882
|
+
report.orphanStates.push(orphanAudio);
|
|
883
|
+
if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
|
|
884
|
+
report.inconsistentScenes.push({
|
|
885
|
+
scene: { bookId, chapterId, sceneId },
|
|
886
|
+
issue: orphanAudio.type
|
|
887
|
+
});
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
// Check orphan assets
|
|
892
|
+
const orphanAssets = await checkOrphanAssets(redis, bookId, chapterId, sceneId);
|
|
893
|
+
if (orphanAssets) {
|
|
894
|
+
report.orphanAssets.push(...orphanAssets);
|
|
895
|
+
if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
|
|
896
|
+
report.inconsistentScenes.push({
|
|
897
|
+
scene: { bookId, chapterId, sceneId },
|
|
898
|
+
issue: 'orphan_assets'
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
// Check orphan GENERATING states (audit d9d67a3). Pushed BEFORE the
|
|
904
|
+
// partial-build checks: those de-duplicate by sceneId, and the orphan
|
|
905
|
+
// repair must win over a PROGRESS_TO_* resurrection attempt.
|
|
906
|
+
const orphanGenerating = await checkOrphanGeneratingState(redis, bookId, chapterId, sceneId);
|
|
907
|
+
if (orphanGenerating) {
|
|
908
|
+
report.orphanStates.push(...orphanGenerating);
|
|
909
|
+
for (const o of orphanGenerating) {
|
|
910
|
+
report.inconsistentScenes.push({
|
|
911
|
+
scene: { bookId, chapterId, sceneId },
|
|
912
|
+
stage: o.stage,
|
|
913
|
+
issue: 'orphan_generating'
|
|
914
|
+
});
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
// Check partial builds
|
|
919
|
+
const partial = await checkPartialBuilds(redis, bookId, chapterId, sceneId);
|
|
920
|
+
if (partial) {
|
|
921
|
+
report.partialBuilds.push(partial);
|
|
922
|
+
if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
|
|
923
|
+
report.inconsistentScenes.push({
|
|
924
|
+
scene: { bookId, chapterId, sceneId },
|
|
925
|
+
issue: partial.type
|
|
926
|
+
});
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
// Check stale locks
|
|
931
|
+
const staleLocks = await checkStaleLocks(redis, bookId, chapterId, sceneId);
|
|
932
|
+
if (staleLocks) {
|
|
933
|
+
report.staleLocks.push(...staleLocks);
|
|
934
|
+
if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
|
|
935
|
+
report.inconsistentScenes.push({
|
|
936
|
+
scene: { bookId, chapterId, sceneId },
|
|
937
|
+
issue: 'stale_locks'
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
// Check stale dispatch leases
|
|
943
|
+
const staleLeases = await checkStaleDispatchLeases(redis, bookId, chapterId, sceneId);
|
|
944
|
+
if (staleLeases) {
|
|
945
|
+
report.staleLocks.push(...staleLeases);
|
|
946
|
+
for (const lease of staleLeases) {
|
|
947
|
+
if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId && s.issue === 'stale_dispatch_lease')) {
|
|
948
|
+
report.inconsistentScenes.push({
|
|
949
|
+
scene: { bookId, chapterId, sceneId },
|
|
950
|
+
issue: 'stale_dispatch_lease'
|
|
951
|
+
});
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
// T7.6: Check audio-orch invariants
|
|
957
|
+
const audioOrchViolations = await checkAudioOrchInvariants(redis, bookId, chapterId, sceneId);
|
|
958
|
+
if (audioOrchViolations) {
|
|
959
|
+
for (const v of audioOrchViolations) {
|
|
960
|
+
report.inconsistentScenes.push({
|
|
961
|
+
scene: v.scene,
|
|
962
|
+
issue: v.type
|
|
963
|
+
});
|
|
964
|
+
}
|
|
965
|
+
log(`[INVARIANT] ${bookId}/${chapterId}/${sceneId}: ${audioOrchViolations.length} audio-orch violations`);
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
// Video-orch invariants (зеркало аудио)
|
|
969
|
+
const videoOrchViolations = await checkVideoOrchInvariants(redis, bookId, chapterId, sceneId);
|
|
970
|
+
if (videoOrchViolations) {
|
|
971
|
+
for (const v of videoOrchViolations) {
|
|
972
|
+
report.inconsistentScenes.push({
|
|
973
|
+
scene: v.scene,
|
|
974
|
+
issue: v.type
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
log(`[INVARIANT] ${bookId}/${chapterId}/${sceneId}: ${videoOrchViolations.length} video-orch violations`);
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
return report;
|
|
981
|
+
} catch (err) {
|
|
982
|
+
error(`Reconciliation error for ${bookId}/${chapterId}/${sceneId}: ${err.message}`);
|
|
983
|
+
return report;
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
// ======================================================
|
|
988
|
+
// RECOMMENDATIONS & AUTO-FIXES
|
|
989
|
+
// ======================================================
|
|
990
|
+
|
|
991
|
+
/**
|
|
992
|
+
* Get recommendations for fixing inconsistencies.
|
|
993
|
+
*/
|
|
994
|
+
function getFixRecommendations(inconsistentScenes) {
|
|
995
|
+
return inconsistentScenes.map(item => {
|
|
996
|
+
const { scene, issue } = item;
|
|
997
|
+
|
|
998
|
+
switch (issue) {
|
|
999
|
+
case 'orphan_video_state':
|
|
1000
|
+
return {
|
|
1001
|
+
scene,
|
|
1002
|
+
action: 'REGENERATE_MISSING_ASSET',
|
|
1003
|
+
reason: 'VIDEO_READY but no video file',
|
|
1004
|
+
safeToExecute: true
|
|
1005
|
+
};
|
|
1006
|
+
case 'orphan_image_state':
|
|
1007
|
+
return {
|
|
1008
|
+
scene,
|
|
1009
|
+
action: 'REGENERATE_MISSING_ASSET',
|
|
1010
|
+
reason: 'IMAGE_READY but no image file',
|
|
1011
|
+
safeToExecute: true
|
|
1012
|
+
};
|
|
1013
|
+
case 'orphan_audio_state':
|
|
1014
|
+
return {
|
|
1015
|
+
scene,
|
|
1016
|
+
action: 'REGENERATE_MISSING_ASSET',
|
|
1017
|
+
reason: 'AUDIO_READY but no audio file',
|
|
1018
|
+
safeToExecute: true
|
|
1019
|
+
};
|
|
1020
|
+
case 'partial_audio_only':
|
|
1021
|
+
return {
|
|
1022
|
+
scene,
|
|
1023
|
+
action: 'PROGRESS_TO_IMAGE',
|
|
1024
|
+
reason: 'Audio ready but image not started',
|
|
1025
|
+
safeToExecute: true
|
|
1026
|
+
};
|
|
1027
|
+
case 'partial_image_video_missing':
|
|
1028
|
+
return {
|
|
1029
|
+
scene,
|
|
1030
|
+
action: 'PROGRESS_TO_VIDEO',
|
|
1031
|
+
reason: 'Image ready but video not started',
|
|
1032
|
+
safeToExecute: true
|
|
1033
|
+
};
|
|
1034
|
+
case 'stale_locks':
|
|
1035
|
+
return {
|
|
1036
|
+
scene,
|
|
1037
|
+
action: 'RELEASE_STALE_LOCKS',
|
|
1038
|
+
reason: 'Locks exist without heartbeat',
|
|
1039
|
+
safeToExecute: true
|
|
1040
|
+
};
|
|
1041
|
+
case 'orphan_assets':
|
|
1042
|
+
return {
|
|
1043
|
+
scene,
|
|
1044
|
+
action: 'RECOVER_ORPHAN_ASSETS',
|
|
1045
|
+
reason: 'Registry has assets but files are missing',
|
|
1046
|
+
safeToExecute: true
|
|
1047
|
+
};
|
|
1048
|
+
case 'stuck_state':
|
|
1049
|
+
return {
|
|
1050
|
+
scene,
|
|
1051
|
+
action: 'MOVE_TO_PENDING',
|
|
1052
|
+
reason: `Stuck in ${item.state} for ${item.ageMinutes} minutes`,
|
|
1053
|
+
safeToExecute: true
|
|
1054
|
+
};
|
|
1055
|
+
case 'orphan_generating':
|
|
1056
|
+
// Audit d9d67a3: GENERATING with no lease/dispatch-meta/
|
|
1057
|
+
// in-flight marker/active PG task — the generation is proven
|
|
1058
|
+
// dead. Stage-scoped MOVE_TO_PENDING performs the strict
|
|
1059
|
+
// FSM-safe repair GENERATING → DIRTY → PENDING.
|
|
1060
|
+
return {
|
|
1061
|
+
scene,
|
|
1062
|
+
stage: item.stage,
|
|
1063
|
+
action: 'MOVE_TO_PENDING',
|
|
1064
|
+
reason: `Orphan GENERATING (${item.stage}): no lease/dispatch/in-flight/PG task`,
|
|
1065
|
+
safeToExecute: true
|
|
1066
|
+
};
|
|
1067
|
+
case 'counter_drift':
|
|
1068
|
+
return {
|
|
1069
|
+
scene,
|
|
1070
|
+
stage: item.stage,
|
|
1071
|
+
action: 'RECONCILE_COUNTER_DRIFT',
|
|
1072
|
+
reason: `Counter drift: ${item.drift} (leases=${item.leaseCount}, counter=${item.counterValue})`,
|
|
1073
|
+
safeToExecute: true
|
|
1074
|
+
};
|
|
1075
|
+
case 'stale_dispatch_lease':
|
|
1076
|
+
return {
|
|
1077
|
+
scene,
|
|
1078
|
+
action: 'RELEASE_STALE_LEASE',
|
|
1079
|
+
reason: `Stale dispatch lease detected`,
|
|
1080
|
+
safeToExecute: true
|
|
1081
|
+
};
|
|
1082
|
+
default:
|
|
1083
|
+
return {
|
|
1084
|
+
scene,
|
|
1085
|
+
action: 'REVIEW_MANUALLY',
|
|
1086
|
+
reason: issue,
|
|
1087
|
+
safeToExecute: false
|
|
1088
|
+
};
|
|
1089
|
+
}
|
|
1090
|
+
});
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
/**
|
|
1094
|
+
* Fix actions that resurrect work (re-add a scene to the active index /
|
|
1095
|
+
* re-mark stages for generation). Audit d9d67a3: they must NEVER run for a
|
|
1096
|
+
* book the user explicitly cancelled — the cancellation tombstone / cancel
|
|
1097
|
+
* flag takes precedence over any reconciliation recommendation.
|
|
1098
|
+
*/
|
|
1099
|
+
const RESURRECTION_ACTIONS = new Set([
|
|
1100
|
+
'REGENERATE_MISSING_ASSET',
|
|
1101
|
+
'PROGRESS_TO_IMAGE',
|
|
1102
|
+
'PROGRESS_TO_VIDEO',
|
|
1103
|
+
'RECOVER_ORPHAN_ASSETS',
|
|
1104
|
+
'RELEASE_STALE_LEASE',
|
|
1105
|
+
]);
|
|
1106
|
+
|
|
1107
|
+
/**
|
|
1108
|
+
* Apply a fix recommendation.
|
|
1109
|
+
* Returns { success: boolean, action, scene, details }
|
|
1110
|
+
*/
|
|
1111
|
+
async function applyFix(redis, fix) {
|
|
1112
|
+
const { scene, action } = fix;
|
|
1113
|
+
|
|
1114
|
+
try {
|
|
1115
|
+
// Cancellation precedence (audit d9d67a3): a fresh tombstone/cancel
|
|
1116
|
+
// flag blocks every resurrection path. State-hygiene repairs that do
|
|
1117
|
+
// NOT re-add the scene to the active index (stage-scoped
|
|
1118
|
+
// MOVE_TO_PENDING) are still allowed — they remove a ghost, not
|
|
1119
|
+
// resurrect work.
|
|
1120
|
+
if (RESURRECTION_ACTIONS.has(action) &&
|
|
1121
|
+
await isBookGenerationCancelled(redis, scene.bookId)) {
|
|
1122
|
+
return {
|
|
1123
|
+
success: false,
|
|
1124
|
+
action,
|
|
1125
|
+
scene,
|
|
1126
|
+
details: 'skipped: book generation cancelled (tombstone/cancel flag precedence)'
|
|
1127
|
+
};
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
switch (action) {
|
|
1131
|
+
case 'MOVE_TO_PENDING': {
|
|
1132
|
+
// S-5: FSM-safe rollback via the injected orchestration seam
|
|
1133
|
+
const rollbackStageToPending = orchestrationSeams.getOrchestrationSeam('rollbackStageToPending');
|
|
1134
|
+
|
|
1135
|
+
// ── Stage-scoped orphan GENERATING repair (audit d9d67a3) ──
|
|
1136
|
+
if (fix.stage) {
|
|
1137
|
+
// Strict re-verification right before the write (TOCTOU):
|
|
1138
|
+
// a concurrent dispatch may have acquired the stage
|
|
1139
|
+
// between detection and fix. A live stage is NEVER
|
|
1140
|
+
// rolled back.
|
|
1141
|
+
const evidence = await dispatchEngine.getDispatchEvidence(
|
|
1142
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId, fix.stage);
|
|
1143
|
+
if (evidence.alive) {
|
|
1144
|
+
return {
|
|
1145
|
+
success: false, action, scene,
|
|
1146
|
+
details: `repair aborted: generation alive (${evidence.reason})`
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
const currentStates = await state.getAssetStates(
|
|
1150
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1151
|
+
if (currentStates[fix.stage] !== state.AssetState.GENERATING) {
|
|
1152
|
+
return {
|
|
1153
|
+
success: false, action, scene,
|
|
1154
|
+
details: `repair aborted: state now ${currentStates[fix.stage]}`
|
|
1155
|
+
};
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
// FSM-safe path GENERATING → DIRTY → PENDING (a direct
|
|
1159
|
+
// GENERATING → PENDING is forbidden). Errors are NOT
|
|
1160
|
+
// swallowed (rollbackStageToPending logs + journals +
|
|
1161
|
+
// counts failures itself).
|
|
1162
|
+
const rollback = await rollbackStageToPending(
|
|
1163
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId,
|
|
1164
|
+
fix.stage, null, 'orphan_generating_repair');
|
|
1165
|
+
if (!rollback.changed) {
|
|
1166
|
+
return { success: false, action, scene, details: rollback.reason || 'rollback_failed' };
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
await journal.appendSceneEvent(redis, scene.bookId, scene.chapterId, scene.sceneId,
|
|
1170
|
+
'AUTO_RECOVER', 'PENDING',
|
|
1171
|
+
{ stage: fix.stage, fromState: 'generating', recoveredBy: 'reconciliation-engine' }
|
|
1172
|
+
).catch(() => {});
|
|
1173
|
+
|
|
1174
|
+
// Make the repaired stage dispatchable again — but never
|
|
1175
|
+
// for a cancelled book (tombstone/cancel flag precedence:
|
|
1176
|
+
// only an explicit new generation clears them).
|
|
1177
|
+
if (!(await isBookGenerationCancelled(redis, scene.bookId))) {
|
|
1178
|
+
await runtimeScheduler.addSceneToActiveIndex(
|
|
1179
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
return {
|
|
1183
|
+
success: true, action, scene,
|
|
1184
|
+
details: `${fix.stage}: GENERATING → DIRTY → PENDING (orphan repair)`
|
|
1185
|
+
};
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
// ── Legacy stuck_state path: whole-scene DIRTY ──
|
|
1189
|
+
// Mark all per-asset states as DIRTY for redispatch
|
|
1190
|
+
// M5: Route through orchestrator.markDirtyScene instead of direct state.setAssetState
|
|
1191
|
+
// S-5: asset-state writer via the injected orchestration seam
|
|
1192
|
+
await orchestrationSeams.getOrchestrationSeam('markDirtyScene')(redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1193
|
+
|
|
1194
|
+
// Remove from active index to avoid immediate re-scheduling
|
|
1195
|
+
await runtimeScheduler.removeSceneFromActiveIndex(redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1196
|
+
|
|
1197
|
+
// Extract previous state from fix reason (format: "Stuck in <state> for <n> minutes")
|
|
1198
|
+
const fromState = fix.reason?.match(/Stuck in (\w+)/)?.[1] || 'unknown';
|
|
1199
|
+
|
|
1200
|
+
// Log to journal
|
|
1201
|
+
await journal.appendSceneEvent(redis, scene.bookId, scene.chapterId, scene.sceneId,
|
|
1202
|
+
'AUTO_RECOVER', 'DIRTY',
|
|
1203
|
+
{ fromState, recoveredBy: 'reconciliation-engine' }
|
|
1204
|
+
);
|
|
1205
|
+
|
|
1206
|
+
return { success: true, action, scene, details: 'moved to DIRTY + assets DIRTY' };
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
case 'RELEASE_STALE_LOCKS': {
|
|
1210
|
+
let removedCount = 0;
|
|
1211
|
+
for (const lockPrefix of LOCK_KEYS) {
|
|
1212
|
+
const lockKey = `${lockPrefix}:${scene.bookId}:${scene.chapterId}:${scene.sceneId}`;
|
|
1213
|
+
const removed = await redis.del(lockKey);
|
|
1214
|
+
removedCount += removed;
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
return { success: removedCount > 0, action, scene, details: `removed ${removedCount} locks` };
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
case 'RELEASE_STALE_LEASE': {
|
|
1221
|
+
const dispatchEngine = require('./dispatch-engine');
|
|
1222
|
+
let removedCount = 0;
|
|
1223
|
+
const clearedDispatchIds = [];
|
|
1224
|
+
|
|
1225
|
+
// Release all dispatch leases for this scene through the
|
|
1226
|
+
// canonical cancellation path (lease + owned quota + renewal
|
|
1227
|
+
// timer + completion marker). A bare `del` would leak the
|
|
1228
|
+
// quota slot and the in-memory renewal timer, and could race a
|
|
1229
|
+
// concurrent re-acquire (no token check).
|
|
1230
|
+
for (const stage of mediaRegistry.listMediaTypes()) {
|
|
1231
|
+
const result = await dispatchEngine.cancelActiveDispatch(
|
|
1232
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId, stage,
|
|
1233
|
+
'stale_lease_recovery'
|
|
1234
|
+
);
|
|
1235
|
+
if (result.cancelled) {
|
|
1236
|
+
removedCount++;
|
|
1237
|
+
if (result.dispatchId) clearedDispatchIds.push(result.dispatchId);
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
// Defensive de-duplication (audit c8b79f6 §2): purge the hub
|
|
1242
|
+
// queue/running copies owned by the cancelled dispatch(es) so a
|
|
1243
|
+
// recovered scene cannot accumulate duplicate job copies that a
|
|
1244
|
+
// worker would later drain. Best-effort — hub unavailability
|
|
1245
|
+
// must not block state recovery. O-9: the hub call goes
|
|
1246
|
+
// through the HubCancelPort (hub HTTP stays host-side; the
|
|
1247
|
+
// cancelActiveDispatch loop above stays a direct
|
|
1248
|
+
// dispatch-engine call — Redis domain operation).
|
|
1249
|
+
if (clearedDispatchIds.length > 0) {
|
|
1250
|
+
try {
|
|
1251
|
+
await require('./hub-cancel-port').hubCancelOp('clearHubDispatches')(clearedDispatchIds, { context: 'STALE_LEASE_RECOVERY' });
|
|
1252
|
+
} catch (hubErr) {
|
|
1253
|
+
warn(`RELEASE_STALE_LEASE hub cleanup failed (non-fatal): ${hubErr.message}`);
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
// Mark per-asset states as DIRTY for redispatch
|
|
1258
|
+
// M5: Route through orchestrator.markDirtyScene instead of direct state.setAssetState
|
|
1259
|
+
// M5 Шаг 3: syncLinearState уже внутри markDirtyScene
|
|
1260
|
+
// S-5: asset-state writer via the injected orchestration seam
|
|
1261
|
+
const markDirtyScene = orchestrationSeams.getOrchestrationSeam('markDirtyScene');
|
|
1262
|
+
await markDirtyScene(redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1263
|
+
|
|
1264
|
+
// Add back to active index
|
|
1265
|
+
await runtimeScheduler.addSceneToActiveIndex(redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1266
|
+
|
|
1267
|
+
return { success: removedCount > 0, action, scene, details: `released ${removedCount} stale leases` };
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
case 'REGENERATE_MISSING_ASSET': {
|
|
1271
|
+
// Derive asset type from action reason or issue field
|
|
1272
|
+
const reasons = (fix.reason + ' ' + (fix.issue || '')).toLowerCase();
|
|
1273
|
+
const pendingState =
|
|
1274
|
+
reasons.includes('audio') || reasons.includes('tts')
|
|
1275
|
+
? 'audio_pending'
|
|
1276
|
+
: reasons.includes('video')
|
|
1277
|
+
? 'video_pending'
|
|
1278
|
+
: reasons.includes('image') || reasons.includes('iu')
|
|
1279
|
+
? 'image_pending'
|
|
1280
|
+
: null;
|
|
1281
|
+
|
|
1282
|
+
if (!pendingState) {
|
|
1283
|
+
return { success: false, action, scene, details: `unknown asset type (reason: ${fix.reason})` };
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
// M5 Шаг 4: Route through orchestrator.setScenePending (which includes syncLinearState)
|
|
1287
|
+
// S-5: asset-state writer via the injected orchestration seam
|
|
1288
|
+
const setScenePending = orchestrationSeams.getOrchestrationSeam('setScenePending');
|
|
1289
|
+
const assetType = pendingState === 'audio_pending' ? 'audio'
|
|
1290
|
+
: pendingState === 'image_pending' ? 'image'
|
|
1291
|
+
: pendingState === 'video_pending' ? 'video'
|
|
1292
|
+
: null;
|
|
1293
|
+
if (assetType) {
|
|
1294
|
+
await setScenePending(redis, scene.bookId, scene.chapterId, scene.sceneId, assetType);
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
await runtimeScheduler.addSceneToActiveIndex(
|
|
1298
|
+
redis,
|
|
1299
|
+
scene.bookId,
|
|
1300
|
+
scene.chapterId,
|
|
1301
|
+
scene.sceneId
|
|
1302
|
+
);
|
|
1303
|
+
|
|
1304
|
+
return { success: true, action, scene, details: `marked as ${pendingState}` };
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
case 'PROGRESS_TO_IMAGE': {
|
|
1308
|
+
// M5 Шаг 4: Route through orchestrator.setScenePending (includes syncLinearState)
|
|
1309
|
+
// S-5: asset-state writer via the injected orchestration seam
|
|
1310
|
+
await orchestrationSeams.getOrchestrationSeam('setScenePending')(redis, scene.bookId, scene.chapterId, scene.sceneId, 'image');
|
|
1311
|
+
|
|
1312
|
+
await runtimeScheduler.addSceneToActiveIndex(
|
|
1313
|
+
redis,
|
|
1314
|
+
scene.bookId,
|
|
1315
|
+
scene.chapterId,
|
|
1316
|
+
scene.sceneId
|
|
1317
|
+
);
|
|
1318
|
+
|
|
1319
|
+
return { success: true, action, scene, details: 'progressed to IMAGE_PENDING' };
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
case 'RECONCILE_COUNTER_DRIFT': {
|
|
1323
|
+
// Get the current lease count and fix counter
|
|
1324
|
+
const stage = fix.stage || 'audio';
|
|
1325
|
+
const leaseCount = await counterReconciliation.countActiveLeasesByStage(redis, stage);
|
|
1326
|
+
|
|
1327
|
+
await counterReconciliation.correctCounterWithLua(redis, stage, leaseCount);
|
|
1328
|
+
|
|
1329
|
+
return { success: true, action, scene, details: `counter corrected to ${leaseCount}` };
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
case 'PROGRESS_TO_VIDEO': {
|
|
1333
|
+
// M5 Шаг 4: Route through orchestrator.setScenePending (includes syncLinearState)
|
|
1334
|
+
// S-5: asset-state writer via the injected orchestration seam
|
|
1335
|
+
await orchestrationSeams.getOrchestrationSeam('setScenePending')(redis, scene.bookId, scene.chapterId, scene.sceneId, 'video');
|
|
1336
|
+
|
|
1337
|
+
await runtimeScheduler.addSceneToActiveIndex(
|
|
1338
|
+
redis,
|
|
1339
|
+
scene.bookId,
|
|
1340
|
+
scene.chapterId,
|
|
1341
|
+
scene.sceneId
|
|
1342
|
+
);
|
|
1343
|
+
|
|
1344
|
+
return { success: true, action, scene, details: 'progressed to VIDEO_PENDING' };
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
case 'RECOVER_ORPHAN_ASSETS': {
|
|
1348
|
+
// Clear registry entries for missing assets
|
|
1349
|
+
const assets = await persist('registry.getSceneAssetsRedis')(redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1350
|
+
if (assets) {
|
|
1351
|
+
if (assets.audio && assets.audio.canonical) {
|
|
1352
|
+
await persist('registry.registerSceneAudioRedis')(redis, scene.bookId, scene.chapterId, scene.sceneId, {
|
|
1353
|
+
canonicalPath: assets.audio.canonical,
|
|
1354
|
+
ready: false
|
|
1355
|
+
});
|
|
1356
|
+
}
|
|
1357
|
+
if (assets.image && assets.image.path) {
|
|
1358
|
+
await persist('registry.registerSceneImageRedis')(redis, scene.bookId, scene.chapterId, scene.sceneId, {
|
|
1359
|
+
path: assets.image.path,
|
|
1360
|
+
ready: false
|
|
1361
|
+
});
|
|
1362
|
+
}
|
|
1363
|
+
if (assets.video && assets.video.path) {
|
|
1364
|
+
await persist('registry.registerSceneVideoRedis')(redis, scene.bookId, scene.chapterId, scene.sceneId, {
|
|
1365
|
+
path: assets.video.path,
|
|
1366
|
+
ready: false
|
|
1367
|
+
});
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
// M5 Шаг 4: Route through orchestrator.setScenePending (includes syncLinearState)
|
|
1372
|
+
// S-5: asset-state writer via the injected orchestration seam
|
|
1373
|
+
await orchestrationSeams.getOrchestrationSeam('setScenePending')(redis, scene.bookId, scene.chapterId, scene.sceneId, 'audio');
|
|
1374
|
+
|
|
1375
|
+
return { success: true, action, scene, details: 'registry marked for recovery' };
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
default:
|
|
1379
|
+
return { success: false, action, scene, details: 'unknown_action' };
|
|
1380
|
+
}
|
|
1381
|
+
} catch (err) {
|
|
1382
|
+
error(`Fix execution error for ${scene.bookId}/${scene.chapterId}/${scene.sceneId}: ${err.message}`);
|
|
1383
|
+
return { success: false, action, scene, details: err.message };
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
// ======================================================
|
|
1388
|
+
// BULK RECONCILIATION
|
|
1389
|
+
// ======================================================
|
|
1390
|
+
|
|
1391
|
+
/**
|
|
1392
|
+
* Reconcile all scenes for a specific book.
|
|
1393
|
+
*/
|
|
1394
|
+
async function reconcileBook(redis, bookId) {
|
|
1395
|
+
log(`Starting book reconciliation: ${bookId}`);
|
|
1396
|
+
|
|
1397
|
+
const report = new ReconciliationReport();
|
|
1398
|
+
let scanned = 0;
|
|
1399
|
+
|
|
1400
|
+
// Scan event journals for scenes
|
|
1401
|
+
const journalPattern = `animastor:event-journal:${bookId}:*`;
|
|
1402
|
+
let cursor = 0;
|
|
1403
|
+
|
|
1404
|
+
do {
|
|
1405
|
+
const result = await redis.scan(cursor, 'MATCH', journalPattern, 'COUNT', 200);
|
|
1406
|
+
cursor = parseInt(result[0], 10);
|
|
1407
|
+
const keys = result[1];
|
|
1408
|
+
|
|
1409
|
+
for (const key of keys) {
|
|
1410
|
+
const parts = key.split(':');
|
|
1411
|
+
if (parts.length >= 5) {
|
|
1412
|
+
const chapterId = parts[3];
|
|
1413
|
+
const sceneId = parts[4];
|
|
1414
|
+
|
|
1415
|
+
const sceneReport = await reconcileScene(redis, bookId, chapterId, sceneId);
|
|
1416
|
+
|
|
1417
|
+
report.orphanStates.push(...sceneReport.orphanStates);
|
|
1418
|
+
report.orphanAssets.push(...sceneReport.orphanAssets);
|
|
1419
|
+
report.partialBuilds.push(...sceneReport.partialBuilds);
|
|
1420
|
+
report.staleLocks.push(...sceneReport.staleLocks);
|
|
1421
|
+
report.inconsistentScenes.push(...sceneReport.inconsistentScenes);
|
|
1422
|
+
|
|
1423
|
+
scanned++;
|
|
1424
|
+
}
|
|
1425
|
+
}
|
|
1426
|
+
} while (cursor !== 0);
|
|
1427
|
+
|
|
1428
|
+
log(`Reconciliation complete: ${scanned} scenes scanned, ${report.inconsistentScenes.length} inconsistencies found`);
|
|
1429
|
+
|
|
1430
|
+
return report;
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
/**
|
|
1434
|
+
* Reconcile all scenes across all books.
|
|
1435
|
+
*/
|
|
1436
|
+
async function reconcileAll(redis) {
|
|
1437
|
+
log('Starting full system reconciliation');
|
|
1438
|
+
|
|
1439
|
+
const report = new ReconciliationReport();
|
|
1440
|
+
let scanned = 0;
|
|
1441
|
+
|
|
1442
|
+
// Find all scenes via event journal
|
|
1443
|
+
const journalPattern = 'animastor:event-journal:*';
|
|
1444
|
+
let cursor = 0;
|
|
1445
|
+
|
|
1446
|
+
do {
|
|
1447
|
+
const result = await redis.scan(cursor, 'MATCH', journalPattern, 'COUNT', 500);
|
|
1448
|
+
cursor = parseInt(result[0], 10);
|
|
1449
|
+
const keys = result[1];
|
|
1450
|
+
|
|
1451
|
+
for (const key of keys) {
|
|
1452
|
+
const parts = key.split(':');
|
|
1453
|
+
if (parts.length >= 5) {
|
|
1454
|
+
const bookId = parts[2];
|
|
1455
|
+
const chapterId = parts[3];
|
|
1456
|
+
const sceneId = parts[4];
|
|
1457
|
+
|
|
1458
|
+
const sceneReport = await reconcileScene(redis, bookId, chapterId, sceneId);
|
|
1459
|
+
|
|
1460
|
+
report.orphanStates.push(...sceneReport.orphanStates);
|
|
1461
|
+
report.orphanAssets.push(...sceneReport.orphanAssets);
|
|
1462
|
+
report.partialBuilds.push(...sceneReport.partialBuilds);
|
|
1463
|
+
report.staleLocks.push(...sceneReport.staleLocks);
|
|
1464
|
+
report.inconsistentScenes.push(...sceneReport.inconsistentScenes);
|
|
1465
|
+
|
|
1466
|
+
scanned++;
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
} while (cursor !== 0);
|
|
1470
|
+
|
|
1471
|
+
// Check counter drift once globally (leases vs counters are system-wide, not per-scene)
|
|
1472
|
+
for (const stage of mediaRegistry.listMediaTypes()) {
|
|
1473
|
+
const driftCheck = await counterReconciliation.getCounterWithDriftCheck(redis, stage);
|
|
1474
|
+
if (driftCheck && !driftCheck.correct) {
|
|
1475
|
+
report.inconsistentScenes.push({
|
|
1476
|
+
scene: { bookId: '*', chapterId: '*', sceneId: '*' },
|
|
1477
|
+
stage,
|
|
1478
|
+
drift: driftCheck.drift,
|
|
1479
|
+
leaseCount: driftCheck.leaseCount,
|
|
1480
|
+
counterValue: driftCheck.counterValue,
|
|
1481
|
+
issue: 'counter_drift'
|
|
1482
|
+
});
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
log(`Full reconciliation complete: ${scanned} scenes scanned, ${report.inconsistentScenes.length} inconsistencies found`);
|
|
1487
|
+
|
|
1488
|
+
return report;
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
/**
|
|
1492
|
+
* Get current metrics from reconciliation engine.
|
|
1493
|
+
*/
|
|
1494
|
+
async function getMetrics(redis) {
|
|
1495
|
+
return {
|
|
1496
|
+
activeChecks: 'all_scenes_via_journal_scan'
|
|
1497
|
+
};
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
// ======================================================
|
|
1501
|
+
// T6: ЕДИНЫЙ RECONCILIATION-ЦИКЛ
|
|
1502
|
+
// ======================================================
|
|
1503
|
+
// Заменяет 4 несогласованных механизма восстановления:
|
|
1504
|
+
// startup-recovery, audio-recovery, cleanup-service, reconcileAll
|
|
1505
|
+
//
|
|
1506
|
+
// Фазы:
|
|
1507
|
+
// A. Result/error key recovery (из audio-recovery.cjs)
|
|
1508
|
+
// B. Cleanup expired locks (из cleanup-service.cjs)
|
|
1509
|
+
// C. Startup-specific (из startup-recovery.js) — только при startup:true
|
|
1510
|
+
// D. Full scene reconciliation (из reconciliation-engine)
|
|
1511
|
+
//
|
|
1512
|
+
// Каждый прогон пишет RECOVERY_STARTED/RECOVERY_COMPLETED в event-journal.
|
|
1513
|
+
// Один распределённый CLEANUP_LOCK на цикл — прогоны не пересекаются.
|
|
1514
|
+
|
|
1515
|
+
/**
|
|
1516
|
+
* Единый reconciliation-цикл. Выполняет все фазы восстановления.
|
|
1517
|
+
*
|
|
1518
|
+
* @param {Object} redis
|
|
1519
|
+
* @param {Object} deps — зависимости { state, config, postgres, orchestrator, taskHandler, ... }
|
|
1520
|
+
* @param {Object} [options]
|
|
1521
|
+
* @param {boolean} [options.startup=false] — первый прогон при старте (доп. шаги C)
|
|
1522
|
+
* @param {string} [options.scope] — если указан, обработать только одну книгу/сцену
|
|
1523
|
+
* Формат: "bookId" (книга), "bookId:chapterId:sceneId" (одна сцена)
|
|
1524
|
+
* @returns {Promise<{ok:boolean, phases:string[], summary:object}>}
|
|
1525
|
+
*/
|
|
1526
|
+
async function reconcileCycle(redis, deps = {}, options = {}) {
|
|
1527
|
+
const { startup = false, scope = null } = options;
|
|
1528
|
+
const startTime = Date.now();
|
|
1529
|
+
const phases = [];
|
|
1530
|
+
const summary = { totalScanned: 0, itemsRecovered: 0, staleLocks: 0, errors: [] };
|
|
1531
|
+
|
|
1532
|
+
// ── Acquire distributed CLEANUP_LOCK ──
|
|
1533
|
+
const lockKey = config.REDIS.CLEANUP_LOCK || 'animastor:cleanup-lock';
|
|
1534
|
+
const lockToken = `reconcile-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
|
|
1535
|
+
const lockAcquired = await redis.set(lockKey, lockToken, 'NX', 'EX', 120);
|
|
1536
|
+
if (!lockAcquired) {
|
|
1537
|
+
log('Cycle skipped — CLEANUP_LOCK held by another instance');
|
|
1538
|
+
return { ok: false, reason: 'lock_held', phases: ['lock_skipped'], summary };
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
const releaseLock = async () => {
|
|
1542
|
+
try {
|
|
1543
|
+
const current = await redis.get(lockKey);
|
|
1544
|
+
if (current === lockToken) await redis.del(lockKey);
|
|
1545
|
+
} catch (_) {}
|
|
1546
|
+
};
|
|
1547
|
+
|
|
1548
|
+
try {
|
|
1549
|
+
// ── Journal: RECOVERY_STARTED ──
|
|
1550
|
+
try {
|
|
1551
|
+
await journal.appendSceneEvent(redis, '_system', '_recovery', '_cycle',
|
|
1552
|
+
journal.EventType.RECOVERY_STARTED, 'INITIATED',
|
|
1553
|
+
{ startup, scope, phases: [] }
|
|
1554
|
+
);
|
|
1555
|
+
} catch (_) {}
|
|
1556
|
+
|
|
1557
|
+
// ══════════════════════════════════════════════
|
|
1558
|
+
// PHASE A: Result/error key recovery (audio-recovery)
|
|
1559
|
+
// ══════════════════════════════════════════════
|
|
1560
|
+
if (deps.taskHandler) {
|
|
1561
|
+
try {
|
|
1562
|
+
const aCount = await recoverResultKeys(redis, deps, scope);
|
|
1563
|
+
summary.itemsRecovered += aCount;
|
|
1564
|
+
phases.push(`result_recovery:${aCount}`);
|
|
1565
|
+
} catch (err) {
|
|
1566
|
+
warn(`Phase A failed: ${err.message}`);
|
|
1567
|
+
summary.errors.push(`result_recovery: ${err.message}`);
|
|
1568
|
+
}
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
// ══════════════════════════════════════════════
|
|
1572
|
+
// PHASE B1: Stalled audio chunks watchdog
|
|
1573
|
+
// ══════════════════════════════════════════════
|
|
1574
|
+
try {
|
|
1575
|
+
const b1Count = await checkStalledAudioScenes(redis, deps);
|
|
1576
|
+
if (b1Count > 0) phases.push(`stalled_audio:${b1Count}`);
|
|
1577
|
+
} catch (err) {
|
|
1578
|
+
warn(`Phase B1 failed: ${err.message}`);
|
|
1579
|
+
summary.errors.push(`stalled_audio: ${err.message}`);
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
// ══════════════════════════════════════════════
|
|
1583
|
+
// PHASE B2: Stalled video groups watchdog
|
|
1584
|
+
// ══════════════════════════════════════════════
|
|
1585
|
+
try {
|
|
1586
|
+
const b2Count = await checkStalledVideoScenes(redis, deps);
|
|
1587
|
+
if (b2Count > 0) phases.push(`stalled_video:${b2Count}`);
|
|
1588
|
+
} catch (err) {
|
|
1589
|
+
warn(`Phase B2 failed: ${err.message}`);
|
|
1590
|
+
summary.errors.push(`stalled_video: ${err.message}`);
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
// ══════════════════════════════════════════════
|
|
1594
|
+
// PHASE C: Startup-specific (startup-recovery)
|
|
1595
|
+
// ══════════════════════════════════════════════
|
|
1596
|
+
if (startup) {
|
|
1597
|
+
// C0: Recover Redis chunks from disk
|
|
1598
|
+
if (typeof deps.recoverAllBooksFromDisk === 'function') {
|
|
1599
|
+
try {
|
|
1600
|
+
await deps.recoverAllBooksFromDisk();
|
|
1601
|
+
phases.push('chunk_recovery:ok');
|
|
1602
|
+
} catch (err) {
|
|
1603
|
+
warn(`Phase C0 failed: ${err.message}`);
|
|
1604
|
+
summary.errors.push(`chunk_recovery: ${err.message}`);
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
// C1: Audio-orch state recovery
|
|
1609
|
+
try {
|
|
1610
|
+
const c1Count = await recoverAudioOrchStates(redis, deps);
|
|
1611
|
+
summary.itemsRecovered += c1Count;
|
|
1612
|
+
phases.push(`audio_orch:${c1Count}`);
|
|
1613
|
+
} catch (err) {
|
|
1614
|
+
warn(`Phase C1 failed: ${err.message}`);
|
|
1615
|
+
summary.errors.push(`audio_orch: ${err.message}`);
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
// C1b: Video-orch state recovery
|
|
1619
|
+
try {
|
|
1620
|
+
const c1bCount = await recoverVideoOrchStates(redis, deps);
|
|
1621
|
+
summary.itemsRecovered += c1bCount;
|
|
1622
|
+
phases.push(`video_orch:${c1bCount}`);
|
|
1623
|
+
} catch (err) {
|
|
1624
|
+
warn(`Phase C1b failed: ${err.message}`);
|
|
1625
|
+
summary.errors.push(`video_orch: ${err.message}`);
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
// C2: Version staleness check
|
|
1629
|
+
try {
|
|
1630
|
+
const c2Count = await checkVersionStaleness(redis, deps);
|
|
1631
|
+
if (c2Count > 0) phases.push(`version_stale:${c2Count}`);
|
|
1632
|
+
} catch (err) {
|
|
1633
|
+
warn(`Phase C2 failed: ${err.message}`);
|
|
1634
|
+
summary.errors.push(`version_stale: ${err.message}`);
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
// C3 removed (S1.4): was log-only PNG scan, no state mutation.
|
|
1638
|
+
// recoverIuImagesFromDisk() too expensive for every 60s cycle without value.
|
|
1639
|
+
|
|
1640
|
+
// C4: Reconcile missing scene counters from PG
|
|
1641
|
+
try {
|
|
1642
|
+
const c4Count = await reconcileMissingSceneState(redis, deps);
|
|
1643
|
+
if (c4Count > 0) phases.push(`counter_reconcile:${c4Count}`);
|
|
1644
|
+
} catch (err) {
|
|
1645
|
+
warn(`Phase C4 failed: ${err.message}`);
|
|
1646
|
+
summary.errors.push(`counter_reconcile: ${err.message}`);
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
// C5: Resume incomplete sessions (optional — requires runBackgroundWindowGeneration)
|
|
1650
|
+
if (typeof deps.resumeIncompleteSessions === 'function' && typeof deps.runBackgroundWindowGeneration === 'function') {
|
|
1651
|
+
try {
|
|
1652
|
+
await deps.resumeIncompleteSessions(log, deps.runBackgroundWindowGeneration);
|
|
1653
|
+
phases.push('session_resume');
|
|
1654
|
+
} catch (err) {
|
|
1655
|
+
warn(`Phase C5 failed: ${err.message}`);
|
|
1656
|
+
summary.errors.push(`session_resume: ${err.message}`);
|
|
1657
|
+
}
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
// ══════════════════════════════════════════════
|
|
1662
|
+
// PHASE C6: Restore durable layer-config from book.json (Кирпич №2)
|
|
1663
|
+
// ══════════════════════════════════════════════
|
|
1664
|
+
// Сервис layer-config владеет форматом: заполняет ТОЛЬКО отсутствующие
|
|
1665
|
+
// Redis-ключи из durable-копии (нормализуя значения). В норме — no-op.
|
|
1666
|
+
// Запускается каждый цикл (не только startup), чтобы heal удалений
|
|
1667
|
+
// ключа cleanBookRedisKeys на cache-clear / import / recovery, пока
|
|
1668
|
+
// книга существует — конфиг не пропадает из Redis до рестарта.
|
|
1669
|
+
|
|
1670
|
+
if (!scope) {
|
|
1671
|
+
// O-10: through the LayerConfigPort (host adapter:
|
|
1672
|
+
// storage/layer-config-adapter). The pre-O-10 optional-load
|
|
1673
|
+
// semantics are preserved EXACTLY: when the host service
|
|
1674
|
+
// cannot be loaded, this phase is skipped with a warn (a
|
|
1675
|
+
// missing module ≠ failed recovery). The outer try/catch
|
|
1676
|
+
// wraps the PORT call — the adapter's lazy call-time
|
|
1677
|
+
// resolver throws at the same point the pre-O-10 inner
|
|
1678
|
+
// require did, and the caller's catch handles it
|
|
1679
|
+
// identically (the pre-O-10 inner try set layerConfig to
|
|
1680
|
+
// null and skipped silently; the C6 error path below is
|
|
1681
|
+
// reached only when the CALL fails, same as before).
|
|
1682
|
+
try {
|
|
1683
|
+
const c6Count = await require('./layer-config-port').layerConfigOp('restoreFromBooks')(redis);
|
|
1684
|
+
if (c6Count > 0) phases.push(`layer_config_restore:${c6Count}`);
|
|
1685
|
+
} catch (err) {
|
|
1686
|
+
warn(`Phase C6 failed: ${err.message}`);
|
|
1687
|
+
summary.errors.push(`layer_config_restore: ${err.message}`);
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
// ══════════════════════════════════════════════
|
|
1692
|
+
// PHASE C7: Retry pending entity purges (deleted scenes/units)
|
|
1693
|
+
// ══════════════════════════════════════════════
|
|
1694
|
+
// The entity-delete cleanup records a marker in the pending-purge set
|
|
1695
|
+
// whenever a purge is only partially complete. Retry here keeps the
|
|
1696
|
+
// "delete = gone everywhere" guarantee without hiding failures.
|
|
1697
|
+
if (deps.entityCleanup && typeof deps.entityCleanup.retryPendingPurges === 'function') {
|
|
1698
|
+
try {
|
|
1699
|
+
const pCount = await deps.entityCleanup.retryPendingPurges();
|
|
1700
|
+
if (pCount > 0) phases.push(`pending_purge:${pCount}`);
|
|
1701
|
+
} catch (err) {
|
|
1702
|
+
warn(`Phase C7 failed: ${err.message}`);
|
|
1703
|
+
summary.errors.push(`pending_purge: ${err.message}`);
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
// ══════════════════════════════════════════════
|
|
1708
|
+
// PHASE C8: WORK_TO_DO rebuild (Option E) — startup-only
|
|
1709
|
+
// ══════════════════════════════════════════════
|
|
1710
|
+
// Авто-возобновление прерванной генерации после потери Redis. Только
|
|
1711
|
+
// первый цикл (startup:true) — периодический цикл работу НЕ возрождает
|
|
1712
|
+
// (политика Recon #4 §10.3: без user action генерация не стартует).
|
|
1713
|
+
if (startup && !scope) {
|
|
1714
|
+
try {
|
|
1715
|
+
const c7Count = await rebuildWorkList(redis);
|
|
1716
|
+
if (c7Count > 0) phases.push(`worklist_rebuild:${c7Count}`);
|
|
1717
|
+
} catch (err) {
|
|
1718
|
+
warn(`Phase C8 failed: ${err.message}`);
|
|
1719
|
+
summary.errors.push(`worklist_rebuild: ${err.message}`);
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
|
|
1723
|
+
// ══════════════════════════════════════════════
|
|
1724
|
+
// PHASE D: Full scene reconciliation
|
|
1725
|
+
// ══════════════════════════════════════════════
|
|
1726
|
+
if (!scope || scope.includes(':')) {
|
|
1727
|
+
try {
|
|
1728
|
+
const dReport = await reconcileAll(redis);
|
|
1729
|
+
summary.totalScanned = dReport.orphanStates.length + dReport.orphanAssets.length;
|
|
1730
|
+
phases.push(`reconcile:${dReport.inconsistentScenes.length}_issues`);
|
|
1731
|
+
// Auto-fix safe issues
|
|
1732
|
+
const fixes = getFixRecommendations(dReport.inconsistentScenes);
|
|
1733
|
+
for (const fix of fixes.filter(f => f.safeToExecute)) {
|
|
1734
|
+
try {
|
|
1735
|
+
await applyFix(redis, fix);
|
|
1736
|
+
} catch (fixErr) {
|
|
1737
|
+
warn(`Auto-fix failed for ${fix.scene.sceneId}: ${fixErr.message}`);
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
} catch (err) {
|
|
1741
|
+
warn(`Phase D failed: ${err.message}`);
|
|
1742
|
+
summary.errors.push(`reconcile: ${err.message}`);
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
|
|
1746
|
+
// ── Journal: RECOVERY_COMPLETED ──
|
|
1747
|
+
try {
|
|
1748
|
+
await journal.appendSceneEvent(redis, '_system', '_recovery', '_cycle',
|
|
1749
|
+
journal.EventType.RECOVERY_COMPLETED, 'DONE',
|
|
1750
|
+
{ startup, scope, phases, summary, elapsedMs: Date.now() - startTime }
|
|
1751
|
+
);
|
|
1752
|
+
} catch (_) {}
|
|
1753
|
+
|
|
1754
|
+
log(`Cycle complete [${phases.join(', ')}] in ${Date.now() - startTime}ms`);
|
|
1755
|
+
return { ok: true, phases, summary };
|
|
1756
|
+
|
|
1757
|
+
} finally {
|
|
1758
|
+
await releaseLock();
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
|
|
1762
|
+
// ── PHASE A: Recover result/error keys ─────────────────
|
|
1763
|
+
// Из audio-recovery.cjs: сканирует animastor:result:* и animastor:error:*,
|
|
1764
|
+
// доигрывает потерянные результаты через taskHandler.handleTaskResult.
|
|
1765
|
+
// Для error-ключей вызывает orchestrator.failStage.
|
|
1766
|
+
async function recoverResultKeys(redis, deps, scope) {
|
|
1767
|
+
const taskHandler = deps.taskHandler;
|
|
1768
|
+
if (!taskHandler || typeof taskHandler.handleTaskResult !== 'function') return 0;
|
|
1769
|
+
|
|
1770
|
+
let recovered = 0;
|
|
1771
|
+
|
|
1772
|
+
// Отклонения dispatch-identity — терминальные для результата: dispatch
|
|
1773
|
+
// уже финализирован/отменён, файл писать нельзя. Ключ удаляем, чтобы
|
|
1774
|
+
// recovery не ретраил его каждые 60 сек до истечения 1-часового TTL
|
|
1775
|
+
// (бывший бесконечный цикл по _g2.._g5 после no_active_dispatch).
|
|
1776
|
+
const TERMINAL_REJECTIONS = new Set([
|
|
1777
|
+
'no_active_dispatch',
|
|
1778
|
+
'stale_dispatch',
|
|
1779
|
+
'missing_dispatch_id',
|
|
1780
|
+
'metadata_missing_dispatch_id',
|
|
1781
|
+
'dispatch_metadata_changed',
|
|
1782
|
+
'lease_token_mismatch',
|
|
1783
|
+
]);
|
|
1784
|
+
|
|
1785
|
+
// Scan animastor:result:*
|
|
1786
|
+
let cursor = '0';
|
|
1787
|
+
do {
|
|
1788
|
+
const [nextCursor, keys] = await redis.scan(cursor, 'MATCH', 'animastor:result:*', 'COUNT', 200);
|
|
1789
|
+
cursor = nextCursor;
|
|
1790
|
+
for (const key of keys) {
|
|
1791
|
+
try {
|
|
1792
|
+
const raw = await redis.get(key);
|
|
1793
|
+
if (!raw) continue;
|
|
1794
|
+
|
|
1795
|
+
let job_id, result_base64, buildId, dispatchId;
|
|
1796
|
+
|
|
1797
|
+
// Try JSON (new format: { job_id, result_base64, build_id })
|
|
1798
|
+
if (raw.startsWith('{')) {
|
|
1799
|
+
try {
|
|
1800
|
+
const data = JSON.parse(raw);
|
|
1801
|
+
job_id = data.job_id;
|
|
1802
|
+
result_base64 = data.result_base64;
|
|
1803
|
+
buildId = data.build_id;
|
|
1804
|
+
dispatchId = data.dispatch_id;
|
|
1805
|
+
} catch (_) {
|
|
1806
|
+
await redis.del(key);
|
|
1807
|
+
continue;
|
|
1808
|
+
}
|
|
1809
|
+
} else {
|
|
1810
|
+
// Data URL fallback (old format)
|
|
1811
|
+
result_base64 = raw;
|
|
1812
|
+
const keyParts = key.split(':');
|
|
1813
|
+
const buildPart = keyParts[2] || '';
|
|
1814
|
+
const combinedId = `${keyParts[3] || ''}_${keyParts[4] || ''}_${keyParts[5] || ''}`;
|
|
1815
|
+
job_id = `${combinedId}:${keyParts[6] || 'audio'}`;
|
|
1816
|
+
buildId = buildPart;
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
if (!job_id || !result_base64) continue;
|
|
1820
|
+
|
|
1821
|
+
// Scope filter
|
|
1822
|
+
if (scope && !job_id.startsWith(scope)) continue;
|
|
1823
|
+
|
|
1824
|
+
if (!buildId || !dispatchId) {
|
|
1825
|
+
warn(`Skipping legacy result without build/dispatch identity: ${job_id}`);
|
|
1826
|
+
continue;
|
|
1827
|
+
}
|
|
1828
|
+
try {
|
|
1829
|
+
await taskHandler.handleTaskResult(job_id, result_base64, buildId, dispatchId);
|
|
1830
|
+
} catch (rejectErr) {
|
|
1831
|
+
if (rejectErr && TERMINAL_REJECTIONS.has(rejectErr.code)) {
|
|
1832
|
+
warn(`Result ${job_id} terminally rejected (${rejectErr.code}) — dropping result key, no infinite retry`);
|
|
1833
|
+
await redis.del(key);
|
|
1834
|
+
} else {
|
|
1835
|
+
throw rejectErr;
|
|
1836
|
+
}
|
|
1837
|
+
continue;
|
|
1838
|
+
}
|
|
1839
|
+
await redis.del(key);
|
|
1840
|
+
recovered++;
|
|
1841
|
+
} catch (itemErr) {
|
|
1842
|
+
warn(`Result recovery item failed: ${itemErr.message}`);
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
} while (cursor !== '0');
|
|
1846
|
+
|
|
1847
|
+
// Scan animastor:error:* (T3: форвард ошибок, которые не дошли до backend)
|
|
1848
|
+
cursor = '0';
|
|
1849
|
+
do {
|
|
1850
|
+
const [nextCursor, keys] = await redis.scan(cursor, 'MATCH', 'animastor:error:*', 'COUNT', 200);
|
|
1851
|
+
cursor = nextCursor;
|
|
1852
|
+
for (const key of keys) {
|
|
1853
|
+
try {
|
|
1854
|
+
const raw = await redis.get(key);
|
|
1855
|
+
if (!raw) continue;
|
|
1856
|
+
|
|
1857
|
+
let job_id, buildId, reason, dispatchId;
|
|
1858
|
+
try {
|
|
1859
|
+
const data = JSON.parse(raw);
|
|
1860
|
+
job_id = data.job_id;
|
|
1861
|
+
buildId = data.build_id;
|
|
1862
|
+
reason = data.reason;
|
|
1863
|
+
dispatchId = data.dispatch_id;
|
|
1864
|
+
} catch (_) {
|
|
1865
|
+
await redis.del(key);
|
|
1866
|
+
continue;
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
if (!job_id) continue;
|
|
1870
|
+
if (scope && !job_id.startsWith(scope)) continue;
|
|
1871
|
+
|
|
1872
|
+
// Parse job_id to get stage and scene info
|
|
1873
|
+
const jobSchema = require('@animastor/contracts').jobProtocolV2;
|
|
1874
|
+
const parsed = jobSchema.parseJobId(job_id);
|
|
1875
|
+
if (parsed) {
|
|
1876
|
+
const stageMap = { audio_chunk: 'audio', iu_image: 'image', scene_image: 'image', scene_video: 'video' };
|
|
1877
|
+
const stage = stageMap[parsed.kind];
|
|
1878
|
+
if (stage && deps.orchestrator) {
|
|
1879
|
+
await deps.orchestrator.failStage(redis, parsed.bookId, parsed.chapterId, parsed.sceneId,
|
|
1880
|
+
stage, buildId, reason || 'recovered_error', { dispatchId });
|
|
1881
|
+
recovered++;
|
|
1882
|
+
}
|
|
1883
|
+
}
|
|
1884
|
+
await redis.del(key);
|
|
1885
|
+
} catch (itemErr) {
|
|
1886
|
+
warn(`Error recovery item failed: ${itemErr.message}`);
|
|
1887
|
+
}
|
|
1888
|
+
}
|
|
1889
|
+
} while (cursor !== '0');
|
|
1890
|
+
|
|
1891
|
+
return recovered;
|
|
1892
|
+
}
|
|
1893
|
+
|
|
1894
|
+
// ── PHASE C1: Recover audio-orch states ────────────────
|
|
1895
|
+
// Из startup-recovery.js: не-терминальные фазы → FAILED,
|
|
1896
|
+
// MERGING → DONE если файл есть, иначе FAILED.
|
|
1897
|
+
async function recoverAudioOrchStates(redis, deps) {
|
|
1898
|
+
// O-7: through the AudioFsmPort (host adapter: storage/audio-fsm-adapter).
|
|
1899
|
+
// The pre-O-7 optional-load semantics are preserved EXACTLY: when the
|
|
1900
|
+
// audio-orchestrator host service cannot be required, this recovery leg
|
|
1901
|
+
// returns 0 (missing module ≠ recoverable state). The try/catch wraps
|
|
1902
|
+
// the ADAPTER's lazy require — the port itself stays fail-fast.
|
|
1903
|
+
let audioOrch;
|
|
1904
|
+
try {
|
|
1905
|
+
audioOrch = require('./audio-fsm-port').audioFsm();
|
|
1906
|
+
} catch (_) {
|
|
1907
|
+
return 0;
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
const allStates = await audioOrch.scanAllStates(redis);
|
|
1911
|
+
if (allStates.length === 0) return 0;
|
|
1912
|
+
|
|
1913
|
+
let recovered = 0;
|
|
1914
|
+
const OUTPUT_DIR = config.OUTPUT_DIR;
|
|
1915
|
+
|
|
1916
|
+
for (const entry of allStates) {
|
|
1917
|
+
const { bookId, chapterId, sceneId, state: orchState } = entry;
|
|
1918
|
+
const phase = orchState.phase;
|
|
1919
|
+
const buildId = orchState.build_id || 'default';
|
|
1920
|
+
const mergedPath = syncPath.join(OUTPUT_DIR, buildId, artifactNaming.sceneAudioName(bookId, chapterId, sceneId));
|
|
1921
|
+
|
|
1922
|
+
switch (phase) {
|
|
1923
|
+
case 'GENERATING':
|
|
1924
|
+
case 'WAITING_CHUNKS':
|
|
1925
|
+
// ── RECOVERY: сначала проверяем, все ли чанки на диске ──
|
|
1926
|
+
// Если все чанки на месте — доигрываем merge, не FAIL'им.
|
|
1927
|
+
// Иначе FAILED → re-dispatch.
|
|
1928
|
+
const expectedChunkCount = parseInt(orchState.expected_count || '0', 10);
|
|
1929
|
+
const chunks = lazyHostBinding('media.audio.chunks');
|
|
1930
|
+
const presentChunks = chunks.findExistingSceneChunks(bookId, chapterId, sceneId, buildId, expectedChunkCount > 0 ? expectedChunkCount : null);
|
|
1931
|
+
if (presentChunks.length > 0) {
|
|
1932
|
+
const actualCount = expectedChunkCount > 0 ? expectedChunkCount : presentChunks.length;
|
|
1933
|
+
log(`[AUDIO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: ${phase} — ${presentChunks.length}/${actualCount} chunks on disk, driving merge`);
|
|
1934
|
+
// Восстанавливаем expected_count если он был 0
|
|
1935
|
+
if (expectedChunkCount === 0 && presentChunks.length > 0) {
|
|
1936
|
+
orchState.expected_count = presentChunks.length;
|
|
1937
|
+
await audioOrch.setState(redis, bookId, chapterId, sceneId, orchState);
|
|
1938
|
+
}
|
|
1939
|
+
try {
|
|
1940
|
+
await audioOrch.completeChunk(redis, bookId, chapterId, sceneId, 'recovery', buildId, {
|
|
1941
|
+
audio: deps.audio || lazyHostBinding('media.audio'),
|
|
1942
|
+
orchestrator: deps.orchestrator,
|
|
1943
|
+
dispatchId: 'startup-recovery',
|
|
1944
|
+
});
|
|
1945
|
+
recovered++;
|
|
1946
|
+
break;
|
|
1947
|
+
} catch (mergeErr) {
|
|
1948
|
+
warn(`[AUDIO-ORCH] Recover merge failed for ${bookId}/${chapterId}/${sceneId}: ${mergeErr.message} — falling through to FAILED`);
|
|
1949
|
+
}
|
|
1950
|
+
}
|
|
1951
|
+
log(`[AUDIO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: ${phase} → FAILED (${presentChunks.length}/${expectedChunkCount || '?'} chunks on disk)`);
|
|
1952
|
+
await audioOrch.setFailed(redis, bookId, chapterId, sceneId, 'restart_recovery');
|
|
1953
|
+
// F1: sync asset.audio → FAILED after audioOrch.setFailed (R3 follow-up)
|
|
1954
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'audio', state.AssetState.FAILED);
|
|
1955
|
+
// T6: Также сбрасываем asset state, чтобы scheduler передиспатчил
|
|
1956
|
+
if (deps.orchestrator) {
|
|
1957
|
+
await deps.orchestrator.markDirtyScene(redis, bookId, chapterId, sceneId, ['audio']);
|
|
1958
|
+
}
|
|
1959
|
+
recovered++;
|
|
1960
|
+
break;
|
|
1961
|
+
case 'MERGING':
|
|
1962
|
+
if (syncFs.existsSync(mergedPath)) {
|
|
1963
|
+
log(`[AUDIO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: MERGING → DONE`);
|
|
1964
|
+
await audioOrch.setDone(redis, bookId, chapterId, sceneId);
|
|
1965
|
+
// R3: sync asset state after audioOrch.setDone — asset.audio → READY
|
|
1966
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'audio', state.AssetState.READY);
|
|
1967
|
+
} else {
|
|
1968
|
+
log(`[AUDIO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: MERGING → FAILED`);
|
|
1969
|
+
await audioOrch.setFailed(redis, bookId, chapterId, sceneId, 'restart_merge_missing');
|
|
1970
|
+
// F1: sync asset.audio → FAILED after audioOrch.setFailed (R3 follow-up)
|
|
1971
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'audio', state.AssetState.FAILED);
|
|
1972
|
+
if (deps.orchestrator) {
|
|
1973
|
+
await deps.orchestrator.markDirtyScene(redis, bookId, chapterId, sceneId, ['audio']);
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
recovered++;
|
|
1977
|
+
break;
|
|
1978
|
+
case 'PLACEHOLDER_READY':
|
|
1979
|
+
// Leave as is, scheduler will dispatch
|
|
1980
|
+
break;
|
|
1981
|
+
case 'DONE':
|
|
1982
|
+
case 'FAILED':
|
|
1983
|
+
// Terminal, leave as is
|
|
1984
|
+
break;
|
|
1985
|
+
default:
|
|
1986
|
+
await audioOrch.deleteState(redis, bookId, chapterId, sceneId);
|
|
1987
|
+
break;
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
|
|
1991
|
+
if (recovered > 0) log(`[AUDIO-ORCH] ${recovered} non-terminal states recovered`);
|
|
1992
|
+
return recovered;
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
// ── PHASE C1b: Recover video-orch states ───────────────
|
|
1996
|
+
// Зеркало recoverAudioOrchStates: не-терминальные фазы → FAILED (re-dispatch),
|
|
1997
|
+
// MERGING/WAITING_CHUNKS с полным набором групп → доигрываем merge.
|
|
1998
|
+
async function recoverVideoOrchStates(redis, deps) {
|
|
1999
|
+
// O-8: through the VideoFsmPort (host adapter: storage/video-fsm-adapter).
|
|
2000
|
+
// The pre-O-8 optional-load semantics are preserved EXACTLY: when the
|
|
2001
|
+
// video-orchestrator host service cannot be required, this recovery leg
|
|
2002
|
+
// returns 0 (missing module ≠ recoverable state). The try/catch wraps
|
|
2003
|
+
// the ADAPTER's lazy require — the port itself stays fail-fast.
|
|
2004
|
+
let videoOrch;
|
|
2005
|
+
try {
|
|
2006
|
+
videoOrch = require('./video-fsm-port').videoFsm();
|
|
2007
|
+
} catch (_) {
|
|
2008
|
+
return 0;
|
|
2009
|
+
}
|
|
2010
|
+
|
|
2011
|
+
const allStates = await videoOrch.scanAllStates(redis);
|
|
2012
|
+
if (allStates.length === 0) return 0;
|
|
2013
|
+
|
|
2014
|
+
let recovered = 0;
|
|
2015
|
+
const OUTPUT_DIR = config.OUTPUT_DIR;
|
|
2016
|
+
|
|
2017
|
+
for (const entry of allStates) {
|
|
2018
|
+
const { bookId, chapterId, sceneId, state: orchState } = entry;
|
|
2019
|
+
const phase = orchState.phase;
|
|
2020
|
+
const buildId = orchState.build_id || 'default';
|
|
2021
|
+
const suffixes = videoOrch.groupSuffixes(orchState);
|
|
2022
|
+
const mergedPath = syncPath.join(OUTPUT_DIR, buildId, artifactNaming.sceneVideoName(bookId, chapterId, sceneId));
|
|
2023
|
+
|
|
2024
|
+
switch (phase) {
|
|
2025
|
+
case 'GENERATING':
|
|
2026
|
+
case 'WAITING_CHUNKS':
|
|
2027
|
+
// ── RECOVERY: сначала проверяем, все ли группы на диске ──
|
|
2028
|
+
const presentCount = suffixes.filter(s => videoOrch.isGroupFileValid(buildId, bookId, chapterId, sceneId, s)).length;
|
|
2029
|
+
if (presentCount === suffixes.length && suffixes.length > 0) {
|
|
2030
|
+
log(`[VIDEO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: ${phase} — ${presentCount}/${suffixes.length} groups on disk, driving merge`);
|
|
2031
|
+
try {
|
|
2032
|
+
await videoOrch.completeGroup(redis, bookId, chapterId, sceneId, '', buildId, {
|
|
2033
|
+
orchestrator: deps.orchestrator,
|
|
2034
|
+
dispatchId: 'startup-recovery',
|
|
2035
|
+
});
|
|
2036
|
+
recovered++;
|
|
2037
|
+
break;
|
|
2038
|
+
} catch (mergeErr) {
|
|
2039
|
+
warn(`[VIDEO-ORCH] Recover merge failed for ${bookId}/${chapterId}/${sceneId}: ${mergeErr.message} — falling through to FAILED`);
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
2042
|
+
log(`[VIDEO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: ${phase} → FAILED (${presentCount}/${suffixes.length} groups on disk)`);
|
|
2043
|
+
await videoOrch.setFailed(redis, bookId, chapterId, sceneId, 'restart_recovery');
|
|
2044
|
+
// F1: sync asset.video → FAILED после videoOrch.setFailed
|
|
2045
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'video', state.AssetState.FAILED);
|
|
2046
|
+
if (deps.orchestrator) {
|
|
2047
|
+
await deps.orchestrator.markDirtyScene(redis, bookId, chapterId, sceneId, ['video']);
|
|
2048
|
+
}
|
|
2049
|
+
recovered++;
|
|
2050
|
+
break;
|
|
2051
|
+
case 'MERGING':
|
|
2052
|
+
if (syncFs.existsSync(mergedPath)) {
|
|
2053
|
+
log(`[VIDEO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: MERGING → DONE`);
|
|
2054
|
+
await videoOrch.setDone(redis, bookId, chapterId, sceneId);
|
|
2055
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'video', state.AssetState.READY);
|
|
2056
|
+
} else {
|
|
2057
|
+
log(`[VIDEO-ORCH] Recover ${bookId}/${chapterId}/${sceneId}: MERGING → FAILED`);
|
|
2058
|
+
await videoOrch.setFailed(redis, bookId, chapterId, sceneId, 'restart_merge_missing');
|
|
2059
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'video', state.AssetState.FAILED);
|
|
2060
|
+
if (deps.orchestrator) {
|
|
2061
|
+
await deps.orchestrator.markDirtyScene(redis, bookId, chapterId, sceneId, ['video']);
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
recovered++;
|
|
2065
|
+
break;
|
|
2066
|
+
case 'DONE':
|
|
2067
|
+
case 'FAILED':
|
|
2068
|
+
// Terminal — leave as is
|
|
2069
|
+
break;
|
|
2070
|
+
default:
|
|
2071
|
+
await videoOrch.deleteState(redis, bookId, chapterId, sceneId);
|
|
2072
|
+
break;
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
if (recovered > 0) log(`[VIDEO-ORCH] ${recovered} non-terminal states recovered`);
|
|
2077
|
+
return recovered;
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
// ── PHASE C2: Version staleness check ───────────────────
|
|
2081
|
+
// Из startup-recovery.js: для stale-ассетов → markDirtyScene
|
|
2082
|
+
// O-2: the raw deps.postgres handle is GONE — the book-wide version scan is
|
|
2083
|
+
// the sceneVersions.getVersionStalenessRows() port op (SQL lives in the host
|
|
2084
|
+
// adapter verbatim). Pre-O-2 a missing deps.postgres skipped the phase; the
|
|
2085
|
+
// unwired-port throw hits the same catch → warn + return 0.
|
|
2086
|
+
async function checkVersionStaleness(redis, deps) {
|
|
2087
|
+
const { orchestrator } = deps;
|
|
2088
|
+
if (!orchestrator) return 0;
|
|
2089
|
+
|
|
2090
|
+
try {
|
|
2091
|
+
const result = { rows: await persist('sceneVersions.getVersionStalenessRows')() };
|
|
2092
|
+
|
|
2093
|
+
const sceneMap = new Map();
|
|
2094
|
+
for (const row of result.rows) {
|
|
2095
|
+
const key = `${row.book_id}|${row.chapter_id}|${row.scene_id}`;
|
|
2096
|
+
if (!sceneMap.has(key)) {
|
|
2097
|
+
sceneMap.set(key, { bookId: row.book_id, chapterId: row.chapter_id, sceneId: row.scene_id, stale: false });
|
|
2098
|
+
}
|
|
2099
|
+
const entry = sceneMap.get(key);
|
|
2100
|
+
if ((row.scene_content_version != null && row.content_version != null && row.scene_content_version < row.content_version) ||
|
|
2101
|
+
(row.scene_audio_config_version != null && row.audio_config_version != null && row.scene_audio_config_version < row.audio_config_version)) {
|
|
2102
|
+
entry.stale = true;
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
|
|
2106
|
+
let outdated = 0;
|
|
2107
|
+
for (const entry of sceneMap.values()) {
|
|
2108
|
+
if (entry.stale) {
|
|
2109
|
+
log(`[VERSION-STALE] ${entry.bookId}/${entry.chapterId}/${entry.sceneId}`);
|
|
2110
|
+
await orchestrator.markDirtyScene(redis, entry.bookId, entry.chapterId, entry.sceneId);
|
|
2111
|
+
outdated++;
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
return outdated;
|
|
2115
|
+
} catch (err) {
|
|
2116
|
+
warn(`Version staleness check failed: ${err.message}`);
|
|
2117
|
+
return 0;
|
|
2118
|
+
}
|
|
2119
|
+
}
|
|
2120
|
+
|
|
2121
|
+
// ── PHASE C3: REMOVED in S1.4 (2026-07-19) ─────────────
|
|
2122
|
+
// Был log-only PNG scan, сканировал всю OUTPUT_DIR по 60s. Не мутировал состояние,
|
|
2123
|
+
// только логировал счётчик сцен с IU images. Полная walks FS без пользы для reconcile.
|
|
2124
|
+
// Аналогичная функция recoverIuImagesFromDisk осталась в services/startup-recovery.js
|
|
2125
|
+
// (вызывается явно, не на каждый reconcile-цикл).
|
|
2126
|
+
|
|
2127
|
+
// ── PHASE C4: Reconcile missing scene counters from PG ─
|
|
2128
|
+
// Из startup-recovery.js: логирует книги с PG записями без Redis-счётчиков.
|
|
2129
|
+
// O-2: the raw deps.postgres handle is GONE — the DISTINCT book scan is the
|
|
2130
|
+
// same listBooks() port op the worklist rebuild uses (identical SQL, host
|
|
2131
|
+
// adapter). Unwired port → throw → catch → return 0 (pre-O-2: no
|
|
2132
|
+
// deps.postgres → return 0).
|
|
2133
|
+
async function reconcileMissingSceneState(redis, deps) {
|
|
2134
|
+
try {
|
|
2135
|
+
const bookResult = { rows: await persist('listBooks')() };
|
|
2136
|
+
let count = 0;
|
|
2137
|
+
for (const row of bookResult.rows) {
|
|
2138
|
+
const totalKey = `animastor:book-scenes:${row.book_id}:total`;
|
|
2139
|
+
const totalRaw = await redis.get(totalKey);
|
|
2140
|
+
if (!totalRaw || parseInt(totalRaw, 10) === 0) {
|
|
2141
|
+
log(`[COUNTER-LOG-ONLY] Book ${row.book_id} has PG records but no Redis counters`);
|
|
2142
|
+
count++;
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
return count;
|
|
2146
|
+
} catch (err) {
|
|
2147
|
+
warn(`Missing scene state check failed: ${err.message}`);
|
|
2148
|
+
return 0;
|
|
2149
|
+
}
|
|
2150
|
+
}
|
|
2151
|
+
|
|
2152
|
+
// ── PHASE C7: WORK_TO_DO rebuild (Option E) ──────────
|
|
2153
|
+
// Cathedral Option E — см. docs/architecture/work-list-rebuild-design.md
|
|
2154
|
+
// (Recon #4 design proof). После полной потери Redis пересчитывает WORK_TO_DO
|
|
2155
|
+
// из PG + FS и МАТЕРИАЛИЗУЕТ runtime-состояние: per-asset Redis states + SADD
|
|
2156
|
+
// в active index. Генерацию НЕ запускает — это делает обычный scheduler tick
|
|
2157
|
+
// (единственный владелец execution, SCHEDULER_TICK_LOCK).
|
|
2158
|
+
//
|
|
2159
|
+
// Доказанные в Recon #4 инварианты:
|
|
2160
|
+
// 1. Предикат НЕ читает Redis asset state (C0 over-marks READY/READY/READY
|
|
2161
|
+
// для любой сцены с .mp3) — только PG + FS-проба.
|
|
2162
|
+
// 2. States пишутся ДО SADD: scheduler решает «что диспатчить» по Redis
|
|
2163
|
+
// states (shouldScheduleAssets), пустой ключ = NEW = «диспатчить всё».
|
|
2164
|
+
// 3. build_id пинится из manifest.book_id — та же формула, что в attemptDispatch.
|
|
2165
|
+
// 4. Книги с cancellation tombstone (Operation #1) пропускаются fail-closed.
|
|
2166
|
+
// 5. Фаза последняя среди state-write C-фаз и идёт ПОСЛЕ C6 (layer-config
|
|
2167
|
+
// восстановлен — нужен для enabled-слоёв).
|
|
2168
|
+
// 6. Ошибки PG — fail-closed: без PG предикат невычислим, фаза прерывается
|
|
2169
|
+
// (нельзя воскресить отменённую работу или потерять dirty-маркеры).
|
|
2170
|
+
async function rebuildWorkList(redis) {
|
|
2171
|
+
// Все зависимости — lazy require на момент вызова, чтобы фаза была
|
|
2172
|
+
// самодостаточной (не зависит от того, с какими зависимостями был
|
|
2173
|
+
// загружен reconciliation-engine) и чтобы unit-тесты не могли подменить
|
|
2174
|
+
// storage/state через require.cache.
|
|
2175
|
+
|
|
2176
|
+
// §32.30: former host requires resolve lazily through the host-bindings seam.
|
|
2177
|
+
const state = lazyHostBinding('state');
|
|
2178
|
+
// O-3: scene content via the SceneDataPort (host adapter:
|
|
2179
|
+
// storage/scene-data-adapter) — the Book Model facade require left this
|
|
2180
|
+
// function. loadBook/collectScenes consumed as `sceneData(op)(...)`.
|
|
2181
|
+
const sceneData = require('./scene-data-port').sceneDataOp;
|
|
2182
|
+
// O-2: PG persistence (scene-assets + cancel tombstone + the two raw
|
|
2183
|
+
// barrel queries) via the PersistencePort; the storage barrel require
|
|
2184
|
+
// left this function.
|
|
2185
|
+
const persist = require('./persistence-port').persist;
|
|
2186
|
+
// O-10: per-book layer config arrives ONLY through the LayerConfigPort
|
|
2187
|
+
// (host adapter: storage/layer-config-adapter) — the layer-config host
|
|
2188
|
+
// service (../services/layer-config) left this function.
|
|
2189
|
+
const layerConfigOp = require('./layer-config-port').layerConfigOp;
|
|
2190
|
+
// O-4: placeholder audio arrives ONLY through the PlaceholderAudioPort
|
|
2191
|
+
// (host adapter: storage/placeholder-audio-adapter) — the ffmpeg/fs
|
|
2192
|
+
// host service left this function.
|
|
2193
|
+
const placeholderAudioOp = require('./placeholder-audio-port').placeholderAudioOp;
|
|
2194
|
+
const sceneWindow = require('./scene-window');
|
|
2195
|
+
const runtimeScheduler = require('./runtime-scheduler');
|
|
2196
|
+
// S-5: asset-state writers via the injected orchestration seams
|
|
2197
|
+
const markDirtyScene = orchestrationSeams.getOrchestrationSeam('markDirtyScene');
|
|
2198
|
+
const setScenePending = orchestrationSeams.getOrchestrationSeam('setScenePending');
|
|
2199
|
+
|
|
2200
|
+
const AssetState = state.AssetState;
|
|
2201
|
+
const STAGES = mediaRegistry.listMediaTypes();
|
|
2202
|
+
|
|
2203
|
+
// 0. Fail-closed: книги с cancellation tombstone (Operation #1)
|
|
2204
|
+
const cancelled = new Set();
|
|
2205
|
+
const cancelRows = await persist('cancel.getAllCancelled')();
|
|
2206
|
+
for (const r of cancelRows) cancelled.add(r.book_id);
|
|
2207
|
+
|
|
2208
|
+
// 1. Книги с историей генерации — из PG (у книги без PG-строк нет работы)
|
|
2209
|
+
// O-2: the raw barrel query moved verbatim to the host adapter
|
|
2210
|
+
// (storage/runtime-persistence-adapter.listBooks).
|
|
2211
|
+
const books = (await persist('listBooks')()).map(r => r.book_id);
|
|
2212
|
+
|
|
2213
|
+
let scenesAdded = 0;
|
|
2214
|
+
for (const bookId of books) {
|
|
2215
|
+
if (cancelled.has(bookId)) {
|
|
2216
|
+
log(`[WORKLIST] ${bookId}: skipped (cancellation tombstone)`);
|
|
2217
|
+
continue;
|
|
2218
|
+
}
|
|
2219
|
+
|
|
2220
|
+
try {
|
|
2221
|
+
const loadedBook = sceneData('loadBook')(bookId);
|
|
2222
|
+
if (!loadedBook) {
|
|
2223
|
+
log(`[WORKLIST] ${bookId}: no book.json on disk — skipped`);
|
|
2224
|
+
continue;
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
const buildId = loadedBook?.manifest?.build_id || 'default';
|
|
2228
|
+
const buildDir = syncPath.join(config.OUTPUT_DIR, buildId);
|
|
2229
|
+
const layerCfg = await layerConfigOp('get')(redis, bookId);
|
|
2230
|
+
const enabled = Object.fromEntries(
|
|
2231
|
+
mediaRegistry.listMediaTypes().map(t => [t, layerCfg[t + '_enabled'] !== false])
|
|
2232
|
+
);
|
|
2233
|
+
|
|
2234
|
+
// 2. Dirty-маркеры из PG (audit gap 2): каждый marker РЕАЛЬНО влияет
|
|
2235
|
+
// на WORK_TO_DO. Раньше dirtyScenes собирался, но не использовался —
|
|
2236
|
+
// dirty_unit_ids и per-asset статусы (stale/failed/pending) теряли
|
|
2237
|
+
// regeneration intent. Теперь:
|
|
2238
|
+
// - staleAssets: ready-строка с version mismatch → PENDING (как раньше)
|
|
2239
|
+
// - dirtyStages: scene_assets.status IN (stale,failed,pending) → PENDING
|
|
2240
|
+
// даже при валидном файле на диске (leftover от прошлого билда)
|
|
2241
|
+
// - unitDirtyScenes: dirty_unit_ids → image PENDING (force-regen юнитов)
|
|
2242
|
+
const staleAssets = {}; // "ch:sc:asset" → ready, но version mismatch
|
|
2243
|
+
const dirtyStages = {}; // "ch:sc:asset" → status IN (stale,failed,pending)
|
|
2244
|
+
const unitDirtyScenes = new Set(); // "ch:sc" → dirty_unit_ids (image force-regen)
|
|
2245
|
+
const dirtyRows = await persist('sceneAssets.getDirtyScenesByVersion')(bookId);
|
|
2246
|
+
for (const row of dirtyRows) {
|
|
2247
|
+
const contentStale = row.scene_content_version != null && row.content_version != null &&
|
|
2248
|
+
row.scene_content_version < row.content_version;
|
|
2249
|
+
const audioCfgStale = row.scene_audio_config_version != null && row.audio_config_version != null &&
|
|
2250
|
+
row.scene_audio_config_version < row.audio_config_version;
|
|
2251
|
+
if (row.status === 'ready' &&
|
|
2252
|
+
(contentStale || (row.asset_type === 'audio' && audioCfgStale))) {
|
|
2253
|
+
staleAssets[`${row.chapter_id}:${row.scene_id}:${row.asset_type}`] = true;
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
// 2b. dirty_unit_ids (granular force-regen) → image; per-asset статусы
|
|
2258
|
+
// (stale/failed/pending) → PENDING. asset_type обязателен, чтобы
|
|
2259
|
+
// знать, КАКАЯ стадия грязная (раньше терялся).
|
|
2260
|
+
// O-2: dirty-marker reads via the PersistencePort. Pre-O-2 these
|
|
2261
|
+
// were two raw barrel queries — the SQL moved verbatim into the
|
|
2262
|
+
// host adapter op `sceneAssets.getDirtyMarkers(bookId)` which
|
|
2263
|
+
// returns { unitScenes, assetStages } in the same row shapes.
|
|
2264
|
+
const markers = await persist('sceneAssets.getDirtyMarkers')(bookId);
|
|
2265
|
+
for (const row of markers.unitScenes) unitDirtyScenes.add(`${row.chapter_id}:${row.scene_id}`);
|
|
2266
|
+
for (const row of markers.assetStages) dirtyStages[`${row.chapter_id}:${row.scene_id}:${row.asset_type}`] = true;
|
|
2267
|
+
|
|
2268
|
+
// 3. Пройти все сцены книги: FS-проба + предикат §4 (Recon #3)
|
|
2269
|
+
const scenes = sceneData('collectScenes')(loadedBook);
|
|
2270
|
+
for (const s of scenes) {
|
|
2271
|
+
const chapterId = s.chapter_id;
|
|
2272
|
+
const sceneId = s.scene_id;
|
|
2273
|
+
const key = `${chapterId}:${sceneId}`;
|
|
2274
|
+
|
|
2275
|
+
const fsStatus = await sceneWindow.getSceneFilesStatus(buildDir, bookId, chapterId, sceneId);
|
|
2276
|
+
let audioReal = false;
|
|
2277
|
+
if (fsStatus.audio.exists) {
|
|
2278
|
+
try { audioReal = await placeholderAudioOp('hasRealAudio')(bookId, chapterId, sceneId, buildId); }
|
|
2279
|
+
catch (err) { warn(`Rebuild: hasRealAudio probe failed for ${bookId}/${chapterId}/${sceneId}: ${err.message}`); }
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
const stateMap = {};
|
|
2283
|
+
let needsWork = false;
|
|
2284
|
+
for (const stage of STAGES) {
|
|
2285
|
+
if (!enabled[stage]) continue; // disabled: PENDING не пишем (isInFlight-override в shouldScheduleAssets)
|
|
2286
|
+
const fileValid = fsStatus[stage].exists && (stage !== 'audio' || audioReal);
|
|
2287
|
+
const stale = !!staleAssets[`${key}:${stage}`];
|
|
2288
|
+
const dirty = !!dirtyStages[`${key}:${stage}`];
|
|
2289
|
+
const unitDirty = stage === 'image' && unitDirtyScenes.has(key);
|
|
2290
|
+
if (fileValid && !stale && !dirty && !unitDirty) {
|
|
2291
|
+
stateMap[stage] = AssetState.READY;
|
|
2292
|
+
} else {
|
|
2293
|
+
stateMap[stage] = AssetState.PENDING;
|
|
2294
|
+
needsWork = true;
|
|
2295
|
+
}
|
|
2296
|
+
}
|
|
2297
|
+
|
|
2298
|
+
if (!needsWork) continue;
|
|
2299
|
+
|
|
2300
|
+
// 4. Материализовать states ДО SADD — через SAFE facade (audit gap 1).
|
|
2301
|
+
// Нельзя downgrade-ить выполняющуюся генерацию:
|
|
2302
|
+
// GENERATING/PENDING/PLACEHOLDER → НЕ трогаем (in-flight/очередь
|
|
2303
|
+
// владеют стадией; recovery не имеет права их откатывать).
|
|
2304
|
+
// PENDING-таргеты → orchestrator.setScenePending (валидация переходов:
|
|
2305
|
+
// GENERATING→PENDING rejected самой таблицей переходов).
|
|
2306
|
+
// READY-таргеты → guarded restore-fact write (S2.4): только из
|
|
2307
|
+
// NEW/DIRTY/FAILED/READY, никогда поверх GENERATING/PENDING.
|
|
2308
|
+
// C0 over-mark (READY без файла) → READY→DIRTY (markDirtyScene) →
|
|
2309
|
+
// →PENDING (setScenePending) — self-heal P1 через валидные переходы.
|
|
2310
|
+
const currentStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
2311
|
+
for (const [stage, target] of Object.entries(stateMap)) {
|
|
2312
|
+
const current = currentStates[stage] || state.AssetState.NEW;
|
|
2313
|
+
// GENERATING/PENDING — выполняющаяся/очередная генерация:
|
|
2314
|
+
// recovery не имеет права downgrade-ить её (audit gap 1).
|
|
2315
|
+
if (current === state.AssetState.GENERATING ||
|
|
2316
|
+
current === state.AssetState.PENDING) {
|
|
2317
|
+
log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: ${current} — untouched by recovery (in-flight/queued)`);
|
|
2318
|
+
continue;
|
|
2319
|
+
}
|
|
2320
|
+
if (target === state.AssetState.PENDING) {
|
|
2321
|
+
if (current === state.AssetState.READY || current === state.AssetState.PLACEHOLDER) {
|
|
2322
|
+
// C0 over-mark (P1, READY без файла) / placeholder-аудио:
|
|
2323
|
+
// прямой READY→PENDING / PLACEHOLDER→PENDING невалиден;
|
|
2324
|
+
// READY→DIRTY / PLACEHOLDER→DIRTY валиден → затем DIRTY→PENDING.
|
|
2325
|
+
// TOCTOU-guard (audit follow-up): scheduler tick работает
|
|
2326
|
+
// конкурентно (runtime-loop стартует ДО startup-цикла),
|
|
2327
|
+
// markVersionStaleDirty → dispatch может успеть выставить
|
|
2328
|
+
// GENERATING между нашим guard-read и markDirtyScene (который
|
|
2329
|
+
// пишет DIRTY через unsafe без re-validation). Re-read прямо
|
|
2330
|
+
// перед записью: если состояние изменилось — стадией уже
|
|
2331
|
+
// владеет кто-то другой, recovery отступает. setScenePending
|
|
2332
|
+
// ниже сам re-reads + validates (вторая линия обороны).
|
|
2333
|
+
const fresh = (await state.getAssetStates(redis, bookId, chapterId, sceneId))[stage] || state.AssetState.NEW;
|
|
2334
|
+
if (fresh !== current) {
|
|
2335
|
+
log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: state changed ${current}→${fresh} during rebuild — untouched (owner present)`);
|
|
2336
|
+
continue;
|
|
2337
|
+
}
|
|
2338
|
+
log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: ${current} → DIRTY → PENDING (valid transitions)`);
|
|
2339
|
+
await markDirtyScene(redis, bookId, chapterId, sceneId, [stage]);
|
|
2340
|
+
await setScenePending(redis, bookId, chapterId, sceneId, stage);
|
|
2341
|
+
} else {
|
|
2342
|
+
// NEW/DIRTY/FAILED → PENDING — валидные переходы.
|
|
2343
|
+
await setScenePending(redis, bookId, chapterId, sceneId, stage);
|
|
2344
|
+
}
|
|
2345
|
+
} else {
|
|
2346
|
+
// target === READY — restore-fact (файл валиден на диске):
|
|
2347
|
+
// guarded unsafe write, S2.4 (restore path). Пропускаются
|
|
2348
|
+
// GENERATING/PENDING (см. guard выше); из остальных состояний
|
|
2349
|
+
// (NEW/DIRTY/FAILED/PLACEHOLDER/READY) восстановить READY
|
|
2350
|
+
// безопасно — файл доказан FS-пробой.
|
|
2351
|
+
if (current !== state.AssetState.READY) {
|
|
2352
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, stage, state.AssetState.READY);
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
await runtimeScheduler.addSceneToActiveIndex(redis, bookId, chapterId, sceneId);
|
|
2357
|
+
scenesAdded++;
|
|
2358
|
+
}
|
|
2359
|
+
} catch (err) {
|
|
2360
|
+
warn(`Rebuild failed for book ${bookId}: ${err.message}`);
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2363
|
+
|
|
2364
|
+
if (scenesAdded > 0) log(`[WORKLIST] ${scenesAdded} scene(s) re-added for generation resume`);
|
|
2365
|
+
return scenesAdded;
|
|
2366
|
+
}
|
|
2367
|
+
|
|
2368
|
+
// ======================================================
|
|
2369
|
+
// EXPORTS
|
|
2370
|
+
// ======================================================
|
|
2371
|
+
|
|
2372
|
+
module.exports = {
|
|
2373
|
+
reconcileScene,
|
|
2374
|
+
reconcileBook,
|
|
2375
|
+
reconcileAll,
|
|
2376
|
+
reconcileCycle,
|
|
2377
|
+
recoverResultKeys,
|
|
2378
|
+
recoverAudioOrchStates,
|
|
2379
|
+
recoverVideoOrchStates,
|
|
2380
|
+
checkVersionStaleness,
|
|
2381
|
+
reconcileMissingSceneState,
|
|
2382
|
+
rebuildWorkList,
|
|
2383
|
+
checkStalledAudioScenes,
|
|
2384
|
+
checkStalledVideoScenes,
|
|
2385
|
+
|
|
2386
|
+
checkAudioOrchInvariants,
|
|
2387
|
+
checkVideoOrchInvariants,
|
|
2388
|
+
|
|
2389
|
+
checkOrphanVideoState,
|
|
2390
|
+
checkOrphanImageState,
|
|
2391
|
+
checkOrphanAudioState,
|
|
2392
|
+
checkOrphanAssets,
|
|
2393
|
+
checkOrphanGeneratingState,
|
|
2394
|
+
checkPartialBuilds,
|
|
2395
|
+
checkStaleLocks,
|
|
2396
|
+
checkStaleDispatchLeases,
|
|
2397
|
+
|
|
2398
|
+
isBookGenerationCancelled,
|
|
2399
|
+
resolveBookBuildId,
|
|
2400
|
+
|
|
2401
|
+
getFixRecommendations,
|
|
2402
|
+
applyFix,
|
|
2403
|
+
getMetrics,
|
|
2404
|
+
ReconciliationReport
|
|
2405
|
+
};
|