@animastor/orchestration 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +75 -0
- package/package.json +51 -0
- package/src/host/host-bindings.js +138 -0
- package/src/index.js +99 -0
- package/src/orchestration/index.js +40 -0
- package/src/orchestration/orchestrator.js +630 -0
- package/src/orchestration/runtime-result-consumer.js +54 -0
- package/src/orchestration/scene-callbacks.js +542 -0
- package/src/orchestration/scene-orchestrator.js +601 -0
- package/src/orchestration/scene-restoration.js +129 -0
- package/src/orchestration/scene-utils.js +36 -0
- package/src/runtime/active-scenes-index.js +164 -0
- package/src/runtime/audio-fsm-port.js +187 -0
- package/src/runtime/circuit-breaker.js +588 -0
- package/src/runtime/counter-reconciliation.js +308 -0
- package/src/runtime/dispatch-engine.js +1768 -0
- package/src/runtime/failure-taxonomy.js +125 -0
- package/src/runtime/hub-cancel-port.js +122 -0
- package/src/runtime/layer-config-port.js +136 -0
- package/src/runtime/lease-manager.js +607 -0
- package/src/runtime/persistence-port.js +162 -0
- package/src/runtime/placeholder-audio-port.js +117 -0
- package/src/runtime/progress-events-port.js +121 -0
- package/src/runtime/reconciliation-engine.js +2405 -0
- package/src/runtime/retry-budget-manager.js +229 -0
- package/src/runtime/runtime-metrics.js +427 -0
- package/src/runtime/runtime-result-emitter.js +123 -0
- package/src/runtime/runtime-scheduler.js +629 -0
- package/src/runtime/scene-data-port.js +114 -0
- package/src/runtime/scene-window.js +827 -0
- package/src/runtime/video-fsm-port.js +189 -0
- package/src/runtime/worker-health.js +270 -0
- package/test/counter-reconciliation.test.js +164 -0
- package/test/mocks/redis-mock.js +221 -0
- package/test/test-bindings.js +60 -0
|
@@ -0,0 +1,630 @@
|
|
|
1
|
+
// ======================================================
|
|
2
|
+
// ORCHESTRATOR — единый владелец жизненного цикла генерации
|
|
3
|
+
// ======================================================
|
|
4
|
+
// Шаг 0 (см. docs-claude/03_Orchestrator.md §12): тонкий фасад поверх уже
|
|
5
|
+
// существующих функций. На этом шаге команды НИЧЕГО не меняют в логике —
|
|
6
|
+
// только дают единую точку входа, через которую постепенно (по одному
|
|
7
|
+
// вызывающему за коммит) будут проходить все писатели lifecycle-состояния.
|
|
8
|
+
//
|
|
9
|
+
// Контракт пяти команд (цели последующих шагов в скобках):
|
|
10
|
+
// markDirty(deps, redis, bookId, buildId, dirtyScenes, layerCfg) → { marked }
|
|
11
|
+
// planScene(redis, bookId, chapterId, sceneId) → { stages, allDone } (Д.2: сделать чистой)
|
|
12
|
+
// beginStage(redis, scene, loadedBook, buildId, stage) → dispatchResult (Д.1: GENERATING уже есть)
|
|
13
|
+
// completeStage(redis, bookId, chapterId, sceneId, stage, buildId) → void (Д.1: idempotent по token)
|
|
14
|
+
// reconcile(redis, bookId, chapterId, sceneId) → reconcileResult (Д.3: диск как факт)
|
|
15
|
+
//
|
|
16
|
+
// Зависимости подтягиваются lazy-require внутри тел команд — это сознательный
|
|
17
|
+
// компромисс Шага 0, чтобы не углублять существующий цикл orchestration и
|
|
18
|
+
// runtime (диспетчер уже делал отложенный импорт orchestration-фасада).
|
|
19
|
+
// Развязка интерфейсом — отдельная задача (после К.4).
|
|
20
|
+
|
|
21
|
+
// S-2: media-type knowledge (stage lists, fail-event map) resolved via registry
|
|
22
|
+
const mediaRegistry = require('@animastor/generation').mediaRegistry;
|
|
23
|
+
// S-5: pure FSM-writer ownership (markDirtyScene/setScene*) — re-exported from
|
|
24
|
+
// the state layer; bodies moved verbatim (see state/scene-state-ops.js header).
|
|
25
|
+
const { lazyHostBinding } = require('../host/host-bindings');
|
|
26
|
+
// §32.30: former host requires resolve lazily through the host-bindings seam.
|
|
27
|
+
const stateOps = lazyHostBinding('stateOps');
|
|
28
|
+
const state = lazyHostBinding('state');
|
|
29
|
+
const journal = lazyHostBinding('journal');
|
|
30
|
+
const runtimeConfig = lazyHostBinding('config');
|
|
31
|
+
|
|
32
|
+
// ── markDirty ─────────────────────────────────────────
|
|
33
|
+
// Единственный способ объявить «нужна регенерация». Делегирует в
|
|
34
|
+
// deps.bookDiff.markDirtyScenes — это метод DI-инстанса (book-diff.cjs —
|
|
35
|
+
// фабрика), поэтому инстанс приходит через deps, а не через require.
|
|
36
|
+
async function markDirty(deps, redis, bookId, buildId, dirtyScenes, layerCfg) {
|
|
37
|
+
if (!deps || !deps.bookDiff || typeof deps.bookDiff.markDirtyScenes !== 'function') {
|
|
38
|
+
throw new Error('orchestrator.markDirty: deps.bookDiff.markDirtyScenes is required');
|
|
39
|
+
}
|
|
40
|
+
return deps.bookDiff.markDirtyScenes(redis, bookId, buildId, dirtyScenes, layerCfg);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// ── planScene ─────────────────────────────────────────
|
|
44
|
+
// Решение «что генерировать». Делегирует в shouldScheduleAssets — теперь это
|
|
45
|
+
// ЧИСТАЯ функция (Д.2): только читает per-asset состояния и layer-config, ничего
|
|
46
|
+
// не пишет. Version-stale reset вынесен в явный пред-проход attemptDispatch
|
|
47
|
+
// (detectVersionStale → markVersionStaleDirty).
|
|
48
|
+
async function planScene(redis, bookId, chapterId, sceneId) {
|
|
49
|
+
const scheduler = require('../runtime/runtime-scheduler');
|
|
50
|
+
return scheduler.shouldScheduleAssets(redis, bookId, chapterId, sceneId);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// ── beginStage ────────────────────────────────────────
|
|
54
|
+
// Старт стадии: lease + quota + per-asset GENERATING. Делегирует в
|
|
55
|
+
// dispatch-engine.dispatchStage (который сам зовёт назад orchestration.dispatchStage
|
|
56
|
+
// для исполнения генерации).
|
|
57
|
+
async function beginStage(redis, scene, loadedBook, buildId, stage) {
|
|
58
|
+
const dispatchEngine = require('../runtime/dispatch-engine');
|
|
59
|
+
// S-4: filename grammar composed from the canonical owner (bytes unchanged)
|
|
60
|
+
const artifactNaming = require('@animastor/generation').artifactNaming;
|
|
61
|
+
const bookId = scene.book_id;
|
|
62
|
+
const chapterId = scene.chapter_id;
|
|
63
|
+
const sceneId = scene.scene_id;
|
|
64
|
+
// T8: syncLinearState удалён — per-asset state единственный source of truth
|
|
65
|
+
const result = await dispatchEngine.dispatchStage(redis, bookId, chapterId, sceneId, stage, loadedBook, buildId);
|
|
66
|
+
return result;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ── completeStage ─────────────────────────────────────
|
|
70
|
+
// Завершение стадии: колбэк завершения + release lease/quota (markDispatchCompleted),
|
|
71
|
+
// в той же error-safe связке (try/finally), что сейчас повторена в task-handler.cjs
|
|
72
|
+
// в шести местах. Фасад даёт ОДНУ реализацию этой пары.
|
|
73
|
+
// ЦЕЛЬ Д.1: идемпотентность по dispatch-token (повторный вызов безвреден).
|
|
74
|
+
//
|
|
75
|
+
// ======================================================
|
|
76
|
+
// T1: Callback completion contract
|
|
77
|
+
// --------------------------------------
|
|
78
|
+
// 1. Handler вызывается, возвращает { ok, retryable, reason, artifact }
|
|
79
|
+
// 2. READY/DIRTY пишется ТОЛЬКО после handler.ok === true
|
|
80
|
+
// 3. PG markReady вызывается ПОСЛЕ версионного gate (fail-closed)
|
|
81
|
+
// 4. Исключение handler → failure, не success
|
|
82
|
+
// 5. ok:false → НЕ ставит READY, не пишет PG
|
|
83
|
+
// ======================================================
|
|
84
|
+
async function completeStage(redis, bookId, chapterId, sceneId, stage, buildId, dispatchId) {
|
|
85
|
+
const callbacks = require('./scene-callbacks');
|
|
86
|
+
const dispatchEngine = require('../runtime/dispatch-engine');
|
|
87
|
+
const { log, warn, error } = require('./scene-utils');
|
|
88
|
+
// O-2: PG persistence via the PersistencePort (host adapter:
|
|
89
|
+
// storage/runtime-persistence-adapter) — the scene-assets repository and
|
|
90
|
+
// the database handle left this file. SQL lives host-side.
|
|
91
|
+
const persist = require('../runtime/persistence-port').persist;
|
|
92
|
+
|
|
93
|
+
// S-2: resolve the completion handler through the registry-driven map.
|
|
94
|
+
// Fallback to the direct named handlers keeps backward compatibility with
|
|
95
|
+
// partial stubs of scene-callbacks (older test contracts inject only the
|
|
96
|
+
// three named handlers) — in production getStageHandler is always present.
|
|
97
|
+
const handler = (callbacks.getStageHandler && callbacks.getStageHandler(stage))
|
|
98
|
+
|| (callbacks.STAGE_HANDLERS && callbacks.STAGE_HANDLERS[stage])
|
|
99
|
+
|| (stage === 'audio' && callbacks.handleAudioCompleted)
|
|
100
|
+
|| (stage === 'image' && callbacks.handleImageCompleted)
|
|
101
|
+
|| (stage === 'video' && callbacks.handleVideoCompleted);
|
|
102
|
+
if (!handler) {
|
|
103
|
+
throw new Error(`orchestrator.completeStage: unknown stage '${stage}'`);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const identity = await dispatchEngine.verifyDispatchIdentity(
|
|
107
|
+
redis,
|
|
108
|
+
bookId,
|
|
109
|
+
chapterId,
|
|
110
|
+
sceneId,
|
|
111
|
+
stage,
|
|
112
|
+
dispatchId
|
|
113
|
+
);
|
|
114
|
+
if (!identity.valid) {
|
|
115
|
+
warn(`[COMPLETE-STAGE] Rejected ${bookId}/${chapterId}/${sceneId} ${stage}: ${identity.reason}`);
|
|
116
|
+
return { completed: false, reason: identity.reason };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
let handlerResult;
|
|
120
|
+
let handlerError = null;
|
|
121
|
+
let handlerOk = false; // T2: true если handler вернул ok:true
|
|
122
|
+
|
|
123
|
+
let phaseCompleted = false;
|
|
124
|
+
let phaseReason = 'handler_rejected';
|
|
125
|
+
|
|
126
|
+
try {
|
|
127
|
+
try {
|
|
128
|
+
handlerResult = await handler(redis, bookId, chapterId, sceneId, buildId);
|
|
129
|
+
} catch (err) {
|
|
130
|
+
// T1.7: Исключение handler → failure path, не success
|
|
131
|
+
handlerError = err;
|
|
132
|
+
error(`completeStage: handler threw for ${bookId}/${chapterId}/${sceneId} ${stage}: ${err.message}`);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// ── Check handler result (T1.6) ────────────────────
|
|
136
|
+
if (handlerError || !handlerResult || handlerResult.ok !== true) {
|
|
137
|
+
const reason = handlerResult?.reason || (handlerError ? handlerError.message : 'unknown');
|
|
138
|
+
phaseReason = reason;
|
|
139
|
+
handlerOk = false;
|
|
140
|
+
log(`[COMPLETE-STAGE] ${bookId}/${chapterId}/${sceneId} ${stage}: rejected (reason=${reason}) — NOT setting READY`);
|
|
141
|
+
} else {
|
|
142
|
+
handlerOk = true;
|
|
143
|
+
|
|
144
|
+
// ── Handler succeeded — version gate (T1.11: fail-closed) ────
|
|
145
|
+
// VERSION-GATE: сцена без строки в `scenes` (legacy-книга, импорт
|
|
146
|
+
// без version-схемы) НЕ блокируется fail-closed — иначе video/audio
|
|
147
|
+
// никогда не получат READY (строка scene_assets создаётся только
|
|
148
|
+
// markReady, который вызывается ПОСЛЕ gate — замкнутый круг).
|
|
149
|
+
// Для таких сцен gate считается пройденным; строки версий
|
|
150
|
+
// создаются лениво (ensureSceneRow) при следующем bump.
|
|
151
|
+
let shouldWriteReady = false;
|
|
152
|
+
try {
|
|
153
|
+
// O-2: version gate reads via the PersistencePort — the raw
|
|
154
|
+
// database handle and the repo left this file; the SQL moved
|
|
155
|
+
// verbatim into the host adapter (sceneVersions.getSceneVersions).
|
|
156
|
+
const sceneRows = await persist('sceneVersions.getSceneVersions')(bookId, chapterId, sceneId);
|
|
157
|
+
|
|
158
|
+
if (sceneRows.length === 1) {
|
|
159
|
+
const sv = sceneRows[0];
|
|
160
|
+
const asset = await persist('sceneAssets.getAsset')(bookId, chapterId, sceneId, stage, buildId);
|
|
161
|
+
|
|
162
|
+
if (asset) {
|
|
163
|
+
const contentVersionConfirmed =
|
|
164
|
+
asset.scene_content_version != null &&
|
|
165
|
+
sv.content_version != null &&
|
|
166
|
+
asset.scene_content_version >= sv.content_version;
|
|
167
|
+
const audioVersionConfirmed =
|
|
168
|
+
stage !== 'audio' || (
|
|
169
|
+
asset.scene_audio_config_version != null &&
|
|
170
|
+
sv.audio_config_version != null &&
|
|
171
|
+
asset.scene_audio_config_version >= sv.audio_config_version
|
|
172
|
+
);
|
|
173
|
+
shouldWriteReady = contentVersionConfirmed && audioVersionConfirmed;
|
|
174
|
+
}
|
|
175
|
+
} else if (sceneRows.length === 0) {
|
|
176
|
+
// Legacy-книга без version-схемы — READY разрешён (сцена
|
|
177
|
+
// не может устареть относительно несуществующей версии).
|
|
178
|
+
shouldWriteReady = true;
|
|
179
|
+
}
|
|
180
|
+
} catch (pgErr) {
|
|
181
|
+
// T1.11: Fail-closed — ошибка PG не разрешает READY
|
|
182
|
+
shouldWriteReady = false;
|
|
183
|
+
warn(`[VERSION-GATE] PG query failed for ${bookId}/${chapterId}/${sceneId}: ${pgErr.message} — blocking READY (fail-closed)`);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
if (shouldWriteReady) {
|
|
187
|
+
// T1.10: PG markReady ПОСЛЕ version gate
|
|
188
|
+
const artifactPath = handlerResult.artifact?.path;
|
|
189
|
+
if (!artifactPath) {
|
|
190
|
+
handlerOk = false;
|
|
191
|
+
phaseReason = 'artifact_path_missing';
|
|
192
|
+
} else {
|
|
193
|
+
await persist('sceneAssets.markReady')(bookId, chapterId, sceneId, stage, artifactPath);
|
|
194
|
+
log(`[PG-${stage.toUpperCase()}-READY] ${bookId}/${chapterId}/${sceneId}: status=ready`);
|
|
195
|
+
|
|
196
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, stage, state.AssetState.READY);
|
|
197
|
+
log(`[COMPLETE-STAGE] ${bookId}/${chapterId}/${sceneId}: ${stage} → READY`);
|
|
198
|
+
phaseCompleted = true;
|
|
199
|
+
phaseReason = null;
|
|
200
|
+
}
|
|
201
|
+
} else {
|
|
202
|
+
// T2: version gate stale — dispatch успешен, но artifact устарел.
|
|
203
|
+
// Финализируем как success (диспетчеризация выполнена), просто не ставим READY.
|
|
204
|
+
log(`[VERSION-GATE] ${bookId}/${chapterId}/${sceneId}: ${stage} stale — DIRTY instead of READY`);
|
|
205
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, stage, state.AssetState.DIRTY);
|
|
206
|
+
phaseCompleted = false;
|
|
207
|
+
phaseReason = 'stale';
|
|
208
|
+
// handlerOk остаётся true — финализируем как success
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
} catch (completionErr) {
|
|
212
|
+
handlerOk = false;
|
|
213
|
+
phaseCompleted = false;
|
|
214
|
+
phaseReason = `completion_error:${completionErr.message}`;
|
|
215
|
+
error(`completeStage failed for ${bookId}/${chapterId}/${sceneId} ${stage}: ${completionErr.message}`);
|
|
216
|
+
} finally {
|
|
217
|
+
// T2: finalizeDispatch — handlerOk определяет success/failure
|
|
218
|
+
// Если handler вернул ok:true (handlerOk = true) — всегда success,
|
|
219
|
+
// даже при version gate stale. Failure только при ok:false/error handler.
|
|
220
|
+
//
|
|
221
|
+
// Fix (video-retry-targeted-investigation-2026-08-26): pass the
|
|
222
|
+
// taxonomy-classified failureType instead of the implicit 'transient'
|
|
223
|
+
// default — a rejected completion (e.g. 'invalid_asset_state', which is
|
|
224
|
+
// PERMANENT by failure-taxonomy) must not consume the transient retry
|
|
225
|
+
// budget and must not be retried like a transient GPU failure.
|
|
226
|
+
try {
|
|
227
|
+
const outcome = handlerOk ? 'success' : 'failure';
|
|
228
|
+
const failureTaxonomy = require('../runtime/failure-taxonomy');
|
|
229
|
+
await dispatchEngine.finalizeDispatch(redis, bookId, chapterId, sceneId, stage, {
|
|
230
|
+
outcome,
|
|
231
|
+
dispatchId,
|
|
232
|
+
reason: handlerOk ? (phaseReason || undefined) : (phaseReason || 'handler_rejected'),
|
|
233
|
+
failureType: handlerOk ? undefined : failureTaxonomy.classifyFailure(new Error(phaseReason || 'handler_rejected')).type,
|
|
234
|
+
});
|
|
235
|
+
} catch (dispErr) {
|
|
236
|
+
warn(`completeStage: finalizeDispatch(${stage}) failed: ${dispErr.message}`);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
return { completed: phaseCompleted, reason: phaseReason };
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// ── failStage ─────────────────────────────────────────
|
|
244
|
+
// T3 консолидации: единственный способ зафиксировать «генерация упала».
|
|
245
|
+
// Вызывается endpoint'ом /gpu/task/error (ошибка воркера, форвард через
|
|
246
|
+
// gpu-hub, включая worker_timeout) и внутренними обработчиками сбоев.
|
|
247
|
+
// Делает: asset-state → FAILED (с валидацией перехода — поздняя ошибка
|
|
248
|
+
// после успешного retry не сбивает READY), событие *_FAILED в journal,
|
|
249
|
+
// release lease+quota через идемпотентный markDispatchCompleted.
|
|
250
|
+
// После FAILED (если redispatch=true, по умолчанию) ассет переводится в
|
|
251
|
+
// PENDING — планировщик передиспатчит на следующем тике под защитой
|
|
252
|
+
// circuit-breaker и retry-budget (dispatch-engine). Это зеркалит проверенный
|
|
253
|
+
// путь исчерпания merge-ретраев в task-handler; сам failStage retry-политику
|
|
254
|
+
// не содержит.
|
|
255
|
+
async function failStage(redis, bookId, chapterId, sceneId, stage, buildId, reason = 'unknown', { redispatch = true, dispatchId } = {}) {
|
|
256
|
+
const dispatchEngine = require('../runtime/dispatch-engine');
|
|
257
|
+
const { log, warn } = require('./scene-utils');
|
|
258
|
+
|
|
259
|
+
// S-2: fail event type map — derived from journal event types.
|
|
260
|
+
// Maps media type → failure event type for the event journal.
|
|
261
|
+
// S-2: fail event types derived from journal event types
|
|
262
|
+
const FAIL_EVENT_TYPES = {};
|
|
263
|
+
for (const t of mediaRegistry.listMediaTypes()) {
|
|
264
|
+
const key = `${t.toUpperCase()}_FAILED`;
|
|
265
|
+
if (journal.EventType[key]) FAIL_EVENT_TYPES[t] = journal.EventType[key];
|
|
266
|
+
}
|
|
267
|
+
const eventType = FAIL_EVENT_TYPES[stage];
|
|
268
|
+
if (!eventType) {
|
|
269
|
+
throw new Error(`orchestrator.failStage: unknown stage '${stage}'`);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
const identity = await dispatchEngine.verifyDispatchIdentity(
|
|
273
|
+
redis,
|
|
274
|
+
bookId,
|
|
275
|
+
chapterId,
|
|
276
|
+
sceneId,
|
|
277
|
+
stage,
|
|
278
|
+
dispatchId
|
|
279
|
+
);
|
|
280
|
+
if (!identity.valid) {
|
|
281
|
+
log(`[FAIL-STAGE] Rejected ${bookId}/${chapterId}/${sceneId} ${stage}: ${identity.reason}`);
|
|
282
|
+
return { failed: false, reason: identity.reason };
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
try {
|
|
286
|
+
const states = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
287
|
+
const current = states ? states[stage] : null;
|
|
288
|
+
const check = state.validateAssetTransition(current, state.AssetState.FAILED);
|
|
289
|
+
|
|
290
|
+
if (!check.valid) {
|
|
291
|
+
// Например READY→FAILED: результат уже принят (поздний/повторный
|
|
292
|
+
// сигнал ошибки) — фиксируем в журнале, состояние не трогаем.
|
|
293
|
+
log(`[FAIL-STAGE] ${bookId}/${chapterId}/${sceneId}: ${stage} ${current}→failed отклонён (${check.reason}) — ignoring late error (reason=${reason})`);
|
|
294
|
+
await journal.appendSceneEvent(redis, bookId, chapterId, sceneId,
|
|
295
|
+
journal.EventType.INVALID_STATE_CALLBACK, current,
|
|
296
|
+
{ stage, reason, attempted: 'failed', ignored: true }).catch(() => {});
|
|
297
|
+
return { failed: false, reason: check.reason, current };
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, stage, state.AssetState.FAILED);
|
|
301
|
+
// F2: sync audio-orch/video-orch phase → FAILED through the orchestrator facade
|
|
302
|
+
try {
|
|
303
|
+
// O-7: the audio FSM sync rides the AudioFsmPort (host adapter:
|
|
304
|
+
// storage/audio-fsm-adapter) — the audio-orchestrator host
|
|
305
|
+
// service left this file.
|
|
306
|
+
// O-8: the video FSM sync rides the VideoFsmPort (host adapter:
|
|
307
|
+
// storage/video-fsm-adapter) — the video-orchestrator host
|
|
308
|
+
// service left this file.
|
|
309
|
+
if (stage === 'audio') {
|
|
310
|
+
await require('../runtime/audio-fsm-port').audioFsmOp('setFailed')(redis, bookId, chapterId, sceneId, reason);
|
|
311
|
+
} else if (stage === 'video') {
|
|
312
|
+
await require('../runtime/video-fsm-port').videoFsmOp('setFailed')(redis, bookId, chapterId, sceneId, reason);
|
|
313
|
+
}
|
|
314
|
+
} catch (_) {}
|
|
315
|
+
log(`[FAIL-STAGE] ${bookId}/${chapterId}/${sceneId}: ${stage} → FAILED (reason=${reason})`);
|
|
316
|
+
await journal.appendSceneEvent(redis, bookId, chapterId, sceneId,
|
|
317
|
+
eventType, state.AssetState.FAILED, { stage, reason, buildId }).catch(() => {});
|
|
318
|
+
|
|
319
|
+
// T8: syncLinearState удалён — per-asset state единственный source of truth
|
|
320
|
+
if (redispatch) {
|
|
321
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, stage, state.AssetState.PENDING);
|
|
322
|
+
log(`[FAIL-STAGE] ${bookId}/${chapterId}/${sceneId}: ${stage} → PENDING (re-dispatch queued)`);
|
|
323
|
+
}
|
|
324
|
+
return { failed: true, reason, redispatch };
|
|
325
|
+
} finally {
|
|
326
|
+
// T2: finalizeDispatch('failure') — recordFailure + consumeRetryBudget,
|
|
327
|
+
// НЕ recordSuccess, НЕ DISPATCH_COMPLETED.
|
|
328
|
+
try {
|
|
329
|
+
const failureTaxonomy = require('../runtime/failure-taxonomy');
|
|
330
|
+
const failureClass = failureTaxonomy.classifyFailure(new Error(reason || 'unknown'));
|
|
331
|
+
await dispatchEngine.finalizeDispatch(redis, bookId, chapterId, sceneId, stage, {
|
|
332
|
+
outcome: 'failure',
|
|
333
|
+
dispatchId,
|
|
334
|
+
reason: reason || 'unknown',
|
|
335
|
+
failureType: failureClass.type
|
|
336
|
+
});
|
|
337
|
+
} catch (dispErr) {
|
|
338
|
+
warn(`failStage: finalizeDispatch(${stage}) failed: ${dispErr.message}`);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// ── rollbackStageToPending ───────────────────────────
|
|
344
|
+
// FSM-валидный откат стадии в PENDING после dispatch, который НЕ отправил
|
|
345
|
+
// ни одного GPU job (no_jobs_sent / cancelled / dispatch_error).
|
|
346
|
+
//
|
|
347
|
+
// Прямой переход generating→pending ЗАПРЕЩЁН FSM (scene-state.js), поэтому
|
|
348
|
+
// GENERATING откатывается двухшаговым штатным путём:
|
|
349
|
+
// GENERATING → DIRTY → PENDING
|
|
350
|
+
// (PLACEHOLDER → DIRTY → PENDING — аналогично; NEW/DIRTY/FAILED → PENDING
|
|
351
|
+
// напрямую). READY не трогаем — результат уже принят.
|
|
352
|
+
//
|
|
353
|
+
// Ошибки rollback НЕ проглатываются: error-лог + TRANSITION_FAILED в journal
|
|
354
|
+
// + счётчик stateRollbackFailures. Иначе сцена навсегда остаётся в GENERATING
|
|
355
|
+
// (ghost, forensic audit 6929ba5: no_jobs_sent → setScenePending rejected →
|
|
356
|
+
// rejection проглочен → вечный GENERATING).
|
|
357
|
+
async function rollbackStageToPending(redis, bookId, chapterId, sceneId, asset, buildId = null, reason = 'dispatch_cancelled') {
|
|
358
|
+
const { log, error } = require('./scene-utils');
|
|
359
|
+
|
|
360
|
+
const states = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
361
|
+
const current = states?.[asset];
|
|
362
|
+
|
|
363
|
+
if (current === state.AssetState.PENDING) {
|
|
364
|
+
return { changed: false, alreadyPending: true, reason: 'already_pending' };
|
|
365
|
+
}
|
|
366
|
+
if (current === state.AssetState.READY) {
|
|
367
|
+
// Результат уже принят — откат повредил бы завершённую работу.
|
|
368
|
+
log(`[ROLLBACK] ${bookId}/${chapterId}/${sceneId} ${asset}: already READY — rollback skipped (reason=${reason})`);
|
|
369
|
+
return { changed: false, reason: 'already_ready' };
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// Только FSM-валидные рёбра (AssetTransitions):
|
|
373
|
+
// GENERATING/PLACEHOLDER → DIRTY → PENDING
|
|
374
|
+
// NEW/DIRTY/FAILED → PENDING
|
|
375
|
+
const path = (current === state.AssetState.GENERATING || current === state.AssetState.PLACEHOLDER)
|
|
376
|
+
? [state.AssetState.DIRTY, state.AssetState.PENDING]
|
|
377
|
+
: [state.AssetState.PENDING];
|
|
378
|
+
|
|
379
|
+
try {
|
|
380
|
+
let from = current;
|
|
381
|
+
for (const next of path) {
|
|
382
|
+
const check = state.validateAssetTransition(from, next);
|
|
383
|
+
if (!check.valid) {
|
|
384
|
+
throw new Error(`invalid_transition:${from}->${next}`);
|
|
385
|
+
}
|
|
386
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, asset, next);
|
|
387
|
+
from = next;
|
|
388
|
+
}
|
|
389
|
+
log(`[ROLLBACK] ${bookId}/${chapterId}/${sceneId} ${asset}: ${current} → ${path.join(' → ')} (reason=${reason})`);
|
|
390
|
+
await journal.appendSceneEvent(redis, bookId, chapterId, sceneId,
|
|
391
|
+
journal.EventType.SCENE_PENDING, state.AssetState.PENDING,
|
|
392
|
+
{ asset, buildId, rollback: true, via: path.join('->'), reason }).catch(() => {});
|
|
393
|
+
return { changed: true, from: current, path };
|
|
394
|
+
} catch (err) {
|
|
395
|
+
// НЕ глотаем: без явной диагностики сцена останется в GENERATING навсегда.
|
|
396
|
+
error(`[ROLLBACK] FAILED ${bookId}/${chapterId}/${sceneId} ${asset}: ${current} → pending (reason=${reason}): ${err.message} — stage may be stuck in GENERATING`);
|
|
397
|
+
try {
|
|
398
|
+
await journal.appendSceneEvent(redis, bookId, chapterId, sceneId,
|
|
399
|
+
journal.EventType.TRANSITION_FAILED, current,
|
|
400
|
+
{ asset, attempted: 'rollback_to_pending', via: path.join('->'), reason, error: err.message });
|
|
401
|
+
} catch (_) {}
|
|
402
|
+
try {
|
|
403
|
+
const runtimeMetrics = require('../runtime/runtime-metrics');
|
|
404
|
+
await runtimeMetrics.incrementCounter(redis, 'stateRollbackFailures');
|
|
405
|
+
} catch (_) {}
|
|
406
|
+
return { changed: false, reason: `rollback_failed:${err.message}` };
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
// ── completeStageWithoutVideo ────────────────────────
|
|
411
|
+
// When video is disabled by layer config, mark video as READY
|
|
412
|
+
// and complete remaining scene lifecycle (cleanup, slide).
|
|
413
|
+
// Callback function handles cleanup only — state is set here in the facade.
|
|
414
|
+
async function completeStageWithoutVideo(redis, loadedBook, bookId, chapterId, sceneId, buildId) {
|
|
415
|
+
const callbacks = require('./scene-callbacks');
|
|
416
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'video', state.AssetState.READY);
|
|
417
|
+
await callbacks.completeSceneWithoutVideo(redis, loadedBook, bookId, chapterId, sceneId, buildId);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// ── completeStageWithoutImage ─────────────────────────
|
|
421
|
+
// When image is disabled by layer config, mark image as READY
|
|
422
|
+
// and complete remaining scene lifecycle.
|
|
423
|
+
async function completeStageWithoutImage(redis, loadedBook, bookId, chapterId, sceneId, buildId) {
|
|
424
|
+
const callbacks = require('./scene-callbacks');
|
|
425
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, 'image', state.AssetState.READY);
|
|
426
|
+
await callbacks.completeSceneWithoutImage(redis, loadedBook, bookId, chapterId, sceneId, buildId);
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// ── reconcile ─────────────────────────────────────────
|
|
430
|
+
// Сверка фактов (PG/диск) с состоянием. Делегирует в reconciliation-engine.
|
|
431
|
+
// ЦЕЛЬ Д.3: диск становится источником ФАКТА, решение принимает фасад.
|
|
432
|
+
async function reconcile(redis, bookId, chapterId, sceneId) {
|
|
433
|
+
const reconciliationEngine = require('../runtime/reconciliation-engine');
|
|
434
|
+
return reconciliationEngine.reconcileScene(redis, bookId, chapterId, sceneId);
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// ── resetScenes ───────────────────────────────────────
|
|
438
|
+
// T4 консолидации: единственная команда для reset'а сцен при регенерации.
|
|
439
|
+
// Собирает ритуал из /regenerate роута (очистка iu-progress/iu-in-flight,
|
|
440
|
+
// снятие lease, чистка очередей gpu-hub, markDirty) в одну команду фасада.
|
|
441
|
+
//
|
|
442
|
+
// Параметры:
|
|
443
|
+
// scenes — [{chapter_id, scene_id}] — список сцен для reset'а
|
|
444
|
+
// options.cleanPngUnitIds — мапу {chapterId_sceneId: [unitId, ...]} для удаления stale PNG
|
|
445
|
+
// options.bookDiff — DI-инстанс book-diff (передаётся из route)
|
|
446
|
+
// options.readdToActiveIndex — вернуть сцены планировщику после reset (default true)
|
|
447
|
+
//
|
|
448
|
+
// Что делает:
|
|
449
|
+
// 1. force-dispatch флаг
|
|
450
|
+
// 2. событие SCENE_RESET в journal
|
|
451
|
+
// 3. удаление из active index
|
|
452
|
+
// 4. снятие dispatch leases только для сбрасываемых стадий
|
|
453
|
+
// 5. очистка очередей gpu-hub (HTTP DELETE /queue/clear)
|
|
454
|
+
// 6. pre-delete stale PNG для указанных unit_id
|
|
455
|
+
// 7. очистка iu-progress + iu-in-flight
|
|
456
|
+
// 8. markDirty (через bookDiff.markDirtyScenes)
|
|
457
|
+
// 9. добавление сцен обратно в active index
|
|
458
|
+
// 10. событие SCENE_RESET_COMPLETED в journal
|
|
459
|
+
async function resetScenes(redis, bookId, buildId, scenes, layerCfg, options = {}) {
|
|
460
|
+
// force — зарезервировано для будущего использования (например, пропуск
|
|
461
|
+
// version-проверок на PG при полном reset). Сейчас resetScenes всегда
|
|
462
|
+
// очищает всё безусловно — это корректно для регенерации.
|
|
463
|
+
const {
|
|
464
|
+
scope = 'WHOLE_BOOK',
|
|
465
|
+
chapterId = null,
|
|
466
|
+
sceneId = null,
|
|
467
|
+
bookDiff = null,
|
|
468
|
+
cleanPngUnitIds = null,
|
|
469
|
+
readdToActiveIndex = true,
|
|
470
|
+
} = options;
|
|
471
|
+
const { log, warn } = require('./scene-utils');
|
|
472
|
+
|
|
473
|
+
if (!scenes || scenes.length === 0) {
|
|
474
|
+
log('[RESET-SCENES] No scenes to reset');
|
|
475
|
+
return { marked: 0, reset_scenes: 0 };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
const scopeDisplay = `${scope}${chapterId ? '/' + chapterId : ''}${sceneId ? '/' + sceneId : ''}`;
|
|
479
|
+
log(`[RESET-SCENES] ${bookId}: ${scenes.length} scenes, scope=${scopeDisplay}`);
|
|
480
|
+
|
|
481
|
+
// 1. Force-dispatch flag (T1: TTL из TIMEOUTS)
|
|
482
|
+
await redis.set(
|
|
483
|
+
`animastor:force-dispatch:${bookId}`, '1',
|
|
484
|
+
'EX', runtimeConfig.TIMEOUTS.FORCE_DISPATCH_TTL_S
|
|
485
|
+
);
|
|
486
|
+
|
|
487
|
+
// 2. Event journal — start
|
|
488
|
+
await journal.appendSceneEvent(redis, bookId, chapterId || bookId, sceneId || 'all',
|
|
489
|
+
journal.EventType.SCENE_RESET, 'INITIATED',
|
|
490
|
+
{ scenes_count: scenes.length, scope, force: !!options.force }
|
|
491
|
+
).catch(() => {});
|
|
492
|
+
|
|
493
|
+
// 3. Remove from active index
|
|
494
|
+
const scheduler = require('../runtime/runtime-scheduler');
|
|
495
|
+
await scheduler.removeScenesFromActiveIndex(redis, bookId, scenes);
|
|
496
|
+
|
|
497
|
+
// 4. Clear dispatch leases only for stages requested by this regeneration.
|
|
498
|
+
// A parallel Image request must not cancel an already-running Audio lease
|
|
499
|
+
// for the same scene.
|
|
500
|
+
const dispatchEngine = require('../runtime/dispatch-engine');
|
|
501
|
+
const leaseResetScenes = scenes.map(ds => ({
|
|
502
|
+
chapter_id: ds.chapter_id,
|
|
503
|
+
scene_id: ds.scene_id,
|
|
504
|
+
stages: (ds.dirty_layers || mediaRegistry.listMediaTypes()).filter(stage =>
|
|
505
|
+
layerCfg?.[`${stage}_enabled`] !== false
|
|
506
|
+
),
|
|
507
|
+
}));
|
|
508
|
+
const resetStagesByScene = new Map(leaseResetScenes.map(s => [
|
|
509
|
+
`${s.chapter_id}:${s.scene_id}`,
|
|
510
|
+
s.stages,
|
|
511
|
+
]));
|
|
512
|
+
const cancellation = await dispatchEngine.clearLeasesForScenes(redis, bookId, leaseResetScenes);
|
|
513
|
+
|
|
514
|
+
// 5. Clear only jobs belonging to the cancelled dispatches.
|
|
515
|
+
// O-9: hub-queue cleanup goes through the HubCancelPort (the tier must
|
|
516
|
+
// not own hub HTTP); lease clearing above stays a direct dispatch-engine
|
|
517
|
+
// call (Redis domain operation).
|
|
518
|
+
await require('../runtime/hub-cancel-port').hubCancelOp('clearHubDispatches')(cancellation.dispatchIds, {
|
|
519
|
+
context: 'RESET-SCENES',
|
|
520
|
+
warn,
|
|
521
|
+
});
|
|
522
|
+
|
|
523
|
+
// 6. Pre-delete stale PNGs for specific unit IDs
|
|
524
|
+
const path = require('path');
|
|
525
|
+
const fs = require('fs');
|
|
526
|
+
if (cleanPngUnitIds && typeof cleanPngUnitIds === 'object' && buildId) {
|
|
527
|
+
const buildDir = path.join(runtimeConfig.OUTPUT_DIR, buildId);
|
|
528
|
+
if (fs.existsSync(buildDir)) {
|
|
529
|
+
let deletedCount = 0;
|
|
530
|
+
for (const ds of scenes) {
|
|
531
|
+
const sceneKey = `${ds.chapter_id}_${ds.scene_id}`;
|
|
532
|
+
const unitIds = cleanPngUnitIds[sceneKey] || [];
|
|
533
|
+
for (const uid of unitIds) {
|
|
534
|
+
const pngPath = path.join(buildDir, artifactNaming.sceneImageName(bookId, ds.chapter_id, ds.scene_id, uid));
|
|
535
|
+
try { if (fs.existsSync(pngPath)) { fs.unlinkSync(pngPath); deletedCount++; } } catch (_) {}
|
|
536
|
+
const strippedUid = uid.replace(/^iu/, '');
|
|
537
|
+
const previewPath = path.join(buildDir, artifactNaming.sceneImagePreviewName(bookId, ds.chapter_id, ds.scene_id, uid));
|
|
538
|
+
try { if (fs.existsSync(previewPath)) { fs.unlinkSync(previewPath); } } catch (_) {}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
if (deletedCount > 0) log(`[RESET-SCENES] Deleted ${deletedCount} stale PNG files`);
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// 7. Clear image-specific IU progress only for scenes whose image stage is reset.
|
|
546
|
+
for (const ds of scenes) {
|
|
547
|
+
const resetStages = resetStagesByScene.get(`${ds.chapter_id}:${ds.scene_id}`) || [];
|
|
548
|
+
if (!resetStages.includes('image')) continue;
|
|
549
|
+
|
|
550
|
+
const progKey = `animastor:iu-progress:${bookId}:${ds.chapter_id}:${ds.scene_id}:image`;
|
|
551
|
+
try { await redis.del(progKey); } catch (_) {}
|
|
552
|
+
|
|
553
|
+
try {
|
|
554
|
+
let cursor = '0';
|
|
555
|
+
const scenePrefix = artifactNaming.iuScanPrefix(bookId, ds.chapter_id, ds.scene_id);
|
|
556
|
+
do {
|
|
557
|
+
const [nextCursor, keys] = await redis.scan(cursor, 'MATCH', `animastor:iu-in-flight:${scenePrefix}*`, 'COUNT', 50);
|
|
558
|
+
cursor = nextCursor;
|
|
559
|
+
if (keys.length > 0) await redis.del(...keys);
|
|
560
|
+
} while (cursor !== '0');
|
|
561
|
+
} catch (_) {}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// 8. markDirty (через bookDiff.markDirtyScenes — обязательный DI-инстанс из route)
|
|
565
|
+
// R7: bookDiff обязателен — неатомарный fallback удалён.
|
|
566
|
+
if (!bookDiff || typeof bookDiff.markDirtyScenes !== 'function') {
|
|
567
|
+
throw new Error(`resetScenes: bookDiff.markDirtyScenes is required for book ${bookId}`);
|
|
568
|
+
}
|
|
569
|
+
// W1: try/catch вокруг markDirty гарантирует, что addSceneToActiveIndex
|
|
570
|
+
// (шаг 9) выполнится даже при runtime-ошибке (PG failure, Redis timeout).
|
|
571
|
+
// Сцена без dirty не страшна; сцена вне active index — scheduler её не
|
|
572
|
+
// увидит, и она зависнет навсегда. Programming error (bookDiff missing)
|
|
573
|
+
// валидируется выше и падает жёстко — это контракт, а не runtime-сбой.
|
|
574
|
+
let marked = { marked: 0 };
|
|
575
|
+
try {
|
|
576
|
+
marked = await markDirty({ bookDiff }, redis, bookId, buildId, scenes, layerCfg);
|
|
577
|
+
} catch (mdErr) {
|
|
578
|
+
warn(`[RESET-SCENES] markDirty failed for ${bookId}: ${mdErr.message} — continuing to re-add scenes to active index`);
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
// 9. Re-add to active index (планировщик подберёт на следующем тике).
|
|
582
|
+
// Selective generation can defer this until its task registry is ready.
|
|
583
|
+
// Гарантированно выполняется даже при ошибке markDirty (шаг 8).
|
|
584
|
+
if (readdToActiveIndex) {
|
|
585
|
+
for (const ds of scenes) {
|
|
586
|
+
await scheduler.addSceneToActiveIndex(redis, bookId, ds.chapter_id, ds.scene_id);
|
|
587
|
+
}
|
|
588
|
+
log(`[RESET-SCENES] ${bookId}: ${scenes.length} scenes re-added to active index`);
|
|
589
|
+
} else {
|
|
590
|
+
log(`[RESET-SCENES] ${bookId}: activation deferred for ${scenes.length} scenes`);
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
// 10. Event journal — completion
|
|
594
|
+
await journal.appendSceneEvent(redis, bookId, chapterId || bookId, sceneId || 'all',
|
|
595
|
+
journal.EventType.SCENE_RESET_COMPLETED, 'DONE',
|
|
596
|
+
{ scenes_count: scenes.length, marked: marked.marked }
|
|
597
|
+
).catch(() => {});
|
|
598
|
+
|
|
599
|
+
return { ...marked, reset_scenes: scenes.length };
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// ── S-5: pure FSM-writer ownership moved to the state layer ──
|
|
603
|
+
// markDirtyScene / setScenePending / setSceneGenerating / setSceneAllReady /
|
|
604
|
+
// setScenePlaceholder now live in state/scene-state-ops.js (bodies verbatim;
|
|
605
|
+
// canonical owner next to the asset-state store + journal). They are
|
|
606
|
+
// RE-EXPORTED below so the facade command surface and every existing caller
|
|
607
|
+
// (deps.orchestrator.*, orchestration index spread, seam wiring) stay
|
|
608
|
+
// byte-compatible. rollbackStageToPending remains a facade-owned function:
|
|
609
|
+
// it additionally reports the runtime-owned stateRollbackFailures metric.
|
|
610
|
+
|
|
611
|
+
// §32.30: the FSM-writer re-exports resolve the stateOps binding lazily via
|
|
612
|
+
// getters — the facade module must stay loadable before the composition root
|
|
613
|
+
// binds host modules (byte-compatible surface, deferred resolution).
|
|
614
|
+
module.exports = {
|
|
615
|
+
markDirty,
|
|
616
|
+
get markDirtyScene() { return stateOps.markDirtyScene; },
|
|
617
|
+
planScene,
|
|
618
|
+
beginStage,
|
|
619
|
+
completeStage,
|
|
620
|
+
failStage,
|
|
621
|
+
completeStageWithoutVideo,
|
|
622
|
+
completeStageWithoutImage,
|
|
623
|
+
get setScenePending() { return stateOps.setScenePending; },
|
|
624
|
+
get setSceneGenerating() { return stateOps.setSceneGenerating; },
|
|
625
|
+
get setSceneAllReady() { return stateOps.setSceneAllReady; },
|
|
626
|
+
get setScenePlaceholder() { return stateOps.setScenePlaceholder; },
|
|
627
|
+
rollbackStageToPending,
|
|
628
|
+
reconcile,
|
|
629
|
+
resetScenes,
|
|
630
|
+
};
|