@animastor/orchestration 0.3.1 → 0.3.2

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## 0.3.2 (2026-11-02) — эпоха dispatch в audio-orch, без ложного FAILED
+
+Основание: `reports/2026-11-02/gpu-e2e-installer-025-logs.md` §6.1, §8 п.3, п.5 —
+состояние audio-orch отменённой попытки (11:09) пережило отмену, новый
+dispatch (12:25) унаследовал его `started_at`, и watchdog через 28 с после
+dispatch'а пометил здоровый прогон FAILED (`chunk_stall_timeout:4619s`), затем
+`[FAIL-STAGE] Rejected … missing_dispatch_id` и `[INVARIANT] 1 audio-orch violations`.
+Описание: animastor-docs `architecture/COMMAND_ONLY_GENERATION.md` §«Эпоха dispatch».
+
+### Changed (поведение)
+
+- `scene-orchestrator.executeAudioDispatch` открывает новую эпоху на каждом
+  dispatch: `setWaitingChunks(…, { dispatchId, buildId, expectedCount })`.
+  Хост пишет `started_at=now`, `dispatch_id`, `expected_count`,
+  `chunks_received=0` и убирает `fail_reason`/`failed_at`/`last_chunk_at`
+  прошлой попытки. Хост без поддержки 5-го аргумента ведёт себя как раньше.
+- `reconciliation-engine.checkStalledAudioScenes`: запись проваливается, только
+  если её `dispatch_id` равен живому `dispatch-meta` сцены. Запись без
+  `dispatch_id` или другого dispatch — `stale_state_ignored` (одна info-строка
+  на запись, не FAILED). `failWaitingScene` получает `dispatchId` записи.
+- `orchestrator.failStage` без `dispatchId` → `{failed:false,
+  reason:'stale_state_ignored'}` (info), а не `Rejected … missing_dispatch_id`.
+
+Fence, command-gate (Stage 155) и retry-политика не меняются.
+
 ## 0.3.1 (2026-11-01) — результат GPU обрабатывается один раз
 
 Основание: `reports/2026-11-01/gpu-audio-image-run-analysis.md`, P1 — каждый

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@animastor/orchestration",
-    "version": "0.3.1",
+    "version": "0.3.2",
     "description": "Animastor Orchestration — the generation orchestration/runtime contour of the Animastor backend, physically extracted from the host (§32.30): the unified generation lifecycle owner (orchestrator facade, stage executor, scene callbacks/restoration), the dispatch engine and its lease/quota/retry managers, the reconciliation engine and counter reconciliation, the scene window/scheduler/worker-health, the runtime-result emitter/consumer, and the eight O-2..O-10 port contracts (Persistence, SceneData, PlaceholderAudio, ProgressEvents, AudioFsm, VideoFsm, HubCancel, LayerConfig). The package reaches the host (runtime config slices, Redis-domain state layer, media facades, the S-5 seam registry, the artifact root, the PW-2 workspace resolver) ONLY through the composition-root host-bindings seam — no backend/src require, no storage/book/routes/services, no Redis, no PG, no Express inside.",
     "keywords": [
         "animastor",

package/src/orchestration/orchestrator.js

@@ -288,6 +288,14 @@ async function failStage(redis, bookId, chapterId, sceneId, stage, buildId, reas
         throw new Error(`orchestrator.failStage: unknown stage '${stage}'`);
     }
 
+    // GPU E2E 2026-11-02 (§6.1): a fail signal without dispatch_id comes
+    // from a legacy / stale record (no epoch) — it can never fail the live
+    // dispatch. A regular no-op, not a reject.
+    if (!dispatchId || typeof dispatchId !== 'string') {
+        log(`[FAIL-STAGE] stale_state_ignored ${bookId}/${chapterId}/${sceneId} ${stage}: no dispatch_id (reason=${reason})`);
+        return { failed: false, reason: 'stale_state_ignored' };
+    }
+
     const identity = await dispatchEngine.verifyDispatchIdentity(
         redis,
         bookId,

package/src/orchestration/scene-orchestrator.js

@@ -280,7 +280,16 @@ async function executeAudioDispatch(redis, scene, loadedBook, buildId, dispatchI
     // jobs, иначе первый чанк может вернуться быстрее чем setWaitingChunks выполнится,
     // и completeChunk увидит фазу GENERATING → early return → чанк на диске есть,
     // но merge не запускается → retry exhausting → failStage → re-dispatch → цикл.
-    await audioFsmOp('setWaitingChunks')(redis, bookId, chapterId, sceneId);
+    //
+    // GPU E2E 2026-11-02 (§6.1): every dispatch opens a NEW audio-orch epoch
+    // (started_at=now, dispatch_id, expected_count; fail_reason/failed_at/
+    // last_chunk_at of an earlier or cancelled attempt are dropped). The stall
+    // watchdog compares this dispatch_id with the live dispatch metadata.
+    await audioFsmOp('setWaitingChunks')(redis, bookId, chapterId, sceneId, {
+        dispatchId,
+        buildId,
+        expectedCount: segments.buildSegments(sceneData).length,
+    });
 
     const result = await audio.generateSceneAudio(redis, sceneData, bookData, buildId, bookId, dispatchId);
 

package/src/runtime/reconciliation-engine.js

@@ -473,6 +473,33 @@ async function checkPartialBuilds(redis, bookId, chapterId, sceneId) {
 // Важно: watchdog НЕ конкурирует с gpu-hub timeout — они ловят
 // разные сценарии (стоп прогресса vs мёртвый воркер).
 
+// GPU E2E 2026-11-02 (§6.1) — dispatch epoch of an audio-orch record.
+// current=true only when the record carries a dispatch_id AND it equals the
+// live dispatch metadata of the scene's audio stage. A metadata read error is
+// "cannot prove" → not current (the watchdog never fails on a guess).
+async function _audioDispatchEpoch(redis, bookId, chapterId, sceneId, orchState) {
+    if (!orchState || !orchState.dispatch_id) return { current: false, reason: 'no_dispatch_id' };
+    let meta = null;
+    try {
+        meta = await require('./dispatch-engine').getDispatchMetadata(redis, bookId, chapterId, sceneId, 'audio');
+    } catch (_) {
+        return { current: false, reason: 'dispatch_metadata_unreadable' };
+    }
+    if (!meta || !meta.dispatch_id) return { current: false, reason: 'no_active_dispatch' };
+    if (meta.dispatch_id !== orchState.dispatch_id) return { current: false, reason: 'other_dispatch' };
+    return { current: true, reason: 'current_dispatch' };
+}
+
+// One info line per stale record (scene + its epoch), not one per cycle.
+const _staleAudioOrchLogged = new Set();
+function _logStaleAudioOrchOnce(sceneRef, orchState, reason) {
+    const k = `${sceneRef}|${orchState.dispatch_id || ''}|${orchState.started_at || ''}`;
+    if (_staleAudioOrchLogged.has(k)) return;
+    if (_staleAudioOrchLogged.size > 5000) _staleAudioOrchLogged.clear();
+    _staleAudioOrchLogged.add(k);
+    log(`[STALLED-AUDIO] ${sceneRef} — stale_state_ignored (${reason}): the record is not of the live dispatch, not failing it`);
+}
+
 async function checkStalledAudioScenes(redis, deps) {
     // O-7: the audio FSM is consumed through the AudioFsmPort (host adapter:
     // storage/audio-fsm-adapter). The pre-O-7 optional-load semantics are
@@ -513,12 +540,24 @@ async function checkStalledAudioScenes(redis, deps) {
             const age = Date.now() - threshold;
             if (age < AUDIO_CHUNK_STALL_MS) continue;
 
+            // GPU E2E 2026-11-02 (§6.1): the record must belong to the LIVE
+            // dispatch. A record without dispatch_id (legacy / pre-epoch) or
+            // of another dispatch is a stale epoch — failing it marked a
+            // healthy run FAILED (`chunk_stall_timeout:4619s`). It is ignored
+            // (one info line per record), never failed.
+            const epoch = await _audioDispatchEpoch(redis, bookId, chapterId, sceneId, orchState);
+            if (!epoch.current) {
+                _logStaleAudioOrchOnce(`${bookId}/${chapterId}/${sceneId}`, orchState, epoch.reason);
+                continue;
+            }
+
             const reason = `chunk_stall_timeout:${Math.round(age / 1000)}s`;
-            warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${reason} (last_chunk_at=${!!lastChunkAt} started_at=${!!startedAt})`);
+            warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${reason} (last_chunk_at=${!!lastChunkAt} started_at=${!!startedAt} dispatch=${String(orchState.dispatch_id).slice(0, 24)})`);
 
             try {
                 await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId, reason, {
-                    orchestrator: deps.orchestrator
+                    orchestrator: deps.orchestrator,
+                    dispatchId: orchState.dispatch_id,
                 });
                 stalled++;
             } catch (err) {
@@ -551,7 +590,7 @@ async function checkStalledAudioScenes(redis, deps) {
                     warn(`[STALLED-AUDIO] completeChunk recovery failed: ${err.message}`);
                     try {
                         await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
-                            `recovery_fallback:${err.message}`, { orchestrator: deps.orchestrator });
+                            `recovery_fallback:${err.message}`, { orchestrator: deps.orchestrator, dispatchId: orchState.dispatch_id });
                         stalled++;
                     } catch (fsErr) {
                         warn(`[STALLED-AUDIO] failWaitingScene fallback failed: ${fsErr.message}`);
@@ -561,7 +600,7 @@ async function checkStalledAudioScenes(redis, deps) {
                 warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${presentCount}/${expectedCount} on disk, no last_chunk_at. Partial chunks, failing scene.`);
                 try {
                     await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
-                        `partial_chunks:${presentCount}/${expectedCount}_no_timestamps`, { orchestrator: deps.orchestrator });
+                        `partial_chunks:${presentCount}/${expectedCount}_no_timestamps`, { orchestrator: deps.orchestrator, dispatchId: orchState.dispatch_id });
                     stalled++;
                 } catch (err) {
                     warn(`[STALLED-AUDIO] failWaitingScene failed: ${err.message}`);