@animastor/orchestration 0.3.0 → 0.3.2

package/CHANGELOG.md

@@ -1,5 +1,54 @@
 # Changelog
 
+## 0.3.2 (2026-11-02) — эпоха dispatch в audio-orch, без ложного FAILED
+
+Основание: `reports/2026-11-02/gpu-e2e-installer-025-logs.md` §6.1, §8 п.3, п.5 —
+состояние audio-orch отменённой попытки (11:09) пережило отмену, новый
+dispatch (12:25) унаследовал его `started_at`, и watchdog через 28 с после
+dispatch'а пометил здоровый прогон FAILED (`chunk_stall_timeout:4619s`), затем
+`[FAIL-STAGE] Rejected … missing_dispatch_id` и `[INVARIANT] 1 audio-orch violations`.
+Описание: animastor-docs `architecture/COMMAND_ONLY_GENERATION.md` §«Эпоха dispatch».
+
+### Changed (поведение)
+
+- `scene-orchestrator.executeAudioDispatch` открывает новую эпоху на каждом
+  dispatch: `setWaitingChunks(…, { dispatchId, buildId, expectedCount })`.
+  Хост пишет `started_at=now`, `dispatch_id`, `expected_count`,
+  `chunks_received=0` и убирает `fail_reason`/`failed_at`/`last_chunk_at`
+  прошлой попытки. Хост без поддержки 5-го аргумента ведёт себя как раньше.
+- `reconciliation-engine.checkStalledAudioScenes`: запись проваливается, только
+  если её `dispatch_id` равен живому `dispatch-meta` сцены. Запись без
+  `dispatch_id` или другого dispatch — `stale_state_ignored` (одна info-строка
+  на запись, не FAILED). `failWaitingScene` получает `dispatchId` записи.
+- `orchestrator.failStage` без `dispatchId` → `{failed:false,
+  reason:'stale_state_ignored'}` (info), а не `Rejected … missing_dispatch_id`.
+
+Fence, command-gate (Stage 155) и retry-политика не меняются.
+
+## 0.3.1 (2026-11-01) — результат GPU обрабатывается один раз
+
+Основание: `reports/2026-11-01/gpu-audio-image-run-analysis.md`, P1 — каждый
+результат обрабатывался дважды (HTTP-push hub → backend, затем Phase A
+reconcile подбирал оставшийся `animastor:result:*` мимо
+`animastor:result-processed:*`). Описание: animastor-docs
+`architecture/GPU_HUB_CONTRACT.md` §«Result ACK».
+
+### Changed (поведение)
+
+- `reconciliation-engine.recoverResultKeys` (Phase A): перед
+  `handleTaskResult` читает маркер `animastor:result-processed:{job}:{build}`.
+  Значение `done:<dispatch_id>` того же dispatch, что в result-ключе, —
+  ключ удаляется молча (без лога), handler не вызывается.
+  Маркер `'1'` (в работе / старый формат), маркер другого dispatch
+  (перегенерация audio/video в пределах TTL) или его отсутствие — как
+  раньше, через fenced handler.
+- Терминальный отказ (`no_active_dispatch` и др.) для дубля доставленного
+  результата (маркер есть) — уровень info; для сироты (маркера нет) — warn.
+- `orchestrator.resetScenes` вместе с `iu-progress` удаляет
+  `animastor:iu-progress-ids:*` (множество IU×fence, хост task-handler).
+
+Fence, dedup-ключ маршрута и command-gate (Stage 155) не меняются.
+
 ## 0.3.0 (2026-10-31) — генерация только по явной команде
 
 Правило владельца: GPU-генерация запускается только кнопками «аудио» /

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@animastor/orchestration",
-    "version": "0.3.0",
+    "version": "0.3.2",
     "description": "Animastor Orchestration — the generation orchestration/runtime contour of the Animastor backend, physically extracted from the host (§32.30): the unified generation lifecycle owner (orchestrator facade, stage executor, scene callbacks/restoration), the dispatch engine and its lease/quota/retry managers, the reconciliation engine and counter reconciliation, the scene window/scheduler/worker-health, the runtime-result emitter/consumer, and the eight O-2..O-10 port contracts (Persistence, SceneData, PlaceholderAudio, ProgressEvents, AudioFsm, VideoFsm, HubCancel, LayerConfig). The package reaches the host (runtime config slices, Redis-domain state layer, media facades, the S-5 seam registry, the artifact root, the PW-2 workspace resolver) ONLY through the composition-root host-bindings seam — no backend/src require, no storage/book/routes/services, no Redis, no PG, no Express inside.",
     "keywords": [
         "animastor",

package/src/orchestration/orchestrator.js

@@ -288,6 +288,14 @@ async function failStage(redis, bookId, chapterId, sceneId, stage, buildId, reas
         throw new Error(`orchestrator.failStage: unknown stage '${stage}'`);
     }
 
+    // GPU E2E 2026-11-02 (§6.1): a fail signal without dispatch_id comes
+    // from a legacy / stale record (no epoch) — it can never fail the live
+    // dispatch. A regular no-op, not a reject.
+    if (!dispatchId || typeof dispatchId !== 'string') {
+        log(`[FAIL-STAGE] stale_state_ignored ${bookId}/${chapterId}/${sceneId} ${stage}: no dispatch_id (reason=${reason})`);
+        return { failed: false, reason: 'stale_state_ignored' };
+    }
+
     const identity = await dispatchEngine.verifyDispatchIdentity(
         redis,
         bookId,
@@ -575,6 +583,8 @@ async function resetScenes(redis, bookId, buildId, scenes, layerCfg, options = {
 
         const progKey = `animastor:iu-progress:${bookId}:${ds.chapter_id}:${ds.scene_id}:image`;
         try { await redis.del(progKey); } catch (_) {}
+        // SYNC: backend task-handler — the IU×fence set behind the counter.
+        try { await redis.del(`animastor:iu-progress-ids:${bookId}:${ds.chapter_id}:${ds.scene_id}:image`); } catch (_) {}
 
         try {
             let cursor = '0';

package/src/orchestration/scene-orchestrator.js

@@ -280,7 +280,16 @@ async function executeAudioDispatch(redis, scene, loadedBook, buildId, dispatchI
     // jobs, иначе первый чанк может вернуться быстрее чем setWaitingChunks выполнится,
     // и completeChunk увидит фазу GENERATING → early return → чанк на диске есть,
     // но merge не запускается → retry exhausting → failStage → re-dispatch → цикл.
-    await audioFsmOp('setWaitingChunks')(redis, bookId, chapterId, sceneId);
+    //
+    // GPU E2E 2026-11-02 (§6.1): every dispatch opens a NEW audio-orch epoch
+    // (started_at=now, dispatch_id, expected_count; fail_reason/failed_at/
+    // last_chunk_at of an earlier or cancelled attempt are dropped). The stall
+    // watchdog compares this dispatch_id with the live dispatch metadata.
+    await audioFsmOp('setWaitingChunks')(redis, bookId, chapterId, sceneId, {
+        dispatchId,
+        buildId,
+        expectedCount: segments.buildSegments(sceneData).length,
+    });
 
     const result = await audio.generateSceneAudio(redis, sceneData, bookData, buildId, bookId, dispatchId);
 

package/src/runtime/reconciliation-engine.js

@@ -63,6 +63,25 @@ function error(msg) {
     console.error(`${logPrefix} ❌ ${msg}`);
 }
 
+// SYNC: backend/src/helpers/queue-idempotency.cjs — resultDedupKey /
+// readResultDedup (marker `'1'` = in flight, `done:<dispatch_id>` = settled).
+// Read-only here: the result route is the only writer.
+const RESULT_DEDUP_DONE_PREFIX = 'done:';
+async function readResultDedup(redis, jobId, buildId) {
+    let raw = null;
+    try {
+        raw = await redis.get(`animastor:result-processed:${jobId}:${buildId}`);
+    } catch (_) {
+        return { state: 'unknown', dispatchId: null };
+    }
+    if (raw == null || raw === false) return { state: 'none', dispatchId: null };
+    const v = String(raw);
+    if (v.startsWith(RESULT_DEDUP_DONE_PREFIX)) {
+        return { state: 'settled', dispatchId: v.slice(RESULT_DEDUP_DONE_PREFIX.length) || null };
+    }
+    return { state: 'in_flight', dispatchId: null };
+}
+
 // ======================================================
 // RECONCILIATION REPORT
 // ======================================================
@@ -454,6 +473,33 @@ async function checkPartialBuilds(redis, bookId, chapterId, sceneId) {
 // Важно: watchdog НЕ конкурирует с gpu-hub timeout — они ловят
 // разные сценарии (стоп прогресса vs мёртвый воркер).
 
+// GPU E2E 2026-11-02 (§6.1) — dispatch epoch of an audio-orch record.
+// current=true only when the record carries a dispatch_id AND it equals the
+// live dispatch metadata of the scene's audio stage. A metadata read error is
+// "cannot prove" → not current (the watchdog never fails on a guess).
+async function _audioDispatchEpoch(redis, bookId, chapterId, sceneId, orchState) {
+    if (!orchState || !orchState.dispatch_id) return { current: false, reason: 'no_dispatch_id' };
+    let meta = null;
+    try {
+        meta = await require('./dispatch-engine').getDispatchMetadata(redis, bookId, chapterId, sceneId, 'audio');
+    } catch (_) {
+        return { current: false, reason: 'dispatch_metadata_unreadable' };
+    }
+    if (!meta || !meta.dispatch_id) return { current: false, reason: 'no_active_dispatch' };
+    if (meta.dispatch_id !== orchState.dispatch_id) return { current: false, reason: 'other_dispatch' };
+    return { current: true, reason: 'current_dispatch' };
+}
+
+// One info line per stale record (scene + its epoch), not one per cycle.
+const _staleAudioOrchLogged = new Set();
+function _logStaleAudioOrchOnce(sceneRef, orchState, reason) {
+    const k = `${sceneRef}|${orchState.dispatch_id || ''}|${orchState.started_at || ''}`;
+    if (_staleAudioOrchLogged.has(k)) return;
+    if (_staleAudioOrchLogged.size > 5000) _staleAudioOrchLogged.clear();
+    _staleAudioOrchLogged.add(k);
+    log(`[STALLED-AUDIO] ${sceneRef} — stale_state_ignored (${reason}): the record is not of the live dispatch, not failing it`);
+}
+
 async function checkStalledAudioScenes(redis, deps) {
     // O-7: the audio FSM is consumed through the AudioFsmPort (host adapter:
     // storage/audio-fsm-adapter). The pre-O-7 optional-load semantics are
@@ -494,12 +540,24 @@ async function checkStalledAudioScenes(redis, deps) {
             const age = Date.now() - threshold;
             if (age < AUDIO_CHUNK_STALL_MS) continue;
 
+            // GPU E2E 2026-11-02 (§6.1): the record must belong to the LIVE
+            // dispatch. A record without dispatch_id (legacy / pre-epoch) or
+            // of another dispatch is a stale epoch — failing it marked a
+            // healthy run FAILED (`chunk_stall_timeout:4619s`). It is ignored
+            // (one info line per record), never failed.
+            const epoch = await _audioDispatchEpoch(redis, bookId, chapterId, sceneId, orchState);
+            if (!epoch.current) {
+                _logStaleAudioOrchOnce(`${bookId}/${chapterId}/${sceneId}`, orchState, epoch.reason);
+                continue;
+            }
+
             const reason = `chunk_stall_timeout:${Math.round(age / 1000)}s`;
-            warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${reason} (last_chunk_at=${!!lastChunkAt} started_at=${!!startedAt})`);
+            warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${reason} (last_chunk_at=${!!lastChunkAt} started_at=${!!startedAt} dispatch=${String(orchState.dispatch_id).slice(0, 24)})`);
 
             try {
                 await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId, reason, {
-                    orchestrator: deps.orchestrator
+                    orchestrator: deps.orchestrator,
+                    dispatchId: orchState.dispatch_id,
                 });
                 stalled++;
             } catch (err) {
@@ -532,7 +590,7 @@ async function checkStalledAudioScenes(redis, deps) {
                     warn(`[STALLED-AUDIO] completeChunk recovery failed: ${err.message}`);
                     try {
                         await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
-                            `recovery_fallback:${err.message}`, { orchestrator: deps.orchestrator });
+                            `recovery_fallback:${err.message}`, { orchestrator: deps.orchestrator, dispatchId: orchState.dispatch_id });
                         stalled++;
                     } catch (fsErr) {
                         warn(`[STALLED-AUDIO] failWaitingScene fallback failed: ${fsErr.message}`);
@@ -542,7 +600,7 @@ async function checkStalledAudioScenes(redis, deps) {
                 warn(`[STALLED-AUDIO] ${bookId}/${chapterId}/${sceneId} — ${presentCount}/${expectedCount} on disk, no last_chunk_at. Partial chunks, failing scene.`);
                 try {
                     await audioOrch.failWaitingScene(redis, bookId, chapterId, sceneId, buildId,
-                        `partial_chunks:${presentCount}/${expectedCount}_no_timestamps`, { orchestrator: deps.orchestrator });
+                        `partial_chunks:${presentCount}/${expectedCount}_no_timestamps`, { orchestrator: deps.orchestrator, dispatchId: orchState.dispatch_id });
                     stalled++;
                 } catch (err) {
                     warn(`[STALLED-AUDIO] failWaitingScene failed: ${err.message}`);
@@ -1947,11 +2005,34 @@ async function recoverResultKeys(redis, deps, scope) {
                     warn(`Skipping legacy result without build/dispatch identity: ${job_id}`);
                     continue;
                 }
+
+                // P1 (2026-11-01): the hub's fallback copy of a delivery the
+                // result route already SETTLED is a duplicate — drop it
+                // silently, never replay it (the replay re-ran the ready chain
+                // and logged false no_active_dispatch warnings). Only a marker
+                // carrying THIS dispatch counts: an in-flight / legacy '1'
+                // marker or one from an older dispatch of the same job_id
+                // (audio/video regeneration within the TTL) is no proof — such
+                // a result goes through the fenced handler exactly as before.
+                const dedup = await readResultDedup(redis, job_id, buildId);
+                if (dedup.state === 'settled' && dedup.dispatchId === String(dispatchId)) {
+                    await redis.del(key);
+                    continue;
+                }
                 try {
                     await taskHandler.handleTaskResult(job_id, result_base64, buildId, dispatchId);
                 } catch (rejectErr) {
                     if (rejectErr && TERMINAL_REJECTIONS.has(rejectErr.code)) {
-                        warn(`Result ${job_id} terminally rejected (${rejectErr.code}) — dropping result key, no infinite retry`);
+                        // A replay of a delivery the route has seen (a marker
+                        // exists) is a duplicate: info. No marker (an orphan
+                        // that never reached the route) or an unreadable
+                        // marker: warn, as before.
+                        const msg = `Result ${job_id} terminally rejected (${rejectErr.code}) — dropping result key, no infinite retry`;
+                        if (dedup.state === 'in_flight' || dedup.state === 'settled') {
+                            log(`${msg} (duplicate of a delivered result)`);
+                        } else {
+                            warn(msg);
+                        }
                         await redis.del(key);
                     } else {
                         throw rejectErr;