@animastor/orchestration 0.1.1 → 0.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@animastor/orchestration",
-    "version": "0.1.1",
+    "version": "0.1.3",
     "description": "Animastor Orchestration — the generation orchestration/runtime contour of the Animastor backend, physically extracted from the host (§32.30): the unified generation lifecycle owner (orchestrator facade, stage executor, scene callbacks/restoration), the dispatch engine and its lease/quota/retry managers, the reconciliation engine and counter reconciliation, the scene window/scheduler/worker-health, the runtime-result emitter/consumer, and the eight O-2..O-10 port contracts (Persistence, SceneData, PlaceholderAudio, ProgressEvents, AudioFsm, VideoFsm, HubCancel, LayerConfig). The package reaches the host (runtime config slices, Redis-domain state layer, media facades, the S-5 seam registry, the artifact root, the PW-2 workspace resolver) ONLY through the composition-root host-bindings seam — no backend/src require, no storage/book/routes/services, no Redis, no PG, no Express inside.",
     "keywords": [
         "animastor",
@@ -32,12 +32,12 @@
     },
     "repository": {
         "type": "git",
-        "url": "git+https://github.com/Animastor/animastor.git",
+        "url": "git+https://github.com/Animastor/animastor-backend.git",
         "directory": "packages/animastor-orchestration"
     },
-    "homepage": "https://github.com/Animastor/animastor/tree/main/packages/animastor-orchestration#readme",
+    "homepage": "https://github.com/Animastor/animastor-backend/tree/main/packages/animastor-orchestration#readme",
     "bugs": {
-        "url": "https://github.com/Animastor/animastor/issues"
+        "url": "https://github.com/Animastor/animastor-backend/issues"
     },
     "dependencies": {
         "@animastor/contracts": "^0.1.0",

package/src/orchestration/scene-orchestrator.js

@@ -93,17 +93,28 @@ async function startScene(redis, scene, loadedBook, buildId) {
 // открывал circuit breaker.
 //
 // Контракт: перед отправкой любого GPU job ассет обязан пройти
-//   NEW → PENDING (здесь) → GENERATING (сразу ниже),
+//   NEW|DIRTY → PENDING (здесь) → GENERATING (сразу ниже),
 // либо уже находиться в состоянии, из которого GENERATING достижим.
 // Если переход невозможен — dispatch ПРЕРЫВАЕТСЯ ДО отправки job.
+//
+// Stage 50: DIRTY включён в нормализацию. DIRTY → GENERATING запрещён FSM
+// (scene-state.js: DIRTY → [PENDING, PLACEHOLDER]), поэтому до этого фикса
+// dirty-ассет (version-stale pre-pass, C1/C1b recovery, RELEASE_STALE_LEASE,
+// markDirtyScene) не мог перегенерироваться через планировщик вообще: guard
+// отклонял setSceneGenerating(dirty), rejection проглатывался в
+// attemptDispatch — ровно тот `INVALID state=dirty attempted=generating`
+// из journal (Case A, stage 49). Комментарий теста stage-dispatch-lifecycle
+// («scheduler's version-stale pre-pass owns DIRTY→PENDING») описывал
+// нормализацию, которой не существовало. PENDING-хоп здесь — это и есть
+// недостающая половина контракта.
 async function ensureStageDispatchable(redis, bookId, chapterId, sceneId, stage) {
     const states = await state.getAssetStates(redis, bookId, chapterId, sceneId);
-    if (states?.[stage] === state.AssetState.NEW) {
+    if (states?.[stage] === state.AssetState.NEW || states?.[stage] === state.AssetState.DIRTY) {
         const init = await setScenePending(redis, bookId, chapterId, sceneId, stage);
         if (!init.changed) {
             return { ok: false, reason: `pending_init_failed:${init.reason}` };
         }
-        log(`STAGE_INIT: ${bookId}/${chapterId}/${sceneId} ${stage}: NEW → PENDING`);
+        log(`STAGE_INIT: ${bookId}/${chapterId}/${sceneId} ${stage}: ${states[stage]} → PENDING`);
     }
     const gen = await setSceneGenerating(redis, bookId, chapterId, sceneId, stage);
     if (!gen.changed) {
@@ -112,6 +123,34 @@ async function ensureStageDispatchable(redis, bookId, chapterId, sceneId, stage)
     return { ok: true };
 }
 
+// ── Stage 50: terminal audio-orch record vs requested regeneration ──
+// DONE — терминальная фаза (VALID_TRANSITIONS[DONE] = []), но per-asset DIRTY
+// поверх неё — явный запрос на перегенерацию (version-stale pre-pass, C1/C1b
+// recovery, RELEASE_STALE_LEASE, completeStage version-gate stale). Без сброса
+// записи dispatch навсегда упирается в DONE guard (already_done) — состояние
+// DONE+DIRTY зависает навсегда (2 сцены в production, stage 49). Хил:
+// терминальная запись удаляется, дальше ensureStageDispatchable проводит
+// DIRTY → PENDING → GENERATING, а no_state-лега переинициализирует FSM.
+// Не-DONE записи (WAITING_CHUNKS/MERGING/FAILED) НЕ трогаются — их ведут
+// штатные recovery-леги (Фаза 4: чистится только DONE).
+async function clearTerminalAudioOrchForDirty(redis, bookId, chapterId, sceneId) {
+    let orchState;
+    try {
+        orchState = await audioFsmOp('getState')(redis, bookId, chapterId, sceneId);
+    } catch (err) {
+        warn(`AUDIO_DONE_DIRTY_HEAL: getState failed for ${bookId}/${chapterId}/${sceneId}: ${err.message}`);
+        return false;
+    }
+    if (!orchState || orchState.phase !== audioFsmPhases().DONE) return false;
+
+    const states = await state.getAssetStates(redis, bookId, chapterId, sceneId);
+    if (states?.audio !== state.AssetState.DIRTY) return false;
+
+    log(`AUDIO_DONE_DIRTY_HEAL: ${bookId}/${chapterId}/${sceneId} — regeneration requested over terminal DONE record — clearing audio-orch state for re-dispatch`);
+    await audioFsmOp('deleteState')(redis, bookId, chapterId, sceneId);
+    return true;
+}
+
 // T3: Каждый executor возвращает { dispatched: true/false, jobs, reason }
 async function executeAudioDispatch(redis, scene, loadedBook, buildId, dispatchId) {
     const bookId = scene.book_id;
@@ -125,6 +164,10 @@ async function executeAudioDispatch(redis, scene, loadedBook, buildId, dispatchI
         buildId, dispatchType: 'orchestrator'
     });
 
+    // Stage 50: heal the DONE×DIRTY deadlock BEFORE the lifecycle guard —
+    // a DIRTY request over a terminal DONE record must regenerate, not skip.
+    await clearTerminalAudioOrchForDirty(redis, bookId, chapterId, sceneId);
+
     // T7: через фасад — единый владелец GENERATING (audio)
     // Lifecycle guard: abort BEFORE any GPU job if the transition chain fails.
     const dispatchable = await ensureStageDispatchable(redis, bookId, chapterId, sceneId, 'audio');
@@ -591,6 +634,7 @@ async function dispatchStage(redis, scene, loadedBook, buildId, overrideStage, d
 
 module.exports = {
     ensureStageDispatchable,
+    clearTerminalAudioOrchForDirty,
     dispatchStage,
     restoreSceneChunkStatus,
     handleAudioCompleted,

package/src/runtime/dispatch-engine.js

@@ -24,6 +24,9 @@ const runtimeMetrics = require('./runtime-metrics');
 // domain state over the injected redis parameter).
 const circuitBreaker = require('./circuit-breaker');
 const retryBudget = require('./retry-budget-manager');
+// O-2 (Stage 50): the dispatch-time scene-version stamp is a PG-side effect
+// consumed through the PersistencePort (host adapter owns the SQL).
+const persist = require('./persistence-port').persist;
 const crypto = require('crypto');
 // Phase 5: Runtime Result Contract — dispatch finalizations are reported to
 // orchestration through the contracts seam (runtime → contract → injected
@@ -864,7 +867,22 @@ async function dispatchStage(redis, bookId, chapterId, sceneId, stage, loadedBoo
         dispatchId,
         leaseKey: lease.leaseKey,
         started_at: metadata.started_at
-    });        // Step 6: Get orchestrator to perform dispatch
+    });
+
+    // Step 5.5 (Stage 50): stamp the scene's CURRENT content/audio-config
+    // versions onto the asset row — «версия, под которую стартовала генерация».
+    // completeStage's version gate then compares exactly that, so a regen
+    // started for the current content reaches READY (convergence of the
+    // DONE×DIRTY / version-stale flows), while a version bump DURING the
+    // generation still fails the gate (T1.11 semantics preserved). Best-effort:
+    // dispatch is a Redis-domain operation; a PG outage must not block it —
+    // the gate stays fail-closed and the next dispatch re-stamps.
+    try {
+        await persist('sceneAssets.stampSceneVersionsForDispatch')(bookId, chapterId, sceneId, stage);
+    } catch (stampErr) {
+        warn(`DISPATCH_VERSION_STAMP_FAILED: ${bookId}/${chapterId}/${sceneId}:${stage}: ${stampErr.message}`);
+    }
+        // Step 6: Get orchestrator to perform dispatch
         try {
             // S-5: the stage executor entry (scene-orchestrator.dispatchStage) is
             // injected by the composition root — runtime must not import

package/src/runtime/persistence-port.js

@@ -59,6 +59,7 @@ const OPS = [
 
     'sceneAssets.getAsset',
     'sceneAssets.markReady',
+    'sceneAssets.stampSceneVersionsForDispatch',
     'sceneAssets.getDirtyUnitIds',
     'sceneAssets.setDirtyUnitIds',
     'sceneAssets.clearDirtyUnitIds',

package/src/runtime/reconciliation-engine.js

@@ -345,15 +345,23 @@ async function checkOrphanGeneratingState(redis, bookId, chapterId, sceneId) {
 /**
  * Check for partially built scenes.
  * T8: использует per-asset state вместо линейного.
+ *
+ * Stage 50: генерация в полёте — не «partial build». Раньше registry мог
+ * ещё не содержать image (registerSceneImage пишется по завершении), а
+ * ассет уже PENDING/GENERATING с живым lease — отчёт partial_audio_only
+ * вёл к PROGRESS_TO_IMAGE → setScenePending(generating→pending rejected)
+ * каждый цикл (issue №3 stage 49: ch-1bb5123c/sc-45c789c6).
  */
 async function checkPartialBuilds(redis, bookId, chapterId, sceneId) {
     const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
     const assets = await persist('registry.getSceneAssetsRedis')(redis, bookId, chapterId, sceneId);
 
+    const isInFlight = s => s === state.AssetState.PENDING || s === state.AssetState.GENERATING;
+
     // Audio READY/PLACEHOLDER but no image asset
     const audioReady = assetStates && assetStates.audio &&
         (assetStates.audio === state.AssetState.READY || assetStates.audio === state.AssetState.PLACEHOLDER);
-    if (audioReady && !assets?.image) {
+    if (audioReady && !assets?.image && !isInFlight(assetStates.image)) {
         return {
             type: 'partial_audio_only',
             scene: { bookId, chapterId, sceneId },
@@ -366,7 +374,7 @@ async function checkPartialBuilds(redis, bookId, chapterId, sceneId) {
     // Image READY but no video asset
     const imageReady = assetStates && assetStates.image &&
         (assetStates.image === state.AssetState.READY || assetStates.image === state.AssetState.PLACEHOLDER);
-    if (imageReady && !assets?.video) {
+    if (imageReady && !assets?.video && !isInFlight(assetStates.video)) {
         return {
             type: 'partial_image_video_missing',
             scene: { bookId, chapterId, sceneId },
@@ -607,8 +615,21 @@ async function checkStalledVideoScenes(redis, deps) {
 // ======================================================
 // Проверяет соответствие между audio-orch phase и asset state:
 //   phase=DONE     ⇔ asset.audio = READY
-//   phase=FAILED   ⇒ asset.audio = FAILED (или PENDING — после re-dispatch)
+//   phase=FAILED   ⇒ asset.audio ∈ {FAILED, PENDING, DIRTY}
 //   промежуточные  ⇒ asset.audio ∈ {PENDING, GENERATING, DIRTY}
+//
+// Stage 47 (FAILED + DIRTY): DIRTY — штатный итог для phase=FAILED.
+// Два писателя приводят к FAILED на стороне оркестратора:
+//   • failStage(redispatch=true) → asset = PENDING (FAILED пишется первым и
+//     тут же перезаписывается), redispatch=false → asset = FAILED;
+//   • startup C1/C1b recovery → setFailed, затем markDirtyScene → asset =
+//     DIRTY (+ PG scene_assets.status='stale'). Это сделано нарочно:
+//     shouldScheduleAssets() перескакивает READY/FAILED и передиспатчитивает
+//     только PENDING/DIRTY, т.е. asset=FAILED застрял бы навсегда, а
+//     recommendation 'mark_dirty' (см. ветку ниже) есть ничто иное, как
+//     путь к допустимому состоянию. Проверка до stage 47 отвергала именно
+//     тот итог, к которому сама же рекомендовала прийти, — ложный
+//     audio_orch_invariant_failed на каждой успешной recovery.
 
 async function checkAudioOrchInvariants(redis, bookId, chapterId, sceneId) {
     // O-7: through the AudioFsmPort (host adapter: storage/audio-fsm-adapter)
@@ -633,17 +654,27 @@ async function checkAudioOrchInvariants(redis, bookId, chapterId, sceneId) {
                 phase,
                 audioState,
                 expected: 'READY or PLACEHOLDER',
-                recommendation: 'run_completeStage'
+                // Stage 50: former value 'run_completeStage' was dead AND
+                // destructive metadata — completing the stage would write READY
+                // and erase exactly the dirty marker that schedules the pending
+                // regeneration. Re-dispatch (stage 50 guard-heal) owns the repair.
+                recommendation: 'review_manually'
             });
         }
     } else if (phase === PHASES.FAILED) {
-        if (audioState !== state.AssetState.FAILED && audioState !== state.AssetState.PENDING) {
+        // Stage 47: DIRTY is a legitimate FAILED-phase asset state (see the
+        // header of this section). Everything else — READY above all — stays
+        // a real contradiction: the orchestrator failed while the asset claims
+        // its result is accepted.
+        if (audioState !== state.AssetState.FAILED &&
+            audioState !== state.AssetState.PENDING &&
+            audioState !== state.AssetState.DIRTY) {
             violations.push({
                 type: 'audio_orch_invariant_failed',
                 scene: { bookId, chapterId, sceneId },
                 phase,
                 audioState,
-                expected: 'FAILED or PENDING',
+                expected: 'FAILED, PENDING or DIRTY',
                 recommendation: 'mark_dirty'
             });
         }
@@ -668,7 +699,8 @@ async function checkAudioOrchInvariants(redis, bookId, chapterId, sceneId) {
 // VIDEO-ORCH INVARIANT CHECKS (зеркало checkAudioOrchInvariants)
 // ======================================================
 //   phase=DONE     ⇒ asset.video = READY (или PLACEHOLDER)
-//   phase=FAILED   ⇒ asset.video = FAILED (или PENDING — после re-dispatch)
+//   phase=FAILED   ⇒ asset.video ∈ {FAILED, PENDING, DIRTY}  (stage 47 —
+//                    зеркало аудио: C1b recovery заканчивается markDirtyScene)
 //   промежуточные  ⇒ asset.video = PENDING|GENERATING|DIRTY
 
 async function checkVideoOrchInvariants(redis, bookId, chapterId, sceneId) {
@@ -694,17 +726,23 @@ async function checkVideoOrchInvariants(redis, bookId, chapterId, sceneId) {
                 phase,
                 videoState,
                 expected: 'READY or PLACEHOLDER',
-                recommendation: 'run_completeStage'
+                // Stage 50: mirror of the audio fix — 'run_completeStage' was
+                // dead + destructive; manual review only, re-dispatch owns it.
+                recommendation: 'review_manually'
             });
         }
     } else if (phase === PHASES.FAILED) {
-        if (videoState !== state.AssetState.FAILED && videoState !== state.AssetState.PENDING) {
+        // Stage 47: mirror of the audio branch — DIRTY is the deliberate C1b
+        // recovery outcome (markDirtyScene → asset DIRTY for re-dispatch).
+        if (videoState !== state.AssetState.FAILED &&
+            videoState !== state.AssetState.PENDING &&
+            videoState !== state.AssetState.DIRTY) {
             violations.push({
                 type: 'video_orch_invariant_failed',
                 scene: { bookId, chapterId, sceneId },
                 phase,
                 videoState,
-                expected: 'FAILED or PENDING',
+                expected: 'FAILED, PENDING or DIRTY',
                 recommendation: 'mark_dirty'
             });
         }
@@ -1079,6 +1117,23 @@ function getFixRecommendations(inconsistentScenes) {
                     reason: `Stale dispatch lease detected`,
                     safeToExecute: true
                 };
+            // Stage 50: explicit (no longer default-fallback) — a terminal
+            // audio/video-orch phase conflicting with the asset state is
+            // NEVER auto-repaired here. In particular run_completeStage would
+            // write READY and erase the dirty marker that schedules the
+            // pending regeneration (stage 49: 2 stuck production scenes).
+            // The DONE×DIRTY deadlock is healed by re-dispatch (stage 50
+            // guard-heal in scene-orchestrator); everything else is manual.
+            case 'audio_orch_invariant_done':
+            case 'video_orch_invariant_done':
+            case 'audio_orch_invariant_failed':
+            case 'video_orch_invariant_failed':
+                return {
+                    scene,
+                    action: 'REVIEW_MANUALLY',
+                    reason: `${issue}: terminal orch phase conflicts with asset state — re-dispatch heals DONE×DIRTY, other combinations need manual review`,
+                    safeToExecute: false
+                };
             default:
                 return {
                     scene,
@@ -2083,6 +2138,19 @@ async function recoverVideoOrchStates(redis, deps) {
 // the sceneVersions.getVersionStalenessRows() port op (SQL lives in the host
 // adapter verbatim). Pre-O-2 a missing deps.postgres skipped the phase; the
 // unwired-port throw hits the same catch → warn + return 0.
+//
+// Stage 50: per-asset и READY-only. Раньше сцена помечалась целиком
+// (markDirtyScene со ВСЕМИ ассетами по умолчанию) при каждом старте:
+//   • unsafeRestoreAssetState пишет DIRTY поверх GENERATING/PENDING — убивал
+//     бы живую генерацию при рестарте;
+//   • READY→DIRTY на не-stale ассетах — лишняя регенерация;
+//   • повторная маркировка уже-DIRTY ассетов — неидемпотентный счётчик.
+// Теперь stale-типы считаются построчно (audio: content ИЛИ audio_config;
+// image/video: content), пишется только READY→DIRTY (FSM-легальный переход),
+// прежний READY-артефакт не удаляется — он остаётся отдаваемым до успешной
+// замены (файл и registry не трогаются; completeStage version-gate пропустит
+// READY только после успешной перегенерации). Повторный прогон: ассет уже
+// DIRTY → 0 маркировок (идемпотентность), счётчик считает только факты записи.
 async function checkVersionStaleness(redis, deps) {
     const { orchestrator } = deps;
     if (!orchestrator) return 0;
@@ -2094,22 +2162,43 @@ async function checkVersionStaleness(redis, deps) {
         for (const row of result.rows) {
             const key = `${row.book_id}|${row.chapter_id}|${row.scene_id}`;
             if (!sceneMap.has(key)) {
-                sceneMap.set(key, { bookId: row.book_id, chapterId: row.chapter_id, sceneId: row.scene_id, stale: false });
+                sceneMap.set(key, { bookId: row.book_id, chapterId: row.chapter_id, sceneId: row.scene_id, staleAssets: new Set() });
             }
             const entry = sceneMap.get(key);
-            if ((row.scene_content_version != null && row.content_version != null && row.scene_content_version < row.content_version) ||
-                (row.scene_audio_config_version != null && row.audio_config_version != null && row.scene_audio_config_version < row.audio_config_version)) {
-                entry.stale = true;
+            const contentStale = row.scene_content_version != null && row.content_version != null &&
+                row.scene_content_version < row.content_version;
+            const audioConfigStale = row.scene_audio_config_version != null && row.audio_config_version != null &&
+                row.scene_audio_config_version < row.audio_config_version;
+            if (row.asset_type === 'audio') {
+                if (contentStale || audioConfigStale) entry.staleAssets.add('audio');
+            } else if (row.asset_type === 'image' || row.asset_type === 'video') {
+                if (contentStale) entry.staleAssets.add(row.asset_type);
             }
         }
 
         let outdated = 0;
         for (const entry of sceneMap.values()) {
-            if (entry.stale) {
-                log(`[VERSION-STALE] ${entry.bookId}/${entry.chapterId}/${entry.sceneId}`);
-                await orchestrator.markDirtyScene(redis, entry.bookId, entry.chapterId, entry.sceneId);
-                outdated++;
+            if (entry.staleAssets.size === 0) continue;
+
+            let assetStates;
+            try {
+                assetStates = await state.getAssetStates(redis, entry.bookId, entry.chapterId, entry.sceneId);
+            } catch (stateErr) {
+                warn(`Version staleness: asset-state read failed for ${entry.bookId}/${entry.chapterId}/${entry.sceneId}: ${stateErr.message}`);
+                continue;
             }
+
+            const readyStale = [...entry.staleAssets]
+                .filter(t => assetStates?.[t] === state.AssetState.READY);
+            if (readyStale.length === 0) {
+                // Уже DIRTY/GENERATING/FAILED — регенерация инициирована или идёт;
+                // повторная маркировка была бы неидемпотентной и опасной.
+                continue;
+            }
+
+            log(`[VERSION-STALE] ${entry.bookId}/${entry.chapterId}/${entry.sceneId} → ${readyStale.join(', ')} (READY → DIRTY, previous artifact preserved until replacement)`);
+            await orchestrator.markDirtyScene(redis, entry.bookId, entry.chapterId, entry.sceneId, readyStale);
+            outdated++;
         }
         return outdated;
     } catch (err) {

package/src/runtime/runtime-scheduler.js

@@ -571,6 +571,8 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
 
     let dispatched = 0;
     let throttled = 0;
+    let anyCompleted = false;
+    const failures = [];
 
     // Dispatch ALL eligible stages in this tick.
     // Concurrency limits per stage type are handled by dispatch engine.
@@ -589,10 +591,29 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
             dispatched++;
         } else if (result.reason === 'backpressure' || result.reason === 'throttled') {
             throttled++;
+        } else if (result.completed) {
+            // Stage finished inside the executor (already_done / cache_hit /
+            // no_segments) — not a failure, nothing to dispatch.
+            anyCompleted = true;
+        } else {
+            // Stage 50: раньше per-stage отказ (circuit_open, retry budget,
+            // lease, generating_transition_failed, …) молча проглатывался и
+            // tick писал «undefined for <scene>» — ошибки исчезали из
+            // observability. Теперь причина всплывает в tick-сводку.
+            failures.push(`${stage}:${result.reason || 'unknown'}`);
         }
     }
 
-    return { dispatched, throttled, stages: stages.length };
+    if (dispatched > 0) {
+        return { dispatched, throttled, stages: stages.length };
+    }
+    if (failures.length > 0) {
+        return { dispatched: 0, throttled, stages: stages.length, success: false, reason: failures.join(', ') };
+    }
+    if (anyCompleted) {
+        return { dispatched: 0, throttled, completed: true, reason: 'stage_completed' };
+    }
+    return { dispatched: 0, throttled, stages: stages.length, success: true, reason: 'no_dispatch' };
 }
 
 // ======================================================
@@ -615,6 +636,7 @@ module.exports = {
     // Scheduling
     shouldScheduleAssets,
     detectVersionStale,
+    attemptDispatch,
     markVersionStaleDirty,
     getLayerConfig,
     checkChunksHaveImages,