@animastor/orchestration 0.1.7 → 0.2.0

package/src/runtime/runtime-scheduler.js

@@ -388,6 +388,7 @@ async function tick(redis, loadedBooks = {}) {
             throttled: 0,
             skipped: 0,
             completed: 0,
+            deferred: 0,
             errors: []
         };
 
@@ -466,6 +467,10 @@ async function tick(redis, loadedBooks = {}) {
                     summary.skipped++;
                 } else if (result.throttled > 0) {
                     summary.throttled += result.throttled;
+                } else if (result.deferred > 0) {
+                    // Stage 118 (M9): a bounded deferral is a normal, quiet
+                    // state — it must not pollute the tick error list.
+                    summary.deferred += result.deferred;
                 } else if (result.reason === 'backpressure' || result.reason === 'throttled') {
                     summary.throttled++;
                 } else if (result.success) {
@@ -480,7 +485,7 @@ async function tick(redis, loadedBooks = {}) {
         }
 
         log(`=== TICK COMPLETE ===`);
-        log(`Active: ${summary.totalActive}, Dispatched: ${summary.dispatched}, Skipped: ${summary.skipped}, Throttled: ${summary.throttled}, Completed: ${summary.completed}`);
+        log(`Active: ${summary.totalActive}, Dispatched: ${summary.dispatched}, Skipped: ${summary.skipped}, Throttled: ${summary.throttled}, Completed: ${summary.completed}, Deferred: ${summary.deferred}`);
 
         if (summary.errors.length > 0) {
             log(`Errors: ${summary.errors.length}`);
@@ -572,6 +577,9 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
     let dispatched = 0;
     let throttled = 0;
     let anyCompleted = false;
+    // Stage 118 (M9): bounded deferrals are counted separately — they are
+    // neither a dispatch nor an error.
+    let deferred = 0;
     const failures = [];
 
     // Dispatch ALL eligible stages in this tick.
@@ -595,6 +603,12 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
             // Stage finished inside the executor (already_done / cache_hit /
             // no_segments) — not a failure, nothing to dispatch.
             anyCompleted = true;
+        } else if (result.deferred) {
+            // Stage 118 (M9): a bounded deferral is NOT an error — nothing
+            // happened, the stage is waiting for capacity (or is already
+            // idempotently complete). Reporting it as a failure filled the
+            // tick summary with "errors" 12 times per minute per scene.
+            deferred++;
         } else {
             // Stage 50: раньше per-stage отказ (circuit_open, retry budget,
             // lease, generating_transition_failed, …) молча проглатывался и
@@ -607,6 +621,9 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
     if (dispatched > 0) {
         return { dispatched, throttled, stages: stages.length };
     }
+    if (deferred > 0 && failures.length === 0) {
+        return { dispatched: 0, throttled, deferred, stages: stages.length, success: true, reason: 'deferred' };
+    }
     if (failures.length > 0) {
         return { dispatched: 0, throttled, stages: stages.length, success: false, reason: failures.join(', ') };
     }

package/src/runtime/scene-window.js

@@ -158,6 +158,34 @@ async function getSceneFilesStatus(buildDir, bookId, chapterId, sceneId) {
         result.image.exists = files.some(f => f.startsWith(imagePrefix) && f.endsWith('.png'));
     } catch (_) {}
 
+    // ── Stage 143a (phase 2a · B1) ───────────────────────────────────
+    // A versioned-only build keeps its IU images under {build}/.versions/**,
+    // which this scan can never see. Only when the LEGACY scan came up empty
+    // do we ask the image facade for a published pointer; the facade is
+    // flag-gated, so flag OFF answers null with ZERO PostgreSQL calls and
+    // this function behaves exactly as before (condition 2).
+    //
+    // A pointer-store ERROR is logged and treated as "no proof" — image
+    // stays NOT existing. It is never read as "the file is there" (that
+    // would write a false READY) and never throws: this is an advisory
+    // checker whose five callers (checkSceneContentCache,
+    // restoreChunkStatusForScene, reconcileWindowStatuses, scene-restoration,
+    // rebuildWorkList) must keep running on a PG outage.
+    if (!result.image.exists) {
+        try {
+            const imageModule = lazyHostBinding('media.image');
+            if (imageModule && typeof imageModule.resolveVersionedSceneProof === 'function') {
+                const buildId = path.basename(path.resolve(buildDir));
+                const proof = await imageModule.resolveVersionedSceneProof(
+                    OUTPUT_DIR, buildId, bookId, chapterId, sceneId);
+                if (proof) result.image.exists = true;
+            }
+        } catch (err) {
+            warn(`image pointer proof unavailable for ${bookId}/${chapterId}/${sceneId} (${err.message}) — image treated as absent`);
+            result.image.exists = false;
+        }
+    }
+
     // Check video file
     const videoPath = path.join(buildDir, artifactNaming.sceneVideoName(bookId, chapterId, sceneId));
     result.video.exists = fs.existsSync(videoPath);

package/test/capacity-deferral.test.js

@@ -0,0 +1,265 @@
+// ======================================================
+// Stage 116 (D3) — CAPACITY DEFERRAL (bounded retry/backoff)
+// ======================================================
+// Regression net for the Stage 115 defect D3: a stage that could not enqueue
+// anything (no live compatible worker, or every job already pending in the
+// lane) was re-dispatched on EVERY scheduler tick (SCHEDULER_TICK_MS = 5 s).
+// Each attempt created a lease + dispatch metadata, the executor sent
+// nothing, the dispatch was cancelled and the FSM was rolled back — an
+// unbounded dispatch/cancel loop with lease churn, FSM churn and unbounded
+// event-journal growth.
+//
+// The fix: a bounded per-stage deferral. While the deferral window is open
+// the dispatch engine returns WITHOUT any side effect (no lease, no metadata,
+// no FSM write, no journal event), the stage stays PENDING and recoverable,
+// and the deferral is cleared by a successful dispatch or an explicit cancel.
+//
+// Isolated: in-memory Redis mock + stubbed host bindings. No production
+// Redis/PG, no hub, no network.
+
+const { expect } = require('chai');
+const { createMockRedis } = require('./mocks/redis-mock');
+const { bindHostModules, clearHostBindings } = require('../src/host/host-bindings');
+
+const BOOK = 'test-stage116-book';
+const CHAPTER = 'ch-stage116';
+const SCENE = 'sc-stage116';
+const BUILD = 'build_test-stage116-book';
+
+describe('Stage 116 (D3): capacity deferral (bounded retry/backoff)', () => {
+    let redis;
+    let dispatchEngine;
+    let journalEvents;
+    let executorCalls;
+    let executorResult;
+    let rollbacks;
+
+    beforeEach(async () => {
+        redis = createMockRedis();
+        journalEvents = [];
+        executorCalls = 0;
+        rollbacks = [];
+        executorResult = { dispatched: false, jobs: 0, reason: 'no_worker_available', retryable: true };
+
+        clearHostBindings();
+        bindHostModules({
+            journal: () => ({
+                appendSceneEvent: async (r, b, c, s, type, state, details) => {
+                    journalEvents.push({ type, state, details });
+                    return true;
+                },
+            }),
+            seams: () => ({
+                getOrchestrationSeam: (name) => {
+                    if (name === 'dispatchStage') {
+                        return async () => {
+                            executorCalls++;
+                            return executorResult;
+                        };
+                    }
+                    if (name === 'rollbackStageToPending') {
+                        return async (r, b, c, s, stage, buildId, reason) => {
+                            rollbacks.push({ stage, reason });
+                            return { changed: true };
+                        };
+                    }
+                    throw new Error(`[stage116-test] seam not wired: ${name}`);
+                },
+            }),
+        });
+
+        dispatchEngine = require('../src/runtime/dispatch-engine');
+    });
+
+    afterEach(() => {
+        clearHostBindings();
+    });
+
+    const dispatch = () => dispatchEngine.dispatchStage(
+        redis, BOOK, CHAPTER, SCENE, 'image', { book_id: BOOK }, BUILD
+    );
+
+    it('a stage that enqueued nothing is deferred and reported as retryable', async () => {
+        const result = await dispatch();
+        expect(result.dispatched).to.equal(false);
+        expect(result.reason).to.equal('no_worker_available');
+        expect(result.retryable).to.equal(true);
+        expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(true);
+    });
+
+    it('a deferred stage is NOT re-dispatched on the next tick (no side effects)', async () => {
+        await dispatch();                       // attempt 1 — really attempted
+        const callsAfterFirst = executorCalls;
+        const journalAfterFirst = journalEvents.length;
+
+        // the next scheduler tick (5 s later) must be a no-op
+        const second = await dispatch();
+        expect(second.dispatched).to.equal(false);
+        expect(second.reason).to.equal('deferred_capacity');
+        expect(second.retryable).to.equal(true);
+        expect(executorCalls).to.equal(callsAfterFirst);          // executor NOT called
+        expect(journalEvents.length).to.equal(journalAfterFirst); // no journal event
+    });
+
+    it('100 scheduler ticks inside the deferral window ⇒ ONE attempt (bounded churn)', async () => {
+        await dispatch();
+        for (let i = 0; i < 99; i++) await dispatch();
+        expect(executorCalls).to.equal(1);
+        // the journal must not grow with the tick count (bounded, not unbounded)
+        const journalAfterLoop = journalEvents.length;
+        expect(journalAfterLoop).to.be.lessThan(10);
+    });
+
+    it('the deferral window expires ⇒ the stage is dispatched again', async () => {
+        await dispatch();
+        expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(true);
+        // simulate the TTL elapsing (the mock keeps keys until told otherwise)
+        await redis.del(dispatchEngine.deferredStageKey(BOOK, CHAPTER, SCENE, 'image'));
+        expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(false);
+
+        executorResult = { dispatched: true, jobs: 2, reason: null };
+        const result = await dispatch();
+        expect(result.dispatched).to.equal(true);
+        expect(executorCalls).to.equal(2);
+    });
+
+    it('a successful dispatch clears the deferral', async () => {
+        await dispatch();                       // deferred
+        expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(true);
+        await redis.del(dispatchEngine.deferredStageKey(BOOK, CHAPTER, SCENE, 'image'));
+        executorResult = { dispatched: true, jobs: 1, reason: null };
+        await dispatch();
+        expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(false);
+    });
+
+    it('an explicit cancel clears the deferral (the user asked for the work)', async () => {
+        await dispatch();
+        expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(true);
+        await dispatchEngine.cancelActiveDispatch(redis, BOOK, CHAPTER, SCENE, 'image', 'user_cancel');
+        expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(false);
+    });
+
+    it('a deferred attempt does NOT consume the retry budget (it is not a failure)', async () => {
+        const budgetKey = `animastor:retry-budget:${BOOK}:${CHAPTER}:${SCENE}:image`;
+        await redis.set(budgetKey, '3');
+        await dispatch();
+        const after = await redis.get(budgetKey);
+        expect(after).to.equal('3');     // unchanged — no budget consumed
+    });
+
+    it('a deferred attempt is not reported as a failure (retryable, not fatal)', async () => {
+        const result = await dispatch();
+        expect(result.dispatched).to.equal(false);
+        expect(result.reason).to.equal('no_worker_available');
+        expect(result.retryable).to.equal(true);
+        // the dispatch engine must not have marked the stage as failed
+        expect(result.result && result.result.retryable).to.equal(true);
+    });
+
+    it('the deferral is per stage — another stage of the same scene is not blocked', async () => {
+        await dispatch();                       // image stage deferred
+        expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(true);
+        expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'video')).to.equal(false);
+        expect(await dispatchEngine.isStageDeferred(redis, 'other-book', CHAPTER, SCENE, 'image')).to.equal(false);
+    });
+
+    it('deferrable outcomes are classified explicitly (capacity only)', () => {
+        expect(dispatchEngine.isDeferrableOutcome('no_worker_available')).to.equal(true);
+        expect(dispatchEngine.isDeferrableOutcome('duplicate_pending')).to.equal(true);
+        expect(dispatchEngine.isDeferrableOutcome('ambiguous_post')).to.equal(true);
+        // Stage 118 (M4): 'already_completed' is NOT a capacity gap — the
+        // logical task is already complete. It stays deferrable (the stage
+        // must not spin) but in its OWN bucket with its own window.
+        expect(dispatchEngine.isCapacityOutcome('already_completed')).to.equal(false);
+        expect(dispatchEngine.isIdempotentOutcome('already_completed')).to.equal(true);
+        expect(dispatchEngine.isDeferrableOutcome('already_completed')).to.equal(true);
+        // the generic "sent nothing" outcomes keep their historic contract
+        // (immediate retry) — the audit-6929ba5 regressions depend on it
+        expect(dispatchEngine.isDeferrableOutcome('no_jobs_sent')).to.equal(false);
+        expect(dispatchEngine.isDeferrableOutcome('no_jobs_generated')).to.equal(false);
+        expect(dispatchEngine.isDeferrableOutcome('send_failed')).to.equal(false);
+        expect(dispatchEngine.isDeferrableOutcome('already_ready')).to.equal(false);
+        expect(dispatchEngine.isDeferrableOutcome('circuit_open')).to.equal(false);
+    });
+
+    it('the deferral window is bounded (not a permanent block)', () => {
+        expect(dispatchEngine.DEFERRED_STAGE_TTL_S).to.be.a('number');
+        expect(dispatchEngine.DEFERRED_STAGE_TTL_S).to.be.greaterThan(0);
+        expect(dispatchEngine.DEFERRED_STAGE_TTL_S).to.be.lessThanOrEqual(600);
+        // the key lives under the runtime namespace (backend-owned family)
+        expect(dispatchEngine.DEFERRED_STAGE_KEY_PREFIX).to.equal('animastor:runtime:deferred:');
+    });
+
+    // ── Stage 118 (audit 117) ─────────────────────────────────────────────
+    describe('Stage 118: audit 117 corrections', () => {
+        it('M2: force mode does NOT bypass the deferral', async () => {
+            await dispatch();                       // deferred (capacity)
+            expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(true);
+            const callsBefore = executorCalls;
+            // The force-dispatch flag lives for 120 s and is set for the WHOLE
+            // book by /regenerate — bypassing the backoff there reproduced the
+            // very churn D3 removes (≈24 full attempts per scene).
+            const forced = await dispatchEngine.dispatchStage(
+                redis, BOOK, CHAPTER, SCENE, 'image', { book_id: BOOK }, BUILD, { force: true }
+            );
+            expect(forced.dispatched).to.equal(false);
+            expect(forced.reason).to.equal('deferred_capacity');
+            expect(forced.deferred).to.equal(true);
+            expect(executorCalls).to.equal(callsBefore);   // no side effect
+        });
+
+        it('M4: the idempotent suppression gets its OWN (longer) window', async () => {
+            executorResult = { dispatched: false, jobs: 0, reason: 'already_completed', retryable: true };
+            await dispatch();
+            expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(true);
+            expect(dispatchEngine.deferralWindowFor('already_completed'))
+                .to.equal(dispatchEngine.DEFERRED_IDEMPOTENT_TTL_S);
+            expect(dispatchEngine.deferralWindowFor('no_worker_available'))
+                .to.equal(dispatchEngine.DEFERRED_STAGE_TTL_S);
+            expect(dispatchEngine.DEFERRED_IDEMPOTENT_TTL_S)
+                .to.be.greaterThan(dispatchEngine.DEFERRED_STAGE_TTL_S);
+        });
+
+        it('M1: a book_id containing ":" cannot reach into another book key space', async () => {
+            const bookA = 'a';
+            const bookB = 'a:b';
+            const chapter = 'ch';
+            const scene = 'sc';
+            // the deferral key of book 'a' / chapter 'b' must not collide
+            // with the key of book 'a:b'
+            executorResult = { dispatched: false, jobs: 0, reason: 'no_worker_available', retryable: true };
+            await dispatchEngine.dispatchStage(redis, bookA, chapter, scene, 'image', { book_id: bookA }, BUILD);
+            expect(await dispatchEngine.isStageDeferred(redis, bookA, chapter, scene, 'image')).to.equal(true);
+            expect(await dispatchEngine.isStageDeferred(redis, bookB, chapter, scene, 'image')).to.equal(false);
+            expect(dispatchEngine.deferredStageKey(bookA, chapter, scene, 'image'))
+                .to.not.equal(dispatchEngine.deferredStageKey(bookB, chapter, scene, 'image'));
+        });
+
+        it('M2: an explicit intent clears the backoff of a book / a scene', async () => {
+            await dispatch();
+            expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(true);
+            const cleared = await dispatchEngine.clearDeferralsForBook(redis, BOOK);
+            expect(cleared).to.be.greaterThan(0);
+            expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(false);
+
+            await dispatch();
+            expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(true);
+            await dispatchEngine.clearDeferralsForScene(redis, BOOK, CHAPTER, SCENE);
+            expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(false);
+            // a sibling book is untouched
+            await dispatchEngine.dispatchStage(redis, 'other-book', CHAPTER, SCENE, 'image', { book_id: 'other-book' }, BUILD);
+            await dispatchEngine.clearDeferralsForBook(redis, 'other-book');
+            expect(await dispatchEngine.isStageDeferred(redis, BOOK, CHAPTER, SCENE, 'image')).to.equal(false);
+        });
+
+        it('M3: several deferrable reasons are classified (the dominant one is reported)', () => {
+            const { classifyNotSentReasons } = require('../src/runtime/dispatch-outcomes');
+            expect(classifyNotSentReasons(new Map([['no_worker_available', 3], ['duplicate_pending', 1]])))
+                .to.equal('no_worker_available');
+            // a real failure keeps the historic semantics
+            expect(classifyNotSentReasons(new Map([['no_worker_available', 3], ['send_failed', 1]])))
+                .to.equal(null);
+            expect(classifyNotSentReasons(new Map())).to.equal(null);
+        });
+    });
+});