@animastor/orchestration 0.1.3 → 0.1.5

package/src/runtime/dispatch-outcomes.js

@@ -0,0 +1,124 @@
+// ======================================================
+// DISPATCH OUTCOMES — the shared taxonomy of "nothing was enqueued"
+//
+// Stage 118 (audit 117 · D3/M4/M5): the classification of an executor
+// outcome lives in ONE module so the dispatch engine (runtime tier) and the
+// stage executors (orchestration tier) cannot drift apart.
+//
+// Buckets
+// ───────
+// • capacity   — the GPU capacity is not there right now:
+//                'no_worker_available' | 'duplicate_pending' | 'ambiguous_post'
+//                ⇒ a BOUNDED retry (deferral window) is the right response.
+//
+// • idempotent — the logical task is ALREADY COMPLETE (the completion marker
+//                exists): 'already_completed'.
+//                This is NOT a capacity problem and MUST NOT be classified as
+//                one (audit 117 · M4): mixing it into the capacity bucket
+//                turned "this task will never run" into an endless retry
+//                loop. It gets its own, much longer backoff window and its
+//                own reporting class.
+//
+// The host side (@animastor/generation → backend/src/helpers/queue-idempotency)
+// mirrors these sets for the transport-level reasons it produces; the two
+// lists must stay in sync (see queue-idempotency.cjs · CAPACITY_REASONS).
+// ======================================================
+
+const CAPACITY_OUTCOMES = new Set([
+    'no_worker_available',   // no live, type-compatible worker
+    'duplicate_pending',     // the job is already pending in the lane
+    'ambiguous_post',        // the POST outcome was ambiguous (the hub may have taken the task)
+    // Stage 129: the intent-fence step could not run (Redis refused the atomic
+    // path / the client cannot run it). Nothing was enqueued and the send IS
+    // retryable, so it belongs to the bounded-retry bucket.
+    'intent_receipt_unavailable',
+]);
+
+const IDEMPOTENT_OUTCOMES = new Set([
+    'already_completed',     // the logical task is already complete (artifact missing)
+]);
+
+/**
+ * Fence rejections (Stage 129, audit 128 · F5): the backend refused this
+ * dispatch because it belongs to a generation the intent record already
+ * superseded, or because the dispatch/intent pair is inconsistent.
+ *
+ * Retrying the SAME generation can never succeed, so they are NOT retryable —
+ * but they must also not fall into the historic "immediate retry on the next
+ * tick" bucket, which would turn a stale send into a 5-second dispatch →
+ * cancel → rollback loop. They get the long (idempotent) deferral window and
+ * stay visible in the logs/metrics.
+ */
+const FENCE_REJECTED_OUTCOMES = new Set([
+    'stale_intent',
+    'receipt_conflict',
+    'intent_not_active',
+    'intent_corrupt',
+    'intent_missing',
+]);
+
+/** Is this outcome a CAPACITY gap (bounded retry with a short window)? */
+function isCapacityOutcome(reason) {
+    return CAPACITY_OUTCOMES.has(String(reason || ''));
+}
+
+/** Is this outcome an idempotent suppression (the task is already done)? */
+function isIdempotentOutcome(reason) {
+    return IDEMPOTENT_OUTCOMES.has(String(reason || ''));
+}
+
+/** Is this outcome a fence rejection (terminal for this generation)? */
+function isFenceRejectedOutcome(reason) {
+    return FENCE_REJECTED_OUTCOMES.has(String(reason || ''));
+}
+
+/** Is this outcome deferrable at all (capacity, idempotent OR fence-rejected)? */
+function isDeferrableOutcome(reason) {
+    return isCapacityOutcome(reason) || isIdempotentOutcome(reason) || isFenceRejectedOutcome(reason);
+}
+
+/**
+ * Classify the reasons why NOTHING was enqueued for a stage.
+ *
+ * Stage 118 (audit 117 · M3): the Stage 116 implementation required EXACTLY
+ * ONE distinct reason (`notSentReasons.size === 1`). A scene whose units
+ * failed for two DIFFERENT deferrable reasons (e.g. one unit gated by
+ * "no live worker" and another suppressed as "duplicate pending") was
+ * therefore NOT deferred and kept the 5-second dispatch/cancel churn.
+ *
+ * The rule now: EVERY reason must be deferrable; the most frequent one is
+ * reported. A single non-deferrable reason (a real failure) keeps the
+ * historic semantics (an immediate retry on the next tick) — the
+ * audit-6929ba5 contract depends on it.
+ *
+ * @param {Map<string, number>|Object<string, number>} reasons
+ * @returns {string|null} the dominant deferrable reason, or null.
+ */
+function classifyNotSentReasons(reasons) {
+    const entries = reasons instanceof Map
+        ? [...reasons.entries()]
+        : Object.entries(reasons || {});
+    if (entries.length === 0) return null;
+    if (!entries.every(([reason]) => isDeferrableOutcome(reason))) return null;
+    let dominant = null;
+    let dominantCount = -1;
+    for (const [reason, count] of entries) {
+        const n = Number(count) || 0;
+        if (n > dominantCount) {
+            dominant = reason;
+            dominantCount = n;
+        }
+    }
+    return dominant;
+}
+
+module.exports = {
+    CAPACITY_OUTCOMES,
+    IDEMPOTENT_OUTCOMES,
+    FENCE_REJECTED_OUTCOMES,
+    isCapacityOutcome,
+    isIdempotentOutcome,
+    isFenceRejectedOutcome,
+    isDeferrableOutcome,
+    classifyNotSentReasons,
+};

package/src/runtime/hub-cancel-port.js

@@ -30,6 +30,12 @@
 //                        a cancelled dispatch still owns on the GPU Hub
 //                        (orchestrator resetScenes cancel leg,
 //                        reconciliation RELEASE_STALE_LEASE recovery leg).
+//                        Accepts EITHER an array of dispatch ids (one
+//                        `?dispatch_id=` DELETE per id) OR an empty list with
+//                        `options.book_id` (Stage 122 · N1 — a single
+//                        book-scoped `?book_id=` sweep, used by the
+//                        regenerate post-release lane sweep; audit 121 found
+//                        the empty-list + book_id form was a silent no-op).
 //                        Returns { requested, cleared, failed }; hub
 //                        unavailability is non-fatal per-id (warn + count),
 //                        never throws for a single failed id.

package/src/runtime/persistence-port.js

@@ -82,6 +82,12 @@ const OPS = [
     'registry.registerSceneImageRedis',
     'registry.registerSceneVideoRedis',
 
+    // Stage 122 (N2 · audit 121): the video send path frees an ORPHANED
+    // completion marker through this op (Redis-domain, `redis`-first like the
+    // registry.*Redis ops) — the orchestration tier cannot require the backend
+    // queue-idempotency helper directly.
+    'idem.releaseTaskDoneIfOrphan',
+
     'filesystem.getSceneAudioPath',
 
     'listBooks',

package/src/runtime/reconciliation-engine.js

@@ -15,9 +15,12 @@ const fs = require('fs').promises;
 const syncFs = require('fs');
 const syncPath = require('path');
 
-const { lazyHostBinding } = require('../host/host-bindings');
+const { lazyHostBinding, hostBinding } = require('../host/host-bindings');
 // §32.30: former host requires resolve lazily through the host-bindings seam.
 const state = lazyHostBinding('state');
+// §32.30: the artifact root resolves through the composition-root seam at
+// call time (Stage 96: former hardcoded artifact root closed here).
+const artifactRoot = () => hostBinding('artifactRoot');
 // S-4: filename grammar composed from the canonical owner (bytes unchanged)
 const artifactNaming = require('@animastor/generation').artifactNaming;
 // S-2: registered stage list resolved from media registry
@@ -121,7 +124,7 @@ async function checkOrphanVideoState(redis, bookId, chapterId, sceneId) {
     }
 
     const buildId = resolveBookBuildId(bookId);
-    const videoPath = `/data/output/${buildId}/${bookId}_${chapterId}_${sceneId}.mp4`;
+    const videoPath = `${artifactRoot()}/${buildId}/${bookId}_${chapterId}_${sceneId}.mp4`;
 
     try {
         await fs.access(videoPath);
@@ -154,14 +157,39 @@ async function checkOrphanImageState(redis, bookId, chapterId, sceneId) {
     // Orphan-check uses the canonical book build identity (audit d9d67a3).
     const buildId = resolveBookBuildId(bookId);
     const imageModule = lazyHostBinding('media.image');
-    const imageInfo = imageModule.resolveCanonicalSceneImage(
-        '/data/output',
+    let imageInfo = imageModule.resolveCanonicalSceneImage(
+        artifactRoot(),
         buildId,
         bookId,
         chapterId,
         sceneId,
     );
 
+    // ── Stage 143a (phase 2a · B2) ───────────────────────────────────
+    // Legacy scan first (byte-identical legacy behavior). A versioned-only
+    // build never writes the canonical file, so with the scan empty we ask
+    // the image facade for the published pointer of this scene build — the
+    // facade is flag-gated, flag OFF answers null with ZERO PG calls.
+    //
+    // An UNREADABLE pointer store ABORTS this reconcile pass (logged,
+    // `pass_incomplete`) instead of being read as "no publication": that
+    // reading would recommend regenerating a scene that is already
+    // published. No orphan issue is emitted on the aborted pass, so no
+    // regeneration is launched.
+    if (!imageInfo) {
+        try {
+            if (typeof imageModule.resolveVersionedSceneProof === 'function') {
+                imageInfo = await imageModule.resolveVersionedSceneProof(
+                    artifactRoot(), buildId, bookId, chapterId, sceneId);
+            }
+        } catch (pgErr) {
+            error(`versioned image proof unavailable for ${bookId}/${chapterId}/${sceneId}: ${pgErr.message} — pass skipped, no regeneration launched`);
+            const err = new Error(`reconcile_pass_incomplete: image proof unavailable for ${bookId}/${chapterId}/${sceneId} (${pgErr.message})`);
+            err.code = 'pass_incomplete';
+            throw err;
+        }
+    }
+
     if (!imageInfo) {
         return {
             type: 'orphan_image_state',
@@ -248,11 +276,37 @@ async function checkOrphanAssets(redis, bookId, chapterId, sceneId) {
         try {
             await fs.access(assets.image.path);
         } catch (err) {
-            orphanAssets.push({
-                type: 'missing_image_file',
-                scene: { bookId, chapterId, sceneId },
-                path: assets.image.path
-            });
+            // ── Stage 143a (phase 2a · B2) ───────────────────────────
+            // With the flag ON a published pointer is the authority: a LIVE
+            // versioned file for this scene build explains the missing
+            // canonical file (versioned-only build) → no issue is emitted.
+            // Flag OFF ⇒ the facade answers null with ZERO PG calls and the
+            // legacy issue is reported exactly as before.
+            // An UNREADABLE pointer store aborts the pass (logged) instead of
+            // emitting a regeneration recommendation on a guess.
+            let coveredByPointer = false;
+            const imageModule = lazyHostBinding('media.image');
+            try {
+                if (typeof imageModule.resolveVersionedSceneProof === 'function') {
+                    const buildId = resolveBookBuildId(bookId);
+                    coveredByPointer = !!(await imageModule.resolveVersionedSceneProof(
+                        artifactRoot(), buildId, bookId, chapterId, sceneId));
+                }
+            } catch (pgErr) {
+                error(`versioned image proof unavailable for ${bookId}/${chapterId}/${sceneId}: ${pgErr.message} — pass skipped, no recommendation emitted`);
+                const e = new Error(`reconcile_pass_incomplete: image proof unavailable for ${bookId}/${chapterId}/${sceneId} (${pgErr.message})`);
+                e.code = 'pass_incomplete';
+                throw e;
+            }
+            if (coveredByPointer) {
+                log(`missing_image_file suppressed: ${bookId}/${chapterId}/${sceneId} is covered by a published versioned pointer`);
+            } else {
+                orphanAssets.push({
+                    type: 'missing_image_file',
+                    scene: { bookId, chapterId, sceneId },
+                    path: assets.image.path
+                });
+            }
         }
     }
 

package/src/runtime/runtime-scheduler.js

@@ -388,6 +388,7 @@ async function tick(redis, loadedBooks = {}) {
             throttled: 0,
             skipped: 0,
             completed: 0,
+            deferred: 0,
             errors: []
         };
 
@@ -466,6 +467,10 @@ async function tick(redis, loadedBooks = {}) {
                     summary.skipped++;
                 } else if (result.throttled > 0) {
                     summary.throttled += result.throttled;
+                } else if (result.deferred > 0) {
+                    // Stage 118 (M9): a bounded deferral is a normal, quiet
+                    // state — it must not pollute the tick error list.
+                    summary.deferred += result.deferred;
                 } else if (result.reason === 'backpressure' || result.reason === 'throttled') {
                     summary.throttled++;
                 } else if (result.success) {
@@ -480,7 +485,7 @@ async function tick(redis, loadedBooks = {}) {
         }
 
         log(`=== TICK COMPLETE ===`);
-        log(`Active: ${summary.totalActive}, Dispatched: ${summary.dispatched}, Skipped: ${summary.skipped}, Throttled: ${summary.throttled}, Completed: ${summary.completed}`);
+        log(`Active: ${summary.totalActive}, Dispatched: ${summary.dispatched}, Skipped: ${summary.skipped}, Throttled: ${summary.throttled}, Completed: ${summary.completed}, Deferred: ${summary.deferred}`);
 
         if (summary.errors.length > 0) {
             log(`Errors: ${summary.errors.length}`);
@@ -572,6 +577,9 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
     let dispatched = 0;
     let throttled = 0;
     let anyCompleted = false;
+    // Stage 118 (M9): bounded deferrals are counted separately — they are
+    // neither a dispatch nor an error.
+    let deferred = 0;
     const failures = [];
 
     // Dispatch ALL eligible stages in this tick.
@@ -595,6 +603,12 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
             // Stage finished inside the executor (already_done / cache_hit /
             // no_segments) — not a failure, nothing to dispatch.
             anyCompleted = true;
+        } else if (result.deferred) {
+            // Stage 118 (M9): a bounded deferral is NOT an error — nothing
+            // happened, the stage is waiting for capacity (or is already
+            // idempotently complete). Reporting it as a failure filled the
+            // tick summary with "errors" 12 times per minute per scene.
+            deferred++;
         } else {
             // Stage 50: раньше per-stage отказ (circuit_open, retry budget,
             // lease, generating_transition_failed, …) молча проглатывался и
@@ -607,6 +621,9 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
     if (dispatched > 0) {
         return { dispatched, throttled, stages: stages.length };
     }
+    if (deferred > 0 && failures.length === 0) {
+        return { dispatched: 0, throttled, deferred, stages: stages.length, success: true, reason: 'deferred' };
+    }
     if (failures.length > 0) {
         return { dispatched: 0, throttled, stages: stages.length, success: false, reason: failures.join(', ') };
     }

package/src/runtime/scene-window.js

@@ -158,6 +158,34 @@ async function getSceneFilesStatus(buildDir, bookId, chapterId, sceneId) {
         result.image.exists = files.some(f => f.startsWith(imagePrefix) && f.endsWith('.png'));
     } catch (_) {}
 
+    // ── Stage 143a (phase 2a · B1) ───────────────────────────────────
+    // A versioned-only build keeps its IU images under {build}/.versions/**,
+    // which this scan can never see. Only when the LEGACY scan came up empty
+    // do we ask the image facade for a published pointer; the facade is
+    // flag-gated, so flag OFF answers null with ZERO PostgreSQL calls and
+    // this function behaves exactly as before (condition 2).
+    //
+    // A pointer-store ERROR is logged and treated as "no proof" — image
+    // stays NOT existing. It is never read as "the file is there" (that
+    // would write a false READY) and never throws: this is an advisory
+    // checker whose five callers (checkSceneContentCache,
+    // restoreChunkStatusForScene, reconcileWindowStatuses, scene-restoration,
+    // rebuildWorkList) must keep running on a PG outage.
+    if (!result.image.exists) {
+        try {
+            const imageModule = lazyHostBinding('media.image');
+            if (imageModule && typeof imageModule.resolveVersionedSceneProof === 'function') {
+                const buildId = path.basename(path.resolve(buildDir));
+                const proof = await imageModule.resolveVersionedSceneProof(
+                    OUTPUT_DIR, buildId, bookId, chapterId, sceneId);
+                if (proof) result.image.exists = true;
+            }
+        } catch (err) {
+            warn(`image pointer proof unavailable for ${bookId}/${chapterId}/${sceneId} (${err.message}) — image treated as absent`);
+            result.image.exists = false;
+        }
+    }
+
     // Check video file
     const videoPath = path.join(buildDir, artifactNaming.sceneVideoName(bookId, chapterId, sceneId));
     result.video.exists = fs.existsSync(videoPath);

package/test/artifact-root-binding.test.js

@@ -0,0 +1,174 @@
+// ======================================================
+// STAGE 96 — ARTIFACT ROOT BINDING (поведение закрытых литералов)
+// ======================================================
+// Четыре production-литерала '/data/output' в orchestration-пакете
+// (scene-callbacks.js:281,393 · reconciliation-engine.js:124,158)
+// переведены на существующий шов artifactRoot (host-bindings, §32.30).
+//
+// Тесты проверяют ПОВЕДЕНИЕ на свежих исходниках пакета (src/, а не
+// npm-копия в backend/node_modules):
+//   — путь строится из привязанного корня, а не из литерала;
+//   — при корне '/data/output' (контракт сохранённых данных PG/Redis)
+//     результирующая строка байт-в-байт равна прежней.
+// ======================================================
+
+const { expect } = require('chai');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { bindHostModules } = require('../src/host/host-bindings');
+const { createMockRedis } = require('./mocks/redis-mock');
+
+const engine = require('../src/runtime/reconciliation-engine');
+const callbacks = require('../src/orchestration/scene-callbacks');
+
+const BOOK = 'b1';
+const CHAPTER = 'ch1';
+const SCENE = 'sc1';
+
+let tmpRoot;
+let redis;
+let currentStates;
+
+const imageStub = {
+    resolveCanonicalSceneImage: () => null,
+    getImageMetadata: async () => null,
+};
+const videoStub = {
+    validateVideoFile: () => ({ valid: false, duration: null, metadata: null }),
+};
+
+const stateStub = {
+    AssetState: {
+        NEW: 'new', DIRTY: 'dirty', PENDING: 'pending',
+        GENERATING: 'generating', READY: 'ready',
+        FAILED: 'failed', PLACEHOLDER: 'placeholder',
+    },
+    getAssetStates: async () => ({ audio: 'new', ...currentStates }),
+};
+
+const failFast = (name) => () => {
+    throw new Error(`[ARTIFACT-ROOT-TEST] ${name} not wired`);
+};
+
+function wire(artifactRootValue) {
+    bindHostModules({
+        state: () => stateStub,
+        media: () => ({ audio: {}, image: imageStub, video: videoStub }),
+        journal: () => ({ appendSceneEvent: async () => {} }),
+        artifactRoot: () => artifactRootValue,
+    });
+}
+
+describe('Stage 96 — artifactRoot-шов вместо литералов', () => {
+    before(() => {
+        tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'stage96-artifact-root-'));
+        fs.mkdirSync(path.join(tmpRoot, 'default'), { recursive: true });
+        redis = createMockRedis();
+        wire(tmpRoot);
+    });
+
+    after(() => {
+        // Вернуть дефолтный fail-fast, как в test/test-bindings.js
+        bindHostModules({
+            state: failFast('state'),
+            media: failFast('media'),
+            journal: failFast('journal'),
+            artifactRoot: failFast('artifactRoot'),
+        });
+        fs.rmSync(tmpRoot, { recursive: true, force: true });
+    });
+
+    describe('reconciliation-engine — checkOrphanVideoState', () => {
+        it('missingFile строится из привязанного корня, а не из литерала', async () => {
+            currentStates = { video: 'ready' };
+            const res = await engine.checkOrphanVideoState(redis, BOOK, CHAPTER, SCENE);
+            expect(res).to.be.an('object');
+            expect(res.missingFile).to.equal(`${tmpRoot}/default/${BOOK}_${CHAPTER}_${SCENE}.mp4`);
+        });
+
+        it('null, когда файл существует ПОД ПРИВЯЗАННЫМ корнем', async () => {
+            const file = path.join(tmpRoot, 'default', `${BOOK}_${CHAPTER}_${SCENE}.mp4`);
+            fs.writeFileSync(file, 'x');
+            try {
+                currentStates = { video: 'ready' };
+                expect(await engine.checkOrphanVideoState(redis, BOOK, CHAPTER, SCENE)).to.be.null;
+            } finally {
+                fs.rmSync(file, { force: true });
+            }
+        });
+    });
+
+    describe('reconciliation-engine — checkOrphanImageState', () => {
+        it('resolveCanonicalSceneImage получает привязанный корень', async () => {
+            let seenRoot = null;
+            imageStub.resolveCanonicalSceneImage = (root) => { seenRoot = root; return null; };
+            try {
+                currentStates = { image: 'ready' };
+                const res = await engine.checkOrphanImageState(redis, BOOK, CHAPTER, SCENE);
+                expect(seenRoot).to.equal(tmpRoot);
+                expect(res).to.be.an('object');
+                expect(res.type).to.equal('orphan_image_state');
+            } finally {
+                imageStub.resolveCanonicalSceneImage = () => null;
+            }
+        });
+    });
+
+    describe('scene-callbacks — handleVideoCompleted', () => {
+        it('validateVideoFile получает путь под привязанным корнем', async () => {
+            let seenPath = null;
+            videoStub.validateVideoFile = (p) => {
+                seenPath = p;
+                return { valid: false, duration: null, metadata: null };
+            };
+            try {
+                currentStates = { video: 'generating' };
+                const res = await callbacks.handleVideoCompleted(redis, BOOK, CHAPTER, SCENE, 'build1');
+                expect(seenPath).to.equal(`${tmpRoot}/build1/${BOOK}_${CHAPTER}_${SCENE}.mp4`);
+                expect(res).to.deep.include({ ok: false, retryable: true, reason: 'video_invalid' });
+            } finally {
+                videoStub.validateVideoFile = () => ({ valid: false, duration: null, metadata: null });
+            }
+        });
+    });
+
+    describe('scene-callbacks — handleImageCompleted', () => {
+        it('resolveCanonicalSceneImage получает привязанный корень', async () => {
+            let seenRoot = null;
+            imageStub.resolveCanonicalSceneImage = (root) => { seenRoot = root; return null; };
+            try {
+                currentStates = { image: 'generating' };
+                const res = await callbacks.handleImageCompleted(redis, BOOK, CHAPTER, SCENE, 'build1');
+                expect(seenRoot).to.equal(tmpRoot);
+                expect(res).to.deep.include({ ok: false, retryable: true, reason: 'image_not_found' });
+            } finally {
+                imageStub.resolveCanonicalSceneImage = () => null;
+            }
+        });
+    });
+
+    describe('обратная совместимость (контракт контейнерных путей)', () => {
+        it('при корне /data/output строка байт-в-байт равна прежнему литералу', async () => {
+            wire('/data/output');
+            try {
+                let seenPath = null;
+                videoStub.validateVideoFile = (p) => {
+                    seenPath = p;
+                    return { valid: false, duration: null, metadata: null };
+                };
+                currentStates = { video: 'generating' };
+                await callbacks.handleVideoCompleted(redis, BOOK, CHAPTER, SCENE, 'build1');
+                // ровно то, что до Stage 96 формировал шаблон
+                expect(seenPath).to.equal(`/data/output/build1/${BOOK}_${CHAPTER}_${SCENE}.mp4`);
+
+                currentStates = { video: 'ready' };
+                const orphan = await engine.checkOrphanVideoState(redis, BOOK, CHAPTER, SCENE);
+                expect(orphan.missingFile).to.equal(`/data/output/default/${BOOK}_${CHAPTER}_${SCENE}.mp4`);
+            } finally {
+                videoStub.validateVideoFile = () => ({ valid: false, duration: null, metadata: null });
+                wire(tmpRoot);
+            }
+        });
+    });
+});