@animastor/orchestration 0.1.7 → 0.3.0

package/src/runtime/command-gate.js

@@ -0,0 +1,270 @@
+// ======================================================
+// COMMAND GATE — «генерация только по явной команде»
+// ======================================================
+// Правило владельца (2026-10-31, разбор reports/2026-10-31/
+// auto-image-task-ba-incident.md §6/§8): GPU-генерация запускается ТОЛЬКО
+// кнопками «аудио» / «картинки» / «всё». Никаких автозапусков следующих
+// стадий, никаких фоновых запусков, задачи только на включённые слои.
+//
+// ИНВАРИАНТ. Сцена×стадия попадает в active index / dispatch ТОЛЬКО когда
+// одновременно:
+//   1. есть АКТИВНАЯ задача реестра (generation-progress, создаётся явной
+//      командой POST /regenerate), в targets которой есть эта сцена и чей
+//      type — эта стадия;
+//   2. слой стадии включён в layer-config книги;
+//   3. у книги нет отмены (cancel flag в Redis / tombstone в PG).
+//
+// Этот модуль — единственное место, где инвариант вычисляется. Им пользуются:
+//   • active-scenes-index.addActiveScene  — запись в active index;
+//   • runtime-scheduler.shouldScheduleAssets — выбор стадий для dispatch;
+//   • reconciliation-engine — ремонтные фиксы, которые раньше ставили работу;
+//   • scene-window.startScene — оконный старт сцены.
+//
+// Реестр задач читается ТОЛЬКО через ProgressEventsPort (O-5), отмена — через
+// PersistencePort (O-2) и cancel flag scene-window; Redis-ключи хост-сервисов
+// здесь не появляются. Ошибка чтения реестра — fail-closed (стадия НЕ
+// авторизована): без доказанной команды работы нет.
+//
+// Docs: animastor-docs/architecture/COMMAND_ONLY_GENERATION.md
+// ======================================================
+
+const mediaRegistry = require('@animastor/generation').mediaRegistry;
+const { lazyHostBinding } = require('../host/host-bindings');
+const state = lazyHostBinding('state');
+const progressEventsOp = require('./progress-events-port').progressEventsOp;
+const persist = require('./persistence-port').persist;
+
+const logPrefix = '[COMMAND-GATE]';
+
+function log(msg) {
+    console.log(`${logPrefix} ${msg}`);
+}
+
+function warn(msg) {
+    console.warn(`${logPrefix} ⚠️ ${msg}`);
+}
+
+// ======================================================
+// LAYER CONFIG
+// ======================================================
+
+/**
+ * Read the book layer config (Redis `animastor:layer-config:<book>`).
+ * Missing/broken config → every registered layer enabled (registry default).
+ */
+async function getLayerConfig(redis, bookId) {
+    const layerRaw = await redis.get(`animastor:layer-config:${bookId}`);
+    if (layerRaw) {
+        try { return JSON.parse(layerRaw); } catch (_) { /* fall through to defaults */ }
+    }
+    const defaults = {};
+    for (const t of mediaRegistry.listMediaTypes()) { defaults[`${t}_enabled`] = true; }
+    return defaults;
+}
+
+function isLayerEnabled(layerCfg, stage) {
+    return !layerCfg || layerCfg[`${stage}_enabled`] !== false;
+}
+
+// ======================================================
+// CANCELLATION
+// ======================================================
+
+/**
+ * True if the user explicitly cancelled generation for this book (Redis
+ * cancel flag and/or PG tombstone). A PG error reads as "not cancelled"
+ * (the Redis flag is still honoured) — the same semantics the reconcile
+ * resurrection guard always had.
+ */
+async function isBookCancelled(redis, bookId) {
+    try {
+        // Lazy: scene-window requires active-scenes-index, which requires
+        // this module — resolve at call time to keep the module graph acyclic.
+        if (await require('./scene-window').isCancelled(redis, bookId)) return true;
+    } catch (_) { /* flag unreadable — fall through to tombstone */ }
+    try {
+        return !!(await persist('cancel.isCancelled')(bookId));
+    } catch (_) {
+        return false;
+    }
+}
+
+// ======================================================
+// AUTHORIZATION
+// ======================================================
+
+/** Per-pass memo (one scheduler tick / one reconcile pass). */
+function createGateCache() {
+    return { layer: new Map(), cancelled: new Map() };
+}
+
+async function memo(cache, bucket, bookId, fn) {
+    if (!cache) return fn();
+    if (cache[bucket].has(bookId)) return cache[bucket].get(bookId);
+    const value = await fn();
+    cache[bucket].set(bookId, value);
+    return value;
+}
+
+const EMPTY_TASK_STATE = Object.freeze({ managed: false, activeTypes: new Set() });
+
+/**
+ * Which stages of one scene are authorized by an explicit command right now.
+ *
+ * @returns {Promise<{stages:Set<string>, denied:Object<string,string>,
+ *   cancelled:boolean, layerCfg:object, taskState:{managed:boolean, activeTypes:Set<string>}}>}
+ *   `denied[stage]` ∈ 'cancelled' | 'no_active_task' | 'layer_disabled'.
+ */
+async function getAuthorizedStages(redis, bookId, chapterId, sceneId, cache = null) {
+    const cancelled = await memo(cache, 'cancelled', bookId, () => isBookCancelled(redis, bookId));
+    const layerCfg = await memo(cache, 'layer', bookId, () => getLayerConfig(redis, bookId));
+
+    let taskState = EMPTY_TASK_STATE;
+    try {
+        const ts = await progressEventsOp('getSceneTaskState')(redis, bookId, chapterId, sceneId);
+        if (ts && ts.activeTypes instanceof Set) taskState = ts;
+    } catch (err) {
+        // Fail-closed: an unreadable registry proves no command.
+        warn(`task registry unreadable for ${bookId}/${chapterId}/${sceneId}: ${err.message} — no stage authorized`);
+    }
+
+    const stages = new Set();
+    const denied = {};
+    for (const stage of mediaRegistry.listMediaTypes()) {
+        if (cancelled) { denied[stage] = 'cancelled'; continue; }
+        if (!taskState.activeTypes.has(stage)) { denied[stage] = 'no_active_task'; continue; }
+        if (!isLayerEnabled(layerCfg, stage)) { denied[stage] = 'layer_disabled'; continue; }
+        stages.add(stage);
+    }
+    return { stages, denied, cancelled, layerCfg, taskState };
+}
+
+/** Is ONE stage of a scene authorized? */
+async function isStageAuthorized(redis, bookId, chapterId, sceneId, stage, cache = null) {
+    const auth = await getAuthorizedStages(redis, bookId, chapterId, sceneId, cache);
+    return auth.stages.has(stage);
+}
+
+function describeDenied(denied) {
+    return Object.entries(denied || {}).map(([s, r]) => `${s}:${r}`).join(',') || 'none';
+}
+
+// ======================================================
+// STATE HYGIENE WITHOUT WORK
+// ======================================================
+
+/**
+ * GENERATING → DIRTY (FSM-valid edge). Used where a dead/unsent generation
+ * must be released WITHOUT re-queueing it: without a command a stage may not
+ * sit in PENDING (PENDING is "queued work" — the incident's mine).
+ * Only GENERATING is touched; every other state is left as-is.
+ */
+async function rollbackGeneratingToDirty(redis, bookId, chapterId, sceneId, stage, reason) {
+    const states = await state.getAssetStates(redis, bookId, chapterId, sceneId);
+    const current = states?.[stage];
+    if (current !== state.AssetState.GENERATING) {
+        return { changed: false, reason: `state_${current || 'new'}` };
+    }
+    const check = state.validateAssetTransition(current, state.AssetState.DIRTY);
+    if (!check.valid) {
+        return { changed: false, reason: `invalid_transition:${current}->dirty` };
+    }
+    await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, stage, state.AssetState.DIRTY);
+    log(`${bookId}/${chapterId}/${sceneId} ${stage}: generating → dirty (reason=${reason}) — not re-queued`);
+    return { changed: true, from: current, to: state.AssetState.DIRTY };
+}
+
+// ======================================================
+// NO WORKER → BLOCKED (no auto-retry)
+// ======================================================
+
+/**
+ * A stage could not be enqueued because no live worker of its type exists.
+ * The command that asked for it is BLOCKED (task registry + PG) with the
+ * reason «нет воркера <type>»; the stage loses its authorization, so the
+ * scheduler never retries it. Only a NEW command retries.
+ * Storage writes happen host-side behind ProgressEventsPort.blockSceneStage.
+ */
+async function blockStageNoWorker(redis, bookId, chapterId, sceneId, stage) {
+    const reason = `no_worker:${stage}`;
+    let result = { tasks: [] };
+    try {
+        result = (await progressEventsOp('blockSceneStage')(redis, bookId, chapterId, sceneId, stage, reason)) || result;
+    } catch (err) {
+        warn(`blockSceneStage failed for ${bookId}/${chapterId}/${sceneId}:${stage}: ${err.message}`);
+    }
+    const ids = (result.tasks || []).map(t => t.task_id || t).join(',') || 'none';
+    log(`BLOCKED ${bookId}/${chapterId}/${sceneId}:${stage} — нет воркера ${stage}; tasks=[${ids}] — no auto-retry, a new command is required`);
+    return { reason, ...result };
+}
+
+// ======================================================
+// FORCE — only for the scene×stage pairs of the command
+// ======================================================
+// The former `animastor:force-dispatch:<book>` flag bypassed the duplicate/
+// lease guards for the WHOLE book for 120 s (incident §6 item 7). The force
+// set below holds exactly the scene×stage pairs reset by one regeneration.
+
+const FORCE_STAGES_KEY_PREFIX = 'animastor:force-dispatch-stages:';
+
+function forceStagesKey(bookId) {
+    return `${FORCE_STAGES_KEY_PREFIX}${bookId}`;
+}
+
+function forceMember(chapterId, sceneId, stage) {
+    return JSON.stringify([String(chapterId), String(sceneId), String(stage)]);
+}
+
+async function addForceStages(redis, bookId, pairs, ttlSeconds) {
+    const members = [];
+    for (const p of pairs || []) {
+        for (const stage of p.stages || []) members.push(forceMember(p.chapter_id, p.scene_id, stage));
+    }
+    if (members.length === 0) return 0;
+    const key = forceStagesKey(bookId);
+    await redis.sadd(key, ...members);
+    await redis.expire(key, ttlSeconds);
+    return members.length;
+}
+
+/** @returns {Promise<Set<string>>} force members of the book (see forceMember). */
+async function getForceStages(redis, bookId) {
+    try {
+        return new Set(await redis.smembers(forceStagesKey(bookId)) || []);
+    } catch (_) {
+        return new Set();
+    }
+}
+
+/** Stages of one scene present in a book force set. */
+function forcedStagesForScene(forceSet, chapterId, sceneId) {
+    const out = new Set();
+    if (!forceSet || forceSet.size === 0) return out;
+    for (const stage of mediaRegistry.listMediaTypes()) {
+        if (forceSet.has(forceMember(chapterId, sceneId, stage))) out.add(stage);
+    }
+    return out;
+}
+
+async function clearForceStages(redis, bookId) {
+    try { await redis.del(forceStagesKey(bookId)); } catch (_) { /* best effort */ }
+}
+
+module.exports = {
+    getLayerConfig,
+    isLayerEnabled,
+    isBookCancelled,
+    createGateCache,
+    getAuthorizedStages,
+    isStageAuthorized,
+    describeDenied,
+    rollbackGeneratingToDirty,
+    blockStageNoWorker,
+    FORCE_STAGES_KEY_PREFIX,
+    forceStagesKey,
+    forceMember,
+    addForceStages,
+    getForceStages,
+    forcedStagesForScene,
+    clearForceStages,
+};

package/src/runtime/dispatch-engine.js

@@ -498,7 +498,8 @@ async function getDispatchEvidence(redis, bookId, chapterId, sceneId, stage) {
  *
  * @param {Object} redis
  * @param {string} bookId
- * @param {{reason?: string}} [opts]
+ * @param {{reason?: string, target?: 'pending'|'dirty'}} [opts] — target 'dirty'
+ *        (command-only generation, Stop All) releases to DIRTY without re-queueing.
  * @returns {Promise<{repaired: Array<{chapterId, sceneId, stage, path}>}>}
  */
 async function repairOrphanGeneratingStates(redis, bookId, opts = {}) {
@@ -530,6 +531,16 @@ async function repairOrphanGeneratingStates(redis, bookId, opts = {}) {
                 if (states[stage] !== stateModule.AssetState.GENERATING) continue;
                 const evidence = await getDispatchEvidence(redis, bookId, chapterId, sceneId, stage);
                 if (evidence.alive) continue;
+                if (opts.target === 'dirty') {
+                    // Command-only generation (Stop All): release the ghost
+                    // to DIRTY without re-queueing it (no PENDING).
+                    const released = await require('./command-gate').rollbackGeneratingToDirty(
+                        redis, bookId, chapterId, sceneId, stage, reason);
+                    if (released.changed) {
+                        repaired.push({ chapterId, sceneId, stage, path: [stateModule.AssetState.DIRTY] });
+                    }
+                    continue;
+                }
                 const rollback = await rollbackStageToPending(
                     redis, bookId, chapterId, sceneId, stage, null, reason
                 );
@@ -730,10 +741,194 @@ async function shouldSkipDispatch(redis, bookId, chapterId, sceneId, stage) {
  * - { dispatched: true, dispatchId, stage, leaseKey }
  * - { dispatched: false, reason, skip: true/false }
  */
+// ======================================================
+// STAGE 116 (D3) — BOUNDED DEFERRAL (capacity backoff)
+// STAGE 118 (audit 117 · M2/M4) — taxonomy + force handling
+// ======================================================
+// The scheduler ticks every SCHEDULER_TICK_MS (5 s). A stage whose jobs could
+// not be enqueued (no live compatible worker, or every job already pending in
+// the lane) used to be re-dispatched on EVERY tick: lease created → executor
+// sent nothing → 'no_jobs_sent' → dispatch cancelled → FSM rolled back →
+// next tick repeats. That is an unbounded dispatch/cancel loop with FSM
+// churn, lease churn and unbounded event-journal growth (Stage 115 D3).
+//
+// The deferral is a bounded, per-stage backoff:
+//   • checked BEFORE any side effect (no lease, no metadata, no FSM write,
+//     no journal event) ⇒ a deferred stage costs one Redis GET per tick;
+//   • the stage stays PENDING and stays in the active index ⇒ recoverable;
+//   • an explicit cancel clears it (the user asked for the work to run).
+//
+// Stage 118 — three corrections of the Stage 116 implementation:
+//
+// 1. The reason taxonomy now distinguishes CAPACITY from IDEMPOTENT
+//    suppression ('already_completed' is NOT a capacity gap — audit 117 · M4).
+//    The idempotent bucket gets its own, much longer window: the stage is not
+//    "waiting for capacity", it is "already done" — retrying it every minute
+//    for a week is not a fix.
+// 2. `force=true` NO LONGER BYPASSES the deferral (audit 117 · M2). The
+//    force-dispatch flag lives for FORCE_DISPATCH_TTL_S (120 s) and is set for
+//    the WHOLE book by /regenerate, so bypassing the backoff there restored
+//    the very dispatch/cancel churn D3 was built to remove (≈24 full attempts
+//    per scene per regeneration). The explicit intent (a regeneration, a
+//    cancel) clears the deferral at ITS call site — the backoff then applies
+//    again after the next capacity failure.
+// 3. The deferral key encodes its components (audit 117 · M1): a book_id may
+//    contain ':' (the bundle book_id is client-controlled), so a raw
+//    ':'-joined key is ambiguous — `a:b` + chapter `c` collides with book
+//    `a` + chapter `b`.
+const DEFERRED_STAGE_KEY_PREFIX = 'animastor:runtime:deferred:';
+const DEFERRED_STAGE_TTL_S = 60;        // capacity backoff window (bounded retry)
+const DEFERRED_IDEMPOTENT_TTL_S = 900;  // idempotent-suppression window (15 min)
+
+/**
+ * Executor outcomes that mean "nothing could be enqueued — retry later".
+ *
+ * The taxonomy lives in ./dispatch-outcomes (shared with the stage
+ * executors). The generic 'no_jobs_sent' / 'no_jobs_generated' /
+ * 'send_failed' outcomes keep their historic contract (retry on the next
+ * tick — the audit-6929ba5 regression tests rely on it): deferring them
+ * would change the meaning of an executor result that does not necessarily
+ * mean "no capacity".
+ */
+const {
+    CAPACITY_OUTCOMES,
+    IDEMPOTENT_OUTCOMES,
+    FENCE_REJECTED_OUTCOMES,
+    isCapacityOutcome,
+    isIdempotentOutcome,
+    isFenceRejectedOutcome,
+    isDeferrableOutcome,
+} = require('./dispatch-outcomes');
+
+/**
+ * Encode one key component so that ':' (and the glob metacharacters) can
+ * never be mistaken for the separator. encodeURIComponent escapes ':' as
+ * '%3A' and leaves '_' readable.
+ */
+function encodeKeyComponent(value) {
+    return encodeURIComponent(String(value == null ? '' : value));
+}
+
+function deferredStageKey(bookId, chapterId, sceneId, stage) {
+    return `${DEFERRED_STAGE_KEY_PREFIX}`
+        + `${encodeKeyComponent(bookId)}:${encodeKeyComponent(chapterId)}`
+        + `:${encodeKeyComponent(sceneId)}:${encodeKeyComponent(stage)}`;
+}
+
+/** Is this stage currently deferred (backoff active)? */
+async function isStageDeferred(redis, bookId, chapterId, sceneId, stage) {
+    try {
+        const value = await redis.get(deferredStageKey(bookId, chapterId, sceneId, stage));
+        return value != null;
+    } catch (_) {
+        return false;
+    }
+}
+
+/**
+ * Defer a stage for the bounded backoff window.
+ * @param {number} [ttlSeconds] — the capacity window by default; the
+ *        idempotent window for an 'already_completed' suppression.
+ */
+async function deferStage(redis, bookId, chapterId, sceneId, stage, ttlSeconds = DEFERRED_STAGE_TTL_S) {
+    try {
+        await redis.set(deferredStageKey(bookId, chapterId, sceneId, stage), '1', 'EX', ttlSeconds);
+    } catch (_) { /* best effort — a missed deferral only costs one extra attempt */ }
+}
+
+/** Clear the deferral (the stage dispatched, completed or was cancelled). */
+async function clearStageDeferral(redis, bookId, chapterId, sceneId, stage) {
+    try {
+        await redis.del(deferredStageKey(bookId, chapterId, sceneId, stage));
+    } catch (_) { /* best effort */ }
+}
+
+/**
+ * Clear the capacity backoff of EVERY stage of a book.
+ *
+ * Stage 118 (audit 117 · M2/B4): the explicit intents — a cancel, a cache
+ * teardown, a deletion — ask for the work to run again, so the backoff must
+ * not survive them. The deferral key is owned by this package, so the
+ * package clears it (the host only injects the port).
+ *
+ * @returns {Promise<number>} the number of cleared deferrals.
+ */
+async function clearDeferralsForBook(redis, bookId) {
+    if (!redis || !bookId) return 0;
+    return clearDeferralsByPrefix(redis, `${DEFERRED_STAGE_KEY_PREFIX}${encodeKeyComponent(bookId)}:`);
+}
+
+/**
+ * Clear the capacity backoff of every stage of one scene (an explicit
+ * regeneration of that scene).
+ */
+async function clearDeferralsForScene(redis, bookId, chapterId, sceneId) {
+    if (!redis || !bookId || !chapterId || !sceneId) return 0;
+    return clearDeferralsByPrefix(redis, `${DEFERRED_STAGE_KEY_PREFIX}`
+        + `${encodeKeyComponent(bookId)}:${encodeKeyComponent(chapterId)}:${encodeKeyComponent(sceneId)}:`);
+}
+
+async function clearDeferralsByPrefix(redis, prefix) {
+    let cleared = 0;
+    if (!redis || !prefix) return cleared;
+    let cursor = '0';
+    try {
+        do {
+            const [next, keys] = await redis.scan(cursor, 'MATCH', `${prefix}*`, 'COUNT', 200);
+            cursor = next;
+            for (const key of keys || []) {
+                if (!key.startsWith(prefix)) continue;
+                await redis.del(key);
+                cleared++;
+            }
+        } while (cursor !== '0');
+    } catch (_) { /* best effort */ }
+    return cleared;
+}
+
+/**
+ * The backoff window that fits the outcome: a capacity gap retries in
+ * minutes, an idempotent suppression (already complete) in quarters of an
+ * hour — it is not waiting for capacity, it is simply done.
+ */
+function deferralWindowFor(reason) {
+    // Stage 129 (audit 128 · F5): a fence rejection is terminal for THIS
+    // generation — retrying it immediately would only recreate the
+    // dispatch → cancel → rollback churn. It shares the long window with the
+    // idempotent suppressions and stays visible through the log/metric.
+    return (isIdempotentOutcome(reason) || isFenceRejectedOutcome(reason))
+        ? DEFERRED_IDEMPOTENT_TTL_S
+        : DEFERRED_STAGE_TTL_S;
+}
+
 async function dispatchStage(redis, bookId, chapterId, sceneId, stage, loadedBook, buildId, options = {}) {
     const { force = false } = options;
     const dispatchId = generateDispatchToken();
 
+    // ─────────────────────────────────────────────────────────────────────
+    // Stage 116 (D3) / Stage 118 (M2) — BOUNDED DEFERRAL (retry/backoff)
+    // ─────────────────────────────────────────────────────────────────────
+    // A stage that could not enqueue anything (no live compatible worker, or
+    // every job already pending in the lane) must NOT be retried on every
+    // scheduler tick: the tick is 5 s, so the un-deferred path produced an
+    // endless dispatch → cancel → FSM-rollback cycle (lease churn, FSM churn
+    // and UNBOUNDED event-journal growth).
+    //
+    // The deferral is checked BEFORE any side effect (no lease, no dispatch
+    // metadata, no FSM write, no journal event) ⇒ a deferred stage costs one
+    // Redis GET per tick. The stage stays PENDING and stays in the active
+    // index ⇒ it is recoverable: as soon as the deferral window expires (or a
+    // worker appears) the next tick dispatches it again.
+    //
+    // Stage 118 (M2): the check applies in FORCE mode as well. The
+    // force-dispatch flag is per BOOK and lives for 120 s, so bypassing the
+    // backoff there reproduced the very churn D3 removes (≈24 full attempts
+    // per scene per /regenerate). An EXPLICIT intent (a regeneration, a
+    // cancel) clears the deferral at its own call site.
+    if (await isStageDeferred(redis, bookId, chapterId, sceneId, stage)) {
+        return { dispatched: false, reason: 'deferred_capacity', retryable: true, deferred: true, dispatchId };
+    }
+
     log(`DISPATCH_REQUEST: ${bookId}/${chapterId}/${sceneId}:${stage}${force ? ' (force=true)' : ''}`);
 
     // Force mode pre-clear: cancel existing dispatch, release quota ТОЛЬКО при наличии lease
@@ -908,6 +1103,18 @@ async function dispatchStage(redis, bookId, chapterId, sceneId, stage, loadedBoo
             if (!result.dispatched) {
                 log(`DISPATCH_SKIPPED: ${bookId}/${chapterId}/${sceneId}:${stage} - ${result.reason || 'already_done'}`);
                 if (!result.completed) {
+                    // Stage 116 (D3): a stage that enqueued NOTHING is not a
+                    // lost task and not a fatal cancellation — it is a bounded
+                    // deferral. Remember it for a bounded window so the
+                    // scheduler's 5 s tick does not spin.
+                    //
+                    // Stage 118 (M4): the window depends on the bucket — a
+                    // capacity gap retries within a minute, an idempotent
+                    // suppression ('already_completed') is NOT waiting for
+                    // capacity and gets the long window.
+                    if (isDeferrableOutcome(result.reason)) {
+                        await deferStage(redis, bookId, chapterId, sceneId, stage, deferralWindowFor(result.reason));
+                    }
                     await finalizeDispatch(redis, bookId, chapterId, sceneId, stage, {
                         outcome: 'cancelled',
                         dispatchId,
@@ -918,13 +1125,29 @@ async function dispatchStage(redis, bookId, chapterId, sceneId, stage, loadedBoo
                     if (result.rollbackFailed) {
                         error(`STATE_ROLLBACK_FAILED: ${bookId}/${chapterId}/${sceneId}:${stage}: ${result.rollbackReason || 'unknown'} — stage may be stuck in GENERATING`);
                     }
+                } else {
+                    // The stage completed (cache hit / already done) — it is no
+                    // longer waiting for capacity.
+                    await clearStageDeferral(redis, bookId, chapterId, sceneId, stage);
                 }
                 // Cancelled finalize does not touch the circuit breaker — release
                 // the test permit here so a half-open circuit can keep recovering.
                 await releaseTestPermit();
-                return { dispatched: false, dispatchId, reason: result.reason || 'already_done', result };
+                return {
+                    dispatched: false,
+                    dispatchId,
+                    reason: result.reason || 'already_done',
+                    retryable: isDeferrableOutcome(result.reason),
+                    // Stage 118 (M9): the caller (scheduler) must be able to
+                    // tell "nothing happened, try later" from a real error.
+                    deferred: isDeferrableOutcome(result.reason),
+                    result,
+                };
             }
 
+            // The stage really dispatched — capacity deferral is over.
+            await clearStageDeferral(redis, bookId, chapterId, sceneId, stage);
+
             // T6: Start lease renewal for this actively dispatched job
             // Only started after confirming dispatcher returned dispatched:true
             // (not for cache hits, backpressure, or empty executor results)
@@ -1296,6 +1519,9 @@ async function markDispatchFailed(redis, bookId, chapterId, sceneId, stage, erro
  * T6: Останавливает in-memory renewal timer, чтобы избежать утечки.
  */
 async function cancelActiveDispatch(redis, bookId, chapterId, sceneId, stage, reason = 'cancelled') {
+    // Stage 116 (D3): an explicit cancel means the work is wanted again — a
+    // capacity deferral must not survive it.
+    await clearStageDeferral(redis, bookId, chapterId, sceneId, stage);
     const metadata = await getDispatchMetadata(redis, bookId, chapterId, sceneId, stage);
     if (metadata?.dispatch_id) {
         const result = await finalizeDispatch(redis, bookId, chapterId, sceneId, stage, {
@@ -1383,9 +1609,33 @@ async function clearLeasesForScenes(redis, bookId, scenes) {
 }
 
 /**
- * Remove specific cancelled dispatches from GPU Hub.
+ * Remove cancelled dispatches from GPU Hub.
  * Kept here with dispatch cancellation so every caller uses the same auth and
  * error-handling contract.
+ *
+ * Stage 122 (N1): the hub's `DELETE /queue/clear` accepts EITHER a
+ * `dispatch_id` (per-job) OR a `book_id` (whole-book lane sweep). Before this
+ * stage the function only honoured the per-id form; a caller that passed an
+ * EMPTY id list together with `book_id` (the regenerate post-release sweep,
+ * audit 121 · N1) fell straight through the id loop and issued ZERO requests —
+ * a silent NO-OP that left orphan lane entries behind while looking like a
+ * successful sweep.
+ *
+ * The contract now is:
+ *   - ids present        → one `?dispatch_id=` DELETE per id (unchanged);
+ *   - no ids + book_id   → exactly ONE `?book_id=` DELETE (the whole-book sweep
+ *                          the hub already supports and cache/delete paths use);
+ *   - neither            → NO request at all (never a fake HTTP call).
+ *
+ * A book-scoped sweep is checked against the hub's echoed `scope.book_id`:
+ * if the hub answers `ok` but echoes a DIFFERENT scope (or none at all while
+ * reporting a full wipe) it did not honour our ownership and other books may
+ * have been touched — that is counted as `failed` and warned, never as a
+ * success. Failures never throw (best-effort contract, callers keep claims).
+ *
+ * @returns {Promise<{requested:number, cleared:number, failed:number}>}
+ *   `requested` = number of hub requests actually attempted (1 for a
+ *   book-scoped sweep, ids.length for per-id, 0 when there is nothing to do).
  */
 async function clearHubDispatches(dispatchIds, options = {}) {
     const ids = [...new Set((dispatchIds || []).filter(Boolean))];
@@ -1396,16 +1646,59 @@ async function clearHubDispatches(dispatchIds, options = {}) {
     const context = options.context || 'DISPATCH';
     const reportWarning = typeof options.warn === 'function' ? options.warn : warn;
     const fetchImpl = options.fetchImpl || globalThis.fetch;
+    const bookId = options.book_id || options.bookId || null;
     let cleared = 0;
     let failed = 0;
 
+    const authHeaders = () => {
+        const headers = {};
+        if (apiKey) headers['x-api-key'] = apiKey;
+        return headers;
+    };
+
+    // ── Book-scoped sweep (no explicit dispatch ids) ──────────────────────
+    // Exactly one request; the hub filters by record.book_id so only this
+    // book's queue/running/result entries are removed.
+    if (ids.length === 0 && bookId) {
+        try {
+            const response = await fetchImpl(
+                `${hubUrl}/queue/clear?book_id=${encodeURIComponent(bookId)}`,
+                { method: 'DELETE', headers: authHeaders() }
+            );
+            if (!response.ok) {
+                failed++;
+                reportWarning(`${context}: GPU hub book cleanup returned ${response.status} for ${bookId}`);
+            } else {
+                // Verify the hub honoured OUR scope. A mismatched echo means it
+                // may have swept a different book (or everything) — that is a
+                // failure, not a success we can report.
+                let enforcedScope = true;
+                try {
+                    const body = typeof response.json === 'function' ? await response.json() : null;
+                    if (body && body.scope && body.scope.book_id != null && body.scope.book_id !== bookId) {
+                        enforcedScope = false;
+                    }
+                } catch (_) { /* unparseable body — trust the ok status */ }
+                if (!enforcedScope) {
+                    failed++;
+                    reportWarning(`${context}: GPU hub book cleanup echoed a foreign scope for ${bookId} — treated as failed`);
+                } else {
+                    cleared++;
+                }
+            }
+        } catch (err) {
+            failed++;
+            reportWarning(`${context}: GPU hub book cleanup failed for ${bookId}: ${err.message}`);
+        }
+        return { requested: 1, cleared, failed };
+    }
+
+    // ── Per-dispatch cleanup (unchanged) ─────────────────────────────────
     for (const dispatchId of ids) {
         try {
-            const headers = {};
-            if (apiKey) headers['x-api-key'] = apiKey;
             const response = await fetchImpl(
                 `${hubUrl}/queue/clear?dispatch_id=${encodeURIComponent(dispatchId)}`,
-                { method: 'DELETE', headers }
+                { method: 'DELETE', headers: authHeaders() }
             );
             if (!response.ok) {
                 failed++;
@@ -1781,6 +2074,29 @@ module.exports = {
     LEASE_TTLS,
     QUOTAS,
 
+    // Stage 116 (D3) / Stage 118 (M2/M4): bounded deferral (retry/backoff) —
+    // the stage is not retried on every 5 s scheduler tick while it cannot
+    // enqueue. Stage 118: the capacity and the idempotent bucket have their
+    // own windows, and the taxonomy is shared with the stage executors.
+    DEFERRED_STAGE_KEY_PREFIX,
+    DEFERRED_STAGE_TTL_S,
+    DEFERRED_IDEMPOTENT_TTL_S,
+    deferredStageKey,
+    isStageDeferred,
+    deferStage,
+    clearStageDeferral,
+    clearDeferralsForBook,
+    clearDeferralsForScene,
+    deferralWindowFor,
+    encodeKeyComponent,
+    isDeferrableOutcome,
+    isCapacityOutcome,
+    isIdempotentOutcome,
+    isFenceRejectedOutcome,
+    CAPACITY_OUTCOMES,
+    IDEMPOTENT_OUTCOMES,
+    FENCE_REJECTED_OUTCOMES,
+
     // PW-2: dispatch identity generation (crypto-hardened)
     generateDispatchToken
 };