@animastor/orchestration 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/orchestration/orchestrator.js +29 -10
- package/src/orchestration/scene-callbacks.js +3 -55
- package/src/orchestration/scene-orchestrator.js +7 -70
- package/src/runtime/dispatch-engine.js +5 -310
- package/src/runtime/hub-cancel-port.js +0 -6
- package/src/runtime/persistence-port.js +5 -7
- package/src/runtime/reconciliation-engine.js +6 -57
- package/src/runtime/runtime-scheduler.js +1 -18
- package/src/runtime/scene-window.js +0 -28
- package/test/stage146-d1-artifact-naming.test.js +103 -0
- package/test/stage147-a1-null-build-gate.test.js +211 -0
- package/src/runtime/dispatch-outcomes.js +0 -124
- package/test/capacity-deferral.test.js +0 -265
|
@@ -730,194 +730,10 @@ async function shouldSkipDispatch(redis, bookId, chapterId, sceneId, stage) {
|
|
|
730
730
|
* - { dispatched: true, dispatchId, stage, leaseKey }
|
|
731
731
|
* - { dispatched: false, reason, skip: true/false }
|
|
732
732
|
*/
|
|
733
|
-
// ======================================================
|
|
734
|
-
// STAGE 116 (D3) — BOUNDED DEFERRAL (capacity backoff)
|
|
735
|
-
// STAGE 118 (audit 117 · M2/M4) — taxonomy + force handling
|
|
736
|
-
// ======================================================
|
|
737
|
-
// The scheduler ticks every SCHEDULER_TICK_MS (5 s). A stage whose jobs could
|
|
738
|
-
// not be enqueued (no live compatible worker, or every job already pending in
|
|
739
|
-
// the lane) used to be re-dispatched on EVERY tick: lease created → executor
|
|
740
|
-
// sent nothing → 'no_jobs_sent' → dispatch cancelled → FSM rolled back →
|
|
741
|
-
// next tick repeats. That is an unbounded dispatch/cancel loop with FSM
|
|
742
|
-
// churn, lease churn and unbounded event-journal growth (Stage 115 D3).
|
|
743
|
-
//
|
|
744
|
-
// The deferral is a bounded, per-stage backoff:
|
|
745
|
-
// • checked BEFORE any side effect (no lease, no metadata, no FSM write,
|
|
746
|
-
// no journal event) ⇒ a deferred stage costs one Redis GET per tick;
|
|
747
|
-
// • the stage stays PENDING and stays in the active index ⇒ recoverable;
|
|
748
|
-
// • an explicit cancel clears it (the user asked for the work to run).
|
|
749
|
-
//
|
|
750
|
-
// Stage 118 — three corrections of the Stage 116 implementation:
|
|
751
|
-
//
|
|
752
|
-
// 1. The reason taxonomy now distinguishes CAPACITY from IDEMPOTENT
|
|
753
|
-
// suppression ('already_completed' is NOT a capacity gap — audit 117 · M4).
|
|
754
|
-
// The idempotent bucket gets its own, much longer window: the stage is not
|
|
755
|
-
// "waiting for capacity", it is "already done" — retrying it every minute
|
|
756
|
-
// for a week is not a fix.
|
|
757
|
-
// 2. `force=true` NO LONGER BYPASSES the deferral (audit 117 · M2). The
|
|
758
|
-
// force-dispatch flag lives for FORCE_DISPATCH_TTL_S (120 s) and is set for
|
|
759
|
-
// the WHOLE book by /regenerate, so bypassing the backoff there restored
|
|
760
|
-
// the very dispatch/cancel churn D3 was built to remove (≈24 full attempts
|
|
761
|
-
// per scene per regeneration). The explicit intent (a regeneration, a
|
|
762
|
-
// cancel) clears the deferral at ITS call site — the backoff then applies
|
|
763
|
-
// again after the next capacity failure.
|
|
764
|
-
// 3. The deferral key encodes its components (audit 117 · M1): a book_id may
|
|
765
|
-
// contain ':' (the bundle book_id is client-controlled), so a raw
|
|
766
|
-
// ':'-joined key is ambiguous — `a:b` + chapter `c` collides with book
|
|
767
|
-
// `a` + chapter `b`.
|
|
768
|
-
const DEFERRED_STAGE_KEY_PREFIX = 'animastor:runtime:deferred:';
|
|
769
|
-
const DEFERRED_STAGE_TTL_S = 60; // capacity backoff window (bounded retry)
|
|
770
|
-
const DEFERRED_IDEMPOTENT_TTL_S = 900; // idempotent-suppression window (15 min)
|
|
771
|
-
|
|
772
|
-
/**
|
|
773
|
-
* Executor outcomes that mean "nothing could be enqueued — retry later".
|
|
774
|
-
*
|
|
775
|
-
* The taxonomy lives in ./dispatch-outcomes (shared with the stage
|
|
776
|
-
* executors). The generic 'no_jobs_sent' / 'no_jobs_generated' /
|
|
777
|
-
* 'send_failed' outcomes keep their historic contract (retry on the next
|
|
778
|
-
* tick — the audit-6929ba5 regression tests rely on it): deferring them
|
|
779
|
-
* would change the meaning of an executor result that does not necessarily
|
|
780
|
-
* mean "no capacity".
|
|
781
|
-
*/
|
|
782
|
-
const {
|
|
783
|
-
CAPACITY_OUTCOMES,
|
|
784
|
-
IDEMPOTENT_OUTCOMES,
|
|
785
|
-
FENCE_REJECTED_OUTCOMES,
|
|
786
|
-
isCapacityOutcome,
|
|
787
|
-
isIdempotentOutcome,
|
|
788
|
-
isFenceRejectedOutcome,
|
|
789
|
-
isDeferrableOutcome,
|
|
790
|
-
} = require('./dispatch-outcomes');
|
|
791
|
-
|
|
792
|
-
/**
|
|
793
|
-
* Encode one key component so that ':' (and the glob metacharacters) can
|
|
794
|
-
* never be mistaken for the separator. encodeURIComponent escapes ':' as
|
|
795
|
-
* '%3A' and leaves '_' readable.
|
|
796
|
-
*/
|
|
797
|
-
function encodeKeyComponent(value) {
|
|
798
|
-
return encodeURIComponent(String(value == null ? '' : value));
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
function deferredStageKey(bookId, chapterId, sceneId, stage) {
|
|
802
|
-
return `${DEFERRED_STAGE_KEY_PREFIX}`
|
|
803
|
-
+ `${encodeKeyComponent(bookId)}:${encodeKeyComponent(chapterId)}`
|
|
804
|
-
+ `:${encodeKeyComponent(sceneId)}:${encodeKeyComponent(stage)}`;
|
|
805
|
-
}
|
|
806
|
-
|
|
807
|
-
/** Is this stage currently deferred (backoff active)? */
|
|
808
|
-
async function isStageDeferred(redis, bookId, chapterId, sceneId, stage) {
|
|
809
|
-
try {
|
|
810
|
-
const value = await redis.get(deferredStageKey(bookId, chapterId, sceneId, stage));
|
|
811
|
-
return value != null;
|
|
812
|
-
} catch (_) {
|
|
813
|
-
return false;
|
|
814
|
-
}
|
|
815
|
-
}
|
|
816
|
-
|
|
817
|
-
/**
|
|
818
|
-
* Defer a stage for the bounded backoff window.
|
|
819
|
-
* @param {number} [ttlSeconds] — the capacity window by default; the
|
|
820
|
-
* idempotent window for an 'already_completed' suppression.
|
|
821
|
-
*/
|
|
822
|
-
async function deferStage(redis, bookId, chapterId, sceneId, stage, ttlSeconds = DEFERRED_STAGE_TTL_S) {
|
|
823
|
-
try {
|
|
824
|
-
await redis.set(deferredStageKey(bookId, chapterId, sceneId, stage), '1', 'EX', ttlSeconds);
|
|
825
|
-
} catch (_) { /* best effort — a missed deferral only costs one extra attempt */ }
|
|
826
|
-
}
|
|
827
|
-
|
|
828
|
-
/** Clear the deferral (the stage dispatched, completed or was cancelled). */
|
|
829
|
-
async function clearStageDeferral(redis, bookId, chapterId, sceneId, stage) {
|
|
830
|
-
try {
|
|
831
|
-
await redis.del(deferredStageKey(bookId, chapterId, sceneId, stage));
|
|
832
|
-
} catch (_) { /* best effort */ }
|
|
833
|
-
}
|
|
834
|
-
|
|
835
|
-
/**
|
|
836
|
-
* Clear the capacity backoff of EVERY stage of a book.
|
|
837
|
-
*
|
|
838
|
-
* Stage 118 (audit 117 · M2/B4): the explicit intents — a cancel, a cache
|
|
839
|
-
* teardown, a deletion — ask for the work to run again, so the backoff must
|
|
840
|
-
* not survive them. The deferral key is owned by this package, so the
|
|
841
|
-
* package clears it (the host only injects the port).
|
|
842
|
-
*
|
|
843
|
-
* @returns {Promise<number>} the number of cleared deferrals.
|
|
844
|
-
*/
|
|
845
|
-
async function clearDeferralsForBook(redis, bookId) {
|
|
846
|
-
if (!redis || !bookId) return 0;
|
|
847
|
-
return clearDeferralsByPrefix(redis, `${DEFERRED_STAGE_KEY_PREFIX}${encodeKeyComponent(bookId)}:`);
|
|
848
|
-
}
|
|
849
|
-
|
|
850
|
-
/**
|
|
851
|
-
* Clear the capacity backoff of every stage of one scene (an explicit
|
|
852
|
-
* regeneration of that scene).
|
|
853
|
-
*/
|
|
854
|
-
async function clearDeferralsForScene(redis, bookId, chapterId, sceneId) {
|
|
855
|
-
if (!redis || !bookId || !chapterId || !sceneId) return 0;
|
|
856
|
-
return clearDeferralsByPrefix(redis, `${DEFERRED_STAGE_KEY_PREFIX}`
|
|
857
|
-
+ `${encodeKeyComponent(bookId)}:${encodeKeyComponent(chapterId)}:${encodeKeyComponent(sceneId)}:`);
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
async function clearDeferralsByPrefix(redis, prefix) {
|
|
861
|
-
let cleared = 0;
|
|
862
|
-
if (!redis || !prefix) return cleared;
|
|
863
|
-
let cursor = '0';
|
|
864
|
-
try {
|
|
865
|
-
do {
|
|
866
|
-
const [next, keys] = await redis.scan(cursor, 'MATCH', `${prefix}*`, 'COUNT', 200);
|
|
867
|
-
cursor = next;
|
|
868
|
-
for (const key of keys || []) {
|
|
869
|
-
if (!key.startsWith(prefix)) continue;
|
|
870
|
-
await redis.del(key);
|
|
871
|
-
cleared++;
|
|
872
|
-
}
|
|
873
|
-
} while (cursor !== '0');
|
|
874
|
-
} catch (_) { /* best effort */ }
|
|
875
|
-
return cleared;
|
|
876
|
-
}
|
|
877
|
-
|
|
878
|
-
/**
|
|
879
|
-
* The backoff window that fits the outcome: a capacity gap retries in
|
|
880
|
-
* minutes, an idempotent suppression (already complete) in quarters of an
|
|
881
|
-
* hour — it is not waiting for capacity, it is simply done.
|
|
882
|
-
*/
|
|
883
|
-
function deferralWindowFor(reason) {
|
|
884
|
-
// Stage 129 (audit 128 · F5): a fence rejection is terminal for THIS
|
|
885
|
-
// generation — retrying it immediately would only recreate the
|
|
886
|
-
// dispatch → cancel → rollback churn. It shares the long window with the
|
|
887
|
-
// idempotent suppressions and stays visible through the log/metric.
|
|
888
|
-
return (isIdempotentOutcome(reason) || isFenceRejectedOutcome(reason))
|
|
889
|
-
? DEFERRED_IDEMPOTENT_TTL_S
|
|
890
|
-
: DEFERRED_STAGE_TTL_S;
|
|
891
|
-
}
|
|
892
|
-
|
|
893
733
|
async function dispatchStage(redis, bookId, chapterId, sceneId, stage, loadedBook, buildId, options = {}) {
|
|
894
734
|
const { force = false } = options;
|
|
895
735
|
const dispatchId = generateDispatchToken();
|
|
896
736
|
|
|
897
|
-
// ─────────────────────────────────────────────────────────────────────
|
|
898
|
-
// Stage 116 (D3) / Stage 118 (M2) — BOUNDED DEFERRAL (retry/backoff)
|
|
899
|
-
// ─────────────────────────────────────────────────────────────────────
|
|
900
|
-
// A stage that could not enqueue anything (no live compatible worker, or
|
|
901
|
-
// every job already pending in the lane) must NOT be retried on every
|
|
902
|
-
// scheduler tick: the tick is 5 s, so the un-deferred path produced an
|
|
903
|
-
// endless dispatch → cancel → FSM-rollback cycle (lease churn, FSM churn
|
|
904
|
-
// and UNBOUNDED event-journal growth).
|
|
905
|
-
//
|
|
906
|
-
// The deferral is checked BEFORE any side effect (no lease, no dispatch
|
|
907
|
-
// metadata, no FSM write, no journal event) ⇒ a deferred stage costs one
|
|
908
|
-
// Redis GET per tick. The stage stays PENDING and stays in the active
|
|
909
|
-
// index ⇒ it is recoverable: as soon as the deferral window expires (or a
|
|
910
|
-
// worker appears) the next tick dispatches it again.
|
|
911
|
-
//
|
|
912
|
-
// Stage 118 (M2): the check applies in FORCE mode as well. The
|
|
913
|
-
// force-dispatch flag is per BOOK and lives for 120 s, so bypassing the
|
|
914
|
-
// backoff there reproduced the very churn D3 removes (≈24 full attempts
|
|
915
|
-
// per scene per /regenerate). An EXPLICIT intent (a regeneration, a
|
|
916
|
-
// cancel) clears the deferral at its own call site.
|
|
917
|
-
if (await isStageDeferred(redis, bookId, chapterId, sceneId, stage)) {
|
|
918
|
-
return { dispatched: false, reason: 'deferred_capacity', retryable: true, deferred: true, dispatchId };
|
|
919
|
-
}
|
|
920
|
-
|
|
921
737
|
log(`DISPATCH_REQUEST: ${bookId}/${chapterId}/${sceneId}:${stage}${force ? ' (force=true)' : ''}`);
|
|
922
738
|
|
|
923
739
|
// Force mode pre-clear: cancel existing dispatch, release quota ТОЛЬКО при наличии lease
|
|
@@ -1092,18 +908,6 @@ async function dispatchStage(redis, bookId, chapterId, sceneId, stage, loadedBoo
|
|
|
1092
908
|
if (!result.dispatched) {
|
|
1093
909
|
log(`DISPATCH_SKIPPED: ${bookId}/${chapterId}/${sceneId}:${stage} - ${result.reason || 'already_done'}`);
|
|
1094
910
|
if (!result.completed) {
|
|
1095
|
-
// Stage 116 (D3): a stage that enqueued NOTHING is not a
|
|
1096
|
-
// lost task and not a fatal cancellation — it is a bounded
|
|
1097
|
-
// deferral. Remember it for a bounded window so the
|
|
1098
|
-
// scheduler's 5 s tick does not spin.
|
|
1099
|
-
//
|
|
1100
|
-
// Stage 118 (M4): the window depends on the bucket — a
|
|
1101
|
-
// capacity gap retries within a minute, an idempotent
|
|
1102
|
-
// suppression ('already_completed') is NOT waiting for
|
|
1103
|
-
// capacity and gets the long window.
|
|
1104
|
-
if (isDeferrableOutcome(result.reason)) {
|
|
1105
|
-
await deferStage(redis, bookId, chapterId, sceneId, stage, deferralWindowFor(result.reason));
|
|
1106
|
-
}
|
|
1107
911
|
await finalizeDispatch(redis, bookId, chapterId, sceneId, stage, {
|
|
1108
912
|
outcome: 'cancelled',
|
|
1109
913
|
dispatchId,
|
|
@@ -1114,29 +918,13 @@ async function dispatchStage(redis, bookId, chapterId, sceneId, stage, loadedBoo
|
|
|
1114
918
|
if (result.rollbackFailed) {
|
|
1115
919
|
error(`STATE_ROLLBACK_FAILED: ${bookId}/${chapterId}/${sceneId}:${stage}: ${result.rollbackReason || 'unknown'} — stage may be stuck in GENERATING`);
|
|
1116
920
|
}
|
|
1117
|
-
} else {
|
|
1118
|
-
// The stage completed (cache hit / already done) — it is no
|
|
1119
|
-
// longer waiting for capacity.
|
|
1120
|
-
await clearStageDeferral(redis, bookId, chapterId, sceneId, stage);
|
|
1121
921
|
}
|
|
1122
922
|
// Cancelled finalize does not touch the circuit breaker — release
|
|
1123
923
|
// the test permit here so a half-open circuit can keep recovering.
|
|
1124
924
|
await releaseTestPermit();
|
|
1125
|
-
return {
|
|
1126
|
-
dispatched: false,
|
|
1127
|
-
dispatchId,
|
|
1128
|
-
reason: result.reason || 'already_done',
|
|
1129
|
-
retryable: isDeferrableOutcome(result.reason),
|
|
1130
|
-
// Stage 118 (M9): the caller (scheduler) must be able to
|
|
1131
|
-
// tell "nothing happened, try later" from a real error.
|
|
1132
|
-
deferred: isDeferrableOutcome(result.reason),
|
|
1133
|
-
result,
|
|
1134
|
-
};
|
|
925
|
+
return { dispatched: false, dispatchId, reason: result.reason || 'already_done', result };
|
|
1135
926
|
}
|
|
1136
927
|
|
|
1137
|
-
// The stage really dispatched — capacity deferral is over.
|
|
1138
|
-
await clearStageDeferral(redis, bookId, chapterId, sceneId, stage);
|
|
1139
|
-
|
|
1140
928
|
// T6: Start lease renewal for this actively dispatched job
|
|
1141
929
|
// Only started after confirming dispatcher returned dispatched:true
|
|
1142
930
|
// (not for cache hits, backpressure, or empty executor results)
|
|
@@ -1508,9 +1296,6 @@ async function markDispatchFailed(redis, bookId, chapterId, sceneId, stage, erro
|
|
|
1508
1296
|
* T6: Останавливает in-memory renewal timer, чтобы избежать утечки.
|
|
1509
1297
|
*/
|
|
1510
1298
|
async function cancelActiveDispatch(redis, bookId, chapterId, sceneId, stage, reason = 'cancelled') {
|
|
1511
|
-
// Stage 116 (D3): an explicit cancel means the work is wanted again — a
|
|
1512
|
-
// capacity deferral must not survive it.
|
|
1513
|
-
await clearStageDeferral(redis, bookId, chapterId, sceneId, stage);
|
|
1514
1299
|
const metadata = await getDispatchMetadata(redis, bookId, chapterId, sceneId, stage);
|
|
1515
1300
|
if (metadata?.dispatch_id) {
|
|
1516
1301
|
const result = await finalizeDispatch(redis, bookId, chapterId, sceneId, stage, {
|
|
@@ -1598,33 +1383,9 @@ async function clearLeasesForScenes(redis, bookId, scenes) {
|
|
|
1598
1383
|
}
|
|
1599
1384
|
|
|
1600
1385
|
/**
|
|
1601
|
-
* Remove cancelled dispatches from GPU Hub.
|
|
1386
|
+
* Remove specific cancelled dispatches from GPU Hub.
|
|
1602
1387
|
* Kept here with dispatch cancellation so every caller uses the same auth and
|
|
1603
1388
|
* error-handling contract.
|
|
1604
|
-
*
|
|
1605
|
-
* Stage 122 (N1): the hub's `DELETE /queue/clear` accepts EITHER a
|
|
1606
|
-
* `dispatch_id` (per-job) OR a `book_id` (whole-book lane sweep). Before this
|
|
1607
|
-
* stage the function only honoured the per-id form; a caller that passed an
|
|
1608
|
-
* EMPTY id list together with `book_id` (the regenerate post-release sweep,
|
|
1609
|
-
* audit 121 · N1) fell straight through the id loop and issued ZERO requests —
|
|
1610
|
-
* a silent NO-OP that left orphan lane entries behind while looking like a
|
|
1611
|
-
* successful sweep.
|
|
1612
|
-
*
|
|
1613
|
-
* The contract now is:
|
|
1614
|
-
* - ids present → one `?dispatch_id=` DELETE per id (unchanged);
|
|
1615
|
-
* - no ids + book_id → exactly ONE `?book_id=` DELETE (the whole-book sweep
|
|
1616
|
-
* the hub already supports and cache/delete paths use);
|
|
1617
|
-
* - neither → NO request at all (never a fake HTTP call).
|
|
1618
|
-
*
|
|
1619
|
-
* A book-scoped sweep is checked against the hub's echoed `scope.book_id`:
|
|
1620
|
-
* if the hub answers `ok` but echoes a DIFFERENT scope (or none at all while
|
|
1621
|
-
* reporting a full wipe) it did not honour our ownership and other books may
|
|
1622
|
-
* have been touched — that is counted as `failed` and warned, never as a
|
|
1623
|
-
* success. Failures never throw (best-effort contract, callers keep claims).
|
|
1624
|
-
*
|
|
1625
|
-
* @returns {Promise<{requested:number, cleared:number, failed:number}>}
|
|
1626
|
-
* `requested` = number of hub requests actually attempted (1 for a
|
|
1627
|
-
* book-scoped sweep, ids.length for per-id, 0 when there is nothing to do).
|
|
1628
1389
|
*/
|
|
1629
1390
|
async function clearHubDispatches(dispatchIds, options = {}) {
|
|
1630
1391
|
const ids = [...new Set((dispatchIds || []).filter(Boolean))];
|
|
@@ -1635,59 +1396,16 @@ async function clearHubDispatches(dispatchIds, options = {}) {
|
|
|
1635
1396
|
const context = options.context || 'DISPATCH';
|
|
1636
1397
|
const reportWarning = typeof options.warn === 'function' ? options.warn : warn;
|
|
1637
1398
|
const fetchImpl = options.fetchImpl || globalThis.fetch;
|
|
1638
|
-
const bookId = options.book_id || options.bookId || null;
|
|
1639
1399
|
let cleared = 0;
|
|
1640
1400
|
let failed = 0;
|
|
1641
1401
|
|
|
1642
|
-
const authHeaders = () => {
|
|
1643
|
-
const headers = {};
|
|
1644
|
-
if (apiKey) headers['x-api-key'] = apiKey;
|
|
1645
|
-
return headers;
|
|
1646
|
-
};
|
|
1647
|
-
|
|
1648
|
-
// ── Book-scoped sweep (no explicit dispatch ids) ──────────────────────
|
|
1649
|
-
// Exactly one request; the hub filters by record.book_id so only this
|
|
1650
|
-
// book's queue/running/result entries are removed.
|
|
1651
|
-
if (ids.length === 0 && bookId) {
|
|
1652
|
-
try {
|
|
1653
|
-
const response = await fetchImpl(
|
|
1654
|
-
`${hubUrl}/queue/clear?book_id=${encodeURIComponent(bookId)}`,
|
|
1655
|
-
{ method: 'DELETE', headers: authHeaders() }
|
|
1656
|
-
);
|
|
1657
|
-
if (!response.ok) {
|
|
1658
|
-
failed++;
|
|
1659
|
-
reportWarning(`${context}: GPU hub book cleanup returned ${response.status} for ${bookId}`);
|
|
1660
|
-
} else {
|
|
1661
|
-
// Verify the hub honoured OUR scope. A mismatched echo means it
|
|
1662
|
-
// may have swept a different book (or everything) — that is a
|
|
1663
|
-
// failure, not a success we can report.
|
|
1664
|
-
let enforcedScope = true;
|
|
1665
|
-
try {
|
|
1666
|
-
const body = typeof response.json === 'function' ? await response.json() : null;
|
|
1667
|
-
if (body && body.scope && body.scope.book_id != null && body.scope.book_id !== bookId) {
|
|
1668
|
-
enforcedScope = false;
|
|
1669
|
-
}
|
|
1670
|
-
} catch (_) { /* unparseable body — trust the ok status */ }
|
|
1671
|
-
if (!enforcedScope) {
|
|
1672
|
-
failed++;
|
|
1673
|
-
reportWarning(`${context}: GPU hub book cleanup echoed a foreign scope for ${bookId} — treated as failed`);
|
|
1674
|
-
} else {
|
|
1675
|
-
cleared++;
|
|
1676
|
-
}
|
|
1677
|
-
}
|
|
1678
|
-
} catch (err) {
|
|
1679
|
-
failed++;
|
|
1680
|
-
reportWarning(`${context}: GPU hub book cleanup failed for ${bookId}: ${err.message}`);
|
|
1681
|
-
}
|
|
1682
|
-
return { requested: 1, cleared, failed };
|
|
1683
|
-
}
|
|
1684
|
-
|
|
1685
|
-
// ── Per-dispatch cleanup (unchanged) ─────────────────────────────────
|
|
1686
1402
|
for (const dispatchId of ids) {
|
|
1687
1403
|
try {
|
|
1404
|
+
const headers = {};
|
|
1405
|
+
if (apiKey) headers['x-api-key'] = apiKey;
|
|
1688
1406
|
const response = await fetchImpl(
|
|
1689
1407
|
`${hubUrl}/queue/clear?dispatch_id=${encodeURIComponent(dispatchId)}`,
|
|
1690
|
-
{ method: 'DELETE', headers
|
|
1408
|
+
{ method: 'DELETE', headers }
|
|
1691
1409
|
);
|
|
1692
1410
|
if (!response.ok) {
|
|
1693
1411
|
failed++;
|
|
@@ -2063,29 +1781,6 @@ module.exports = {
|
|
|
2063
1781
|
LEASE_TTLS,
|
|
2064
1782
|
QUOTAS,
|
|
2065
1783
|
|
|
2066
|
-
// Stage 116 (D3) / Stage 118 (M2/M4): bounded deferral (retry/backoff) —
|
|
2067
|
-
// the stage is not retried on every 5 s scheduler tick while it cannot
|
|
2068
|
-
// enqueue. Stage 118: the capacity and the idempotent bucket have their
|
|
2069
|
-
// own windows, and the taxonomy is shared with the stage executors.
|
|
2070
|
-
DEFERRED_STAGE_KEY_PREFIX,
|
|
2071
|
-
DEFERRED_STAGE_TTL_S,
|
|
2072
|
-
DEFERRED_IDEMPOTENT_TTL_S,
|
|
2073
|
-
deferredStageKey,
|
|
2074
|
-
isStageDeferred,
|
|
2075
|
-
deferStage,
|
|
2076
|
-
clearStageDeferral,
|
|
2077
|
-
clearDeferralsForBook,
|
|
2078
|
-
clearDeferralsForScene,
|
|
2079
|
-
deferralWindowFor,
|
|
2080
|
-
encodeKeyComponent,
|
|
2081
|
-
isDeferrableOutcome,
|
|
2082
|
-
isCapacityOutcome,
|
|
2083
|
-
isIdempotentOutcome,
|
|
2084
|
-
isFenceRejectedOutcome,
|
|
2085
|
-
CAPACITY_OUTCOMES,
|
|
2086
|
-
IDEMPOTENT_OUTCOMES,
|
|
2087
|
-
FENCE_REJECTED_OUTCOMES,
|
|
2088
|
-
|
|
2089
1784
|
// PW-2: dispatch identity generation (crypto-hardened)
|
|
2090
1785
|
generateDispatchToken
|
|
2091
1786
|
};
|
|
@@ -30,12 +30,6 @@
|
|
|
30
30
|
// a cancelled dispatch still owns on the GPU Hub
|
|
31
31
|
// (orchestrator resetScenes cancel leg,
|
|
32
32
|
// reconciliation RELEASE_STALE_LEASE recovery leg).
|
|
33
|
-
// Accepts EITHER an array of dispatch ids (one
|
|
34
|
-
// `?dispatch_id=` DELETE per id) OR an empty list with
|
|
35
|
-
// `options.book_id` (Stage 122 · N1 — a single
|
|
36
|
-
// book-scoped `?book_id=` sweep, used by the
|
|
37
|
-
// regenerate post-release lane sweep; audit 121 found
|
|
38
|
-
// the empty-list + book_id form was a silent no-op).
|
|
39
33
|
// Returns { requested, cleared, failed }; hub
|
|
40
34
|
// unavailability is non-fatal per-id (warn + count),
|
|
41
35
|
// never throws for a single failed id.
|
|
@@ -24,7 +24,8 @@
|
|
|
24
24
|
// is a routes-side concern and stays there
|
|
25
25
|
// sceneAssets — per-asset status truth + version gates + dirty
|
|
26
26
|
// units (orchestrator completeStage version gate,
|
|
27
|
-
//
|
|
27
|
+
// incl. its A1 build_id IS NULL fallback read
|
|
28
|
+
// getAssetNullBuild, scene-orchestrator dirty reads, scene-window
|
|
28
29
|
// staleness, scene-callbacks markReady/dirty clears,
|
|
29
30
|
// scene-restoration, reconciliation rebuildWorkList);
|
|
30
31
|
// markStale is consumed by state/scene-state-ops
|
|
@@ -58,6 +59,9 @@ const OPS = [
|
|
|
58
59
|
'tasks.hasActiveTaskForScene',
|
|
59
60
|
|
|
60
61
|
'sceneAssets.getAsset',
|
|
62
|
+
// A1 (stage147): version-gate fallback read — same key with
|
|
63
|
+
// build_id IS NULL ONLY (never another build, never "latest any row").
|
|
64
|
+
'sceneAssets.getAssetNullBuild',
|
|
61
65
|
'sceneAssets.markReady',
|
|
62
66
|
'sceneAssets.stampSceneVersionsForDispatch',
|
|
63
67
|
'sceneAssets.getDirtyUnitIds',
|
|
@@ -82,12 +86,6 @@ const OPS = [
|
|
|
82
86
|
'registry.registerSceneImageRedis',
|
|
83
87
|
'registry.registerSceneVideoRedis',
|
|
84
88
|
|
|
85
|
-
// Stage 122 (N2 · audit 121): the video send path frees an ORPHANED
|
|
86
|
-
// completion marker through this op (Redis-domain, `redis`-first like the
|
|
87
|
-
// registry.*Redis ops) — the orchestration tier cannot require the backend
|
|
88
|
-
// queue-idempotency helper directly.
|
|
89
|
-
'idem.releaseTaskDoneIfOrphan',
|
|
90
|
-
|
|
91
89
|
'filesystem.getSceneAudioPath',
|
|
92
90
|
|
|
93
91
|
'listBooks',
|
|
@@ -157,7 +157,7 @@ async function checkOrphanImageState(redis, bookId, chapterId, sceneId) {
|
|
|
157
157
|
// Orphan-check uses the canonical book build identity (audit d9d67a3).
|
|
158
158
|
const buildId = resolveBookBuildId(bookId);
|
|
159
159
|
const imageModule = lazyHostBinding('media.image');
|
|
160
|
-
|
|
160
|
+
const imageInfo = imageModule.resolveCanonicalSceneImage(
|
|
161
161
|
artifactRoot(),
|
|
162
162
|
buildId,
|
|
163
163
|
bookId,
|
|
@@ -165,31 +165,6 @@ async function checkOrphanImageState(redis, bookId, chapterId, sceneId) {
|
|
|
165
165
|
sceneId,
|
|
166
166
|
);
|
|
167
167
|
|
|
168
|
-
// ── Stage 143a (phase 2a · B2) ───────────────────────────────────
|
|
169
|
-
// Legacy scan first (byte-identical legacy behavior). A versioned-only
|
|
170
|
-
// build never writes the canonical file, so with the scan empty we ask
|
|
171
|
-
// the image facade for the published pointer of this scene build — the
|
|
172
|
-
// facade is flag-gated, flag OFF answers null with ZERO PG calls.
|
|
173
|
-
//
|
|
174
|
-
// An UNREADABLE pointer store ABORTS this reconcile pass (logged,
|
|
175
|
-
// `pass_incomplete`) instead of being read as "no publication": that
|
|
176
|
-
// reading would recommend regenerating a scene that is already
|
|
177
|
-
// published. No orphan issue is emitted on the aborted pass, so no
|
|
178
|
-
// regeneration is launched.
|
|
179
|
-
if (!imageInfo) {
|
|
180
|
-
try {
|
|
181
|
-
if (typeof imageModule.resolveVersionedSceneProof === 'function') {
|
|
182
|
-
imageInfo = await imageModule.resolveVersionedSceneProof(
|
|
183
|
-
artifactRoot(), buildId, bookId, chapterId, sceneId);
|
|
184
|
-
}
|
|
185
|
-
} catch (pgErr) {
|
|
186
|
-
error(`versioned image proof unavailable for ${bookId}/${chapterId}/${sceneId}: ${pgErr.message} — pass skipped, no regeneration launched`);
|
|
187
|
-
const err = new Error(`reconcile_pass_incomplete: image proof unavailable for ${bookId}/${chapterId}/${sceneId} (${pgErr.message})`);
|
|
188
|
-
err.code = 'pass_incomplete';
|
|
189
|
-
throw err;
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
|
|
193
168
|
if (!imageInfo) {
|
|
194
169
|
return {
|
|
195
170
|
type: 'orphan_image_state',
|
|
@@ -276,37 +251,11 @@ async function checkOrphanAssets(redis, bookId, chapterId, sceneId) {
|
|
|
276
251
|
try {
|
|
277
252
|
await fs.access(assets.image.path);
|
|
278
253
|
} catch (err) {
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
// legacy issue is reported exactly as before.
|
|
285
|
-
// An UNREADABLE pointer store aborts the pass (logged) instead of
|
|
286
|
-
// emitting a regeneration recommendation on a guess.
|
|
287
|
-
let coveredByPointer = false;
|
|
288
|
-
const imageModule = lazyHostBinding('media.image');
|
|
289
|
-
try {
|
|
290
|
-
if (typeof imageModule.resolveVersionedSceneProof === 'function') {
|
|
291
|
-
const buildId = resolveBookBuildId(bookId);
|
|
292
|
-
coveredByPointer = !!(await imageModule.resolveVersionedSceneProof(
|
|
293
|
-
artifactRoot(), buildId, bookId, chapterId, sceneId));
|
|
294
|
-
}
|
|
295
|
-
} catch (pgErr) {
|
|
296
|
-
error(`versioned image proof unavailable for ${bookId}/${chapterId}/${sceneId}: ${pgErr.message} — pass skipped, no recommendation emitted`);
|
|
297
|
-
const e = new Error(`reconcile_pass_incomplete: image proof unavailable for ${bookId}/${chapterId}/${sceneId} (${pgErr.message})`);
|
|
298
|
-
e.code = 'pass_incomplete';
|
|
299
|
-
throw e;
|
|
300
|
-
}
|
|
301
|
-
if (coveredByPointer) {
|
|
302
|
-
log(`missing_image_file suppressed: ${bookId}/${chapterId}/${sceneId} is covered by a published versioned pointer`);
|
|
303
|
-
} else {
|
|
304
|
-
orphanAssets.push({
|
|
305
|
-
type: 'missing_image_file',
|
|
306
|
-
scene: { bookId, chapterId, sceneId },
|
|
307
|
-
path: assets.image.path
|
|
308
|
-
});
|
|
309
|
-
}
|
|
254
|
+
orphanAssets.push({
|
|
255
|
+
type: 'missing_image_file',
|
|
256
|
+
scene: { bookId, chapterId, sceneId },
|
|
257
|
+
path: assets.image.path
|
|
258
|
+
});
|
|
310
259
|
}
|
|
311
260
|
}
|
|
312
261
|
|
|
@@ -388,7 +388,6 @@ async function tick(redis, loadedBooks = {}) {
|
|
|
388
388
|
throttled: 0,
|
|
389
389
|
skipped: 0,
|
|
390
390
|
completed: 0,
|
|
391
|
-
deferred: 0,
|
|
392
391
|
errors: []
|
|
393
392
|
};
|
|
394
393
|
|
|
@@ -467,10 +466,6 @@ async function tick(redis, loadedBooks = {}) {
|
|
|
467
466
|
summary.skipped++;
|
|
468
467
|
} else if (result.throttled > 0) {
|
|
469
468
|
summary.throttled += result.throttled;
|
|
470
|
-
} else if (result.deferred > 0) {
|
|
471
|
-
// Stage 118 (M9): a bounded deferral is a normal, quiet
|
|
472
|
-
// state — it must not pollute the tick error list.
|
|
473
|
-
summary.deferred += result.deferred;
|
|
474
469
|
} else if (result.reason === 'backpressure' || result.reason === 'throttled') {
|
|
475
470
|
summary.throttled++;
|
|
476
471
|
} else if (result.success) {
|
|
@@ -485,7 +480,7 @@ async function tick(redis, loadedBooks = {}) {
|
|
|
485
480
|
}
|
|
486
481
|
|
|
487
482
|
log(`=== TICK COMPLETE ===`);
|
|
488
|
-
log(`Active: ${summary.totalActive}, Dispatched: ${summary.dispatched}, Skipped: ${summary.skipped}, Throttled: ${summary.throttled}, Completed: ${summary.completed}
|
|
483
|
+
log(`Active: ${summary.totalActive}, Dispatched: ${summary.dispatched}, Skipped: ${summary.skipped}, Throttled: ${summary.throttled}, Completed: ${summary.completed}`);
|
|
489
484
|
|
|
490
485
|
if (summary.errors.length > 0) {
|
|
491
486
|
log(`Errors: ${summary.errors.length}`);
|
|
@@ -577,9 +572,6 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
|
|
|
577
572
|
let dispatched = 0;
|
|
578
573
|
let throttled = 0;
|
|
579
574
|
let anyCompleted = false;
|
|
580
|
-
// Stage 118 (M9): bounded deferrals are counted separately — they are
|
|
581
|
-
// neither a dispatch nor an error.
|
|
582
|
-
let deferred = 0;
|
|
583
575
|
const failures = [];
|
|
584
576
|
|
|
585
577
|
// Dispatch ALL eligible stages in this tick.
|
|
@@ -603,12 +595,6 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
|
|
|
603
595
|
// Stage finished inside the executor (already_done / cache_hit /
|
|
604
596
|
// no_segments) — not a failure, nothing to dispatch.
|
|
605
597
|
anyCompleted = true;
|
|
606
|
-
} else if (result.deferred) {
|
|
607
|
-
// Stage 118 (M9): a bounded deferral is NOT an error — nothing
|
|
608
|
-
// happened, the stage is waiting for capacity (or is already
|
|
609
|
-
// idempotently complete). Reporting it as a failure filled the
|
|
610
|
-
// tick summary with "errors" 12 times per minute per scene.
|
|
611
|
-
deferred++;
|
|
612
598
|
} else {
|
|
613
599
|
// Stage 50: раньше per-stage отказ (circuit_open, retry budget,
|
|
614
600
|
// lease, generating_transition_failed, …) молча проглатывался и
|
|
@@ -621,9 +607,6 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
|
|
|
621
607
|
if (dispatched > 0) {
|
|
622
608
|
return { dispatched, throttled, stages: stages.length };
|
|
623
609
|
}
|
|
624
|
-
if (deferred > 0 && failures.length === 0) {
|
|
625
|
-
return { dispatched: 0, throttled, deferred, stages: stages.length, success: true, reason: 'deferred' };
|
|
626
|
-
}
|
|
627
610
|
if (failures.length > 0) {
|
|
628
611
|
return { dispatched: 0, throttled, stages: stages.length, success: false, reason: failures.join(', ') };
|
|
629
612
|
}
|
|
@@ -158,34 +158,6 @@ async function getSceneFilesStatus(buildDir, bookId, chapterId, sceneId) {
|
|
|
158
158
|
result.image.exists = files.some(f => f.startsWith(imagePrefix) && f.endsWith('.png'));
|
|
159
159
|
} catch (_) {}
|
|
160
160
|
|
|
161
|
-
// ── Stage 143a (phase 2a · B1) ───────────────────────────────────
|
|
162
|
-
// A versioned-only build keeps its IU images under {build}/.versions/**,
|
|
163
|
-
// which this scan can never see. Only when the LEGACY scan came up empty
|
|
164
|
-
// do we ask the image facade for a published pointer; the facade is
|
|
165
|
-
// flag-gated, so flag OFF answers null with ZERO PostgreSQL calls and
|
|
166
|
-
// this function behaves exactly as before (condition 2).
|
|
167
|
-
//
|
|
168
|
-
// A pointer-store ERROR is logged and treated as "no proof" — image
|
|
169
|
-
// stays NOT existing. It is never read as "the file is there" (that
|
|
170
|
-
// would write a false READY) and never throws: this is an advisory
|
|
171
|
-
// checker whose five callers (checkSceneContentCache,
|
|
172
|
-
// restoreChunkStatusForScene, reconcileWindowStatuses, scene-restoration,
|
|
173
|
-
// rebuildWorkList) must keep running on a PG outage.
|
|
174
|
-
if (!result.image.exists) {
|
|
175
|
-
try {
|
|
176
|
-
const imageModule = lazyHostBinding('media.image');
|
|
177
|
-
if (imageModule && typeof imageModule.resolveVersionedSceneProof === 'function') {
|
|
178
|
-
const buildId = path.basename(path.resolve(buildDir));
|
|
179
|
-
const proof = await imageModule.resolveVersionedSceneProof(
|
|
180
|
-
OUTPUT_DIR, buildId, bookId, chapterId, sceneId);
|
|
181
|
-
if (proof) result.image.exists = true;
|
|
182
|
-
}
|
|
183
|
-
} catch (err) {
|
|
184
|
-
warn(`image pointer proof unavailable for ${bookId}/${chapterId}/${sceneId} (${err.message}) — image treated as absent`);
|
|
185
|
-
result.image.exists = false;
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
|
|
189
161
|
// Check video file
|
|
190
162
|
const videoPath = path.join(buildDir, artifactNaming.sceneVideoName(bookId, chapterId, sceneId));
|
|
191
163
|
result.video.exists = fs.existsSync(videoPath);
|