@animastor/orchestration 0.1.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +156 -0
- package/package.json +3 -2
- package/src/index.js +1 -0
- package/src/orchestration/orchestrator.js +18 -8
- package/src/orchestration/scene-callbacks.js +55 -3
- package/src/orchestration/scene-orchestrator.js +91 -14
- package/src/runtime/active-scenes-index.js +18 -2
- package/src/runtime/command-gate.js +270 -0
- package/src/runtime/dispatch-engine.js +322 -6
- package/src/runtime/dispatch-outcomes.js +124 -0
- package/src/runtime/hub-cancel-port.js +6 -0
- package/src/runtime/persistence-port.js +6 -0
- package/src/runtime/progress-events-port.js +6 -0
- package/src/runtime/reconciliation-engine.js +206 -172
- package/src/runtime/runtime-scheduler.js +88 -61
- package/src/runtime/scene-window.js +51 -2
- package/test/capacity-deferral.test.js +265 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// ======================================================
|
|
2
|
+
// DISPATCH OUTCOMES — the shared taxonomy of "nothing was enqueued"
|
|
3
|
+
//
|
|
4
|
+
// Stage 118 (audit 117 · D3/M4/M5): the classification of an executor
|
|
5
|
+
// outcome lives in ONE module so the dispatch engine (runtime tier) and the
|
|
6
|
+
// stage executors (orchestration tier) cannot drift apart.
|
|
7
|
+
//
|
|
8
|
+
// Buckets
|
|
9
|
+
// ───────
|
|
10
|
+
// • capacity — the GPU capacity is not there right now:
|
|
11
|
+
// 'no_worker_available' | 'duplicate_pending' | 'ambiguous_post'
|
|
12
|
+
// ⇒ a BOUNDED retry (deferral window) is the right response.
|
|
13
|
+
//
|
|
14
|
+
// • idempotent — the logical task is ALREADY COMPLETE (the completion marker
|
|
15
|
+
// exists): 'already_completed'.
|
|
16
|
+
// This is NOT a capacity problem and MUST NOT be classified as
|
|
17
|
+
// one (audit 117 · M4): mixing it into the capacity bucket
|
|
18
|
+
// turned "this task will never run" into an endless retry
|
|
19
|
+
// loop. It gets its own, much longer backoff window and its
|
|
20
|
+
// own reporting class.
|
|
21
|
+
//
|
|
22
|
+
// The host side (@animastor/generation → backend/src/helpers/queue-idempotency)
|
|
23
|
+
// mirrors these sets for the transport-level reasons it produces; the two
|
|
24
|
+
// lists must stay in sync (see queue-idempotency.cjs · CAPACITY_REASONS).
|
|
25
|
+
// ======================================================
|
|
26
|
+
|
|
27
|
+
const CAPACITY_OUTCOMES = new Set([
|
|
28
|
+
'no_worker_available', // no live, type-compatible worker
|
|
29
|
+
'duplicate_pending', // the job is already pending in the lane
|
|
30
|
+
'ambiguous_post', // the POST outcome was ambiguous (the hub may have taken the task)
|
|
31
|
+
// Stage 129: the intent-fence step could not run (Redis refused the atomic
|
|
32
|
+
// path / the client cannot run it). Nothing was enqueued and the send IS
|
|
33
|
+
// retryable, so it belongs to the bounded-retry bucket.
|
|
34
|
+
'intent_receipt_unavailable',
|
|
35
|
+
]);
|
|
36
|
+
|
|
37
|
+
const IDEMPOTENT_OUTCOMES = new Set([
|
|
38
|
+
'already_completed', // the logical task is already complete (artifact missing)
|
|
39
|
+
]);
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Fence rejections (Stage 129, audit 128 · F5): the backend refused this
|
|
43
|
+
* dispatch because it belongs to a generation the intent record already
|
|
44
|
+
* superseded, or because the dispatch/intent pair is inconsistent.
|
|
45
|
+
*
|
|
46
|
+
* Retrying the SAME generation can never succeed, so they are NOT retryable —
|
|
47
|
+
* but they must also not fall into the historic "immediate retry on the next
|
|
48
|
+
* tick" bucket, which would turn a stale send into a 5-second dispatch →
|
|
49
|
+
* cancel → rollback loop. They get the long (idempotent) deferral window and
|
|
50
|
+
* stay visible in the logs/metrics.
|
|
51
|
+
*/
|
|
52
|
+
const FENCE_REJECTED_OUTCOMES = new Set([
|
|
53
|
+
'stale_intent',
|
|
54
|
+
'receipt_conflict',
|
|
55
|
+
'intent_not_active',
|
|
56
|
+
'intent_corrupt',
|
|
57
|
+
'intent_missing',
|
|
58
|
+
]);
|
|
59
|
+
|
|
60
|
+
/** Is this outcome a CAPACITY gap (bounded retry with a short window)? */
|
|
61
|
+
function isCapacityOutcome(reason) {
|
|
62
|
+
return CAPACITY_OUTCOMES.has(String(reason || ''));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Is this outcome an idempotent suppression (the task is already done)? */
|
|
66
|
+
function isIdempotentOutcome(reason) {
|
|
67
|
+
return IDEMPOTENT_OUTCOMES.has(String(reason || ''));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Is this outcome a fence rejection (terminal for this generation)? */
|
|
71
|
+
function isFenceRejectedOutcome(reason) {
|
|
72
|
+
return FENCE_REJECTED_OUTCOMES.has(String(reason || ''));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Is this outcome deferrable at all (capacity, idempotent OR fence-rejected)? */
|
|
76
|
+
function isDeferrableOutcome(reason) {
|
|
77
|
+
return isCapacityOutcome(reason) || isIdempotentOutcome(reason) || isFenceRejectedOutcome(reason);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Classify the reasons why NOTHING was enqueued for a stage.
|
|
82
|
+
*
|
|
83
|
+
* Stage 118 (audit 117 · M3): the Stage 116 implementation required EXACTLY
|
|
84
|
+
* ONE distinct reason (`notSentReasons.size === 1`). A scene whose units
|
|
85
|
+
* failed for two DIFFERENT deferrable reasons (e.g. one unit gated by
|
|
86
|
+
* "no live worker" and another suppressed as "duplicate pending") was
|
|
87
|
+
* therefore NOT deferred and kept the 5-second dispatch/cancel churn.
|
|
88
|
+
*
|
|
89
|
+
* The rule now: EVERY reason must be deferrable; the most frequent one is
|
|
90
|
+
* reported. A single non-deferrable reason (a real failure) keeps the
|
|
91
|
+
* historic semantics (an immediate retry on the next tick) — the
|
|
92
|
+
* audit-6929ba5 contract depends on it.
|
|
93
|
+
*
|
|
94
|
+
* @param {Map<string, number>|Object<string, number>} reasons
|
|
95
|
+
* @returns {string|null} the dominant deferrable reason, or null.
|
|
96
|
+
*/
|
|
97
|
+
function classifyNotSentReasons(reasons) {
|
|
98
|
+
const entries = reasons instanceof Map
|
|
99
|
+
? [...reasons.entries()]
|
|
100
|
+
: Object.entries(reasons || {});
|
|
101
|
+
if (entries.length === 0) return null;
|
|
102
|
+
if (!entries.every(([reason]) => isDeferrableOutcome(reason))) return null;
|
|
103
|
+
let dominant = null;
|
|
104
|
+
let dominantCount = -1;
|
|
105
|
+
for (const [reason, count] of entries) {
|
|
106
|
+
const n = Number(count) || 0;
|
|
107
|
+
if (n > dominantCount) {
|
|
108
|
+
dominant = reason;
|
|
109
|
+
dominantCount = n;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return dominant;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
module.exports = {
|
|
116
|
+
CAPACITY_OUTCOMES,
|
|
117
|
+
IDEMPOTENT_OUTCOMES,
|
|
118
|
+
FENCE_REJECTED_OUTCOMES,
|
|
119
|
+
isCapacityOutcome,
|
|
120
|
+
isIdempotentOutcome,
|
|
121
|
+
isFenceRejectedOutcome,
|
|
122
|
+
isDeferrableOutcome,
|
|
123
|
+
classifyNotSentReasons,
|
|
124
|
+
};
|
|
@@ -30,6 +30,12 @@
|
|
|
30
30
|
// a cancelled dispatch still owns on the GPU Hub
|
|
31
31
|
// (orchestrator resetScenes cancel leg,
|
|
32
32
|
// reconciliation RELEASE_STALE_LEASE recovery leg).
|
|
33
|
+
// Accepts EITHER an array of dispatch ids (one
|
|
34
|
+
// `?dispatch_id=` DELETE per id) OR an empty list with
|
|
35
|
+
// `options.book_id` (Stage 122 · N1 — a single
|
|
36
|
+
// book-scoped `?book_id=` sweep, used by the
|
|
37
|
+
// regenerate post-release lane sweep; audit 121 found
|
|
38
|
+
// the empty-list + book_id form was a silent no-op).
|
|
33
39
|
// Returns { requested, cleared, failed }; hub
|
|
34
40
|
// unavailability is non-fatal per-id (warn + count),
|
|
35
41
|
// never throws for a single failed id.
|
|
@@ -86,6 +86,12 @@ const OPS = [
|
|
|
86
86
|
'registry.registerSceneImageRedis',
|
|
87
87
|
'registry.registerSceneVideoRedis',
|
|
88
88
|
|
|
89
|
+
// Stage 122 (N2 · audit 121): the video send path frees an ORPHANED
|
|
90
|
+
// completion marker through this op (Redis-domain, `redis`-first like the
|
|
91
|
+
// registry.*Redis ops) — the orchestration tier cannot require the backend
|
|
92
|
+
// queue-idempotency helper directly.
|
|
93
|
+
'idem.releaseTaskDoneIfOrphan',
|
|
94
|
+
|
|
89
95
|
'filesystem.getSceneAudioPath',
|
|
90
96
|
|
|
91
97
|
'listBooks',
|
|
@@ -39,6 +39,11 @@
|
|
|
39
39
|
// pre-pass; the getAssetStates callback arrives
|
|
40
40
|
// as an argument exactly as pre-O-5)
|
|
41
41
|
//
|
|
42
|
+
// blockSceneStage — command-only generation (2026-10-31): no live
|
|
43
|
+
// worker of a stage's type → block the command
|
|
44
|
+
// that asked for it (registry + PG are host-side;
|
|
45
|
+
// runtime-scheduler via command-gate)
|
|
46
|
+
//
|
|
42
47
|
// Deliberately NOT in the port: createTasks/listTasks/getTask/updateTask/
|
|
43
48
|
// markCompleted/markCancelled/removeTask/clear (routes-side task CRUD —
|
|
44
49
|
// the HTTP contour owns task lifecycle), getActiveTasksByType,
|
|
@@ -53,6 +58,7 @@ const OPS = [
|
|
|
53
58
|
'getSceneTaskState',
|
|
54
59
|
'hasActiveTasks',
|
|
55
60
|
'reconcileCompletedTasks',
|
|
61
|
+
'blockSceneStage',
|
|
56
62
|
];
|
|
57
63
|
|
|
58
64
|
let impl = null;
|