@animastor/orchestration 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +75 -0
- package/package.json +51 -0
- package/src/host/host-bindings.js +138 -0
- package/src/index.js +99 -0
- package/src/orchestration/index.js +40 -0
- package/src/orchestration/orchestrator.js +630 -0
- package/src/orchestration/runtime-result-consumer.js +54 -0
- package/src/orchestration/scene-callbacks.js +542 -0
- package/src/orchestration/scene-orchestrator.js +601 -0
- package/src/orchestration/scene-restoration.js +129 -0
- package/src/orchestration/scene-utils.js +36 -0
- package/src/runtime/active-scenes-index.js +164 -0
- package/src/runtime/audio-fsm-port.js +187 -0
- package/src/runtime/circuit-breaker.js +588 -0
- package/src/runtime/counter-reconciliation.js +308 -0
- package/src/runtime/dispatch-engine.js +1768 -0
- package/src/runtime/failure-taxonomy.js +125 -0
- package/src/runtime/hub-cancel-port.js +122 -0
- package/src/runtime/layer-config-port.js +136 -0
- package/src/runtime/lease-manager.js +607 -0
- package/src/runtime/persistence-port.js +162 -0
- package/src/runtime/placeholder-audio-port.js +117 -0
- package/src/runtime/progress-events-port.js +121 -0
- package/src/runtime/reconciliation-engine.js +2405 -0
- package/src/runtime/retry-budget-manager.js +229 -0
- package/src/runtime/runtime-metrics.js +427 -0
- package/src/runtime/runtime-result-emitter.js +123 -0
- package/src/runtime/runtime-scheduler.js +629 -0
- package/src/runtime/scene-data-port.js +114 -0
- package/src/runtime/scene-window.js +827 -0
- package/src/runtime/video-fsm-port.js +189 -0
- package/src/runtime/worker-health.js +270 -0
- package/test/counter-reconciliation.test.js +164 -0
- package/test/mocks/redis-mock.js +221 -0
- package/test/test-bindings.js +60 -0
|
@@ -0,0 +1,629 @@
|
|
|
1
|
+
const mediaRegistry = require('@animastor/generation').mediaRegistry;
|
|
2
|
+
// ======================================================
|
|
3
|
+
// Runtime Scheduler - v1.0.0
|
|
4
|
+
// ======================================================
|
|
5
|
+
// The ONLY authority for scene lifecycle progression.
|
|
6
|
+
// Decides WHEN to start stages, NOT HOW.
|
|
7
|
+
// Callbacks only register results - scheduler owns progression.
|
|
8
|
+
|
|
9
|
+
const { lazyHostBinding } = require('../host/host-bindings');
|
|
10
|
+
// §32.30: former host requires resolve lazily through the host-bindings seam.
|
|
11
|
+
const state = lazyHostBinding('state');
|
|
12
|
+
const dispatchEngine = require('./dispatch-engine');
|
|
13
|
+
// S-5: orchestration-owned behavior reaches runtime only via the injected seams
|
|
14
|
+
const orchestrationSeams = lazyHostBinding('seams');
|
|
15
|
+
// O-5: progress events (selective-task reads + completion reconciliation)
|
|
16
|
+
// arrive ONLY through the ProgressEventsPort (host adapter:
|
|
17
|
+
// storage/progress-events-adapter) — the Redis task-registry host service
|
|
18
|
+
// (../services/generation-progress) left this file.
|
|
19
|
+
const progressEventsOp = require('./progress-events-port').progressEventsOp;
|
|
20
|
+
// O-2: persistence arrives ONLY through the PersistencePort (host adapter:
|
|
21
|
+
// storage/runtime-persistence-adapter) — no PG repository or database
|
|
22
|
+
// handles in the runtime tier.
|
|
23
|
+
const persist = require('./persistence-port').persist;
|
|
24
|
+
|
|
25
|
+
const logPrefix = '[SCHEDULER]';
|
|
26
|
+
|
|
27
|
+
function log(msg) {
|
|
28
|
+
console.log(`${logPrefix} ${msg}`);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function warn(msg) {
|
|
32
|
+
console.warn(`${logPrefix} ⚠️ ${msg}`);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function error(msg) {
|
|
36
|
+
console.error(`${logPrefix} ❌ ${msg}`);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// ======================================================
|
|
40
|
+
// HELPERS
|
|
41
|
+
// ======================================================
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Read layer config for a book.
|
|
45
|
+
*/
|
|
46
|
+
async function getLayerConfig(redis, bookId) {
|
|
47
|
+
const layerKey = `animastor:layer-config:${bookId}`;
|
|
48
|
+
const layerRaw = await redis.get(layerKey);
|
|
49
|
+
if (layerRaw) {
|
|
50
|
+
try { return JSON.parse(layerRaw); } catch (_) {}
|
|
51
|
+
}
|
|
52
|
+
// S-2: default layer config derived from registry
|
|
53
|
+
const defaults = {};
|
|
54
|
+
for (const t of mediaRegistry.listMediaTypes()) { defaults[`${t}_enabled`] = true; }
|
|
55
|
+
return defaults;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Check if scene chunks have images marked as ready on disk.
|
|
60
|
+
* Used when image worker is disabled but video needs to verify images exist.
|
|
61
|
+
*/
|
|
62
|
+
async function checkChunksHaveImages(redis, bookId, chapterId, sceneId) {
|
|
63
|
+
const chunkPrefix = `animastor:chunk:${bookId}_${chapterId}_${sceneId}_`;
|
|
64
|
+
let cursor = '0';
|
|
65
|
+
let hasImages = false;
|
|
66
|
+
do {
|
|
67
|
+
const [nextCursor, keys] = await redis.scan(cursor, 'MATCH', `${chunkPrefix}*`, 'COUNT', 50);
|
|
68
|
+
cursor = nextCursor;
|
|
69
|
+
for (const key of keys) {
|
|
70
|
+
const raw = await redis.get(key);
|
|
71
|
+
if (raw) {
|
|
72
|
+
try {
|
|
73
|
+
const ch = JSON.parse(raw);
|
|
74
|
+
if (ch.image === true && ch.image_status === 'ready') {
|
|
75
|
+
hasImages = true;
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
} catch (_) {}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
} while (cursor !== '0' && !hasImages);
|
|
82
|
+
return hasImages;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ======================================================
|
|
86
|
+
// CONFIGURATION
|
|
87
|
+
// ======================================================
|
|
88
|
+
|
|
89
|
+
const SCHEDULER_TICK_MS = 5000; // 5 seconds
|
|
90
|
+
|
|
91
|
+
// ======================================================
|
|
92
|
+
// ACTIVE SCENE INDEX — T8: де-дубликация
|
|
93
|
+
// ======================================================
|
|
94
|
+
// T8: runtime-scheduler больше НЕ владеет своим API active-scenes.
|
|
95
|
+
// Все операции делегируются в active-scenes-index.js — единственный command API.
|
|
96
|
+
// addSceneToActiveIndex/removeSceneToActiveIndex оставлены как обёртки для
|
|
97
|
+
// обратной совместимости. Новый код должен использовать activeScenes API напрямую.
|
|
98
|
+
|
|
99
|
+
const activeScenes = require('./active-scenes-index');
|
|
100
|
+
const ACTIVE_SCENES_KEY = activeScenes.ACTIVE_SCENES_KEY;
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Add scene to active index. T8: делегирует в active-scenes-index.js.
|
|
104
|
+
*/
|
|
105
|
+
async function addSceneToActiveIndex(redis, bookId, chapterId, sceneId) {
|
|
106
|
+
const result = await activeScenes.addActiveScene(redis, bookId, chapterId, sceneId);
|
|
107
|
+
if (result.added) {
|
|
108
|
+
log(`+ ACTIVE: ${result.sceneKey} added to active index`);
|
|
109
|
+
}
|
|
110
|
+
return result;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Remove scene from active index. T8: делегирует в active-scenes-index.js.
|
|
115
|
+
*/
|
|
116
|
+
async function removeSceneFromActiveIndex(redis, bookId, chapterId, sceneId) {
|
|
117
|
+
const result = await activeScenes.removeActiveScene(redis, bookId, chapterId, sceneId);
|
|
118
|
+
if (result.removed) {
|
|
119
|
+
log(`- ACTIVE: ${result.sceneKey} removed from active index`);
|
|
120
|
+
}
|
|
121
|
+
return result;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// M4 (Н.9): removed dead concurrent counter functions — dispatchEngine owns quotas
|
|
125
|
+
|
|
126
|
+
// ======================================================
|
|
127
|
+
// STATE TO STAGE MAPPING
|
|
128
|
+
// ======================================================
|
|
129
|
+
|
|
130
|
+
// Re-export SceneState for convenience
|
|
131
|
+
// S-2: stage mappings derived from registry (lazy getters — resolved on
|
|
132
|
+
// first access so module load order can never observe an empty registry)
|
|
133
|
+
const STATE_TO_STAGE = new Proxy({}, {
|
|
134
|
+
get(_, key) { return mediaRegistry.getMediaType(String(key).replace(/_pending$/, '')) ? String(key).replace(/_pending$/, '') : undefined; },
|
|
135
|
+
ownKeys() { return mediaRegistry.listMediaTypes().map(t => `${t}_pending`); },
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
const STAGE_TO_STATE = new Proxy({}, {
|
|
139
|
+
get(_, stage) { return mediaRegistry.hasMediaType(String(stage)) ? `${stage}_pending` : undefined; },
|
|
140
|
+
ownKeys() { return mediaRegistry.listMediaTypes(); },
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// ======================================================
|
|
144
|
+
// SCHEDULER API
|
|
145
|
+
// ======================================================
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Get list of all active scenes from the index.
|
|
149
|
+
*/
|
|
150
|
+
async function getActiveSceneKeys(redis) {
|
|
151
|
+
return await redis.smembers(ACTIVE_SCENES_KEY);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Remove all active scenes belonging to a specific book.
|
|
156
|
+
* Used when cancelling generation for a book.
|
|
157
|
+
*/
|
|
158
|
+
async function clearBookFromActiveIndex(redis, bookId) {
|
|
159
|
+
const activeKeys = await getActiveSceneKeys(redis);
|
|
160
|
+
let removed = 0;
|
|
161
|
+
for (const sceneKey of activeKeys) {
|
|
162
|
+
if (sceneKey.startsWith(`${bookId}:`)) {
|
|
163
|
+
await redis.srem(ACTIVE_SCENES_KEY, sceneKey);
|
|
164
|
+
removed++;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if (removed > 0) {
|
|
168
|
+
log(`CANCEL: removed ${removed} active scenes for book ${bookId}`);
|
|
169
|
+
}
|
|
170
|
+
return removed;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Remove specific scenes from the active index.
|
|
175
|
+
* Used by regenerate to only remove dirty scenes, preserving other scenes'
|
|
176
|
+
* active index entries so the scheduler continues processing them.
|
|
177
|
+
*
|
|
178
|
+
* @param {Array<{chapter_id:string,scene_id:string}>} scenes - scenes to remove
|
|
179
|
+
*/
|
|
180
|
+
async function removeScenesFromActiveIndex(redis, bookId, scenes) {
|
|
181
|
+
if (!scenes || scenes.length === 0) return 0;
|
|
182
|
+
let removed = 0;
|
|
183
|
+
for (const s of scenes) {
|
|
184
|
+
const sceneKey = `${bookId}:${s.chapter_id}:${s.scene_id}`;
|
|
185
|
+
const result = await redis.srem(ACTIVE_SCENES_KEY, sceneKey);
|
|
186
|
+
if (result > 0) removed++;
|
|
187
|
+
}
|
|
188
|
+
if (removed > 0) {
|
|
189
|
+
log(`REMOVED: removed ${removed} specific scenes from active index for book ${bookId}`);
|
|
190
|
+
}
|
|
191
|
+
return removed;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Parse scene key to extract bookId, chapterId, sceneId.
|
|
196
|
+
*/
|
|
197
|
+
function parseSceneKey(sceneKey) {
|
|
198
|
+
const parts = sceneKey.split(':');
|
|
199
|
+
if (parts.length !== 3) {
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
return {
|
|
203
|
+
bookId: parts[0],
|
|
204
|
+
chapterId: parts[1],
|
|
205
|
+
sceneId: parts[2]
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Determine which asset stages need scheduling for a scene.
|
|
211
|
+
* Uses per-asset states as source of truth.
|
|
212
|
+
* Each worker is independent:
|
|
213
|
+
* - Audio: dispatches if enabled and not ready
|
|
214
|
+
* - Image: dispatches if enabled and not ready (independent of audio)
|
|
215
|
+
* - Video: dispatches if enabled, not ready, and image=ready
|
|
216
|
+
*
|
|
217
|
+
* Returns { stages: string[], allDone: boolean }
|
|
218
|
+
*/
|
|
219
|
+
/**
|
|
220
|
+
* Д.2: Detect PG version-staleness — PURE READ, no writes.
|
|
221
|
+
*
|
|
222
|
+
* R4.2: If asset_version < scene_version in PG, the scene needs regeneration
|
|
223
|
+
* even if Redis per-asset states show it as 'ready' (e.g. Redis was flushed or
|
|
224
|
+
* the version bump happened while Redis was offline). PG is the source of truth
|
|
225
|
+
* for dirty detection; Redis per-asset states are only a runtime cache.
|
|
226
|
+
*
|
|
227
|
+
* @returns {Promise<boolean>} true if any 'ready' asset row is version-stale
|
|
228
|
+
*/
|
|
229
|
+
async function detectVersionStale(redis, bookId, chapterId, sceneId) {
|
|
230
|
+
try {
|
|
231
|
+
// O-2: scenes-table version read via the PersistencePort (host SQL:
|
|
232
|
+
// storage/runtime-persistence-adapter.getAudioConfigVersions — the
|
|
233
|
+
// pre-O-2 raw query moved there verbatim).
|
|
234
|
+
const verResult = await persist('sceneVersions.getAudioConfigVersions')(bookId, chapterId, sceneId);
|
|
235
|
+
|
|
236
|
+
for (const row of verResult) {
|
|
237
|
+
if (row.asset_status === 'ready') {
|
|
238
|
+
if (row.scene_content_version != null && row.content_version != null &&
|
|
239
|
+
row.scene_content_version < row.content_version) {
|
|
240
|
+
log(`[VERSION-DIRTY] ${bookId}/${chapterId}/${sceneId}: content_version stale (asset=${row.scene_content_version} < scene=${row.content_version})`);
|
|
241
|
+
return true;
|
|
242
|
+
}
|
|
243
|
+
if (row.scene_audio_config_version != null && row.audio_config_version != null &&
|
|
244
|
+
row.scene_audio_config_version < row.audio_config_version) {
|
|
245
|
+
log(`[VERSION-DIRTY] ${bookId}/${chapterId}/${sceneId}: audio_config_version stale (asset=${row.scene_audio_config_version} < scene=${row.audio_config_version})`);
|
|
246
|
+
return true;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
} catch (err) {
|
|
251
|
+
// PG query failed — fall back to Redis-only detection (no stale signal)
|
|
252
|
+
warn(`[VERSION-DIRTY] PG version check failed for ${bookId}/${chapterId}/${sceneId}: ${err.message}`);
|
|
253
|
+
}
|
|
254
|
+
return false;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Д.2: Apply the version-stale reset — the explicit WRITE pass.
|
|
259
|
+
*
|
|
260
|
+
* Resets enabled READY assets to DIRTY so the dispatch engine picks them up.
|
|
261
|
+
* Separated from detection so shouldScheduleAssets stays pure (P3 in the
|
|
262
|
+
* state-writers map). Returns the number of assets reset.
|
|
263
|
+
*/
|
|
264
|
+
async function markVersionStaleDirty(redis, bookId, chapterId, sceneId) {
|
|
265
|
+
const layerCfg = await getLayerConfig(redis, bookId);
|
|
266
|
+
const taskState = await progressEventsOp('getSceneTaskState')(redis, bookId, chapterId, sceneId);
|
|
267
|
+
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
268
|
+
const audioEnabled = taskState.managed
|
|
269
|
+
? taskState.activeTypes.has('audio')
|
|
270
|
+
: layerCfg.audio_enabled !== false;
|
|
271
|
+
const imageEnabled = taskState.managed
|
|
272
|
+
? taskState.activeTypes.has('image')
|
|
273
|
+
: layerCfg.image_enabled !== false;
|
|
274
|
+
const videoEnabled = taskState.managed
|
|
275
|
+
? taskState.activeTypes.has('video')
|
|
276
|
+
: layerCfg.video_enabled !== false;
|
|
277
|
+
|
|
278
|
+
log(`[VERSION-DIRTY] ${bookId}/${chapterId}/${sceneId}: PG version mismatch — resetting per-asset states for dispatch`);
|
|
279
|
+
// T8: через фасад — единый владелец DIRTY
|
|
280
|
+
// S-5: asset-state writer via the injected orchestration seam
|
|
281
|
+
const markDirtyScene = orchestrationSeams.getOrchestrationSeam('markDirtyScene');
|
|
282
|
+
const dirtyAssets = [];
|
|
283
|
+
if (audioEnabled && assetStates.audio === state.AssetState.READY) dirtyAssets.push('audio');
|
|
284
|
+
if (imageEnabled && assetStates.image === state.AssetState.READY) dirtyAssets.push('image');
|
|
285
|
+
if (videoEnabled && assetStates.video === state.AssetState.READY) dirtyAssets.push('video');
|
|
286
|
+
if (dirtyAssets.length > 0) {
|
|
287
|
+
await markDirtyScene(redis, bookId, chapterId, sceneId, dirtyAssets);
|
|
288
|
+
}
|
|
289
|
+
return dirtyAssets.length;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
async function shouldScheduleAssets(redis, bookId, chapterId, sceneId) {
|
|
293
|
+
// Д.2: This function is now PURE — it only reads per-asset states and layer
|
|
294
|
+
// config and computes { stages, allDone }. It performs NO writes. The former
|
|
295
|
+
// version-stale READY→DIRTY reset is an explicit pre-pass in attemptDispatch
|
|
296
|
+
// (detectVersionStale + markVersionStaleDirty), so the decision is separated
|
|
297
|
+
// from the mutation (single arbiter — see docs/STATE_WRITERS_MAP.md P3).
|
|
298
|
+
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
299
|
+
const layerCfg = await getLayerConfig(redis, bookId);
|
|
300
|
+
const taskState = await progressEventsOp('getSceneTaskState')(redis, bookId, chapterId, sceneId);
|
|
301
|
+
|
|
302
|
+
// Selective generation tasks are authoritative for their target scenes.
|
|
303
|
+
// Book-wide layer preferences are only a fallback for the initial/legacy
|
|
304
|
+
// generation flow, where no explicit task registry exists.
|
|
305
|
+
const isInFlight = status =>
|
|
306
|
+
status === state.AssetState.PENDING || status === state.AssetState.GENERATING;
|
|
307
|
+
const audioEnabled = taskState.managed
|
|
308
|
+
? taskState.activeTypes.has('audio')
|
|
309
|
+
: layerCfg.audio_enabled !== false || isInFlight(assetStates.audio);
|
|
310
|
+
const imageEnabled = taskState.managed
|
|
311
|
+
? taskState.activeTypes.has('image')
|
|
312
|
+
: layerCfg.image_enabled !== false || isInFlight(assetStates.image);
|
|
313
|
+
const videoEnabled = taskState.managed
|
|
314
|
+
? taskState.activeTypes.has('video')
|
|
315
|
+
: layerCfg.video_enabled !== false || isInFlight(assetStates.video);
|
|
316
|
+
|
|
317
|
+
// Check if all enabled assets are in terminal states
|
|
318
|
+
// (after potential version-stale reset above)
|
|
319
|
+
const allDone = (
|
|
320
|
+
(!audioEnabled || assetStates.audio === state.AssetState.READY || assetStates.audio === state.AssetState.FAILED) &&
|
|
321
|
+
(!imageEnabled || assetStates.image === state.AssetState.READY || assetStates.image === state.AssetState.FAILED) &&
|
|
322
|
+
(!videoEnabled || assetStates.video === state.AssetState.READY || assetStates.video === state.AssetState.FAILED)
|
|
323
|
+
);
|
|
324
|
+
if (allDone) return { stages: [], allDone: true };
|
|
325
|
+
|
|
326
|
+
const stages = [];
|
|
327
|
+
|
|
328
|
+
// Audio: independent — dispatch if not ready and not already generating
|
|
329
|
+
if (audioEnabled &&
|
|
330
|
+
assetStates.audio !== state.AssetState.READY &&
|
|
331
|
+
assetStates.audio !== state.AssetState.FAILED &&
|
|
332
|
+
assetStates.audio !== state.AssetState.GENERATING &&
|
|
333
|
+
assetStates.audio !== state.AssetState.PLACEHOLDER) {
|
|
334
|
+
stages.push('audio');
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
// Image: independent of audio — dispatch if not ready and not already generating
|
|
338
|
+
if (imageEnabled &&
|
|
339
|
+
assetStates.image !== state.AssetState.READY &&
|
|
340
|
+
assetStates.image !== state.AssetState.FAILED &&
|
|
341
|
+
assetStates.image !== state.AssetState.GENERATING) {
|
|
342
|
+
stages.push('image');
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// Video: requires image=ready — if images aren't ready, skip silently
|
|
346
|
+
if (videoEnabled &&
|
|
347
|
+
assetStates.video !== state.AssetState.READY &&
|
|
348
|
+
assetStates.video !== state.AssetState.FAILED &&
|
|
349
|
+
assetStates.video !== state.AssetState.GENERATING) {
|
|
350
|
+
let imageReady = assetStates.image === state.AssetState.READY;
|
|
351
|
+
if (!imageReady && !imageEnabled) {
|
|
352
|
+
imageReady = await checkChunksHaveImages(redis, bookId, chapterId, sceneId);
|
|
353
|
+
}
|
|
354
|
+
if (imageReady) {
|
|
355
|
+
stages.push('video');
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
return { stages, allDone };
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/**
|
|
363
|
+
* Process all active scenes.
|
|
364
|
+
* Returns summary of operations.
|
|
365
|
+
*/
|
|
366
|
+
async function tick(redis, loadedBooks = {}) {
|
|
367
|
+
log('=== RUNTIME SCHEDULER TICK ===');
|
|
368
|
+
|
|
369
|
+
// Acquire tick lock for single-flight protection
|
|
370
|
+
const tickLock = await dispatchEngine.acquireSchedulerTickLock(redis);
|
|
371
|
+
if (!tickLock.acquired) {
|
|
372
|
+
log('TICK_SKIPPED: Another tick is still running');
|
|
373
|
+
return {
|
|
374
|
+
skipped: true,
|
|
375
|
+
reason: 'tick_running',
|
|
376
|
+
timestamp: new Date().toISOString()
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// T6.5: Все тело tick обёрнуто в try/finally для гарантированного
|
|
381
|
+
// освобождения lock даже при исключении (не только в per-scene catch).
|
|
382
|
+
try {
|
|
383
|
+
const activeKeys = await getActiveSceneKeys(redis);
|
|
384
|
+
const summary = {
|
|
385
|
+
totalActive: activeKeys.length,
|
|
386
|
+
processed: 0,
|
|
387
|
+
dispatched: 0,
|
|
388
|
+
throttled: 0,
|
|
389
|
+
skipped: 0,
|
|
390
|
+
completed: 0,
|
|
391
|
+
errors: []
|
|
392
|
+
};
|
|
393
|
+
|
|
394
|
+
const activeBookIds = new Set();
|
|
395
|
+
for (const sceneKey of activeKeys) {
|
|
396
|
+
const parsed = parseSceneKey(sceneKey);
|
|
397
|
+
if (parsed) activeBookIds.add(parsed.bookId);
|
|
398
|
+
}
|
|
399
|
+
for (const bookId of activeBookIds) {
|
|
400
|
+
const completedTasks = await progressEventsOp('reconcileCompletedTasks')(
|
|
401
|
+
redis,
|
|
402
|
+
bookId,
|
|
403
|
+
state.getAssetStates
|
|
404
|
+
);
|
|
405
|
+
for (const task of completedTasks) {
|
|
406
|
+
try {
|
|
407
|
+
await persist('tasks.updateTaskStatus')(task.task_id, 'completed');
|
|
408
|
+
} catch (pgErr) {
|
|
409
|
+
warn(`TASK_COMPLETE_PERSIST_FAILED: ${task.task_id}: ${pgErr.message}`);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Cache force-dispatch flags per book to avoid redundant Redis calls
|
|
415
|
+
const forceCache = new Map();
|
|
416
|
+
// Cache loaded books per tick to avoid redundant disk reads
|
|
417
|
+
const bookCache = new Map();
|
|
418
|
+
|
|
419
|
+
for (const sceneKey of activeKeys) {
|
|
420
|
+
const parsed = parseSceneKey(sceneKey);
|
|
421
|
+
if (!parsed) {
|
|
422
|
+
summary.errors.push(`Invalid scene key: ${sceneKey}`);
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
const { bookId, chapterId, sceneId } = parsed;
|
|
427
|
+
|
|
428
|
+
// Check force-dispatch flag for recently regenerated books (cached per book)
|
|
429
|
+
let force = false;
|
|
430
|
+
if (forceCache.has(bookId)) {
|
|
431
|
+
force = forceCache.get(bookId);
|
|
432
|
+
} else {
|
|
433
|
+
const forceFlag = await redis.get(`animastor:force-dispatch:${bookId}`);
|
|
434
|
+
force = forceFlag === '1';
|
|
435
|
+
forceCache.set(bookId, force);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// Resolve loaded book from cache or load it
|
|
439
|
+
let sceneLoadedBook = loadedBooks[bookId];
|
|
440
|
+
if (!sceneLoadedBook) {
|
|
441
|
+
if (bookCache.has(bookId)) {
|
|
442
|
+
sceneLoadedBook = bookCache.get(bookId);
|
|
443
|
+
} else {
|
|
444
|
+
try {
|
|
445
|
+
// O-3: scene content via the SceneDataPort (host
|
|
446
|
+
// adapter: storage/scene-data-adapter) — the Book
|
|
447
|
+
// Model facade require left the scheduler.
|
|
448
|
+
sceneLoadedBook = require('./scene-data-port').sceneDataOp('loadBook')(bookId);
|
|
449
|
+
bookCache.set(bookId, sceneLoadedBook);
|
|
450
|
+
} catch (_) {
|
|
451
|
+
sceneLoadedBook = null;
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
try {
|
|
457
|
+
const result = await attemptDispatch(redis, bookId, chapterId, sceneId, sceneLoadedBook, force);
|
|
458
|
+
summary.processed++;
|
|
459
|
+
|
|
460
|
+
if (result.completed) {
|
|
461
|
+
summary.completed++;
|
|
462
|
+
} else if (result.dispatched && result.dispatched > 0) {
|
|
463
|
+
summary.dispatched += result.dispatched;
|
|
464
|
+
if (result.throttled > 0) summary.throttled += result.throttled;
|
|
465
|
+
} else if (result.skip) {
|
|
466
|
+
summary.skipped++;
|
|
467
|
+
} else if (result.throttled > 0) {
|
|
468
|
+
summary.throttled += result.throttled;
|
|
469
|
+
} else if (result.reason === 'backpressure' || result.reason === 'throttled') {
|
|
470
|
+
summary.throttled++;
|
|
471
|
+
} else if (result.success) {
|
|
472
|
+
// No action needed
|
|
473
|
+
} else {
|
|
474
|
+
summary.errors.push(`${result.reason} for ${sceneKey}`);
|
|
475
|
+
}
|
|
476
|
+
} catch (err) {
|
|
477
|
+
summary.errors.push(`Error processing ${sceneKey}: ${err.message}`);
|
|
478
|
+
error(`Tick error for ${sceneKey}: ${err.message}`);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
log(`=== TICK COMPLETE ===`);
|
|
483
|
+
log(`Active: ${summary.totalActive}, Dispatched: ${summary.dispatched}, Skipped: ${summary.skipped}, Throttled: ${summary.throttled}, Completed: ${summary.completed}`);
|
|
484
|
+
|
|
485
|
+
if (summary.errors.length > 0) {
|
|
486
|
+
log(`Errors: ${summary.errors.length}`);
|
|
487
|
+
summary.errors.slice(0, 5).forEach(e => log(` - ${e}`));
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
return summary;
|
|
491
|
+
} finally {
|
|
492
|
+
// T6.5: Гарантированное освобождение lock в finally.
|
|
493
|
+
// Если исключение вылетело за пределы per-scene catch (напр. getActiveSceneKeys),
|
|
494
|
+
// lock должен быть отпущен, чтобы scheduler не завис навсегда.
|
|
495
|
+
try {
|
|
496
|
+
await dispatchEngine.releaseSchedulerTickLock(redis, tickLock.token);
|
|
497
|
+
} catch (lockErr) {
|
|
498
|
+
error(`TICK_LOCK_RELEASE_FAILED: ${lockErr.message}`);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Get runtime metrics for debugging.
|
|
505
|
+
*/
|
|
506
|
+
async function getMetrics(redis) {
|
|
507
|
+
const activeCount = await redis.scard(ACTIVE_SCENES_KEY);
|
|
508
|
+
|
|
509
|
+
// M4 (Н.9): quotas are owned by dispatchEngine — delegate to getQuotaStatus
|
|
510
|
+
let quotaStatus = { audio: { current: 0, max: 0 }, image: { current: 0, max: 0 }, video: { current: 0, max: 0 } };
|
|
511
|
+
try {
|
|
512
|
+
quotaStatus = await dispatchEngine.getQuotaStatus(redis);
|
|
513
|
+
} catch (e) {
|
|
514
|
+
warn(`getMetrics: dispatchEngine.getQuotaStatus failed: ${e.message}`);
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
return {
|
|
518
|
+
activeScenes: activeCount,
|
|
519
|
+
concurrent: {
|
|
520
|
+
audio: quotaStatus.audio.current,
|
|
521
|
+
image: quotaStatus.image.current,
|
|
522
|
+
video: quotaStatus.video.current
|
|
523
|
+
},
|
|
524
|
+
limits: {
|
|
525
|
+
audio: quotaStatus.audio.max,
|
|
526
|
+
image: quotaStatus.image.max,
|
|
527
|
+
video: quotaStatus.video.max
|
|
528
|
+
}
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// ======================================================
|
|
533
|
+
// ATTEMPT DISPATCH (NEW: Uses dispatch engine)
|
|
534
|
+
// ======================================================
|
|
535
|
+
|
|
536
|
+
/**
|
|
537
|
+
* Attempt to dispatch scene stages using per-asset states.
|
|
538
|
+
* Supports independent workers: dispatches ALL eligible stages in one tick.
|
|
539
|
+
* When force=true, any existing lease is cleared before dispatch.
|
|
540
|
+
* Returns { dispatched: number, completed: boolean, reason: string }
|
|
541
|
+
*/
|
|
542
|
+
async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, force = false) {
|
|
543
|
+
const sceneKey = `${bookId}:${chapterId}:${sceneId}`;
|
|
544
|
+
|
|
545
|
+
// Resolve build_id from manifest (passed from tick with book cache) or null
|
|
546
|
+
const buildId = loadedBook?.manifest?.build_id || null;
|
|
547
|
+
|
|
548
|
+
// Д.2: Version-stale pre-pass — detect (pure read) then reset (explicit write)
|
|
549
|
+
// BEFORE planning, so a stale 'ready' scene is re-dispatched in the SAME tick.
|
|
550
|
+
// Keeps shouldScheduleAssets pure (decision separated from mutation).
|
|
551
|
+
if (await detectVersionStale(redis, bookId, chapterId, sceneId)) {
|
|
552
|
+
await markVersionStaleDirty(redis, bookId, chapterId, sceneId);
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
// Determine which asset stages need scheduling (from per-asset states)
|
|
556
|
+
const { stages, allDone } = await shouldScheduleAssets(redis, bookId, chapterId, sceneId);
|
|
557
|
+
|
|
558
|
+
if (allDone) {
|
|
559
|
+
await removeSceneFromActiveIndex(redis, bookId, chapterId, sceneId);
|
|
560
|
+
log(`COMPLETE: ${sceneKey} (all assets ready) - removed from active index`);
|
|
561
|
+
return { completed: true, reason: 'all_assets_ready' };
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
if (stages.length === 0) {
|
|
565
|
+
// Assets are in non-dispatchable states (e.g., generating, placeholder)
|
|
566
|
+
// or video skipped because images not ready
|
|
567
|
+
return { success: true, skip: true, reason: 'no_dispatchable_stages' };
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
log(`ATTEMPT_DISPATCH: ${sceneKey} -> stages=[${stages.join(', ')}]${force ? ' (force=true)' : ''}`);
|
|
571
|
+
|
|
572
|
+
let dispatched = 0;
|
|
573
|
+
let throttled = 0;
|
|
574
|
+
|
|
575
|
+
// Dispatch ALL eligible stages in this tick.
|
|
576
|
+
// Concurrency limits per stage type are handled by dispatch engine.
|
|
577
|
+
for (const stage of stages) {
|
|
578
|
+
const result = await dispatchEngine.dispatchStage(
|
|
579
|
+
redis,
|
|
580
|
+
bookId,
|
|
581
|
+
chapterId,
|
|
582
|
+
sceneId,
|
|
583
|
+
stage,
|
|
584
|
+
loadedBook,
|
|
585
|
+
buildId,
|
|
586
|
+
{ force }
|
|
587
|
+
);
|
|
588
|
+
if (result.dispatched) {
|
|
589
|
+
dispatched++;
|
|
590
|
+
} else if (result.reason === 'backpressure' || result.reason === 'throttled') {
|
|
591
|
+
throttled++;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
return { dispatched, throttled, stages: stages.length };
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// ======================================================
|
|
599
|
+
// EXPORTS
|
|
600
|
+
// ======================================================
|
|
601
|
+
|
|
602
|
+
module.exports = {
|
|
603
|
+
// Configuration
|
|
604
|
+
SCHEDULER_TICK_MS,
|
|
605
|
+
|
|
606
|
+
// Active scene index
|
|
607
|
+
ACTIVE_SCENES_KEY,
|
|
608
|
+
addSceneToActiveIndex,
|
|
609
|
+
removeSceneFromActiveIndex,
|
|
610
|
+
clearBookFromActiveIndex,
|
|
611
|
+
removeScenesFromActiveIndex,
|
|
612
|
+
getActiveSceneKeys,
|
|
613
|
+
parseSceneKey,
|
|
614
|
+
|
|
615
|
+
// Scheduling
|
|
616
|
+
shouldScheduleAssets,
|
|
617
|
+
detectVersionStale,
|
|
618
|
+
markVersionStaleDirty,
|
|
619
|
+
getLayerConfig,
|
|
620
|
+
checkChunksHaveImages,
|
|
621
|
+
tick,
|
|
622
|
+
getMetrics,
|
|
623
|
+
|
|
624
|
+
// Re-exports
|
|
625
|
+
get AssetState() { return state.AssetState; },
|
|
626
|
+
STATE_TO_STAGE,
|
|
627
|
+
STAGE_TO_STATE,
|
|
628
|
+
|
|
629
|
+
};
|