@animastor/orchestration 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/orchestration/scene-callbacks.js +2 -2
- package/src/orchestration/scene-orchestrator.js +47 -3
- package/src/runtime/dispatch-engine.js +19 -1
- package/src/runtime/persistence-port.js +1 -0
- package/src/runtime/reconciliation-engine.js +83 -15
- package/src/runtime/runtime-scheduler.js +23 -1
- package/test/artifact-root-binding.test.js +174 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@animastor/orchestration",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.4",
|
|
4
4
|
"description": "Animastor Orchestration — the generation orchestration/runtime contour of the Animastor backend, physically extracted from the host (§32.30): the unified generation lifecycle owner (orchestrator facade, stage executor, scene callbacks/restoration), the dispatch engine and its lease/quota/retry managers, the reconciliation engine and counter reconciliation, the scene window/scheduler/worker-health, the runtime-result emitter/consumer, and the eight O-2..O-10 port contracts (Persistence, SceneData, PlaceholderAudio, ProgressEvents, AudioFsm, VideoFsm, HubCancel, LayerConfig). The package reaches the host (runtime config slices, Redis-domain state layer, media facades, the S-5 seam registry, the artifact root, the PW-2 workspace resolver) ONLY through the composition-root host-bindings seam — no backend/src require, no storage/book/routes/services, no Redis, no PG, no Express inside.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"animastor",
|
|
@@ -278,7 +278,7 @@ async function handleImageCompleted(redis, bookId, chapterId, sceneId, buildId)
|
|
|
278
278
|
log(`IMAGE_CALLBACK: ${bookId}/${chapterId}/${sceneId}`);
|
|
279
279
|
|
|
280
280
|
const sceneImage = image.resolveCanonicalSceneImage(
|
|
281
|
-
|
|
281
|
+
artifactRoot(), buildId, bookId, chapterId, sceneId
|
|
282
282
|
);
|
|
283
283
|
|
|
284
284
|
// Per-asset check
|
|
@@ -390,7 +390,7 @@ async function handleVideoCompleted(redis, bookId, chapterId, sceneId, buildId)
|
|
|
390
390
|
// T1: возвращает { ok, retryable, reason, artifact }
|
|
391
391
|
log(`VIDEO_CALLBACK: ${bookId}/${chapterId}/${sceneId}`);
|
|
392
392
|
|
|
393
|
-
const videoPath =
|
|
393
|
+
const videoPath = `${artifactRoot()}/${buildId}/${bookId}_${chapterId}_${sceneId}.mp4`;
|
|
394
394
|
const { valid, duration, metadata } = video.validateVideoFile(videoPath);
|
|
395
395
|
|
|
396
396
|
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
@@ -93,17 +93,28 @@ async function startScene(redis, scene, loadedBook, buildId) {
|
|
|
93
93
|
// открывал circuit breaker.
|
|
94
94
|
//
|
|
95
95
|
// Контракт: перед отправкой любого GPU job ассет обязан пройти
|
|
96
|
-
// NEW → PENDING (здесь) → GENERATING (сразу ниже),
|
|
96
|
+
// NEW|DIRTY → PENDING (здесь) → GENERATING (сразу ниже),
|
|
97
97
|
// либо уже находиться в состоянии, из которого GENERATING достижим.
|
|
98
98
|
// Если переход невозможен — dispatch ПРЕРЫВАЕТСЯ ДО отправки job.
|
|
99
|
+
//
|
|
100
|
+
// Stage 50: DIRTY включён в нормализацию. DIRTY → GENERATING запрещён FSM
|
|
101
|
+
// (scene-state.js: DIRTY → [PENDING, PLACEHOLDER]), поэтому до этого фикса
|
|
102
|
+
// dirty-ассет (version-stale pre-pass, C1/C1b recovery, RELEASE_STALE_LEASE,
|
|
103
|
+
// markDirtyScene) не мог перегенерироваться через планировщик вообще: guard
|
|
104
|
+
// отклонял setSceneGenerating(dirty), rejection проглатывался в
|
|
105
|
+
// attemptDispatch — ровно тот `INVALID state=dirty attempted=generating`
|
|
106
|
+
// из journal (Case A, stage 49). Комментарий теста stage-dispatch-lifecycle
|
|
107
|
+
// («scheduler's version-stale pre-pass owns DIRTY→PENDING») описывал
|
|
108
|
+
// нормализацию, которой не существовало. PENDING-хоп здесь — это и есть
|
|
109
|
+
// недостающая половина контракта.
|
|
99
110
|
async function ensureStageDispatchable(redis, bookId, chapterId, sceneId, stage) {
|
|
100
111
|
const states = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
101
|
-
if (states?.[stage] === state.AssetState.NEW) {
|
|
112
|
+
if (states?.[stage] === state.AssetState.NEW || states?.[stage] === state.AssetState.DIRTY) {
|
|
102
113
|
const init = await setScenePending(redis, bookId, chapterId, sceneId, stage);
|
|
103
114
|
if (!init.changed) {
|
|
104
115
|
return { ok: false, reason: `pending_init_failed:${init.reason}` };
|
|
105
116
|
}
|
|
106
|
-
log(`STAGE_INIT: ${bookId}/${chapterId}/${sceneId} ${stage}:
|
|
117
|
+
log(`STAGE_INIT: ${bookId}/${chapterId}/${sceneId} ${stage}: ${states[stage]} → PENDING`);
|
|
107
118
|
}
|
|
108
119
|
const gen = await setSceneGenerating(redis, bookId, chapterId, sceneId, stage);
|
|
109
120
|
if (!gen.changed) {
|
|
@@ -112,6 +123,34 @@ async function ensureStageDispatchable(redis, bookId, chapterId, sceneId, stage)
|
|
|
112
123
|
return { ok: true };
|
|
113
124
|
}
|
|
114
125
|
|
|
126
|
+
// ── Stage 50: terminal audio-orch record vs requested regeneration ──
|
|
127
|
+
// DONE — терминальная фаза (VALID_TRANSITIONS[DONE] = []), но per-asset DIRTY
|
|
128
|
+
// поверх неё — явный запрос на перегенерацию (version-stale pre-pass, C1/C1b
|
|
129
|
+
// recovery, RELEASE_STALE_LEASE, completeStage version-gate stale). Без сброса
|
|
130
|
+
// записи dispatch навсегда упирается в DONE guard (already_done) — состояние
|
|
131
|
+
// DONE+DIRTY зависает навсегда (2 сцены в production, stage 49). Хил:
|
|
132
|
+
// терминальная запись удаляется, дальше ensureStageDispatchable проводит
|
|
133
|
+
// DIRTY → PENDING → GENERATING, а no_state-лега переинициализирует FSM.
|
|
134
|
+
// Не-DONE записи (WAITING_CHUNKS/MERGING/FAILED) НЕ трогаются — их ведут
|
|
135
|
+
// штатные recovery-леги (Фаза 4: чистится только DONE).
|
|
136
|
+
async function clearTerminalAudioOrchForDirty(redis, bookId, chapterId, sceneId) {
|
|
137
|
+
let orchState;
|
|
138
|
+
try {
|
|
139
|
+
orchState = await audioFsmOp('getState')(redis, bookId, chapterId, sceneId);
|
|
140
|
+
} catch (err) {
|
|
141
|
+
warn(`AUDIO_DONE_DIRTY_HEAL: getState failed for ${bookId}/${chapterId}/${sceneId}: ${err.message}`);
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
if (!orchState || orchState.phase !== audioFsmPhases().DONE) return false;
|
|
145
|
+
|
|
146
|
+
const states = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
147
|
+
if (states?.audio !== state.AssetState.DIRTY) return false;
|
|
148
|
+
|
|
149
|
+
log(`AUDIO_DONE_DIRTY_HEAL: ${bookId}/${chapterId}/${sceneId} — regeneration requested over terminal DONE record — clearing audio-orch state for re-dispatch`);
|
|
150
|
+
await audioFsmOp('deleteState')(redis, bookId, chapterId, sceneId);
|
|
151
|
+
return true;
|
|
152
|
+
}
|
|
153
|
+
|
|
115
154
|
// T3: Каждый executor возвращает { dispatched: true/false, jobs, reason }
|
|
116
155
|
async function executeAudioDispatch(redis, scene, loadedBook, buildId, dispatchId) {
|
|
117
156
|
const bookId = scene.book_id;
|
|
@@ -125,6 +164,10 @@ async function executeAudioDispatch(redis, scene, loadedBook, buildId, dispatchI
|
|
|
125
164
|
buildId, dispatchType: 'orchestrator'
|
|
126
165
|
});
|
|
127
166
|
|
|
167
|
+
// Stage 50: heal the DONE×DIRTY deadlock BEFORE the lifecycle guard —
|
|
168
|
+
// a DIRTY request over a terminal DONE record must regenerate, not skip.
|
|
169
|
+
await clearTerminalAudioOrchForDirty(redis, bookId, chapterId, sceneId);
|
|
170
|
+
|
|
128
171
|
// T7: через фасад — единый владелец GENERATING (audio)
|
|
129
172
|
// Lifecycle guard: abort BEFORE any GPU job if the transition chain fails.
|
|
130
173
|
const dispatchable = await ensureStageDispatchable(redis, bookId, chapterId, sceneId, 'audio');
|
|
@@ -591,6 +634,7 @@ async function dispatchStage(redis, scene, loadedBook, buildId, overrideStage, d
|
|
|
591
634
|
|
|
592
635
|
module.exports = {
|
|
593
636
|
ensureStageDispatchable,
|
|
637
|
+
clearTerminalAudioOrchForDirty,
|
|
594
638
|
dispatchStage,
|
|
595
639
|
restoreSceneChunkStatus,
|
|
596
640
|
handleAudioCompleted,
|
|
@@ -24,6 +24,9 @@ const runtimeMetrics = require('./runtime-metrics');
|
|
|
24
24
|
// domain state over the injected redis parameter).
|
|
25
25
|
const circuitBreaker = require('./circuit-breaker');
|
|
26
26
|
const retryBudget = require('./retry-budget-manager');
|
|
27
|
+
// O-2 (Stage 50): the dispatch-time scene-version stamp is a PG-side effect
|
|
28
|
+
// consumed through the PersistencePort (host adapter owns the SQL).
|
|
29
|
+
const persist = require('./persistence-port').persist;
|
|
27
30
|
const crypto = require('crypto');
|
|
28
31
|
// Phase 5: Runtime Result Contract — dispatch finalizations are reported to
|
|
29
32
|
// orchestration through the contracts seam (runtime → contract → injected
|
|
@@ -864,7 +867,22 @@ async function dispatchStage(redis, bookId, chapterId, sceneId, stage, loadedBoo
|
|
|
864
867
|
dispatchId,
|
|
865
868
|
leaseKey: lease.leaseKey,
|
|
866
869
|
started_at: metadata.started_at
|
|
867
|
-
});
|
|
870
|
+
});
|
|
871
|
+
|
|
872
|
+
// Step 5.5 (Stage 50): stamp the scene's CURRENT content/audio-config
|
|
873
|
+
// versions onto the asset row — «версия, под которую стартовала генерация».
|
|
874
|
+
// completeStage's version gate then compares exactly that, so a regen
|
|
875
|
+
// started for the current content reaches READY (convergence of the
|
|
876
|
+
// DONE×DIRTY / version-stale flows), while a version bump DURING the
|
|
877
|
+
// generation still fails the gate (T1.11 semantics preserved). Best-effort:
|
|
878
|
+
// dispatch is a Redis-domain operation; a PG outage must not block it —
|
|
879
|
+
// the gate stays fail-closed and the next dispatch re-stamps.
|
|
880
|
+
try {
|
|
881
|
+
await persist('sceneAssets.stampSceneVersionsForDispatch')(bookId, chapterId, sceneId, stage);
|
|
882
|
+
} catch (stampErr) {
|
|
883
|
+
warn(`DISPATCH_VERSION_STAMP_FAILED: ${bookId}/${chapterId}/${sceneId}:${stage}: ${stampErr.message}`);
|
|
884
|
+
}
|
|
885
|
+
// Step 6: Get orchestrator to perform dispatch
|
|
868
886
|
try {
|
|
869
887
|
// S-5: the stage executor entry (scene-orchestrator.dispatchStage) is
|
|
870
888
|
// injected by the composition root — runtime must not import
|
|
@@ -15,9 +15,12 @@ const fs = require('fs').promises;
|
|
|
15
15
|
const syncFs = require('fs');
|
|
16
16
|
const syncPath = require('path');
|
|
17
17
|
|
|
18
|
-
const { lazyHostBinding } = require('../host/host-bindings');
|
|
18
|
+
const { lazyHostBinding, hostBinding } = require('../host/host-bindings');
|
|
19
19
|
// §32.30: former host requires resolve lazily through the host-bindings seam.
|
|
20
20
|
const state = lazyHostBinding('state');
|
|
21
|
+
// §32.30: the artifact root resolves through the composition-root seam at
|
|
22
|
+
// call time (Stage 96: former hardcoded artifact root closed here).
|
|
23
|
+
const artifactRoot = () => hostBinding('artifactRoot');
|
|
21
24
|
// S-4: filename grammar composed from the canonical owner (bytes unchanged)
|
|
22
25
|
const artifactNaming = require('@animastor/generation').artifactNaming;
|
|
23
26
|
// S-2: registered stage list resolved from media registry
|
|
@@ -121,7 +124,7 @@ async function checkOrphanVideoState(redis, bookId, chapterId, sceneId) {
|
|
|
121
124
|
}
|
|
122
125
|
|
|
123
126
|
const buildId = resolveBookBuildId(bookId);
|
|
124
|
-
const videoPath =
|
|
127
|
+
const videoPath = `${artifactRoot()}/${buildId}/${bookId}_${chapterId}_${sceneId}.mp4`;
|
|
125
128
|
|
|
126
129
|
try {
|
|
127
130
|
await fs.access(videoPath);
|
|
@@ -155,7 +158,7 @@ async function checkOrphanImageState(redis, bookId, chapterId, sceneId) {
|
|
|
155
158
|
const buildId = resolveBookBuildId(bookId);
|
|
156
159
|
const imageModule = lazyHostBinding('media.image');
|
|
157
160
|
const imageInfo = imageModule.resolveCanonicalSceneImage(
|
|
158
|
-
|
|
161
|
+
artifactRoot(),
|
|
159
162
|
buildId,
|
|
160
163
|
bookId,
|
|
161
164
|
chapterId,
|
|
@@ -345,15 +348,23 @@ async function checkOrphanGeneratingState(redis, bookId, chapterId, sceneId) {
|
|
|
345
348
|
/**
|
|
346
349
|
* Check for partially built scenes.
|
|
347
350
|
* T8: использует per-asset state вместо линейного.
|
|
351
|
+
*
|
|
352
|
+
* Stage 50: генерация в полёте — не «partial build». Раньше registry мог
|
|
353
|
+
* ещё не содержать image (registerSceneImage пишется по завершении), а
|
|
354
|
+
* ассет уже PENDING/GENERATING с живым lease — отчёт partial_audio_only
|
|
355
|
+
* вёл к PROGRESS_TO_IMAGE → setScenePending(generating→pending rejected)
|
|
356
|
+
* каждый цикл (issue №3 stage 49: ch-1bb5123c/sc-45c789c6).
|
|
348
357
|
*/
|
|
349
358
|
async function checkPartialBuilds(redis, bookId, chapterId, sceneId) {
|
|
350
359
|
const assetStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
351
360
|
const assets = await persist('registry.getSceneAssetsRedis')(redis, bookId, chapterId, sceneId);
|
|
352
361
|
|
|
362
|
+
const isInFlight = s => s === state.AssetState.PENDING || s === state.AssetState.GENERATING;
|
|
363
|
+
|
|
353
364
|
// Audio READY/PLACEHOLDER but no image asset
|
|
354
365
|
const audioReady = assetStates && assetStates.audio &&
|
|
355
366
|
(assetStates.audio === state.AssetState.READY || assetStates.audio === state.AssetState.PLACEHOLDER);
|
|
356
|
-
if (audioReady && !assets?.image) {
|
|
367
|
+
if (audioReady && !assets?.image && !isInFlight(assetStates.image)) {
|
|
357
368
|
return {
|
|
358
369
|
type: 'partial_audio_only',
|
|
359
370
|
scene: { bookId, chapterId, sceneId },
|
|
@@ -366,7 +377,7 @@ async function checkPartialBuilds(redis, bookId, chapterId, sceneId) {
|
|
|
366
377
|
// Image READY but no video asset
|
|
367
378
|
const imageReady = assetStates && assetStates.image &&
|
|
368
379
|
(assetStates.image === state.AssetState.READY || assetStates.image === state.AssetState.PLACEHOLDER);
|
|
369
|
-
if (imageReady && !assets?.video) {
|
|
380
|
+
if (imageReady && !assets?.video && !isInFlight(assetStates.video)) {
|
|
370
381
|
return {
|
|
371
382
|
type: 'partial_image_video_missing',
|
|
372
383
|
scene: { bookId, chapterId, sceneId },
|
|
@@ -646,7 +657,11 @@ async function checkAudioOrchInvariants(redis, bookId, chapterId, sceneId) {
|
|
|
646
657
|
phase,
|
|
647
658
|
audioState,
|
|
648
659
|
expected: 'READY or PLACEHOLDER',
|
|
649
|
-
|
|
660
|
+
// Stage 50: former value 'run_completeStage' was dead AND
|
|
661
|
+
// destructive metadata — completing the stage would write READY
|
|
662
|
+
// and erase exactly the dirty marker that schedules the pending
|
|
663
|
+
// regeneration. Re-dispatch (stage 50 guard-heal) owns the repair.
|
|
664
|
+
recommendation: 'review_manually'
|
|
650
665
|
});
|
|
651
666
|
}
|
|
652
667
|
} else if (phase === PHASES.FAILED) {
|
|
@@ -714,7 +729,9 @@ async function checkVideoOrchInvariants(redis, bookId, chapterId, sceneId) {
|
|
|
714
729
|
phase,
|
|
715
730
|
videoState,
|
|
716
731
|
expected: 'READY or PLACEHOLDER',
|
|
717
|
-
|
|
732
|
+
// Stage 50: mirror of the audio fix — 'run_completeStage' was
|
|
733
|
+
// dead + destructive; manual review only, re-dispatch owns it.
|
|
734
|
+
recommendation: 'review_manually'
|
|
718
735
|
});
|
|
719
736
|
}
|
|
720
737
|
} else if (phase === PHASES.FAILED) {
|
|
@@ -1103,6 +1120,23 @@ function getFixRecommendations(inconsistentScenes) {
|
|
|
1103
1120
|
reason: `Stale dispatch lease detected`,
|
|
1104
1121
|
safeToExecute: true
|
|
1105
1122
|
};
|
|
1123
|
+
// Stage 50: explicit (no longer default-fallback) — a terminal
|
|
1124
|
+
// audio/video-orch phase conflicting with the asset state is
|
|
1125
|
+
// NEVER auto-repaired here. In particular run_completeStage would
|
|
1126
|
+
// write READY and erase the dirty marker that schedules the
|
|
1127
|
+
// pending regeneration (stage 49: 2 stuck production scenes).
|
|
1128
|
+
// The DONE×DIRTY deadlock is healed by re-dispatch (stage 50
|
|
1129
|
+
// guard-heal in scene-orchestrator); everything else is manual.
|
|
1130
|
+
case 'audio_orch_invariant_done':
|
|
1131
|
+
case 'video_orch_invariant_done':
|
|
1132
|
+
case 'audio_orch_invariant_failed':
|
|
1133
|
+
case 'video_orch_invariant_failed':
|
|
1134
|
+
return {
|
|
1135
|
+
scene,
|
|
1136
|
+
action: 'REVIEW_MANUALLY',
|
|
1137
|
+
reason: `${issue}: terminal orch phase conflicts with asset state — re-dispatch heals DONE×DIRTY, other combinations need manual review`,
|
|
1138
|
+
safeToExecute: false
|
|
1139
|
+
};
|
|
1106
1140
|
default:
|
|
1107
1141
|
return {
|
|
1108
1142
|
scene,
|
|
@@ -2107,6 +2141,19 @@ async function recoverVideoOrchStates(redis, deps) {
|
|
|
2107
2141
|
// the sceneVersions.getVersionStalenessRows() port op (SQL lives in the host
|
|
2108
2142
|
// adapter verbatim). Pre-O-2 a missing deps.postgres skipped the phase; the
|
|
2109
2143
|
// unwired-port throw hits the same catch → warn + return 0.
|
|
2144
|
+
//
|
|
2145
|
+
// Stage 50: per-asset и READY-only. Раньше сцена помечалась целиком
|
|
2146
|
+
// (markDirtyScene со ВСЕМИ ассетами по умолчанию) при каждом старте:
|
|
2147
|
+
// • unsafeRestoreAssetState пишет DIRTY поверх GENERATING/PENDING — убивал
|
|
2148
|
+
// бы живую генерацию при рестарте;
|
|
2149
|
+
// • READY→DIRTY на не-stale ассетах — лишняя регенерация;
|
|
2150
|
+
// • повторная маркировка уже-DIRTY ассетов — неидемпотентный счётчик.
|
|
2151
|
+
// Теперь stale-типы считаются построчно (audio: content ИЛИ audio_config;
|
|
2152
|
+
// image/video: content), пишется только READY→DIRTY (FSM-легальный переход),
|
|
2153
|
+
// прежний READY-артефакт не удаляется — он остаётся отдаваемым до успешной
|
|
2154
|
+
// замены (файл и registry не трогаются; completeStage version-gate пропустит
|
|
2155
|
+
// READY только после успешной перегенерации). Повторный прогон: ассет уже
|
|
2156
|
+
// DIRTY → 0 маркировок (идемпотентность), счётчик считает только факты записи.
|
|
2110
2157
|
async function checkVersionStaleness(redis, deps) {
|
|
2111
2158
|
const { orchestrator } = deps;
|
|
2112
2159
|
if (!orchestrator) return 0;
|
|
@@ -2118,22 +2165,43 @@ async function checkVersionStaleness(redis, deps) {
|
|
|
2118
2165
|
for (const row of result.rows) {
|
|
2119
2166
|
const key = `${row.book_id}|${row.chapter_id}|${row.scene_id}`;
|
|
2120
2167
|
if (!sceneMap.has(key)) {
|
|
2121
|
-
sceneMap.set(key, { bookId: row.book_id, chapterId: row.chapter_id, sceneId: row.scene_id,
|
|
2168
|
+
sceneMap.set(key, { bookId: row.book_id, chapterId: row.chapter_id, sceneId: row.scene_id, staleAssets: new Set() });
|
|
2122
2169
|
}
|
|
2123
2170
|
const entry = sceneMap.get(key);
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2171
|
+
const contentStale = row.scene_content_version != null && row.content_version != null &&
|
|
2172
|
+
row.scene_content_version < row.content_version;
|
|
2173
|
+
const audioConfigStale = row.scene_audio_config_version != null && row.audio_config_version != null &&
|
|
2174
|
+
row.scene_audio_config_version < row.audio_config_version;
|
|
2175
|
+
if (row.asset_type === 'audio') {
|
|
2176
|
+
if (contentStale || audioConfigStale) entry.staleAssets.add('audio');
|
|
2177
|
+
} else if (row.asset_type === 'image' || row.asset_type === 'video') {
|
|
2178
|
+
if (contentStale) entry.staleAssets.add(row.asset_type);
|
|
2127
2179
|
}
|
|
2128
2180
|
}
|
|
2129
2181
|
|
|
2130
2182
|
let outdated = 0;
|
|
2131
2183
|
for (const entry of sceneMap.values()) {
|
|
2132
|
-
if (entry.
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2184
|
+
if (entry.staleAssets.size === 0) continue;
|
|
2185
|
+
|
|
2186
|
+
let assetStates;
|
|
2187
|
+
try {
|
|
2188
|
+
assetStates = await state.getAssetStates(redis, entry.bookId, entry.chapterId, entry.sceneId);
|
|
2189
|
+
} catch (stateErr) {
|
|
2190
|
+
warn(`Version staleness: asset-state read failed for ${entry.bookId}/${entry.chapterId}/${entry.sceneId}: ${stateErr.message}`);
|
|
2191
|
+
continue;
|
|
2136
2192
|
}
|
|
2193
|
+
|
|
2194
|
+
const readyStale = [...entry.staleAssets]
|
|
2195
|
+
.filter(t => assetStates?.[t] === state.AssetState.READY);
|
|
2196
|
+
if (readyStale.length === 0) {
|
|
2197
|
+
// Уже DIRTY/GENERATING/FAILED — регенерация инициирована или идёт;
|
|
2198
|
+
// повторная маркировка была бы неидемпотентной и опасной.
|
|
2199
|
+
continue;
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
log(`[VERSION-STALE] ${entry.bookId}/${entry.chapterId}/${entry.sceneId} → ${readyStale.join(', ')} (READY → DIRTY, previous artifact preserved until replacement)`);
|
|
2203
|
+
await orchestrator.markDirtyScene(redis, entry.bookId, entry.chapterId, entry.sceneId, readyStale);
|
|
2204
|
+
outdated++;
|
|
2137
2205
|
}
|
|
2138
2206
|
return outdated;
|
|
2139
2207
|
} catch (err) {
|
|
@@ -571,6 +571,8 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
|
|
|
571
571
|
|
|
572
572
|
let dispatched = 0;
|
|
573
573
|
let throttled = 0;
|
|
574
|
+
let anyCompleted = false;
|
|
575
|
+
const failures = [];
|
|
574
576
|
|
|
575
577
|
// Dispatch ALL eligible stages in this tick.
|
|
576
578
|
// Concurrency limits per stage type are handled by dispatch engine.
|
|
@@ -589,10 +591,29 @@ async function attemptDispatch(redis, bookId, chapterId, sceneId, loadedBook, fo
|
|
|
589
591
|
dispatched++;
|
|
590
592
|
} else if (result.reason === 'backpressure' || result.reason === 'throttled') {
|
|
591
593
|
throttled++;
|
|
594
|
+
} else if (result.completed) {
|
|
595
|
+
// Stage finished inside the executor (already_done / cache_hit /
|
|
596
|
+
// no_segments) — not a failure, nothing to dispatch.
|
|
597
|
+
anyCompleted = true;
|
|
598
|
+
} else {
|
|
599
|
+
// Stage 50: раньше per-stage отказ (circuit_open, retry budget,
|
|
600
|
+
// lease, generating_transition_failed, …) молча проглатывался и
|
|
601
|
+
// tick писал «undefined for <scene>» — ошибки исчезали из
|
|
602
|
+
// observability. Теперь причина всплывает в tick-сводку.
|
|
603
|
+
failures.push(`${stage}:${result.reason || 'unknown'}`);
|
|
592
604
|
}
|
|
593
605
|
}
|
|
594
606
|
|
|
595
|
-
|
|
607
|
+
if (dispatched > 0) {
|
|
608
|
+
return { dispatched, throttled, stages: stages.length };
|
|
609
|
+
}
|
|
610
|
+
if (failures.length > 0) {
|
|
611
|
+
return { dispatched: 0, throttled, stages: stages.length, success: false, reason: failures.join(', ') };
|
|
612
|
+
}
|
|
613
|
+
if (anyCompleted) {
|
|
614
|
+
return { dispatched: 0, throttled, completed: true, reason: 'stage_completed' };
|
|
615
|
+
}
|
|
616
|
+
return { dispatched: 0, throttled, stages: stages.length, success: true, reason: 'no_dispatch' };
|
|
596
617
|
}
|
|
597
618
|
|
|
598
619
|
// ======================================================
|
|
@@ -615,6 +636,7 @@ module.exports = {
|
|
|
615
636
|
// Scheduling
|
|
616
637
|
shouldScheduleAssets,
|
|
617
638
|
detectVersionStale,
|
|
639
|
+
attemptDispatch,
|
|
618
640
|
markVersionStaleDirty,
|
|
619
641
|
getLayerConfig,
|
|
620
642
|
checkChunksHaveImages,
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
// ======================================================
|
|
2
|
+
// STAGE 96 — ARTIFACT ROOT BINDING (поведение закрытых литералов)
|
|
3
|
+
// ======================================================
|
|
4
|
+
// Четыре production-литерала '/data/output' в orchestration-пакете
|
|
5
|
+
// (scene-callbacks.js:281,393 · reconciliation-engine.js:124,158)
|
|
6
|
+
// переведены на существующий шов artifactRoot (host-bindings, §32.30).
|
|
7
|
+
//
|
|
8
|
+
// Тесты проверяют ПОВЕДЕНИЕ на свежих исходниках пакета (src/, а не
|
|
9
|
+
// npm-копия в backend/node_modules):
|
|
10
|
+
// — путь строится из привязанного корня, а не из литерала;
|
|
11
|
+
// — при корне '/data/output' (контракт сохранённых данных PG/Redis)
|
|
12
|
+
// результирующая строка байт-в-байт равна прежней.
|
|
13
|
+
// ======================================================
|
|
14
|
+
|
|
15
|
+
const { expect } = require('chai');
|
|
16
|
+
const fs = require('fs');
|
|
17
|
+
const os = require('os');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const { bindHostModules } = require('../src/host/host-bindings');
|
|
20
|
+
const { createMockRedis } = require('./mocks/redis-mock');
|
|
21
|
+
|
|
22
|
+
const engine = require('../src/runtime/reconciliation-engine');
|
|
23
|
+
const callbacks = require('../src/orchestration/scene-callbacks');
|
|
24
|
+
|
|
25
|
+
const BOOK = 'b1';
|
|
26
|
+
const CHAPTER = 'ch1';
|
|
27
|
+
const SCENE = 'sc1';
|
|
28
|
+
|
|
29
|
+
let tmpRoot;
|
|
30
|
+
let redis;
|
|
31
|
+
let currentStates;
|
|
32
|
+
|
|
33
|
+
const imageStub = {
|
|
34
|
+
resolveCanonicalSceneImage: () => null,
|
|
35
|
+
getImageMetadata: async () => null,
|
|
36
|
+
};
|
|
37
|
+
const videoStub = {
|
|
38
|
+
validateVideoFile: () => ({ valid: false, duration: null, metadata: null }),
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
const stateStub = {
|
|
42
|
+
AssetState: {
|
|
43
|
+
NEW: 'new', DIRTY: 'dirty', PENDING: 'pending',
|
|
44
|
+
GENERATING: 'generating', READY: 'ready',
|
|
45
|
+
FAILED: 'failed', PLACEHOLDER: 'placeholder',
|
|
46
|
+
},
|
|
47
|
+
getAssetStates: async () => ({ audio: 'new', ...currentStates }),
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
const failFast = (name) => () => {
|
|
51
|
+
throw new Error(`[ARTIFACT-ROOT-TEST] ${name} not wired`);
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
function wire(artifactRootValue) {
|
|
55
|
+
bindHostModules({
|
|
56
|
+
state: () => stateStub,
|
|
57
|
+
media: () => ({ audio: {}, image: imageStub, video: videoStub }),
|
|
58
|
+
journal: () => ({ appendSceneEvent: async () => {} }),
|
|
59
|
+
artifactRoot: () => artifactRootValue,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
describe('Stage 96 — artifactRoot-шов вместо литералов', () => {
|
|
64
|
+
before(() => {
|
|
65
|
+
tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'stage96-artifact-root-'));
|
|
66
|
+
fs.mkdirSync(path.join(tmpRoot, 'default'), { recursive: true });
|
|
67
|
+
redis = createMockRedis();
|
|
68
|
+
wire(tmpRoot);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
after(() => {
|
|
72
|
+
// Вернуть дефолтный fail-fast, как в test/test-bindings.js
|
|
73
|
+
bindHostModules({
|
|
74
|
+
state: failFast('state'),
|
|
75
|
+
media: failFast('media'),
|
|
76
|
+
journal: failFast('journal'),
|
|
77
|
+
artifactRoot: failFast('artifactRoot'),
|
|
78
|
+
});
|
|
79
|
+
fs.rmSync(tmpRoot, { recursive: true, force: true });
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe('reconciliation-engine — checkOrphanVideoState', () => {
|
|
83
|
+
it('missingFile строится из привязанного корня, а не из литерала', async () => {
|
|
84
|
+
currentStates = { video: 'ready' };
|
|
85
|
+
const res = await engine.checkOrphanVideoState(redis, BOOK, CHAPTER, SCENE);
|
|
86
|
+
expect(res).to.be.an('object');
|
|
87
|
+
expect(res.missingFile).to.equal(`${tmpRoot}/default/${BOOK}_${CHAPTER}_${SCENE}.mp4`);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('null, когда файл существует ПОД ПРИВЯЗАННЫМ корнем', async () => {
|
|
91
|
+
const file = path.join(tmpRoot, 'default', `${BOOK}_${CHAPTER}_${SCENE}.mp4`);
|
|
92
|
+
fs.writeFileSync(file, 'x');
|
|
93
|
+
try {
|
|
94
|
+
currentStates = { video: 'ready' };
|
|
95
|
+
expect(await engine.checkOrphanVideoState(redis, BOOK, CHAPTER, SCENE)).to.be.null;
|
|
96
|
+
} finally {
|
|
97
|
+
fs.rmSync(file, { force: true });
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
describe('reconciliation-engine — checkOrphanImageState', () => {
|
|
103
|
+
it('resolveCanonicalSceneImage получает привязанный корень', async () => {
|
|
104
|
+
let seenRoot = null;
|
|
105
|
+
imageStub.resolveCanonicalSceneImage = (root) => { seenRoot = root; return null; };
|
|
106
|
+
try {
|
|
107
|
+
currentStates = { image: 'ready' };
|
|
108
|
+
const res = await engine.checkOrphanImageState(redis, BOOK, CHAPTER, SCENE);
|
|
109
|
+
expect(seenRoot).to.equal(tmpRoot);
|
|
110
|
+
expect(res).to.be.an('object');
|
|
111
|
+
expect(res.type).to.equal('orphan_image_state');
|
|
112
|
+
} finally {
|
|
113
|
+
imageStub.resolveCanonicalSceneImage = () => null;
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
describe('scene-callbacks — handleVideoCompleted', () => {
|
|
119
|
+
it('validateVideoFile получает путь под привязанным корнем', async () => {
|
|
120
|
+
let seenPath = null;
|
|
121
|
+
videoStub.validateVideoFile = (p) => {
|
|
122
|
+
seenPath = p;
|
|
123
|
+
return { valid: false, duration: null, metadata: null };
|
|
124
|
+
};
|
|
125
|
+
try {
|
|
126
|
+
currentStates = { video: 'generating' };
|
|
127
|
+
const res = await callbacks.handleVideoCompleted(redis, BOOK, CHAPTER, SCENE, 'build1');
|
|
128
|
+
expect(seenPath).to.equal(`${tmpRoot}/build1/${BOOK}_${CHAPTER}_${SCENE}.mp4`);
|
|
129
|
+
expect(res).to.deep.include({ ok: false, retryable: true, reason: 'video_invalid' });
|
|
130
|
+
} finally {
|
|
131
|
+
videoStub.validateVideoFile = () => ({ valid: false, duration: null, metadata: null });
|
|
132
|
+
}
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
describe('scene-callbacks — handleImageCompleted', () => {
|
|
137
|
+
it('resolveCanonicalSceneImage получает привязанный корень', async () => {
|
|
138
|
+
let seenRoot = null;
|
|
139
|
+
imageStub.resolveCanonicalSceneImage = (root) => { seenRoot = root; return null; };
|
|
140
|
+
try {
|
|
141
|
+
currentStates = { image: 'generating' };
|
|
142
|
+
const res = await callbacks.handleImageCompleted(redis, BOOK, CHAPTER, SCENE, 'build1');
|
|
143
|
+
expect(seenRoot).to.equal(tmpRoot);
|
|
144
|
+
expect(res).to.deep.include({ ok: false, retryable: true, reason: 'image_not_found' });
|
|
145
|
+
} finally {
|
|
146
|
+
imageStub.resolveCanonicalSceneImage = () => null;
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
describe('обратная совместимость (контракт контейнерных путей)', () => {
|
|
152
|
+
it('при корне /data/output строка байт-в-байт равна прежнему литералу', async () => {
|
|
153
|
+
wire('/data/output');
|
|
154
|
+
try {
|
|
155
|
+
let seenPath = null;
|
|
156
|
+
videoStub.validateVideoFile = (p) => {
|
|
157
|
+
seenPath = p;
|
|
158
|
+
return { valid: false, duration: null, metadata: null };
|
|
159
|
+
};
|
|
160
|
+
currentStates = { video: 'generating' };
|
|
161
|
+
await callbacks.handleVideoCompleted(redis, BOOK, CHAPTER, SCENE, 'build1');
|
|
162
|
+
// ровно то, что до Stage 96 формировал шаблон
|
|
163
|
+
expect(seenPath).to.equal(`/data/output/build1/${BOOK}_${CHAPTER}_${SCENE}.mp4`);
|
|
164
|
+
|
|
165
|
+
currentStates = { video: 'ready' };
|
|
166
|
+
const orphan = await engine.checkOrphanVideoState(redis, BOOK, CHAPTER, SCENE);
|
|
167
|
+
expect(orphan.missingFile).to.equal(`/data/output/default/${BOOK}_${CHAPTER}_${SCENE}.mp4`);
|
|
168
|
+
} finally {
|
|
169
|
+
videoStub.validateVideoFile = () => ({ valid: false, duration: null, metadata: null });
|
|
170
|
+
wire(tmpRoot);
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
});
|
|
174
|
+
});
|