@animastor/orchestration 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/package.json +1 -1
- package/src/index.js +1 -0
- package/src/orchestration/orchestrator.js +20 -8
- package/src/orchestration/scene-orchestrator.js +21 -7
- package/src/runtime/active-scenes-index.js +18 -2
- package/src/runtime/command-gate.js +270 -0
- package/src/runtime/dispatch-engine.js +12 -1
- package/src/runtime/progress-events-port.js +6 -0
- package/src/runtime/reconciliation-engine.js +192 -167
- package/src/runtime/runtime-scheduler.js +70 -60
- package/src/runtime/scene-window.js +23 -2
|
@@ -40,6 +40,10 @@ const runtimeScheduler = require('./runtime-scheduler');
|
|
|
40
40
|
const counterReconciliation = require('./counter-reconciliation');
|
|
41
41
|
const dispatchEngine = require('./dispatch-engine');
|
|
42
42
|
const leaseManager = require('./lease-manager');
|
|
43
|
+
// Generation only by explicit command: every fix that used to (re)queue work
|
|
44
|
+
// is gated per scene×stage by the command gate (active task-registry task ∧
|
|
45
|
+
// layer enabled ∧ not cancelled). Without authorization a fix is diagnostics.
|
|
46
|
+
const commandGate = require('./command-gate');
|
|
43
47
|
// S-5: orchestration-owned behavior (FSM-safe facade writers, rollbacks)
|
|
44
48
|
// reaches runtime ONLY through the composition-root-injected seam registry.
|
|
45
49
|
// No orchestration module is imported from this file (S-5).
|
|
@@ -59,6 +63,25 @@ function error(msg) {
|
|
|
59
63
|
console.error(`${logPrefix} ❌ ${msg}`);
|
|
60
64
|
}
|
|
61
65
|
|
|
66
|
+
// SYNC: backend/src/helpers/queue-idempotency.cjs — resultDedupKey /
|
|
67
|
+
// readResultDedup (marker `'1'` = in flight, `done:<dispatch_id>` = settled).
|
|
68
|
+
// Read-only here: the result route is the only writer.
|
|
69
|
+
const RESULT_DEDUP_DONE_PREFIX = 'done:';
|
|
70
|
+
async function readResultDedup(redis, jobId, buildId) {
|
|
71
|
+
let raw = null;
|
|
72
|
+
try {
|
|
73
|
+
raw = await redis.get(`animastor:result-processed:${jobId}:${buildId}`);
|
|
74
|
+
} catch (_) {
|
|
75
|
+
return { state: 'unknown', dispatchId: null };
|
|
76
|
+
}
|
|
77
|
+
if (raw == null || raw === false) return { state: 'none', dispatchId: null };
|
|
78
|
+
const v = String(raw);
|
|
79
|
+
if (v.startsWith(RESULT_DEDUP_DONE_PREFIX)) {
|
|
80
|
+
return { state: 'settled', dispatchId: v.slice(RESULT_DEDUP_DONE_PREFIX.length) || null };
|
|
81
|
+
}
|
|
82
|
+
return { state: 'in_flight', dispatchId: null };
|
|
83
|
+
}
|
|
84
|
+
|
|
62
85
|
// ======================================================
|
|
63
86
|
// RECONCILIATION REPORT
|
|
64
87
|
// ======================================================
|
|
@@ -335,15 +358,7 @@ async function checkOrphanAssets(redis, bookId, chapterId, sceneId) {
|
|
|
335
358
|
* Reconciliation must never resurrect a cancelled book.
|
|
336
359
|
*/
|
|
337
360
|
async function isBookGenerationCancelled(redis, bookId) {
|
|
338
|
-
|
|
339
|
-
const sceneWindow = require('./scene-window');
|
|
340
|
-
if (await sceneWindow.isCancelled(redis, bookId)) return true;
|
|
341
|
-
} catch (_) { /* flag unreadable — fall through to tombstone */ }
|
|
342
|
-
try {
|
|
343
|
-
return await persist('cancel.isCancelled')(bookId);
|
|
344
|
-
} catch (_) {
|
|
345
|
-
return false;
|
|
346
|
-
}
|
|
361
|
+
return commandGate.isBookCancelled(redis, bookId);
|
|
347
362
|
}
|
|
348
363
|
|
|
349
364
|
/**
|
|
@@ -397,9 +412,15 @@ async function checkOrphanGeneratingState(redis, bookId, chapterId, sceneId) {
|
|
|
397
412
|
// ======================================================
|
|
398
413
|
|
|
399
414
|
/**
|
|
400
|
-
* Check for partially built scenes.
|
|
415
|
+
* Check for partially built scenes — DIAGNOSTICS ONLY.
|
|
401
416
|
* T8: использует per-asset state вместо линейного.
|
|
402
417
|
*
|
|
418
|
+
* Command-only generation (2026-10-31): a partial build is NOT an
|
|
419
|
+
* inconsistency and never produces a fix. The former PROGRESS_TO_IMAGE /
|
|
420
|
+
* PROGRESS_TO_VIDEO auto-progression (incident 2026-10-30 BA: reconcile
|
|
421
|
+
* started image after audio READY while image was disabled) is removed —
|
|
422
|
+
* the next stage starts only by an explicit command.
|
|
423
|
+
*
|
|
403
424
|
* Stage 50: генерация в полёте — не «partial build». Раньше registry мог
|
|
404
425
|
* ещё не содержать image (registerSceneImage пишется по завершении), а
|
|
405
426
|
* ассет уже PENDING/GENERATING с живым lease — отчёт partial_audio_only
|
|
@@ -992,9 +1013,7 @@ async function reconcileScene(redis, bookId, chapterId, sceneId) {
|
|
|
992
1013
|
}
|
|
993
1014
|
}
|
|
994
1015
|
|
|
995
|
-
// Check orphan GENERATING states (audit d9d67a3).
|
|
996
|
-
// partial-build checks: those de-duplicate by sceneId, and the orphan
|
|
997
|
-
// repair must win over a PROGRESS_TO_* resurrection attempt.
|
|
1016
|
+
// Check orphan GENERATING states (audit d9d67a3).
|
|
998
1017
|
const orphanGenerating = await checkOrphanGeneratingState(redis, bookId, chapterId, sceneId);
|
|
999
1018
|
if (orphanGenerating) {
|
|
1000
1019
|
report.orphanStates.push(...orphanGenerating);
|
|
@@ -1007,16 +1026,11 @@ async function reconcileScene(redis, bookId, chapterId, sceneId) {
|
|
|
1007
1026
|
}
|
|
1008
1027
|
}
|
|
1009
1028
|
|
|
1010
|
-
//
|
|
1029
|
+
// Partial builds — diagnostics only (command-only generation): kept in
|
|
1030
|
+
// the report, NEVER turned into an inconsistency/fix.
|
|
1011
1031
|
const partial = await checkPartialBuilds(redis, bookId, chapterId, sceneId);
|
|
1012
1032
|
if (partial) {
|
|
1013
1033
|
report.partialBuilds.push(partial);
|
|
1014
|
-
if (!report.inconsistentScenes.find(s => s.scene.sceneId === sceneId)) {
|
|
1015
|
-
report.inconsistentScenes.push({
|
|
1016
|
-
scene: { bookId, chapterId, sceneId },
|
|
1017
|
-
issue: partial.type
|
|
1018
|
-
});
|
|
1019
|
-
}
|
|
1020
1034
|
}
|
|
1021
1035
|
|
|
1022
1036
|
// Check stale locks
|
|
@@ -1109,20 +1123,6 @@ function getFixRecommendations(inconsistentScenes) {
|
|
|
1109
1123
|
reason: 'AUDIO_READY but no audio file',
|
|
1110
1124
|
safeToExecute: true
|
|
1111
1125
|
};
|
|
1112
|
-
case 'partial_audio_only':
|
|
1113
|
-
return {
|
|
1114
|
-
scene,
|
|
1115
|
-
action: 'PROGRESS_TO_IMAGE',
|
|
1116
|
-
reason: 'Audio ready but image not started',
|
|
1117
|
-
safeToExecute: true
|
|
1118
|
-
};
|
|
1119
|
-
case 'partial_image_video_missing':
|
|
1120
|
-
return {
|
|
1121
|
-
scene,
|
|
1122
|
-
action: 'PROGRESS_TO_VIDEO',
|
|
1123
|
-
reason: 'Image ready but video not started',
|
|
1124
|
-
safeToExecute: true
|
|
1125
|
-
};
|
|
1126
1126
|
case 'stale_locks':
|
|
1127
1127
|
return {
|
|
1128
1128
|
scene,
|
|
@@ -1207,12 +1207,15 @@ function getFixRecommendations(inconsistentScenes) {
|
|
|
1207
1207
|
*/
|
|
1208
1208
|
const RESURRECTION_ACTIONS = new Set([
|
|
1209
1209
|
'REGENERATE_MISSING_ASSET',
|
|
1210
|
-
'PROGRESS_TO_IMAGE',
|
|
1211
|
-
'PROGRESS_TO_VIDEO',
|
|
1212
1210
|
'RECOVER_ORPHAN_ASSETS',
|
|
1213
1211
|
'RELEASE_STALE_LEASE',
|
|
1214
1212
|
]);
|
|
1215
1213
|
|
|
1214
|
+
function diagnosticOnly(action, scene, what) {
|
|
1215
|
+
log(`[DIAGNOSTIC] ${action} ${scene.bookId}/${scene.chapterId}/${scene.sceneId}: ${what} — no command authorizes it, nothing queued`);
|
|
1216
|
+
return { success: false, action, scene, diagnostic: true, details: `diagnostic only: ${what} (no command-authorized stage)` };
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1216
1219
|
/**
|
|
1217
1220
|
* Apply a fix recommendation.
|
|
1218
1221
|
* Returns { success: boolean, action, scene, details }
|
|
@@ -1264,6 +1267,23 @@ async function applyFix(redis, fix) {
|
|
|
1264
1267
|
};
|
|
1265
1268
|
}
|
|
1266
1269
|
|
|
1270
|
+
// Command-only generation: without an active command for
|
|
1271
|
+
// this stage the dead generation is released to DIRTY
|
|
1272
|
+
// (ghost removed) but NOT re-queued — no PENDING, no index.
|
|
1273
|
+
if (!(await commandGate.isStageAuthorized(
|
|
1274
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId, fix.stage))) {
|
|
1275
|
+
const released = await commandGate.rollbackGeneratingToDirty(
|
|
1276
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId,
|
|
1277
|
+
fix.stage, 'orphan_generating_no_command');
|
|
1278
|
+
log(`[DIAGNOSTIC] orphan GENERATING ${scene.bookId}/${scene.chapterId}/${scene.sceneId} ${fix.stage}: no command — released to DIRTY, not re-queued`);
|
|
1279
|
+
return {
|
|
1280
|
+
success: released.changed, action, scene, diagnostic: true,
|
|
1281
|
+
details: released.changed
|
|
1282
|
+
? `${fix.stage}: GENERATING → DIRTY (orphan, no command — not re-queued)`
|
|
1283
|
+
: `orphan release skipped: ${released.reason}`
|
|
1284
|
+
};
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1267
1287
|
// FSM-safe path GENERATING → DIRTY → PENDING (a direct
|
|
1268
1288
|
// GENERATING → PENDING is forbidden). Errors are NOT
|
|
1269
1289
|
// swallowed (rollbackStageToPending logs + journals +
|
|
@@ -1280,13 +1300,11 @@ async function applyFix(redis, fix) {
|
|
|
1280
1300
|
{ stage: fix.stage, fromState: 'generating', recoveredBy: 'reconciliation-engine' }
|
|
1281
1301
|
).catch(() => {});
|
|
1282
1302
|
|
|
1283
|
-
// Make the repaired stage dispatchable again —
|
|
1284
|
-
//
|
|
1285
|
-
//
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1289
|
-
}
|
|
1303
|
+
// Make the repaired stage dispatchable again — the active
|
|
1304
|
+
// command authorizes it (the index write is command-gated:
|
|
1305
|
+
// cancellation / missing task / disabled layer deny it).
|
|
1306
|
+
await runtimeScheduler.addSceneToActiveIndex(
|
|
1307
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1290
1308
|
|
|
1291
1309
|
return {
|
|
1292
1310
|
success: true, action, scene,
|
|
@@ -1295,6 +1313,14 @@ async function applyFix(redis, fix) {
|
|
|
1295
1313
|
}
|
|
1296
1314
|
|
|
1297
1315
|
// ── Legacy stuck_state path: whole-scene DIRTY ──
|
|
1316
|
+
// Command-only generation: only under an active command.
|
|
1317
|
+
{
|
|
1318
|
+
const auth = await commandGate.getAuthorizedStages(
|
|
1319
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1320
|
+
if (auth.stages.size === 0) {
|
|
1321
|
+
return diagnosticOnly(action, scene, 'stuck scene');
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1298
1324
|
// Mark all per-asset states as DIRTY for redispatch
|
|
1299
1325
|
// M5: Route through orchestrator.markDirtyScene instead of direct state.setAssetState
|
|
1300
1326
|
// S-5: asset-state writer via the injected orchestration seam
|
|
@@ -1331,12 +1357,22 @@ async function applyFix(redis, fix) {
|
|
|
1331
1357
|
let removedCount = 0;
|
|
1332
1358
|
const clearedDispatchIds = [];
|
|
1333
1359
|
|
|
1360
|
+
// Command-only generation: the lease release + re-queue runs
|
|
1361
|
+
// only for stages under an active command; otherwise the stale
|
|
1362
|
+
// lease is reported (diagnostics) and expires on its own TTL.
|
|
1363
|
+
const auth = await commandGate.getAuthorizedStages(
|
|
1364
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1365
|
+
if (auth.stages.size === 0) {
|
|
1366
|
+
return diagnosticOnly(action, scene, 'stale dispatch lease');
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1334
1369
|
// Release all dispatch leases for this scene through the
|
|
1335
1370
|
// canonical cancellation path (lease + owned quota + renewal
|
|
1336
1371
|
// timer + completion marker). A bare `del` would leak the
|
|
1337
1372
|
// quota slot and the in-memory renewal timer, and could race a
|
|
1338
1373
|
// concurrent re-acquire (no token check).
|
|
1339
1374
|
for (const stage of mediaRegistry.listMediaTypes()) {
|
|
1375
|
+
if (!auth.stages.has(stage)) continue;
|
|
1340
1376
|
const result = await dispatchEngine.cancelActiveDispatch(
|
|
1341
1377
|
redis, scene.bookId, scene.chapterId, scene.sceneId, stage,
|
|
1342
1378
|
'stale_lease_recovery'
|
|
@@ -1363,14 +1399,21 @@ async function applyFix(redis, fix) {
|
|
|
1363
1399
|
}
|
|
1364
1400
|
}
|
|
1365
1401
|
|
|
1366
|
-
// Mark
|
|
1402
|
+
// Mark the in-flight states of the AUTHORIZED stages DIRTY for
|
|
1403
|
+
// redispatch (never a finished READY stage, never a stage
|
|
1404
|
+
// without a command).
|
|
1367
1405
|
// M5: Route through orchestrator.markDirtyScene instead of direct state.setAssetState
|
|
1368
|
-
// M5 Шаг 3: syncLinearState уже внутри markDirtyScene
|
|
1369
1406
|
// S-5: asset-state writer via the injected orchestration seam
|
|
1370
1407
|
const markDirtyScene = orchestrationSeams.getOrchestrationSeam('markDirtyScene');
|
|
1371
|
-
await
|
|
1408
|
+
const currentStates = await state.getAssetStates(redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1409
|
+
const redo = [...auth.stages].filter(stage =>
|
|
1410
|
+
currentStates[stage] === state.AssetState.GENERATING ||
|
|
1411
|
+
currentStates[stage] === state.AssetState.PENDING);
|
|
1412
|
+
if (redo.length > 0) {
|
|
1413
|
+
await markDirtyScene(redis, scene.bookId, scene.chapterId, scene.sceneId, redo);
|
|
1414
|
+
}
|
|
1372
1415
|
|
|
1373
|
-
// Add back to active index
|
|
1416
|
+
// Add back to active index (command-gated write)
|
|
1374
1417
|
await runtimeScheduler.addSceneToActiveIndex(redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
1375
1418
|
|
|
1376
1419
|
return { success: removedCount > 0, action, scene, details: `released ${removedCount} stale leases` };
|
|
@@ -1391,6 +1434,13 @@ async function applyFix(redis, fix) {
|
|
|
1391
1434
|
if (!pendingState) {
|
|
1392
1435
|
return { success: false, action, scene, details: `unknown asset type (reason: ${fix.reason})` };
|
|
1393
1436
|
}
|
|
1437
|
+
{
|
|
1438
|
+
const missingType = pendingState.replace(/_pending$/, '');
|
|
1439
|
+
if (!(await commandGate.isStageAuthorized(
|
|
1440
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId, missingType))) {
|
|
1441
|
+
return diagnosticOnly(action, scene, `${missingType} READY but file missing`);
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1394
1444
|
|
|
1395
1445
|
// M5 Шаг 4: Route through orchestrator.setScenePending (which includes syncLinearState)
|
|
1396
1446
|
// S-5: asset-state writer via the injected orchestration seam
|
|
@@ -1413,21 +1463,6 @@ async function applyFix(redis, fix) {
|
|
|
1413
1463
|
return { success: true, action, scene, details: `marked as ${pendingState}` };
|
|
1414
1464
|
}
|
|
1415
1465
|
|
|
1416
|
-
case 'PROGRESS_TO_IMAGE': {
|
|
1417
|
-
// M5 Шаг 4: Route through orchestrator.setScenePending (includes syncLinearState)
|
|
1418
|
-
// S-5: asset-state writer via the injected orchestration seam
|
|
1419
|
-
await orchestrationSeams.getOrchestrationSeam('setScenePending')(redis, scene.bookId, scene.chapterId, scene.sceneId, 'image');
|
|
1420
|
-
|
|
1421
|
-
await runtimeScheduler.addSceneToActiveIndex(
|
|
1422
|
-
redis,
|
|
1423
|
-
scene.bookId,
|
|
1424
|
-
scene.chapterId,
|
|
1425
|
-
scene.sceneId
|
|
1426
|
-
);
|
|
1427
|
-
|
|
1428
|
-
return { success: true, action, scene, details: 'progressed to IMAGE_PENDING' };
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
1466
|
case 'RECONCILE_COUNTER_DRIFT': {
|
|
1432
1467
|
// Get the current lease count and fix counter
|
|
1433
1468
|
const stage = fix.stage || 'audio';
|
|
@@ -1438,21 +1473,6 @@ async function applyFix(redis, fix) {
|
|
|
1438
1473
|
return { success: true, action, scene, details: `counter corrected to ${leaseCount}` };
|
|
1439
1474
|
}
|
|
1440
1475
|
|
|
1441
|
-
case 'PROGRESS_TO_VIDEO': {
|
|
1442
|
-
// M5 Шаг 4: Route through orchestrator.setScenePending (includes syncLinearState)
|
|
1443
|
-
// S-5: asset-state writer via the injected orchestration seam
|
|
1444
|
-
await orchestrationSeams.getOrchestrationSeam('setScenePending')(redis, scene.bookId, scene.chapterId, scene.sceneId, 'video');
|
|
1445
|
-
|
|
1446
|
-
await runtimeScheduler.addSceneToActiveIndex(
|
|
1447
|
-
redis,
|
|
1448
|
-
scene.bookId,
|
|
1449
|
-
scene.chapterId,
|
|
1450
|
-
scene.sceneId
|
|
1451
|
-
);
|
|
1452
|
-
|
|
1453
|
-
return { success: true, action, scene, details: 'progressed to VIDEO_PENDING' };
|
|
1454
|
-
}
|
|
1455
|
-
|
|
1456
1476
|
case 'RECOVER_ORPHAN_ASSETS': {
|
|
1457
1477
|
// Clear registry entries for missing assets
|
|
1458
1478
|
const assets = await persist('registry.getSceneAssetsRedis')(redis, scene.bookId, scene.chapterId, scene.sceneId);
|
|
@@ -1477,6 +1497,13 @@ async function applyFix(redis, fix) {
|
|
|
1477
1497
|
}
|
|
1478
1498
|
}
|
|
1479
1499
|
|
|
1500
|
+
// Registry fact repair above is hygiene. Re-queueing audio is
|
|
1501
|
+
// work — only under an active audio command.
|
|
1502
|
+
if (!(await commandGate.isStageAuthorized(
|
|
1503
|
+
redis, scene.bookId, scene.chapterId, scene.sceneId, 'audio'))) {
|
|
1504
|
+
log(`[DIAGNOSTIC] RECOVER_ORPHAN_ASSETS ${scene.bookId}/${scene.chapterId}/${scene.sceneId}: registry marked not-ready; audio not re-queued (no command)`);
|
|
1505
|
+
return { success: true, action, scene, diagnostic: true, details: 'registry marked for recovery (no command — nothing queued)' };
|
|
1506
|
+
}
|
|
1480
1507
|
// M5 Шаг 4: Route through orchestrator.setScenePending (includes syncLinearState)
|
|
1481
1508
|
// S-5: asset-state writer via the injected orchestration seam
|
|
1482
1509
|
await orchestrationSeams.getOrchestrationSeam('setScenePending')(redis, scene.bookId, scene.chapterId, scene.sceneId, 'audio');
|
|
@@ -1755,14 +1782,17 @@ async function reconcileCycle(redis, deps = {}, options = {}) {
|
|
|
1755
1782
|
summary.errors.push(`counter_reconcile: ${err.message}`);
|
|
1756
1783
|
}
|
|
1757
1784
|
|
|
1758
|
-
// C5:
|
|
1759
|
-
|
|
1785
|
+
// C5: Interrupted window sessions (command-only generation): the
|
|
1786
|
+
// host marks sessions that were running at shutdown as
|
|
1787
|
+
// interrupted. Nothing is resumed in the background — the user
|
|
1788
|
+
// resumes with a button.
|
|
1789
|
+
if (typeof deps.interruptIncompleteSessions === 'function') {
|
|
1760
1790
|
try {
|
|
1761
|
-
await deps.
|
|
1762
|
-
phases.push(
|
|
1791
|
+
const c5Count = await deps.interruptIncompleteSessions(log);
|
|
1792
|
+
phases.push(`session_interrupt:${c5Count || 0}`);
|
|
1763
1793
|
} catch (err) {
|
|
1764
1794
|
warn(`Phase C5 failed: ${err.message}`);
|
|
1765
|
-
summary.errors.push(`
|
|
1795
|
+
summary.errors.push(`session_interrupt: ${err.message}`);
|
|
1766
1796
|
}
|
|
1767
1797
|
}
|
|
1768
1798
|
}
|
|
@@ -1814,15 +1844,17 @@ async function reconcileCycle(redis, deps = {}, options = {}) {
|
|
|
1814
1844
|
}
|
|
1815
1845
|
|
|
1816
1846
|
// ══════════════════════════════════════════════
|
|
1817
|
-
// PHASE C8:
|
|
1847
|
+
// PHASE C8: asset-state rebuild — startup-only
|
|
1818
1848
|
// ══════════════════════════════════════════════
|
|
1819
|
-
//
|
|
1820
|
-
//
|
|
1821
|
-
//
|
|
1849
|
+
// Command-only generation (2026-10-31): the former WORK_TO_DO rebuild
|
|
1850
|
+
// re-queued work after a restart (PENDING + active index). It now only
|
|
1851
|
+
// restores per-asset FACTS from PG + FS (READY for proven files,
|
|
1852
|
+
// DIRTY for READY without a valid file / version-stale) — it never
|
|
1853
|
+
// writes PENDING and never touches the active index.
|
|
1822
1854
|
if (startup && !scope) {
|
|
1823
1855
|
try {
|
|
1824
1856
|
const c7Count = await rebuildWorkList(redis);
|
|
1825
|
-
if (c7Count > 0) phases.push(`
|
|
1857
|
+
if (c7Count > 0) phases.push(`asset_state_rebuild:${c7Count}`);
|
|
1826
1858
|
} catch (err) {
|
|
1827
1859
|
warn(`Phase C8 failed: ${err.message}`);
|
|
1828
1860
|
summary.errors.push(`worklist_rebuild: ${err.message}`);
|
|
@@ -1934,11 +1966,34 @@ async function recoverResultKeys(redis, deps, scope) {
|
|
|
1934
1966
|
warn(`Skipping legacy result without build/dispatch identity: ${job_id}`);
|
|
1935
1967
|
continue;
|
|
1936
1968
|
}
|
|
1969
|
+
|
|
1970
|
+
// P1 (2026-11-01): the hub's fallback copy of a delivery the
|
|
1971
|
+
// result route already SETTLED is a duplicate — drop it
|
|
1972
|
+
// silently, never replay it (the replay re-ran the ready chain
|
|
1973
|
+
// and logged false no_active_dispatch warnings). Only a marker
|
|
1974
|
+
// carrying THIS dispatch counts: an in-flight / legacy '1'
|
|
1975
|
+
// marker or one from an older dispatch of the same job_id
|
|
1976
|
+
// (audio/video regeneration within the TTL) is no proof — such
|
|
1977
|
+
// a result goes through the fenced handler exactly as before.
|
|
1978
|
+
const dedup = await readResultDedup(redis, job_id, buildId);
|
|
1979
|
+
if (dedup.state === 'settled' && dedup.dispatchId === String(dispatchId)) {
|
|
1980
|
+
await redis.del(key);
|
|
1981
|
+
continue;
|
|
1982
|
+
}
|
|
1937
1983
|
try {
|
|
1938
1984
|
await taskHandler.handleTaskResult(job_id, result_base64, buildId, dispatchId);
|
|
1939
1985
|
} catch (rejectErr) {
|
|
1940
1986
|
if (rejectErr && TERMINAL_REJECTIONS.has(rejectErr.code)) {
|
|
1941
|
-
|
|
1987
|
+
// A replay of a delivery the route has seen (a marker
|
|
1988
|
+
// exists) is a duplicate: info. No marker (an orphan
|
|
1989
|
+
// that never reached the route) or an unreadable
|
|
1990
|
+
// marker: warn, as before.
|
|
1991
|
+
const msg = `Result ${job_id} terminally rejected (${rejectErr.code}) — dropping result key, no infinite retry`;
|
|
1992
|
+
if (dedup.state === 'in_flight' || dedup.state === 'settled') {
|
|
1993
|
+
log(`${msg} (duplicate of a delivered result)`);
|
|
1994
|
+
} else {
|
|
1995
|
+
warn(msg);
|
|
1996
|
+
}
|
|
1942
1997
|
await redis.del(key);
|
|
1943
1998
|
} else {
|
|
1944
1999
|
throw rejectErr;
|
|
@@ -2292,24 +2347,26 @@ async function reconcileMissingSceneState(redis, deps) {
|
|
|
2292
2347
|
}
|
|
2293
2348
|
}
|
|
2294
2349
|
|
|
2295
|
-
// ── PHASE
|
|
2296
|
-
// Cathedral Option E — см. docs/architecture/work-list-rebuild-design.md
|
|
2297
|
-
//
|
|
2298
|
-
// из PG + FS и МАТЕРИАЛИЗУЕТ runtime-состояние: per-asset Redis states + SADD
|
|
2299
|
-
// в active index. Генерацию НЕ запускает — это делает обычный scheduler tick
|
|
2300
|
-
// (единственный владелец execution, SCHEDULER_TICK_LOCK).
|
|
2350
|
+
// ── PHASE C8: asset-state rebuild (formerly WORK_TO_DO, Option E) ──
|
|
2351
|
+
// Cathedral Option E — см. docs/architecture/work-list-rebuild-design.md.
|
|
2352
|
+
// После потери Redis пересчитывает per-asset ФАКТЫ из PG + FS.
|
|
2301
2353
|
//
|
|
2302
|
-
//
|
|
2303
|
-
//
|
|
2304
|
-
//
|
|
2305
|
-
//
|
|
2306
|
-
//
|
|
2307
|
-
//
|
|
2308
|
-
//
|
|
2309
|
-
//
|
|
2310
|
-
//
|
|
2311
|
-
//
|
|
2312
|
-
//
|
|
2354
|
+
// Command-only generation (2026-10-31, animastor-docs COMMAND_ONLY_GENERATION):
|
|
2355
|
+
// фаза БОЛЬШЕ НЕ СОЗДАЁТ РАБОТУ — не пишет PENDING и не делает SADD в active
|
|
2356
|
+
// index. Прерванная генерация возобновляется только кнопкой (задачи реестра
|
|
2357
|
+
// помечаются interrupted хостом на старте).
|
|
2358
|
+
//
|
|
2359
|
+
// Что фаза делает:
|
|
2360
|
+
// 1. Предикат НЕ читает Redis asset state (C0 over-marks READY) — только PG + FS.
|
|
2361
|
+
// 2. Файл валиден и нет dirty-маркеров → READY (guarded restore-fact write),
|
|
2362
|
+
// поверх GENERATING/PENDING не пишет.
|
|
2363
|
+
// 3. Нужна работа (нет файла / stale / dirty-маркер): READY → DIRTY, а NEW при
|
|
2364
|
+
// существующем устаревшем файле → DIRTY (stale-метка, FSM-валидно). Без файла
|
|
2365
|
+
// NEW остаётся NEW; PLACEHOLDER, DIRTY, FAILED, PENDING, GENERATING не
|
|
2366
|
+
// трогаются: работа — только по команде.
|
|
2367
|
+
// 4. Книги с cancellation tombstone пропускаются fail-closed.
|
|
2368
|
+
// 5. Выключенные слои не трогаются.
|
|
2369
|
+
// 6. Ошибки PG — fail-closed (фаза прерывается).
|
|
2313
2370
|
async function rebuildWorkList(redis) {
|
|
2314
2371
|
// Все зависимости — lazy require на момент вызова, чтобы фаза была
|
|
2315
2372
|
// самодостаточной (не зависит от того, с какими зависимостями был
|
|
@@ -2335,10 +2392,8 @@ const state = lazyHostBinding('state');
|
|
|
2335
2392
|
// host service left this function.
|
|
2336
2393
|
const placeholderAudioOp = require('./placeholder-audio-port').placeholderAudioOp;
|
|
2337
2394
|
const sceneWindow = require('./scene-window');
|
|
2338
|
-
|
|
2339
|
-
// S-5: asset-state writers via the injected orchestration seams
|
|
2395
|
+
// S-5: asset-state writer via the injected orchestration seam
|
|
2340
2396
|
const markDirtyScene = orchestrationSeams.getOrchestrationSeam('markDirtyScene');
|
|
2341
|
-
const setScenePending = orchestrationSeams.getOrchestrationSeam('setScenePending');
|
|
2342
2397
|
|
|
2343
2398
|
const AssetState = state.AssetState;
|
|
2344
2399
|
const STAGES = mediaRegistry.listMediaTypes();
|
|
@@ -2423,88 +2478,58 @@ const state = lazyHostBinding('state');
|
|
|
2423
2478
|
}
|
|
2424
2479
|
|
|
2425
2480
|
const stateMap = {};
|
|
2426
|
-
let needsWork = false;
|
|
2427
2481
|
for (const stage of STAGES) {
|
|
2428
|
-
if (!enabled[stage]) continue; // disabled:
|
|
2482
|
+
if (!enabled[stage]) continue; // disabled layer: untouched
|
|
2429
2483
|
const fileValid = fsStatus[stage].exists && (stage !== 'audio' || audioReal);
|
|
2430
2484
|
const stale = !!staleAssets[`${key}:${stage}`];
|
|
2431
2485
|
const dirty = !!dirtyStages[`${key}:${stage}`];
|
|
2432
2486
|
const unitDirty = stage === 'image' && unitDirtyScenes.has(key);
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
stateMap[stage] = AssetState.PENDING;
|
|
2437
|
-
needsWork = true;
|
|
2438
|
-
}
|
|
2487
|
+
stateMap[stage] = (fileValid && !stale && !dirty && !unitDirty)
|
|
2488
|
+
? AssetState.READY
|
|
2489
|
+
: AssetState.DIRTY;
|
|
2439
2490
|
}
|
|
2440
2491
|
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
// 4. Материализовать states ДО SADD — через SAFE facade (audit gap 1).
|
|
2444
|
-
// Нельзя downgrade-ить выполняющуюся генерацию:
|
|
2445
|
-
// GENERATING/PENDING/PLACEHOLDER → НЕ трогаем (in-flight/очередь
|
|
2446
|
-
// владеют стадией; recovery не имеет права их откатывать).
|
|
2447
|
-
// PENDING-таргеты → orchestrator.setScenePending (валидация переходов:
|
|
2448
|
-
// GENERATING→PENDING rejected самой таблицей переходов).
|
|
2449
|
-
// READY-таргеты → guarded restore-fact write (S2.4): только из
|
|
2450
|
-
// NEW/DIRTY/FAILED/READY, никогда поверх GENERATING/PENDING.
|
|
2451
|
-
// C0 over-mark (READY без файла) → READY→DIRTY (markDirtyScene) →
|
|
2452
|
-
// →PENDING (setScenePending) — self-heal P1 через валидные переходы.
|
|
2492
|
+
// 4. Материализовать ФАКТЫ через safe-переходы. Работа не ставится.
|
|
2453
2493
|
const currentStates = await state.getAssetStates(redis, bookId, chapterId, sceneId);
|
|
2494
|
+
let touched = false;
|
|
2454
2495
|
for (const [stage, target] of Object.entries(stateMap)) {
|
|
2455
2496
|
const current = currentStates[stage] || state.AssetState.NEW;
|
|
2456
|
-
// GENERATING/PENDING — выполняющаяся/очередная генерация:
|
|
2457
|
-
// recovery не имеет права downgrade-ить её (audit gap 1).
|
|
2458
2497
|
if (current === state.AssetState.GENERATING ||
|
|
2459
2498
|
current === state.AssetState.PENDING) {
|
|
2460
|
-
log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: ${current} — untouched by
|
|
2499
|
+
log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: ${current} — untouched by rebuild`);
|
|
2461
2500
|
continue;
|
|
2462
2501
|
}
|
|
2463
|
-
if (target === state.AssetState.
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
const fresh = (await state.getAssetStates(redis, bookId, chapterId, sceneId))[stage] || state.AssetState.NEW;
|
|
2477
|
-
if (fresh !== current) {
|
|
2478
|
-
log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: state changed ${current}→${fresh} during rebuild — untouched (owner present)`);
|
|
2479
|
-
continue;
|
|
2480
|
-
}
|
|
2481
|
-
log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: ${current} → DIRTY → PENDING (valid transitions)`);
|
|
2482
|
-
await markDirtyScene(redis, bookId, chapterId, sceneId, [stage]);
|
|
2483
|
-
await setScenePending(redis, bookId, chapterId, sceneId, stage);
|
|
2484
|
-
} else {
|
|
2485
|
-
// NEW/DIRTY/FAILED → PENDING — валидные переходы.
|
|
2486
|
-
await setScenePending(redis, bookId, chapterId, sceneId, stage);
|
|
2487
|
-
}
|
|
2488
|
-
} else {
|
|
2489
|
-
// target === READY — restore-fact (файл валиден на диске):
|
|
2490
|
-
// guarded unsafe write, S2.4 (restore path). Пропускаются
|
|
2491
|
-
// GENERATING/PENDING (см. guard выше); из остальных состояний
|
|
2492
|
-
// (NEW/DIRTY/FAILED/PLACEHOLDER/READY) восстановить READY
|
|
2493
|
-
// безопасно — файл доказан FS-пробой.
|
|
2494
|
-
if (current !== state.AssetState.READY) {
|
|
2495
|
-
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, stage, state.AssetState.READY);
|
|
2502
|
+
if (target === state.AssetState.DIRTY) {
|
|
2503
|
+
// Stale marker only (the work waits for an explicit
|
|
2504
|
+
// command): a READY over-mark, or a NEW stage whose
|
|
2505
|
+
// artifact exists but is outdated/flagged. A stage
|
|
2506
|
+
// with no artifact at all stays NEW; PLACEHOLDER,
|
|
2507
|
+
// DIRTY, FAILED are left as they are.
|
|
2508
|
+
const markable = current === state.AssetState.READY ||
|
|
2509
|
+
(current === state.AssetState.NEW && fsStatus[stage].exists);
|
|
2510
|
+
if (!markable) continue;
|
|
2511
|
+
const fresh = (await state.getAssetStates(redis, bookId, chapterId, sceneId))[stage] || state.AssetState.NEW;
|
|
2512
|
+
if (fresh !== current) {
|
|
2513
|
+
log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: state changed ${current}→${fresh} during rebuild — untouched (owner present)`);
|
|
2514
|
+
continue;
|
|
2496
2515
|
}
|
|
2516
|
+
log(`[WORKLIST] ${bookId}/${chapterId}/${sceneId} ${stage}: ${current} without a valid/current file → DIRTY (stale; regeneration only by command)`);
|
|
2517
|
+
await markDirtyScene(redis, bookId, chapterId, sceneId, [stage]);
|
|
2518
|
+
touched = true;
|
|
2519
|
+
} else if (current !== state.AssetState.READY) {
|
|
2520
|
+
// target === READY — restore-fact (файл доказан FS-пробой).
|
|
2521
|
+
await state.unsafeRestoreAssetState(redis, bookId, chapterId, sceneId, stage, state.AssetState.READY);
|
|
2522
|
+
touched = true;
|
|
2497
2523
|
}
|
|
2498
2524
|
}
|
|
2499
|
-
|
|
2500
|
-
scenesAdded++;
|
|
2525
|
+
if (touched) scenesAdded++;
|
|
2501
2526
|
}
|
|
2502
2527
|
} catch (err) {
|
|
2503
2528
|
warn(`Rebuild failed for book ${bookId}: ${err.message}`);
|
|
2504
2529
|
}
|
|
2505
2530
|
}
|
|
2506
2531
|
|
|
2507
|
-
if (scenesAdded > 0) log(`[WORKLIST] ${scenesAdded} scene(s)
|
|
2532
|
+
if (scenesAdded > 0) log(`[WORKLIST] ${scenesAdded} scene(s) asset-state facts rebuilt (no work queued)`);
|
|
2508
2533
|
return scenesAdded;
|
|
2509
2534
|
}
|
|
2510
2535
|
|