@animastor/orchestration 0.1.1

package/src/runtime/retry-budget-manager.js

@@ -0,0 +1,229 @@
+const mediaRegistry = require('@animastor/generation').mediaRegistry;
+// ======================================================
+// RETRY BUDGET MANAGER (MINIMAL)
+// ======================================================
+// S1.3 (2026-07-19): сокращён с 520 до ~140 строк.
+// Удалены: refillBudgets (мёртвый wildcard-код, Б6), formatBudget,
+// getGlobalBudgetOverview, getSceneBudgets (read-only debug), resetSceneStageBudget,
+// избыточные export'ы.
+//
+// Production usage (dispatch-engine.js):
+//   checkRetryBudget()  — перед dispatch'ем, решает можно ли попробовать
+//   consumeRetryBudget() — после finalizeDispatch('failure')
+//
+// Логика: per-(scene,stage) budget → 0 = stop retry.
+// Глобальный, по-worker, по-type — оставлены как soft caps (TTL, без сложных policies).
+// Этого достаточно для 1–15 concurrent пользователей (см. CAPACITY_AND_COMPLEXITY.md).
+
+const logPrefix = '[RETRY-BUDGET]';
+function log(msg) { console.log(`${logPrefix} ${msg}`); }
+
+// ======================================================
+// CONFIGURATION
+// ======================================================
+
+// S-2: per-scene limits resolved through media registry
+function _perSceneLimit(stage) { return mediaRegistry.resolveRetryBudget(stage, 10); }
+const GLOBAL_LIMITS = { retriesPerMinute: 100, retriesPerWorkerPerMinute: 20 };
+const FAILURE_TYPE_LIMITS = { transient: 50, permanent: 0, infrastructure: 30, orchestration: 20 };
+
+// ======================================================
+// KEYS
+// ======================================================
+
+function getSceneStageBudgetKey(bookId, chapterId, sceneId, stage) {
+    return `animastor:retry-budget:${bookId}:${chapterId}:${sceneId}:${stage}`;
+}
+
+function getGlobalBudgetKey() {
+    return 'animastor:retry-budget:global';
+}
+
+function getFailureTypeBudgetKey(failureType) {
+    return `animastor:retry-budget:type:${failureType}`;
+}
+
+function getWorkerBudgetKey(workerId) {
+    return `animastor:retry-budget:worker:${workerId}`;
+}
+
+// ======================================================
+// BUDGET GETTERS (read current budget value)
+// ======================================================
+
+async function getSceneStageBudget(redis, bookId, chapterId, sceneId, stage) {
+    const key = getSceneStageBudgetKey(bookId, chapterId, sceneId, stage);
+    const current = await redis.get(key);
+    const defaultValue = _perSceneLimit(stage);
+    return parseInt(current || String(defaultValue), 10);
+}
+
+async function getGlobalBudget(redis) {
+    const key = getGlobalBudgetKey();
+    const current = await redis.get(key);
+    return {
+        current: parseInt(current || String(GLOBAL_LIMITS.retriesPerMinute), 10),
+        max: GLOBAL_LIMITS.retriesPerMinute
+    };
+}
+
+async function getFailureTypeBudget(redis, failureType) {
+    const key = getFailureTypeBudgetKey(failureType);
+    const current = await redis.get(key);
+    const maxValue = FAILURE_TYPE_LIMITS[failureType] || 50;
+    return {
+        current: parseInt(current || String(maxValue), 10),
+        max: maxValue,
+        type: failureType
+    };
+}
+
+async function getWorkerBudget(redis, workerId) {
+    const key = getWorkerBudgetKey(workerId);
+    const current = await redis.get(key);
+    const maxValue = GLOBAL_LIMITS.retriesPerWorkerPerMinute;
+    return {
+        current: parseInt(current || String(maxValue), 10),
+        max: maxValue,
+        workerId
+    };
+}
+
+// ======================================================
+// BUDGET CONSUMERS (atomic decrement)
+// ======================================================
+
+async function consumeSceneStageBudget(redis, bookId, chapterId, sceneId, stage) {
+    const key = getSceneStageBudgetKey(bookId, chapterId, sceneId, stage);
+    const current = await redis.get(key);
+    const value = parseInt(current || String(_perSceneLimit(stage)), 10);
+
+    if (value <= 0) {
+        return { consumed: false, remaining: 0, exceeded: true };
+    }
+
+    const remaining = value - 1;
+    await redis.set(key, remaining.toString());
+    log(`BUDGET_CONSUMED: ${bookId}/${chapterId}/${sceneId}:${stage} (remaining: ${remaining})`);
+    return { consumed: true, remaining, exceeded: false };
+}
+
+async function consumeGlobalBudget(redis) {
+    const key = getGlobalBudgetKey();
+    const current = await redis.get(key);
+    const currentValue = parseInt(current || String(GLOBAL_LIMITS.retriesPerMinute), 10);
+
+    if (currentValue <= 0) {
+        return { consumed: false, remaining: 0, exceeded: true, global: true };
+    }
+
+    const remaining = currentValue - 1;
+    await redis.set(key, remaining.toString(), 'EX', 60);
+    return { consumed: true, remaining, exceeded: false, global: true };
+}
+
+async function consumeFailureTypeBudget(redis, failureType) {
+    const key = getFailureTypeBudgetKey(failureType);
+    const current = await redis.get(key);
+    const maxValue = FAILURE_TYPE_LIMITS[failureType] || 50;
+    const currentValue = parseInt(current || String(maxValue), 10);
+
+    if (currentValue <= 0) {
+        return { consumed: false, remaining: 0, exceeded: true, failureType };
+    }
+
+    const remaining = currentValue - 1;
+    await redis.set(key, remaining.toString(), 'EX', 300);
+    return { consumed: true, remaining, exceeded: false, failureType };
+}
+
+async function consumeWorkerBudget(redis, workerId) {
+    if (!workerId) return { consumed: true, skipped: true };
+    const key = getWorkerBudgetKey(workerId);
+    const current = await redis.get(key);
+    const maxValue = GLOBAL_LIMITS.retriesPerWorkerPerMinute;
+    const currentValue = parseInt(current || String(maxValue), 10);
+
+    if (currentValue <= 0) {
+        return { consumed: false, remaining: 0, exceeded: true, workerId };
+    }
+
+    const remaining = currentValue - 1;
+    await redis.set(key, remaining.toString(), 'EX', 60);
+    return { consumed: true, remaining, exceeded: false, workerId };
+}
+
+// ======================================================
+// CHECK + CONSUME (production API)
+// ======================================================
+
+/**
+ * Check if a retry is allowed before dispatching.
+ * Returns: { allowed, reason, budgets }
+ *
+ * Order: scene → failureType → global → worker. First exhausted wins.
+ */
+async function checkRetryBudget(redis, bookId, chapterId, sceneId, stage, failureType, workerId) {
+    const sceneBudget = await getSceneStageBudget(redis, bookId, chapterId, sceneId, stage);
+    const budgets = {
+        sceneStage: { current: sceneBudget, limit: _perSceneLimit(stage), allowed: sceneBudget > 0 }
+    };
+
+    if (sceneBudget <= 0) {
+        return { allowed: false, reason: 'scene_budget_exceeded', budgets };
+    }
+
+    const typeBudget = await getFailureTypeBudget(redis, failureType);
+    budgets.byType = { current: typeBudget.current, max: typeBudget.max, allowed: typeBudget.current > 0 };
+    if (typeBudget.current <= 0 && FAILURE_TYPE_LIMITS[failureType] !== 0) {
+        return { allowed: false, reason: 'type_budget_exceeded', budgets };
+    }
+
+    const globalBudget = await getGlobalBudget(redis);
+    budgets.global = { current: globalBudget.current, max: globalBudget.max, allowed: globalBudget.current > 0 };
+    if (globalBudget.current <= 0) {
+        return { allowed: false, reason: 'global_budget_exceeded', budgets };
+    }
+
+    if (workerId) {
+        const workerBudget = await getWorkerBudget(redis, workerId);
+        budgets.worker = { current: workerBudget.current, max: workerBudget.max, allowed: workerBudget.current > 0 };
+        if (workerBudget.current <= 0) {
+            return { allowed: false, reason: 'worker_budget_exceeded', budgets };
+        }
+    }
+
+    return { allowed: true, reason: 'budget_available', budgets };
+}
+
+/**
+ * Consume retry budget after a failed dispatch finalization (called once per failure).
+ * Returns: { consumed, budgets }
+ */
+async function consumeRetryBudget(redis, bookId, chapterId, sceneId, stage, failureType, workerId) {
+    const budgets = {
+        sceneStage: { ...await consumeSceneStageBudget(redis, bookId, chapterId, sceneId, stage), limit: _perSceneLimit(stage) },
+        byType: await consumeFailureTypeBudget(redis, failureType),
+        global: await consumeGlobalBudget(redis),
+        worker: await consumeWorkerBudget(redis, workerId)
+    };
+    return { consumed: true, budgets };
+}
+
+// ======================================================
+// EXPORTS
+// ======================================================
+
+module.exports = {
+    // Config (для диагностики) — S-2: lazy view over the media registry
+    PER_SCENE_LIMITS: new Proxy({}, {
+        get(_, stage) { return mediaRegistry.resolveRetryBudget(String(stage), undefined); },
+        ownKeys() { return mediaRegistry.listMediaTypes(); },
+    }),
+    GLOBAL_LIMITS,
+    FAILURE_TYPE_LIMITS,
+
+    // Production API
+    checkRetryBudget,
+    consumeRetryBudget
+};

package/src/runtime/runtime-metrics.js

@@ -0,0 +1,427 @@
+// ======================================================
+// RUNTIME METRICS LAYER
+// ======================================================
+// Collects and exports runtime metrics for observability.
+//
+// Metrics stored in Redis:
+// - animastor:runtime:metrics:current
+// - animastor:runtime:metrics:history (sorted set)
+//
+// Metrics tracked:
+// - active scenes
+// - active dispatches/leases
+// - quota utilization
+// - scheduler ticks (count, duration)
+// - recovery actions
+// - stale recoveries
+// - duplicate dispatch skips
+// - counter drift
+
+const counterReconciliation = require('./counter-reconciliation');
+// S-2: media types and quotas resolved from media registry
+const mediaRegistry = require('@animastor/generation').mediaRegistry;
+
+const logPrefix = '[METRICS]';
+
+function log(msg) {
+    console.log(`${logPrefix} ${msg}`);
+}
+
+function warn(msg) {
+    console.warn(`${logPrefix} ⚠️ ${msg}`);
+}
+
+function error(msg) {
+    console.error(`${logPrefix} ❌ ${msg}`);
+}
+
+// ======================================================
+// CONFIGURATION
+// ======================================================
+
+const METRICS_KEY = 'animastor:runtime:metrics:current';
+const METRICS_HISTORY_KEY = 'animastor:runtime:metrics:history';
+const METRICS_RETENTION_LIMIT = 1000;
+
+// ======================================================
+// METRIC DEFINITIONS
+// ======================================================
+
+const METRIC_DEFS = {
+    // Scene metrics
+    activeScenes: { type: 'gauge', description: 'Number of active scenes' },
+    activeAudioScenes: { type: 'gauge', description: 'Active scenes in audio stages' },
+    activeImageScenes: { type: 'gauge', description: 'Active scenes in image stages' },
+    activeVideoScenes: { type: 'gauge', description: 'Active scenes in video stages' },
+
+    // Dispatch/lease metrics
+    activeLeases: { type: 'gauge', description: 'Total active leases' },
+    activeAudioLeases: { type: 'gauge', description: 'Active audio leases' },
+    activeImageLeases: { type: 'gauge', description: 'Active image leases' },
+    activeVideoLeases: { type: 'gauge', description: 'Active video leases' },
+    leaseRenewalsInFlight: { type: 'gauge', description: 'Active lease renewal timers' },
+
+    // Quota metrics
+    quotaAudioUsed: { type: 'gauge', description: 'Active audio dispatches' },
+    quotaAudioMax: { type: 'gauge', description: 'Max concurrent audio' },
+    quotaImageUsed: { type: 'gauge', description: 'Active image dispatches' },
+    quotaImageMax: { type: 'gauge', description: 'Max concurrent image' },
+    quotaVideoUsed: { type: 'gauge', description: 'Active video dispatches' },
+    quotaVideoMax: { type: 'gauge', description: 'Max concurrent video' },
+
+    // Scheduler metrics
+    schedulerTicks: { type: 'counter', description: 'Total scheduler ticks' },
+    schedulerTickDurationAvg: { type: 'gauge', description: 'Average tick duration (ms)' },
+    schedulerLastTickDuration: { type: 'gauge', description: 'Last tick duration (ms)' },
+
+    // Dispatch metrics
+    dispatchesAttempted: { type: 'counter', description: 'Total dispatch attempts' },
+    dispatchesSuccess: { type: 'counter', description: 'Successful dispatches' },
+    dispatchesSkipped: { type: 'counter', description: 'Skipped dispatches (lease exists)' },
+    dispatchesBackpressure: { type: 'counter', description: 'BackpressureBlocked dispatches' },
+
+    // Recovery metrics
+    recoveryActions: { type: 'counter', description: 'Total recovery actions' },
+    staleLeaseRecoveries: { type: 'counter', description: 'Stale lease recoveries' },
+    counterDriftCorrections: { type: 'counter', description: 'Counter drift corrections' },
+    stateRollbackFailures: { type: 'counter', description: 'Failed FSM rollbacks after jobless dispatch cancellation (ghost GENERATING risk)' },
+    iuInFlightStaleCleared: { type: 'counter', description: 'Stale iu-in-flight markers cleared by ownership self-heal' },
+
+    // Event metrics
+    callbackDuplicates: { type: 'counter', description: 'Duplicate callback events' },
+    queueBlocked: { type: 'counter', description: 'Queue blocked events' },
+
+    // Drift metrics
+    counterDriftTotal: { type: 'gauge', description: 'Total counter drift magnitude' }
+};
+
+// ======================================================
+// GETTER FUNCTIONS
+// ======================================================
+
+/**
+ * Get metric definitions.
+ */
+function getMetricDefinitions() {
+    return METRIC_DEFS;
+}
+
+/**
+ * Get current runtime metrics from Redis.
+ */
+async function getCurrentMetrics(redis, additionalData = {}) {
+    const timestamp = Date.now();
+
+    // Get active scenes
+    const activeScenesKey = 'animastor:active-scenes';
+    const activeScenes = await redis.scard(activeScenesKey);
+
+    // Get active leases by stage
+    const leasePatternAudio = 'animastor:dispatch-lease:*:audio';
+    const leasePatternImage = 'animastor:dispatch-lease:*:image';
+    const leasePatternVideo = 'animastor:dispatch-lease:*:video';
+
+    const [audioCount, imageCount, videoCount] = await Promise.all([
+        countPattern(redis, leasePatternAudio),
+        countPattern(redis, leasePatternImage),
+        countPattern(redis, leasePatternVideo)
+    ]);
+
+    // Get active counters
+    const activeKeys = [
+        'animastor:runtime:active-audio',
+        'animastor:runtime:active-image',
+        'animastor:runtime:active-video'
+    ];
+
+    const [audioCounter, imageCounter, videoCounter] = await Promise.all([
+        getCounter(redis, activeKeys[0]),
+        getCounter(redis, activeKeys[1]),
+        getCounter(redis, activeKeys[2])
+    ]);
+
+    // Get quota settings
+    const quotas = Object.fromEntries(mediaRegistry.listMediaTypes().map(t => ['max' + t.charAt(0).toUpperCase() + t.slice(1), mediaRegistry.resolveMaxActive(t)]));
+
+    // Get lease renewal count
+    const renewalKey = 'animastor:runtime:renewal-timers';
+    const renewalCount = await redis.scard(renewalKey);
+
+    // Get scheduler history
+    const schedulerKey = 'animastor:runtime:scheduler:history';
+    const schedulerHistory = await redis.lrange(schedulerKey, -10, -1);
+
+    let schedulerTicks = 0;
+    let totalTickDuration = 0;
+    let lastTickDuration = 0;
+
+    if (schedulerHistory.length > 0) {
+        schedulerTicks = schedulerHistory.length;
+        for (const item of schedulerHistory) {
+            try {
+                const data = JSON.parse(item);
+                totalTickDuration += data.durationMs || 0;
+                lastTickDuration = data.durationMs || 0;
+            } catch (e) {
+                // Skip invalid entries
+            }
+        }
+    }
+
+    // Get recovery counts
+    const recoveryKey = 'animastor:runtime:recovery:actions';
+    const recoveryActions = await redis.scard(recoveryKey);
+
+    const metrics = {
+        // Timestamp
+        timestamp,
+        createdAt: new Date(timestamp).toISOString(),
+
+        // Scenes
+        activeScenes,
+        activeAudioScenes: await getScenesInStage(redis, 'audio_generating'),
+        activeImageScenes: await getScenesInStage(redis, 'image_generating'),
+        activeVideoScenes: await getScenesInStage(redis, 'video_generating'),
+
+        // Leases
+        activeLeases: audioCount + imageCount + videoCount,
+        activeAudioLeases: audioCount,
+        activeImageLeases: imageCount,
+        activeVideoLeases: videoCount,
+        leaseRenewalsInFlight: renewalCount,
+
+        // Quotas
+        quotaAudioUsed: audioCounter,
+        quotaAudioMax: quotas.maxAudio,
+        quotaImageUsed: imageCounter,
+        quotaImageMax: quotas.maxImage,
+        quotaVideoUsed: videoCounter,
+        quotaVideoMax: quotas.maxVideo,
+
+        // Scheduler
+        schedulerTicks,
+        schedulerTickDurationAvg: schedulerTicks > 0 ? Math.round(totalTickDuration / schedulerTicks) : 0,
+        schedulerLastTickDuration: lastTickDuration,
+
+        // Additional data
+        ...additionalData
+    };
+
+    return metrics;
+}
+
+/**
+ * Count keys matching pattern.
+ */
+async function countPattern(redis, pattern) {
+    let cursor = 0;
+    let count = 0;
+
+    do {
+        const result = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 500);
+        cursor = parseInt(result[0], 10);
+        count += result[1].length;
+    } while (cursor !== 0 && count < 10000);
+
+    return count;
+}
+
+/**
+ * Get counter value.
+ */
+async function getCounter(redis, key) {
+    const val = await redis.get(key);
+    return parseInt(val || '0', 10);
+}
+
+/**
+ * Get scenes in a specific stage (stub — scene-state removed).
+ */
+async function getScenesInStage(redis, stage) {
+    return 0;
+}
+
+/**
+ * Store current metrics in Redis.
+ */
+async function storeMetrics(redis, metrics) {
+    // Store current
+    await redis.set(METRICS_KEY, JSON.stringify(metrics));
+
+    // Add to history (sorted set with timestamp as score)
+    const historyKey = METRICS_HISTORY_KEY;
+    await redis.zadd(historyKey, metrics.timestamp, JSON.stringify(metrics));
+
+    // Trim history
+    const count = await redis.zcard(historyKey);
+    if (count > METRICS_RETENTION_LIMIT) {
+        await redis.zremrangebyrank(historyKey, 0, count - METRICS_RETENTION_LIMIT - 1);
+    }
+
+    return { stored: true, key: METRICS_KEY, historyKey, historyLength: count };
+}
+
+/**
+ * Get metrics history.
+ */
+async function getMetricsHistory(redis, limit = 100, offset = 0) {
+    const historyKey = METRICS_HISTORY_KEY;
+    const count = await redis.zcard(historyKey);
+
+    const start = Math.max(0, count - limit - offset);
+    const end = count - offset - 1;
+
+    const range = await redis.zrange(historyKey, start, end);
+    const metrics = range.map(item => {
+        try {
+            return JSON.parse(item);
+        } catch (e) {
+            return null;
+        }
+    }).filter(m => m !== null);
+
+    return {
+        total: count,
+        metrics,
+        returned: metrics.length
+    };
+}
+
+/**
+ * Increment counter metric.
+ */
+async function incrementCounter(redis, name, value = 1) {
+    const key = `${METRICS_KEY}:${name}`;
+    const newValue = await redis.incrby(key, value);
+    return { name, key, value: newValue };
+}
+
+/**
+ * Get counter metric.
+ */
+async function getCounterMetric(redis, name) {
+    const key = `${METRICS_KEY}:${name}`;
+    const val = await redis.get(key);
+    return parseInt(val || '0', 10);
+}
+
+/**
+ * Record scheduler tick.
+ */
+async function recordSchedulerTick(redis, tickMetrics) {
+    const historyKey = 'animastor:runtime:scheduler:history';
+    await redis.rpush(historyKey, JSON.stringify(tickMetrics));
+    await redis.ltrim(historyKey, -1000, -1); // Keep last 1000
+
+    return { recorded: true, historyKey };
+}
+
+/**
+ * Record recovery action.
+ */
+async function recordRecoveryAction(redis, action) {
+    const key = 'animastor:runtime:recovery:actions';
+    const data = {
+        ...action,
+        timestamp: Date.now()
+    };
+    await redis.sadd(key, JSON.stringify(data));
+    return { recorded: true, key, action };
+}
+
+/**
+ * Record duplicate callback.
+ */
+async function recordDuplicateCallback(redis) {
+    return await incrementCounter(redis, 'callbackDuplicates');
+}
+
+/**
+ * Record backpressure-blocked dispatch.
+ */
+async function recordBackpressureBlocked(redis) {
+    return await incrementCounter(redis, 'dispatchesBackpressure');
+}
+
+/**
+ * Record stuck scene detection.
+ */
+async function recordStuckScene(redis, scene) {
+    const key = 'animastor:runtime:stuck-scenes';
+    const data = {
+        ...scene,
+        timestamp: Date.now()
+    };
+    await redis.sadd(key, JSON.stringify(data));
+    return { recorded: true, key, scene };
+}
+
+/**
+ * Get comprehensive metrics report.
+ */
+async function getMetricsReport(redis) {
+    const current = await getCurrentMetrics(redis);
+    const history = await getMetricsHistory(redis, 10);
+    const driftReport = await counterReconciliation.checkForDrift(redis);
+
+    return {
+        current,
+        history: history.metrics,
+        drift: driftReport
+    };
+}
+
+/**
+ * Get runtime health status.
+ */
+async function getHealthStatus(redis) {
+    const current = await getCurrentMetrics(redis);
+
+    // Check if leases are growing faster than counter consumption
+    const leaseGrowth = current.activeLeases > 0 && current.quotaAudioUsed > current.activeLeases;
+    const potentialLeak = current.activeLeases > 100; // Suspiciously high
+
+    return {
+        healthy: !potentialLeak,
+        issues: potentialLeak ? ['high_active_leases'] : [],
+        metrics: current
+    };
+}
+
+// ======================================================
+// EXPORTS
+// ======================================================
+
+module.exports = {
+    // Configuration
+    METRICS_KEY,
+    METRICS_HISTORY_KEY,
+    METRICS_RETENTION_LIMIT,
+
+    // Definitions
+    getMetricDefinitions,
+    METRIC_DEFS,
+
+    // Getters
+    getCurrentMetrics,
+    getMetricsHistory,
+    getMetricsReport,
+    getHealthStatus,
+
+    // Storage
+    storeMetrics,
+    recordSchedulerTick,
+    recordRecoveryAction,
+    recordDuplicateCallback,
+    recordBackpressureBlocked,
+    recordStuckScene,
+
+    // Counters
+    incrementCounter,
+    getCounterMetric,
+
+    // Helper
+    countPattern,
+    getCounter,
+    getScenesInStage
+};