@kbediako/codex-orchestrator 0.1.38 → 0.2.0

package/dist/orchestrator/src/cli/control/providerPollingHealth.js

@@ -0,0 +1,435 @@
+import { logger } from '../../logger.js';
+const DEFAULT_PROVIDER_POLLING_INTERVAL_MS = 15_000;
+const providerPollingHealthStates = new WeakMap();
+export function initializeProviderPollingHealth(providerIssueHandoff, input) {
+    const state = getOrCreateProviderPollingHealthState(providerIssueHandoff, input.intervalMs);
+    state.intervalMs = input.intervalMs;
+    if (input.stuckAfterMs !== undefined) {
+        state.stuckAfterMs = input.stuckAfterMs;
+    }
+    if (input.onUpdate !== undefined) {
+        state.onUpdate = input.onUpdate;
+    }
+    if (input.controlHostOwner !== undefined) {
+        state.controlHostOwner = input.controlHostOwner;
+    }
+    if (state.nextPollAtMs === null && input.intervalMs !== null) {
+        state.nextPollAtMs = Date.now();
+    }
+    if (input.skipInitialUpdate) {
+        return;
+    }
+    queueProviderPollingHealthUpdate(providerIssueHandoff, state);
+}
+export function noteProviderPollingRequest(providerIssueHandoff, input) {
+    const atMs = input.atMs ?? Date.now();
+    const state = getOrCreateProviderPollingHealthState(providerIssueHandoff);
+    const preserveActiveMode = input.preserveActiveMode && state.checking && state.operationStartedAtMs !== null;
+    state.lastMode = preserveActiveMode ? state.lastMode ?? input.mode : input.mode;
+    state.lastRequestedAtMs = atMs;
+    state.queued = input.replaceQueued ? input.queued : state.queued || input.queued;
+    state.updatedAtMs = atMs;
+    queueProviderPollingHealthUpdate(providerIssueHandoff, state, atMs);
+}
+export function markProviderPollingStarted(providerIssueHandoff, input) {
+    const atMs = input.atMs ?? Date.now();
+    const state = getOrCreateProviderPollingHealthState(providerIssueHandoff);
+    state.lastMode = input.mode;
+    state.lastRequestedAtMs = atMs;
+    state.checking = true;
+    state.queued = input.queued ?? false;
+    state.nextPollAtMs = null;
+    state.updatedAtMs = atMs;
+    state.operationStartedAtMs = atMs;
+    state.refreshPhase = `${input.mode}:started`;
+    state.refreshRequestClass = null;
+    state.refreshProviderKeys = null;
+    state.refreshCounts = null;
+    state.stuckAtMs = null;
+    state.reason = null;
+    queueProviderPollingHealthUpdate(providerIssueHandoff, state, atMs);
+}
+export function recordProviderPollingProgress(providerIssueHandoff, input) {
+    const state = providerPollingHealthStates.get(providerIssueHandoff);
+    if (!state) {
+        return;
+    }
+    const atMs = input.atMs ?? Date.now();
+    const previousRequestClass = state.refreshRequestClass;
+    state.refreshPhase = normalizeOptionalString(input.phase) ?? null;
+    if (input.requestClass !== undefined) {
+        state.refreshRequestClass = normalizeOptionalString(input.requestClass) ?? null;
+    }
+    if (input.providerKeys !== undefined) {
+        state.refreshProviderKeys = normalizePollingProviderKeys(input.providerKeys);
+    }
+    else if (input.requestClass !== undefined &&
+        state.refreshRequestClass !== previousRequestClass) {
+        state.refreshProviderKeys = null;
+    }
+    state.refreshCounts = copyFiniteRefreshCounts(input.counts ?? null);
+    state.updatedAtMs = atMs;
+    queueProviderPollingHealthUpdate(providerIssueHandoff, state, atMs);
+}
+export function markProviderPollingCompleted(providerIssueHandoff, input = {}) {
+    const atMs = input.atMs ?? Date.now();
+    const state = getOrCreateProviderPollingHealthState(providerIssueHandoff);
+    const preserveStuckState = isProviderLifecycleStuckError(input.error);
+    state.checking = false;
+    state.lastCompletedAtMs = atMs;
+    if (input.error === undefined) {
+        state.lastSuccessAtMs = atMs;
+        state.lastErrorAtMs = null;
+        state.lastError = null;
+    }
+    else {
+        state.lastErrorAtMs = atMs;
+        state.lastError = normalizePollingError(input.error);
+    }
+    state.nextPollAtMs =
+        preserveStuckState || state.intervalMs === null ? null : atMs + state.intervalMs;
+    state.updatedAtMs = atMs;
+    state.operationStartedAtMs = null;
+    if (preserveStuckState) {
+        if (state.stuckAtMs === null) {
+            state.stuckAtMs =
+                state.stuckAfterMs !== null && state.lastRequestedAtMs !== null
+                    ? state.lastRequestedAtMs + state.stuckAfterMs
+                    : atMs;
+        }
+        state.reason = state.reason ?? buildProviderPollingStuckReason(state);
+    }
+    else {
+        state.refreshPhase = null;
+        state.refreshRequestClass = null;
+        state.refreshProviderKeys = null;
+        state.refreshCounts = null;
+        state.stuckAtMs = null;
+        state.reason = null;
+    }
+    queueProviderPollingHealthUpdate(providerIssueHandoff, state, atMs);
+}
+export function scheduleProviderPolling(providerIssueHandoff, input) {
+    const atMs = input.atMs ?? Date.now();
+    const state = getOrCreateProviderPollingHealthState(providerIssueHandoff);
+    const intervalMs = normalizeScheduledPollingIntervalMs(input.intervalMs, state.intervalMs);
+    state.intervalMs = intervalMs;
+    if (state.stuckAtMs !== null) {
+        state.nextPollAtMs = null;
+        state.reason = state.reason ?? buildProviderPollingStuckReason(state);
+    }
+    else {
+        state.nextPollAtMs = atMs + intervalMs;
+        state.reason = normalizeOptionalString(input.reason) ?? null;
+    }
+    state.updatedAtMs = atMs;
+    state.linearBudget = input.linearBudget ?? null;
+    queueProviderPollingHealthUpdate(providerIssueHandoff, state, atMs);
+}
+export function isProviderPollingStuck(providerIssueHandoff, nowMs = Date.now()) {
+    return readProviderPollingHealth(providerIssueHandoff, nowMs)?.stuck === true;
+}
+export function markProviderPollingStuck(providerIssueHandoff, input = {}) {
+    const atMs = input.atMs ?? Date.now();
+    const state = getOrCreateProviderPollingHealthState(providerIssueHandoff);
+    if (!state.checking || state.operationStartedAtMs === null) {
+        return Promise.resolve();
+    }
+    if (state.stuckAtMs !== null) {
+        return state.updateChain;
+    }
+    state.stuckAtMs =
+        state.stuckAfterMs !== null ? state.operationStartedAtMs + state.stuckAfterMs : atMs;
+    state.lastErrorAtMs = atMs;
+    state.lastError = buildProviderPollingStuckMessage(state);
+    state.reason = buildProviderPollingStuckReason(state);
+    state.updatedAtMs = atMs;
+    return queueProviderPollingHealthUpdate(providerIssueHandoff, state, atMs);
+}
+export function readProviderPollingHealth(providerIssueHandoff, nowMs = Date.now()) {
+    if (!providerIssueHandoff) {
+        return null;
+    }
+    const state = providerPollingHealthStates.get(providerIssueHandoff);
+    if (!state) {
+        return null;
+    }
+    maybeMarkProviderPollingStuck(providerIssueHandoff, state, nowMs);
+    return buildProviderPollingHealthPayload(state, nowMs);
+}
+export async function flushProviderPollingHealthUpdates(providerIssueHandoff) {
+    if (!providerIssueHandoff) {
+        return;
+    }
+    const state = providerPollingHealthStates.get(providerIssueHandoff);
+    if (!state) {
+        return;
+    }
+    await state.updateChain;
+}
+export function resolveControlPollingNextRefreshProjection(input) {
+    const nowMs = input.nowMs ?? Date.now();
+    const nextPollAtMs = parseIsoToMs(input.nextPollAt) ??
+        (typeof input.nextPollInMs === 'number' && Number.isFinite(input.nextPollInMs)
+            ? nowMs + Math.max(0, input.nextPollInMs)
+            : null);
+    const projection = resolveControlPollingNextRefreshProjectionFromMs({
+        checking: input.checking,
+        nextPollAtMs,
+        operationStartedAtMs: parseIsoToMs(input.operationStartedAt),
+        linearBudget: input.linearBudget ?? null,
+        nowMs
+    });
+    return {
+        state: projection.state,
+        at: toIsoTimestamp(projection.atMs),
+        in_ms: projection.inMs
+    };
+}
+function buildProviderPollingHealthPayload(state, nowMs) {
+    const sourceUpdatedAtMs = latestFiniteTimestampMs(parseIsoToMs(state.linearBudget?.observed_at), state.lastSuccessAtMs);
+    const operationElapsedMs = state.checking && state.operationStartedAtMs !== null
+        ? Math.max(0, nowMs - state.operationStartedAtMs)
+        : null;
+    const stuck = state.stuckAtMs !== null;
+    const reason = state.reason ?? (stuck ? buildProviderPollingStuckReason(state) : null);
+    const nextRefresh = resolveControlPollingNextRefreshProjectionFromMs({
+        checking: state.checking,
+        nextPollAtMs: state.nextPollAtMs,
+        operationStartedAtMs: state.operationStartedAtMs,
+        linearBudget: state.linearBudget,
+        nowMs
+    });
+    return {
+        enabled: true,
+        interval_ms: state.intervalMs,
+        checking: state.checking,
+        queued: state.queued,
+        last_mode: state.lastMode,
+        last_requested_at: toIsoTimestamp(state.lastRequestedAtMs),
+        last_completed_at: toIsoTimestamp(state.lastCompletedAtMs),
+        last_success_at: toIsoTimestamp(state.lastSuccessAtMs),
+        last_error_at: toIsoTimestamp(state.lastErrorAtMs),
+        last_error: state.lastError,
+        next_poll_at: toIsoTimestamp(state.nextPollAtMs),
+        next_poll_in_ms: state.nextPollAtMs === null ? null : Math.max(0, state.nextPollAtMs - nowMs),
+        next_refresh_state: nextRefresh.state,
+        next_refresh_at: toIsoTimestamp(nextRefresh.atMs),
+        next_refresh_in_ms: nextRefresh.inMs,
+        source_updated_at: toIsoTimestamp(sourceUpdatedAtMs),
+        updated_at: toIsoTimestamp(state.updatedAtMs),
+        operation_started_at: toIsoTimestamp(state.operationStartedAtMs),
+        operation_elapsed_ms: operationElapsedMs,
+        stalled_after_ms: state.stuckAfterMs,
+        refresh_phase: state.refreshPhase,
+        refresh_request_class: state.refreshRequestClass,
+        refresh_provider_keys: state.refreshProviderKeys ? [...state.refreshProviderKeys] : null,
+        refresh_counts: state.refreshCounts ? { ...state.refreshCounts } : null,
+        stuck,
+        stuck_since_at: toIsoTimestamp(state.stuckAtMs),
+        restart_required: stuck,
+        reason,
+        linear_budget: state.linearBudget,
+        control_host_owner: state.controlHostOwner
+    };
+}
+function resolveControlPollingNextRefreshProjectionFromMs(input) {
+    const cooldownUntilMs = resolveActiveLinearBudgetCooldownUntilMs(input.linearBudget, input.nowMs);
+    if (cooldownUntilMs !== null) {
+        return {
+            state: 'cooldown',
+            atMs: cooldownUntilMs,
+            inMs: Math.max(0, cooldownUntilMs - input.nowMs)
+        };
+    }
+    if (input.checking && input.operationStartedAtMs !== null) {
+        return {
+            state: 'checking',
+            atMs: null,
+            inMs: null
+        };
+    }
+    if (input.nextPollAtMs !== null && Number.isFinite(input.nextPollAtMs)) {
+        return {
+            state: 'scheduled',
+            atMs: input.nextPollAtMs,
+            inMs: Math.max(0, input.nextPollAtMs - input.nowMs)
+        };
+    }
+    return {
+        state: 'unknown',
+        atMs: null,
+        inMs: null
+    };
+}
+function resolveActiveLinearBudgetCooldownUntilMs(budget, nowMs) {
+    const cooldownUntilMs = resolveLinearBudgetCooldownUntilMs(budget);
+    return cooldownUntilMs !== null && cooldownUntilMs > nowMs ? cooldownUntilMs : null;
+}
+function resolveLinearBudgetCooldownUntilMs(budget) {
+    if (!budget) {
+        return null;
+    }
+    const cooldownUntilMs = parseIsoToMs(budget.cooldown_until);
+    if (cooldownUntilMs !== null) {
+        return cooldownUntilMs;
+    }
+    if (typeof budget.retry_after_seconds === 'number' &&
+        Number.isFinite(budget.retry_after_seconds)) {
+        const observedAtMs = parseIsoToMs(budget.observed_at);
+        if (observedAtMs !== null) {
+            return observedAtMs + Math.max(0, Math.ceil(budget.retry_after_seconds * 1000));
+        }
+    }
+    return null;
+}
+function getOrCreateProviderPollingHealthState(providerIssueHandoff, intervalMs = null) {
+    const existing = providerPollingHealthStates.get(providerIssueHandoff);
+    if (existing) {
+        if (intervalMs !== null) {
+            existing.intervalMs = intervalMs;
+        }
+        return existing;
+    }
+    const nextState = {
+        intervalMs,
+        stuckAfterMs: null,
+        checking: false,
+        queued: false,
+        lastMode: null,
+        lastRequestedAtMs: null,
+        lastCompletedAtMs: null,
+        lastSuccessAtMs: null,
+        lastErrorAtMs: null,
+        lastError: null,
+        nextPollAtMs: intervalMs !== null ? Date.now() : null,
+        updatedAtMs: null,
+        operationStartedAtMs: null,
+        refreshPhase: null,
+        refreshRequestClass: null,
+        refreshProviderKeys: null,
+        refreshCounts: null,
+        stuckAtMs: null,
+        reason: null,
+        linearBudget: null,
+        controlHostOwner: null,
+        onUpdate: null,
+        updateChain: Promise.resolve()
+    };
+    providerPollingHealthStates.set(providerIssueHandoff, nextState);
+    return nextState;
+}
+function maybeMarkProviderPollingStuck(providerIssueHandoff, state, nowMs) {
+    if (!state.checking ||
+        state.operationStartedAtMs === null ||
+        state.stuckAfterMs === null ||
+        nowMs - state.operationStartedAtMs < state.stuckAfterMs) {
+        return;
+    }
+    markProviderPollingStuck(providerIssueHandoff, {
+        atMs: nowMs
+    });
+}
+function buildProviderPollingStuckReason(state) {
+    return state.lastMode === 'poll'
+        ? 'provider_poll_lifecycle_stuck'
+        : 'provider_refresh_lifecycle_stuck';
+}
+function buildProviderPollingStuckMessage(state) {
+    const thresholdText = typeof state.stuckAfterMs === 'number' && Number.isFinite(state.stuckAfterMs)
+        ? `${state.stuckAfterMs}ms`
+        : 'the configured safe budget';
+    return `${state.lastMode === 'poll' ? 'Provider poll' : 'Provider refresh'} lifecycle exceeded ${thresholdText}; restart required`;
+}
+function queueProviderPollingHealthUpdate(providerIssueHandoff, state, nowMs = Date.now()) {
+    if (!state.onUpdate) {
+        return state.updateChain;
+    }
+    const payload = buildProviderPollingHealthPayload(state, nowMs);
+    const nextUpdate = state.updateChain
+        .catch(() => undefined)
+        .then(async () => {
+        await state.onUpdate?.(payload);
+    });
+    // Most callers intentionally fire-and-forget polling sync. Attach a no-op handler so those
+    // paths do not surface unhandled rejections while explicit awaiters still observe failures.
+    void nextUpdate.catch(() => undefined);
+    state.updateChain = nextUpdate.catch((error) => {
+        logger.warn(`Failed to sync provider polling health: ${error?.message ?? String(error)}`);
+    });
+    return nextUpdate;
+}
+function toIsoTimestamp(value) {
+    if (value === null || !Number.isFinite(value)) {
+        return null;
+    }
+    return new Date(value).toISOString();
+}
+function parseIsoToMs(value) {
+    if (typeof value !== 'string' || value.trim().length === 0) {
+        return null;
+    }
+    const parsed = Date.parse(value);
+    return Number.isFinite(parsed) ? parsed : null;
+}
+function latestFiniteTimestampMs(...values) {
+    const finiteValues = values.filter((value) => typeof value === 'number' && Number.isFinite(value));
+    if (finiteValues.length === 0) {
+        return null;
+    }
+    return Math.max(...finiteValues);
+}
+function normalizePollingError(error) {
+    if (error instanceof Error && error.message) {
+        return error.message;
+    }
+    if (typeof error === 'string' && error.trim().length > 0) {
+        return error;
+    }
+    return String(error);
+}
+function isProviderLifecycleStuckError(error) {
+    return (error instanceof Error &&
+        (error.name === 'ProviderRefreshLifecycleStuckError' ||
+            error.name === 'ProviderPollLifecycleStuckError' ||
+            error.message === 'provider_refresh_lifecycle_stuck' ||
+            error.message === 'provider_poll_lifecycle_stuck'));
+}
+function normalizeOptionalString(value) {
+    return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
+}
+function copyFiniteRefreshCounts(value) {
+    if (!value) {
+        return null;
+    }
+    const entries = Object.entries(value)
+        .map(([key, count]) => [key.trim(), count])
+        .filter(([key, count]) => key.length > 0 && Number.isFinite(count));
+    return entries.length > 0 ? Object.fromEntries(entries) : null;
+}
+function normalizePollingProviderKeys(value) {
+    if (!Array.isArray(value)) {
+        return null;
+    }
+    const normalized = new Set();
+    for (const entry of value) {
+        const key = normalizeOptionalString(entry);
+        if (!key) {
+            continue;
+        }
+        normalized.add(key);
+        if (normalized.size >= 8) {
+            break;
+        }
+    }
+    return normalized.size > 0 ? [...normalized] : null;
+}
+function normalizeScheduledPollingIntervalMs(value, fallback) {
+    if (Number.isFinite(value) && value > 0) {
+        return value;
+    }
+    if (typeof fallback === 'number' && Number.isFinite(fallback) && fallback > 0) {
+        return fallback;
+    }
+    return DEFAULT_PROVIDER_POLLING_INTERVAL_MS;
+}