ai-maestro 1.0.0

package/src/orchestration/provider-health.mjs

@@ -0,0 +1,566 @@
+import fs from 'fs/promises';
+import path from 'path';
+
+import { CONFIG } from '../config.mjs';
+import { readJsonSafe, stripBom } from '../files.mjs';
+import { redactCredentials, redactObject } from '../format.mjs';
+import { stampSchemaVersion } from '../schema.mjs';
+import { withAccountingLock } from '../task-commands.mjs';
+import {
+  HEALTH_AFFECTING_FAILURE_CLASSES,
+  LONG_COOLDOWN_FAILURE_CLASSES,
+  EVIDENCE_SUMMARY_LIMIT,
+  validateRetryAfterMs
+} from './failure-evidence.mjs';
+
+// 03H1 intentionally reuses withAccountingLock, which is an in-memory
+// promise mutex in one Node process. It protects concurrent calls made by the
+// same process only. It does not provide cross-process mutual exclusion, does
+// not prevent lost updates between separate CLI invocations, and is not an
+// exactly-once guarantee between processes.
+
+export const PROVIDER_HEALTH_FILE = 'PROVIDER_HEALTH.json';
+export const HEALTH_EVENTS_FILE = 'HEALTH_EVENTS.jsonl';
+export const FAILURE_THRESHOLD = 3;
+export const CIRCUIT_STATES = new Set(['CLOSED', 'OPEN', 'HALF_OPEN']);
+export const SHORT_COOLDOWN_INITIAL_MS = 60000;
+export const LONG_COOLDOWN_INITIAL_MS = 1800000;
+export const SHORT_COOLDOWN_MAX_MS = 3600000;
+export const LONG_COOLDOWN_MAX_MS = 14400000;
+
+const VALID_EVENT_TYPES = new Set([
+  'health_observation_recorded',
+  'circuit_opened',
+  'circuit_half_opened',
+  'circuit_closed',
+  'health_snapshot_rebuilt',
+  'failure_classified',
+  'fallback_recommended'
+]);
+
+const inProcessHalfOpenProbes = new Map();
+
+function nowIso(timestamp) {
+  if (timestamp instanceof Date) return timestamp.toISOString();
+  if (typeof timestamp === 'string' && !Number.isNaN(Date.parse(timestamp))) return new Date(timestamp).toISOString();
+  return new Date().toISOString();
+}
+
+function nowMs(timestamp) {
+  return Date.parse(nowIso(timestamp));
+}
+
+function healthPath(projectRoot = process.cwd()) {
+  return path.join(projectRoot, CONFIG.maestroPath, PROVIDER_HEALTH_FILE);
+}
+
+function eventsPath(projectRoot = process.cwd()) {
+  return path.join(projectRoot, CONFIG.maestroPath, HEALTH_EVENTS_FILE);
+}
+
+function emptySnapshot(generatedAt = null) {
+  return {
+    schemaVersion: 1,
+    generatedAt,
+    lastAppliedEventSequence: 0,
+    targets: {}
+  };
+}
+
+function sanitizeString(value) {
+  return redactCredentials(value == null ? '' : String(value));
+}
+
+function normalizeMetadata(metadata = {}) {
+  const safe = redactObject({
+    provider: metadata && metadata.provider != null ? String(metadata.provider) : null,
+    model: metadata && metadata.model != null ? String(metadata.model) : null
+  });
+  return {
+    provider: safe.provider || null,
+    model: safe.model || null
+  };
+}
+
+function normalizeEvidenceSummary(value) {
+  const redacted = redactCredentials(value == null ? '' : String(value));
+  return redacted.length > EVIDENCE_SUMMARY_LIMIT ? redacted.slice(0, EVIDENCE_SUMMARY_LIMIT) : redacted;
+}
+
+function makeEventId(sequence) {
+  return 'health-' + process.pid + '-' + Date.now().toString(36) + '-' + sequence + '-' + Math.random().toString(36).slice(2, 8);
+}
+
+export function deriveHealthTargetId(engineRecord = {}) {
+  if (!engineRecord || typeof engineRecord.id !== 'string' || engineRecord.id.trim() === '') {
+    throw new TypeError('engineRecord.id is required to derive healthTargetId');
+  }
+  return engineRecord.id;
+}
+
+function defaultEntry(healthTargetId, timestamp = null, metadata = {}) {
+  const iso = timestamp ? nowIso(timestamp) : null;
+  return {
+    healthTargetId: sanitizeString(healthTargetId),
+    state: 'CLOSED',
+    consecutiveFailures: 0,
+    consecutiveSuccesses: 0,
+    lastFailureAt: null,
+    lastSuccessAt: null,
+    lastFailureClass: null,
+    openedAt: null,
+    cooldownUntil: null,
+    halfOpenProbeAt: null,
+    consecutiveOpenCycles: 0,
+    observedRetryAfterMs: null,
+    metadata: normalizeMetadata(metadata),
+    updatedAt: iso
+  };
+}
+
+function normalizeEntry(entry, healthTargetId) {
+  const base = defaultEntry(healthTargetId);
+  if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return base;
+  const state = CIRCUIT_STATES.has(entry.state) ? entry.state : 'CLOSED';
+  const normalized = {
+    ...base,
+    ...entry,
+    healthTargetId: sanitizeString(entry.healthTargetId || healthTargetId),
+    state,
+    consecutiveFailures: Number.isInteger(entry.consecutiveFailures) && entry.consecutiveFailures >= 0 ? entry.consecutiveFailures : 0,
+    consecutiveSuccesses: Number.isInteger(entry.consecutiveSuccesses) && entry.consecutiveSuccesses >= 0 ? entry.consecutiveSuccesses : 0,
+    lastFailureAt: typeof entry.lastFailureAt === 'string' ? entry.lastFailureAt : null,
+    lastSuccessAt: typeof entry.lastSuccessAt === 'string' ? entry.lastSuccessAt : null,
+    lastFailureClass: typeof entry.lastFailureClass === 'string' ? entry.lastFailureClass : null,
+    openedAt: typeof entry.openedAt === 'string' ? entry.openedAt : null,
+    cooldownUntil: typeof entry.cooldownUntil === 'string' ? entry.cooldownUntil : null,
+    halfOpenProbeAt: typeof entry.halfOpenProbeAt === 'string' ? entry.halfOpenProbeAt : null,
+    consecutiveOpenCycles: Number.isInteger(entry.consecutiveOpenCycles) && entry.consecutiveOpenCycles >= 0 ? entry.consecutiveOpenCycles : 0,
+    observedRetryAfterMs: typeof entry.observedRetryAfterMs === 'number' && Number.isFinite(entry.observedRetryAfterMs) ? entry.observedRetryAfterMs : null,
+    metadata: normalizeMetadata(entry.metadata || {}),
+    updatedAt: typeof entry.updatedAt === 'string' ? entry.updatedAt : null
+  };
+  return redactObject(normalized);
+}
+
+function normalizeSnapshot(value, generatedAt = null) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return emptySnapshot(generatedAt);
+  const snapshot = {
+    schemaVersion: 1,
+    generatedAt: typeof value.generatedAt === 'string' ? value.generatedAt : generatedAt,
+    lastAppliedEventSequence: Number.isInteger(value.lastAppliedEventSequence) && value.lastAppliedEventSequence >= 0 ? value.lastAppliedEventSequence : 0,
+    targets: {}
+  };
+  const targets = value.targets && typeof value.targets === 'object' && !Array.isArray(value.targets) ? value.targets : {};
+  for (const [targetId, entry] of Object.entries(targets)) {
+    const safeId = sanitizeString(targetId);
+    snapshot.targets[safeId] = normalizeEntry(entry, safeId);
+  }
+  return snapshot;
+}
+
+export function validateProviderHealthSnapshot(value) {
+  const errors = [];
+  if (!value || typeof value !== 'object' || Array.isArray(value)) errors.push('snapshot must be an object');
+  if (value && value.schemaVersion !== 1) errors.push('schemaVersion must be 1');
+  if (value && (!Number.isInteger(value.lastAppliedEventSequence) || value.lastAppliedEventSequence < 0)) errors.push('lastAppliedEventSequence must be a non-negative integer');
+  if (value && (!value.targets || typeof value.targets !== 'object' || Array.isArray(value.targets))) errors.push('targets must be an object');
+  return { ok: errors.length === 0, errors };
+}
+
+export function validateHealthEvent(value) {
+  const errors = [];
+  if (!value || typeof value !== 'object' || Array.isArray(value)) errors.push('event must be an object');
+  if (value && value.schemaVersion !== 1) errors.push('schemaVersion must be 1');
+  if (value && !VALID_EVENT_TYPES.has(value.eventType)) errors.push('eventType is invalid');
+  if (value && (!Number.isInteger(value.sequence) || value.sequence < 1)) errors.push('sequence must be a positive integer');
+  if (value && typeof value.healthTargetId !== 'string') errors.push('healthTargetId must be a string');
+  if (value && value.entry !== undefined && (!value.entry || typeof value.entry !== 'object' || Array.isArray(value.entry))) errors.push('entry must be an object when present');
+  return { ok: errors.length === 0, errors };
+}
+
+async function readHealthFileQueryOnly(filePath, fallback) {
+  try {
+    const content = await fs.readFile(filePath, 'utf-8');
+    return { value: JSON.parse(stripBom(content)), degraded: false, degradedReason: null };
+  } catch (error) {
+    if (error.code === 'ENOENT') {
+      return { value: fallback, degraded: false, degradedReason: null };
+    }
+    if (error instanceof SyntaxError) {
+      return { value: fallback, degraded: true, degradedReason: 'corrupted-json' };
+    }
+    return { value: fallback, degraded: true, degradedReason: 'io-error', errorMessage: error.message };
+  }
+}
+
+async function readSnapshotForUpdate(filePath, timestamp) {
+  try {
+    return normalizeSnapshot(await readJsonSafe(filePath, emptySnapshot(timestamp)), timestamp);
+  } catch (error) {
+    if (error instanceof SyntaxError) return emptySnapshot(timestamp);
+    throw error;
+  }
+}
+
+async function readEventsForUpdate(filePath) {
+  try {
+    const content = await fs.readFile(filePath, 'utf-8');
+    return content.split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line));
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    if (error instanceof SyntaxError) return [];
+    throw error;
+  }
+}
+
+async function appendEvents(filePath, events) {
+  if (!events.length) return;
+  const content = events.map(event => JSON.stringify(redactObject(event))).join('\n') + '\n';
+  await fs.appendFile(filePath, content, { encoding: 'utf8' });
+}
+
+async function writeSnapshotAtomic(filePath, snapshot) {
+  const tempPath = filePath + '.tmp-' + process.pid + '-' + Date.now();
+  await fs.writeFile(tempPath, JSON.stringify(redactObject(snapshot), null, 2) + '\n', { encoding: 'utf8' });
+  await fs.rename(tempPath, filePath);
+}
+
+function cooldownFamily(failureClass) {
+  return LONG_COOLDOWN_FAILURE_CLASSES.has(failureClass) ? 'long' : 'short';
+}
+
+function cooldownBase(failureClass) {
+  return cooldownFamily(failureClass) === 'long' ? LONG_COOLDOWN_INITIAL_MS : SHORT_COOLDOWN_INITIAL_MS;
+}
+
+function cooldownMax(failureClass) {
+  return cooldownFamily(failureClass) === 'long' ? LONG_COOLDOWN_MAX_MS : SHORT_COOLDOWN_MAX_MS;
+}
+
+function computeCooldownMs(failureClass, openCycles) {
+  const cycles = Math.max(1, openCycles || 1);
+  return Math.min(cooldownBase(failureClass) * (2 ** (cycles - 1)), cooldownMax(failureClass));
+}
+
+function openEntry(entry, { failureClass, timestamp, previousState }) {
+  const cycles = previousState === 'HALF_OPEN' || entry.state === 'OPEN' || entry.consecutiveOpenCycles > 0
+    ? entry.consecutiveOpenCycles + 1
+    : 1;
+  const cooldownMs = computeCooldownMs(failureClass, cycles);
+  return {
+    ...entry,
+    state: 'OPEN',
+    consecutiveFailures: Math.max(entry.consecutiveFailures, FAILURE_THRESHOLD),
+    lastFailureAt: timestamp,
+    lastFailureClass: failureClass,
+    openedAt: timestamp,
+    cooldownUntil: new Date(Date.parse(timestamp) + cooldownMs).toISOString(),
+    halfOpenProbeAt: null,
+    consecutiveOpenCycles: cycles,
+    updatedAt: timestamp
+  };
+}
+
+function closeEntry(entry, timestamp, metadata = {}) {
+  return {
+    ...entry,
+    state: 'CLOSED',
+    consecutiveFailures: 0,
+    consecutiveSuccesses: entry.consecutiveSuccesses + 1,
+    lastSuccessAt: timestamp,
+    openedAt: null,
+    cooldownUntil: null,
+    halfOpenProbeAt: null,
+    consecutiveOpenCycles: 0,
+    metadata: normalizeMetadata(metadata || entry.metadata || {}),
+    updatedAt: timestamp
+  };
+}
+
+function lazyState(entry, timestamp = null, { reserveProbe = false } = {}) {
+  const observed = normalizeEntry(entry, entry.healthTargetId);
+  if (observed.state !== 'OPEN' || !observed.cooldownUntil) {
+    return { entry: observed, lazyTransitioned: false, halfOpenProbeAvailable: observed.state === 'HALF_OPEN' && !inProcessHalfOpenProbes.has(observed.healthTargetId) };
+  }
+  const referenceMs = timestamp ? nowMs(timestamp) : Date.now();
+  if (Date.parse(observed.cooldownUntil) > referenceMs) {
+    return { entry: observed, lazyTransitioned: false, halfOpenProbeAvailable: false };
+  }
+  const probeAvailable = !inProcessHalfOpenProbes.has(observed.healthTargetId);
+  const half = {
+    ...observed,
+    state: 'HALF_OPEN',
+    halfOpenProbeAt: probeAvailable && reserveProbe ? new Date(referenceMs).toISOString() : observed.halfOpenProbeAt,
+    updatedAt: new Date(referenceMs).toISOString()
+  };
+  if (reserveProbe && probeAvailable) {
+    inProcessHalfOpenProbes.set(observed.healthTargetId, half.halfOpenProbeAt);
+  }
+  return { entry: half, lazyTransitioned: true, halfOpenProbeAvailable: probeAvailable };
+}
+
+function eventFrom({ sequence, eventType, timestamp, healthTargetId, taskId, runId, orchestrationId, failureClass = null, evidenceSource = null, evidenceSummary = '', previousState = null, nextState = null, cooldownUntil = null, recommendation = null, reason = '', metadata = {}, entry = null, observedRetryAfterMs = null }) {
+  return stampSchemaVersion({
+    sequence,
+    eventId: makeEventId(sequence),
+    eventType,
+    timestamp,
+    healthTargetId: sanitizeString(healthTargetId),
+    taskId: taskId == null ? null : sanitizeString(taskId),
+    runId: runId == null ? null : sanitizeString(runId),
+    orchestrationId: orchestrationId == null ? null : sanitizeString(orchestrationId),
+    failureClass,
+    evidenceSource,
+    evidenceSummary: normalizeEvidenceSummary(evidenceSummary),
+    previousState,
+    nextState,
+    cooldownUntil,
+    recommendation,
+    reason: normalizeEvidenceSummary(reason),
+    metadata: normalizeMetadata(metadata),
+    observedRetryAfterMs,
+    entry: entry ? normalizeEntry(entry, healthTargetId) : null
+  });
+}
+
+function applyEvent(snapshot, event) {
+  if (!validateHealthEvent(event).ok) return snapshot;
+  if (event.sequence <= snapshot.lastAppliedEventSequence) return snapshot;
+  if (event.entry) {
+    const targetId = sanitizeString(event.healthTargetId);
+    snapshot.targets[targetId] = normalizeEntry(event.entry, targetId);
+  }
+  snapshot.lastAppliedEventSequence = event.sequence;
+  snapshot.generatedAt = event.timestamp || snapshot.generatedAt;
+  return snapshot;
+}
+
+export function rebuildSnapshotFromEvents(events = [], options = {}) {
+  const snapshot = normalizeSnapshot(options.baseSnapshot || emptySnapshot(options.generatedAt || null), options.generatedAt || null);
+  const seen = new Set();
+  const ordered = [...events]
+    .filter(event => event && Number.isInteger(event.sequence))
+    .sort((a, b) => a.sequence - b.sequence);
+  for (const event of ordered) {
+    if (seen.has(event.sequence)) continue;
+    seen.add(event.sequence);
+    applyEvent(snapshot, event);
+  }
+  return normalizeSnapshot(snapshot, snapshot.generatedAt);
+}
+
+function catchUpSnapshot(snapshot, events) {
+  const pending = events.filter(event => event && Number.isInteger(event.sequence) && event.sequence > snapshot.lastAppliedEventSequence);
+  if (!pending.length) return snapshot;
+  return rebuildSnapshotFromEvents(pending, { baseSnapshot: snapshot });
+}
+
+function nextSequence(snapshot) {
+  return (snapshot.lastAppliedEventSequence || 0) + 1;
+}
+
+async function updateHealth({ healthTargetId, metadata = {}, timestamp, runId = null, taskId = null, orchestrationId = null, kind, failureClass = null, evidenceSummary = '', evidenceSource = null, retryAfterMs = undefined, reason = '' }) {
+  const iso = nowIso(timestamp);
+  const targetId = sanitizeString(healthTargetId);
+  if (!targetId) throw new TypeError('healthTargetId is required');
+  return withAccountingLock(async () => {
+    await fs.mkdir(path.join(process.cwd(), CONFIG.maestroPath), { recursive: true });
+    const snapshotFile = healthPath();
+    const eventFile = eventsPath();
+    let snapshot = await readSnapshotForUpdate(snapshotFile, iso);
+    snapshot = catchUpSnapshot(snapshot, await readEventsForUpdate(eventFile));
+    const beforePersisted = snapshot.targets[targetId] ? normalizeEntry(snapshot.targets[targetId], targetId) : defaultEntry(targetId, iso, metadata);
+    const lazy = lazyState(beforePersisted, iso);
+    let previous = lazy.entry;
+    const events = [];
+    if (lazy.lazyTransitioned) {
+      previous = { ...previous, halfOpenProbeAt: iso, updatedAt: iso };
+      const sequence = nextSequence(snapshot);
+      const halfEvent = eventFrom({ sequence, eventType: 'circuit_half_opened', timestamp: iso, healthTargetId: targetId, taskId, runId, orchestrationId, previousState: 'OPEN', nextState: 'HALF_OPEN', cooldownUntil: previous.cooldownUntil, reason: 'cooldown elapsed; next real update observes HALF_OPEN', metadata, entry: previous });
+      events.push(halfEvent);
+      applyEvent(snapshot, halfEvent);
+    }
+
+    const previousState = previous.state;
+    let next = { ...previous, metadata: normalizeMetadata(metadata || previous.metadata || {}), updatedAt: iso };
+    let eventType = 'health_observation_recorded';
+    let transitioned = false;
+    let resultReason = reason || 'health observation recorded';
+    const retryAfter = validateRetryAfterMs(retryAfterMs);
+    const observedRetryAfterMs = retryAfter.valid ? retryAfter.value : null;
+    next.observedRetryAfterMs = observedRetryAfterMs;
+
+    if (kind === 'success') {
+      if (previous.state === 'HALF_OPEN') {
+        next = closeEntry(previous, iso, metadata);
+        next.observedRetryAfterMs = observedRetryAfterMs;
+        eventType = 'circuit_closed';
+        transitioned = true;
+        resultReason = 'success in HALF_OPEN closed the circuit';
+        inProcessHalfOpenProbes.delete(targetId);
+      } else {
+        next = closeEntry(previous, iso, metadata);
+        next.consecutiveOpenCycles = previous.state === 'CLOSED' ? previous.consecutiveOpenCycles : 0;
+        next.openedAt = previous.state === 'CLOSED' ? previous.openedAt : null;
+        next.cooldownUntil = previous.state === 'CLOSED' ? previous.cooldownUntil : null;
+        next.state = 'CLOSED';
+        next.observedRetryAfterMs = observedRetryAfterMs;
+      }
+    } else if (kind === 'reset') {
+      next = closeEntry(defaultEntry(targetId, iso, metadata), iso, metadata);
+      next.consecutiveSuccesses = 0;
+      eventType = 'health_snapshot_rebuilt';
+      transitioned = previous.state !== 'CLOSED';
+      resultReason = reason;
+      inProcessHalfOpenProbes.delete(targetId);
+    } else {
+      const affectsHealth = HEALTH_AFFECTING_FAILURE_CLASSES.has(failureClass);
+      next.lastFailureAt = iso;
+      next.lastFailureClass = failureClass;
+      if (affectsHealth) {
+        if (previous.state === 'HALF_OPEN') {
+          next = openEntry(next, { failureClass, timestamp: iso, previousState: 'HALF_OPEN' });
+          eventType = 'circuit_opened';
+          transitioned = true;
+          resultReason = 'failure in HALF_OPEN reopened the circuit';
+          inProcessHalfOpenProbes.delete(targetId);
+        } else if (previous.state === 'CLOSED') {
+          next.consecutiveFailures = previous.consecutiveFailures + 1;
+          next.consecutiveSuccesses = 0;
+          if (next.consecutiveFailures >= FAILURE_THRESHOLD) {
+            next = openEntry(next, { failureClass, timestamp: iso, previousState: 'CLOSED' });
+            eventType = 'circuit_opened';
+            transitioned = true;
+            resultReason = 'failure threshold reached';
+          }
+        } else if (previous.state === 'OPEN') {
+          next = openEntry(next, { failureClass, timestamp: iso, previousState: 'OPEN' });
+          eventType = 'circuit_opened';
+          transitioned = true;
+          resultReason = 'failure observed while OPEN; circuit remains open';
+        }
+      }
+      next.observedRetryAfterMs = observedRetryAfterMs;
+    }
+
+    const sequence = nextSequence(snapshot);
+    const event = eventFrom({
+      sequence,
+      eventType,
+      timestamp: iso,
+      healthTargetId: targetId,
+      taskId,
+      runId,
+      orchestrationId,
+      failureClass,
+      evidenceSource,
+      evidenceSummary,
+      previousState,
+      nextState: next.state,
+      cooldownUntil: next.cooldownUntil,
+      reason: resultReason,
+      metadata,
+      entry: next,
+      observedRetryAfterMs
+    });
+    events.push(event);
+    applyEvent(snapshot, event);
+    snapshot.generatedAt = iso;
+    snapshot = normalizeSnapshot(snapshot, iso);
+    await appendEvents(eventFile, events);
+    await writeSnapshotAtomic(snapshotFile, snapshot);
+    return {
+      previousState,
+      nextState: next.state,
+      transitioned,
+      cooldownUntil: next.cooldownUntil,
+      eventId: event.eventId,
+      sequence: event.sequence,
+      observedRetryAfterMs
+    };
+  });
+}
+
+export async function recordHealthSuccess(input = {}) {
+  return updateHealth({ ...input, kind: 'success' });
+}
+
+export async function recordHealthFailure(input = {}) {
+  return updateHealth({ ...input, kind: 'failure' });
+}
+
+export async function resetHealthManually(input = {}) {
+  if (!input.reason || typeof input.reason !== 'string' || input.reason.trim() === '') {
+    throw new TypeError('reason is required for manual health reset');
+  }
+  return updateHealth({ ...input, kind: 'reset' });
+}
+
+function queryEntry(snapshot, healthTargetId, timestamp = null, { reserveProbe = false } = {}) {
+  const targetId = sanitizeString(healthTargetId);
+  const persisted = snapshot.targets[targetId] ? normalizeEntry(snapshot.targets[targetId], targetId) : defaultEntry(targetId, timestamp, {});
+  const lazy = lazyState(persisted, timestamp, { reserveProbe });
+  return {
+    ...lazy.entry,
+    halfOpenProbeAvailable: lazy.halfOpenProbeAvailable,
+    lazyTransitioned: lazy.lazyTransitioned
+  };
+}
+
+export async function getHealthState(healthTargetId, options = {}) {
+  return withAccountingLock(async () => {
+    const result = await readHealthFileQueryOnly(healthPath(), emptySnapshot(null));
+    const snapshot = normalizeSnapshot(result.value, null);
+    const entry = queryEntry(snapshot, healthTargetId, options.timestamp || null, { reserveProbe: options.reserveProbe === true });
+    return {
+      ...entry,
+      diagnostics: {
+        degraded: result.degraded,
+        degradedReason: result.degradedReason || null,
+        consistency: 'eventually_consistent_same_process_lock_only'
+      }
+    };
+  });
+}
+
+export async function getFullHealthSnapshot(options = {}) {
+  return withAccountingLock(async () => {
+    const result = await readHealthFileQueryOnly(healthPath(), emptySnapshot(null));
+    const snapshot = normalizeSnapshot(result.value, null);
+    const targets = {};
+    for (const [targetId, entry] of Object.entries(snapshot.targets)) {
+      targets[targetId] = queryEntry(snapshot, targetId, options.timestamp || null, { reserveProbe: false });
+    }
+    return {
+      ...snapshot,
+      targets,
+      diagnostics: {
+        degraded: result.degraded,
+        degradedReason: result.degradedReason || null,
+        consistency: 'eventually_consistent_same_process_lock_only',
+        crossProcessLocking: false,
+        queryOnly: true
+      }
+    };
+  });
+}
+
+export function computeNextAttemptTime(healthEntry) {
+  if (!healthEntry || healthEntry.state === 'CLOSED') return null;
+  if (healthEntry.state === 'HALF_OPEN') return null;
+  if (!healthEntry.cooldownUntil) return null;
+  return Date.parse(healthEntry.cooldownUntil) > Date.now() ? healthEntry.cooldownUntil : null;
+}
+
+export function computeNextAttemptTimeAt(healthEntry, timestamp) {
+  if (!healthEntry || healthEntry.state === 'CLOSED') return null;
+  if (healthEntry.state === 'HALF_OPEN') return null;
+  if (!healthEntry.cooldownUntil) return null;
+  return Date.parse(healthEntry.cooldownUntil) > nowMs(timestamp) ? healthEntry.cooldownUntil : null;
+}
+
+export const providerHealthPaths = {
+  healthPath,
+  eventsPath
+};