@pdpp/local-collector 0.1.0-beta.7 → 0.1.0-beta.8

package/dist/polyfill-connectors/src/collector-runner.js

@@ -4,10 +4,12 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { delimiter, join } from "node:path";
 import { createInterface } from "node:readline";
 import { fileURLToPath } from "node:url";
+import { buildAgentVersion } from "./collector-build-info.js";
 import { LocalDeviceClient, } from "./local-device-client.js";
 import { buildLocalDeviceRecordEnvelope, hashCanonicalJson, } from "./local-device-envelope.js";
 import { buildLocalDeviceOutboxId, LocalDeviceOutbox, } from "./local-device-outbox.js";
 import { assertPlacementOrThrow, COLLECTOR_RUNTIME_CAPABILITIES, } from "./runtime-capabilities.js";
+const COLLECTOR_AGENT_VERSION = buildAgentVersion();
 export const COLLECTOR_STDERR_MAX_BYTES = 256 * 1024;
 const COLLECTOR_GAP_DETAILS_MAX_CHARS = 300;
 const KEYED_SECRET_RE = /\b(authorization|bearer|token|password|passwd|cookie|secret|otp|api[_-]?key)\b\s*[:=]\s*["']?[^"',\s}]+/gi;
@@ -17,7 +19,7 @@ const SCAN_BATCH_LIMIT_DETAIL_RE = /enqueued\s+\d+\s+batches\s+>=\s+(?:run batch
 const CONNECTOR_PROTOCOL_DEBUG_DIR_ENV = "PDPP_DEBUG_CONNECTOR_PROTOCOL_DIR";
 export const DEFAULT_COLLECTOR_OUTBOX_POLICY = Object.freeze({
     drainBatchSize: 4,
-    leaseMs: 60_000,
+    leaseMs: 300_000,
     maxAttempts: 5,
     maxDrainDurationMs: 120_000,
     maxDrainIterations: 256,
@@ -25,6 +27,71 @@ export const DEFAULT_COLLECTOR_OUTBOX_POLICY = Object.freeze({
     maxQueueDepth: 10_000,
     retryBackoffMs: 30_000,
 });
+export const DEFAULT_COLLECTOR_AUTO_PRUNE_POLICY = Object.freeze({
+    enabled: true,
+    keepRecentCount: 10_000,
+});
+export function resolveCollectorAutoPrunePolicy(override, env = process.env) {
+    const policy = { ...DEFAULT_COLLECTOR_AUTO_PRUNE_POLICY, ...(override ?? {}) };
+    const enabledRaw = env.PDPP_COLLECTOR_AUTO_PRUNE;
+    if (typeof enabledRaw === "string" && enabledRaw.trim() !== "") {
+        policy.enabled = !DISABLED_ENV_VALUES.has(enabledRaw.trim().toLowerCase());
+    }
+    const keepCount = parseNonNegativeInt(env.PDPP_COLLECTOR_AUTO_PRUNE_KEEP_COUNT);
+    if (keepCount !== null) {
+        policy.keepRecentCount = keepCount;
+    }
+    return policy;
+}
+const DISABLED_ENV_VALUES = new Set(["0", "false", "off", "no"]);
+function parseNonNegativeInt(raw) {
+    if (typeof raw !== "string" || raw.trim() === "") {
+        return null;
+    }
+    const value = Number(raw.trim());
+    return Number.isSafeInteger(value) && value >= 0 ? value : null;
+}
+export function autoPruneSucceededOutbox(input) {
+    if (!input.policy.enabled) {
+        return { enabled: false, matched: 0, pruned: 0 };
+    }
+    const result = input.outbox.pruneSent({
+        dryRun: false,
+        keepCount: input.policy.keepRecentCount,
+        sourceInstanceId: input.sourceInstanceId,
+    });
+    return { enabled: true, matched: result.matched, pruned: result.pruned };
+}
+export const DEFAULT_COLLECTOR_AUTO_COMPACT_POLICY = Object.freeze({
+    enabled: true,
+    minReclaimableBytes: 512 * 1024 * 1024,
+});
+export function resolveCollectorAutoCompactPolicy(override, env = process.env) {
+    const policy = { ...DEFAULT_COLLECTOR_AUTO_COMPACT_POLICY, ...(override ?? {}) };
+    const enabledRaw = env.PDPP_COLLECTOR_AUTO_COMPACT;
+    if (typeof enabledRaw === "string" && enabledRaw.trim() !== "") {
+        policy.enabled = !DISABLED_ENV_VALUES.has(enabledRaw.trim().toLowerCase());
+    }
+    const minBytes = parseNonNegativeInt(env.PDPP_COLLECTOR_AUTO_COMPACT_MIN_RECLAIM_BYTES);
+    if (minBytes !== null) {
+        policy.minReclaimableBytes = minBytes;
+    }
+    return policy;
+}
+export function autoCompactOutboxIfBloated(input) {
+    if (!input.policy.enabled) {
+        return { compacted: false, enabled: false, reason: "disabled", reclaimedBytes: 0 };
+    }
+    const before = input.outbox.pageStats();
+    if (before.reclaimableBytes < input.policy.minReclaimableBytes) {
+        return { compacted: false, enabled: true, reason: "below_threshold", reclaimedBytes: 0 };
+    }
+    if (input.outbox.countNonSucceeded() > 0) {
+        return { compacted: false, enabled: true, reason: "lane_not_quiet", reclaimedBytes: 0 };
+    }
+    const result = input.outbox.compact();
+    return { compacted: true, enabled: true, reason: "compacted", reclaimedBytes: result.reclaimedBytes };
+}
 const PACKAGE_ROOT = fileURLToPath(new URL("..", import.meta.url));
 const REPO_ROOT = join(PACKAGE_ROOT, "..", "..");
 export async function enrollCollector(config) {
@@ -34,6 +101,15 @@ export async function enrollCollector(config) {
         ...(config.deviceLabel ? { deviceLabel: config.deviceLabel } : {}),
     });
 }
+export const COLLECTOR_COVERAGE_STATUSES = [
+    "collected",
+    "inventory_only",
+    "excluded",
+    "deferred",
+    "missing",
+    "unsupported",
+    "unaccounted",
+];
 export class CollectorStateReadError extends Error {
     constructor(message, cause) {
         super(message, { cause });
@@ -44,6 +120,8 @@ export async function runCollectorConnector(config) {
     throwIfAborted(config.abortSignal);
     const satisfiedBindings = assertPlacementOrThrow(config.connector, COLLECTOR_RUNTIME_CAPABILITIES);
     const policy = { ...DEFAULT_COLLECTOR_OUTBOX_POLICY, ...(config.outboxPolicy ?? {}) };
+    const autoPrunePolicy = resolveCollectorAutoPrunePolicy(config.autoPrune);
+    const autoCompactPolicy = resolveCollectorAutoCompactPolicy(config.autoCompact);
     const holderId = config.collectorHolderId ?? randomUUID();
     const outboxPath = config.outboxPath ?? config.queuePath;
     const outbox = new LocalDeviceOutbox({ path: outboxPath });
@@ -52,6 +130,7 @@ export async function runCollectorConnector(config) {
         deviceId: config.deviceId,
         deviceToken: config.deviceToken,
     });
+    let startingHeartbeatSent = false;
     try {
         const recoveredLeases = outbox.recoverExpiredLeases({ sourceInstanceId: config.sourceInstanceId });
         const preScanDrain = await drainCollectorOutbox({
@@ -65,6 +144,7 @@ export async function runCollectorConnector(config) {
         });
         const postDrainSummary = outbox.summary({ sourceInstanceId: config.sourceInstanceId });
         const skipResult = await maybeSkipScanForBacklog({
+            autoPrunePolicy,
             client,
             config,
             outbox,
@@ -83,6 +163,7 @@ export async function runCollectorConnector(config) {
             recordsPending: pendingOutboxWorkCount(postDrainSummary),
         });
         await client.heartbeat({
+            agent_version: COLLECTOR_AGENT_VERSION,
             connector_id: config.connector.connector_id,
             outbox: buildHeartbeatOutboxDiagnostics(postDrainSummary, {
                 backlogOpen: countOpenBacklogGaps(outbox, config.sourceInstanceId),
@@ -91,6 +172,7 @@ export async function runCollectorConnector(config) {
             source_instance_id: config.sourceInstanceId,
             status: "starting",
         });
+        startingHeartbeatSent = true;
         const streamResult = await streamConnectorIntoOutbox({
             ...(config.abortSignal ? { abortSignal: config.abortSignal } : {}),
             batchSize: config.batchSize ?? 100,
@@ -130,11 +212,20 @@ export async function runCollectorConnector(config) {
             deferRecoveredGapCleanup: streamResult.scanBudgetExceeded,
             outbox,
         });
+        const prunedSent = autoPruneSucceededOutbox({
+            outbox,
+            policy: autoPrunePolicy,
+            sourceInstanceId: config.sourceInstanceId,
+        });
+        autoCompactOutboxIfBloated({ outbox, policy: autoCompactPolicy });
         const finalSummary = outbox.summary({ sourceInstanceId: config.sourceInstanceId });
         const recordsPending = pendingOutboxWorkCount(finalSummary);
         if (!checkpointResult.statePutFailed) {
+            const finalDeadLetterError = buildHeartbeatDeadLetterError(outbox, config.sourceInstanceId);
             await client.heartbeat({
+                agent_version: COLLECTOR_AGENT_VERSION,
                 connector_id: config.connector.connector_id,
+                ...(finalDeadLetterError ? { last_error: finalDeadLetterError } : {}),
                 outbox: buildHeartbeatOutboxDiagnostics(finalSummary, {
                     backlogOpen: countOpenBacklogGaps(outbox, config.sourceInstanceId),
                 }),
@@ -144,11 +235,13 @@ export async function runCollectorConnector(config) {
             });
         }
         return {
+            completeness: summarizeCollectorCompleteness(streamResult.coverageByStore),
             done,
             enqueuedBatches: enqueueResult.enqueuedBatches,
             flushedState: checkpointResult.flushedState,
             outboxSummary: finalSummary,
             priorState,
+            prunedSent,
             recordsQueued: enqueueResult.recordsQueued,
             recoveredLeases,
             satisfiedBindings,
@@ -159,10 +252,30 @@ export async function runCollectorConnector(config) {
             streamingBufferHighWaterMark: streamResult.bufferHighWaterMark,
         };
     }
+    catch (error) {
+        if (startingHeartbeatSent) {
+            await emitCorrectiveHeartbeatFromOutbox({ client, config, outbox, policy });
+        }
+        throw error;
+    }
     finally {
         outbox.close();
     }
 }
+async function emitCorrectiveHeartbeatFromOutbox(input) {
+    const summary = input.outbox.summary({ sourceInstanceId: input.config.sourceInstanceId });
+    const deadLetterError = buildHeartbeatDeadLetterError(input.outbox, input.config.sourceInstanceId);
+    await safeHeartbeat(input.client, {
+        connector_id: input.config.connector.connector_id,
+        ...(deadLetterError ? { last_error: deadLetterError } : {}),
+        outbox: buildHeartbeatOutboxDiagnostics(summary, {
+            backlogOpen: countOpenBacklogGaps(input.outbox, input.config.sourceInstanceId),
+        }),
+        records_pending: pendingOutboxWorkCount(summary),
+        source_instance_id: input.config.sourceInstanceId,
+        status: heartbeatStatusForSummary(summary, input.policy),
+    });
+}
 async function maybeSkipScanForBacklog(input) {
     if (!hasScanBlockingOutboxWork(input.outbox, input.config.sourceInstanceId, input.policy)) {
         return null;
@@ -180,6 +293,11 @@ async function maybeSkipScanForBacklog(input) {
             sourceInstanceId: input.config.sourceInstanceId,
         });
     }
+    const prunedSent = autoPruneSucceededOutbox({
+        outbox: input.outbox,
+        policy: input.autoPrunePolicy,
+        sourceInstanceId: input.config.sourceInstanceId,
+    });
     const summaryAfterGap = input.outbox.summary({ sourceInstanceId: input.config.sourceInstanceId });
     const recordsPendingAfterGap = pendingOutboxWorkCount(summaryAfterGap);
     await safeHeartbeat(input.client, {
@@ -192,11 +310,13 @@ async function maybeSkipScanForBacklog(input) {
         status: heartbeatStatusForSummary(summaryAfterGap, input.policy),
     });
     return {
+        completeness: null,
         done: null,
         enqueuedBatches: 0,
         flushedState: null,
         outboxSummary: summaryAfterGap,
         priorState: Object.freeze({}),
+        prunedSent,
         recordsQueued: 0,
         recoveredLeases: input.recoveredLeases,
         satisfiedBindings: input.satisfiedBindings,
@@ -218,6 +338,7 @@ async function readPriorStateOrBlock(input) {
     catch (error) {
         await safeHeartbeat(input.client, {
             connector_id: input.config.connector.connector_id,
+            last_error: { kind: "state_read_failed" },
             records_pending: input.recordsPending,
             source_instance_id: input.config.sourceInstanceId,
             status: "blocked",
@@ -225,6 +346,50 @@ async function readPriorStateOrBlock(input) {
         throw new CollectorStateReadError(`failed to read prior state for ${input.config.sourceInstanceId}: ${error instanceof Error ? error.message : String(error)}`, error);
     }
 }
+const COVERAGE_DIAGNOSTICS_STREAM = "coverage_diagnostics";
+function coverageEntryFromRecord(message) {
+    if (message.stream !== COVERAGE_DIAGNOSTICS_STREAM) {
+        return null;
+    }
+    const data = message.data;
+    const dataStore = isRecord(data) && typeof data.store === "string" && data.store ? data.store : null;
+    const keyStore = typeof message.key === "string" && message.key ? message.key : null;
+    const store = dataStore ?? keyStore;
+    if (!store) {
+        return null;
+    }
+    const rawStatus = isRecord(data) && typeof data.status === "string" ? data.status : null;
+    if (!rawStatus) {
+        return null;
+    }
+    const status = COLLECTOR_COVERAGE_STATUSES.includes(rawStatus)
+        ? rawStatus
+        : "unaccounted";
+    return { status, store };
+}
+export function summarizeCollectorCompleteness(coverageByStore) {
+    if (!coverageByStore || coverageByStore.size === 0) {
+        return null;
+    }
+    const countsByStatus = Object.fromEntries(COLLECTOR_COVERAGE_STATUSES.map((status) => [status, 0]));
+    const unaccountedStores = [];
+    const byStore = {};
+    for (const store of [...coverageByStore.keys()].sort()) {
+        const status = coverageByStore.get(store);
+        byStore[store] = status;
+        countsByStatus[status] += 1;
+        if (status === "unaccounted") {
+            unaccountedStores.push(store);
+        }
+    }
+    return {
+        byStore,
+        countsByStatus,
+        fullyAccounted: unaccountedStores.length === 0,
+        storeCount: coverageByStore.size,
+        unaccountedStores,
+    };
+}
 async function streamConnectorIntoOutbox(input) {
     throwIfAborted(input.abortSignal);
     const child = spawnConnector(input.config.connector, {
@@ -242,6 +407,7 @@ async function streamConnectorIntoOutbox(input) {
     let enqueuedBatches = 0;
     let done = null;
     let scanBudgetExceeded = false;
+    let coverageByStore = null;
     const flushPendingBatch = () => {
         if (pendingRecords.length === 0) {
             return;
@@ -288,8 +454,17 @@ async function streamConnectorIntoOutbox(input) {
             scanBudgetExceeded,
         });
     };
+    const recordCoverageIfPresent = (message) => {
+        const entry = coverageEntryFromRecord(message);
+        if (!entry) {
+            return;
+        }
+        coverageByStore ??= new Map();
+        coverageByStore.set(entry.store, entry.status);
+    };
     const handleMessage = (message) => {
         if (message.type === "RECORD") {
+            recordCoverageIfPresent(message);
             pendingRecords.push(message);
             if (pendingRecords.length > bufferHighWaterMark) {
                 bufferHighWaterMark = pendingRecords.length;
@@ -397,6 +572,7 @@ async function streamConnectorIntoOutbox(input) {
     return {
         bufferedState: Object.freeze(scanBudgetExceeded ? {} : { ...bufferedState }),
         bufferHighWaterMark,
+        coverageByStore,
         done: scanBudgetExceeded ? null : done,
         enqueuedBatches,
         recordsQueued,
@@ -567,7 +743,7 @@ async function recoverResolvedLocalCollectorGaps(input) {
 }
 async function safeHeartbeat(client, request) {
     try {
-        await client.heartbeat(request);
+        await client.heartbeat({ agent_version: COLLECTOR_AGENT_VERSION, ...request });
     }
     catch {
     }
@@ -715,35 +891,53 @@ function hasCheckpointPredecessorBlockingWork(outbox, checkpoint) {
 async function drainClaimedOutboxItem(input, item, result, sentByKind) {
     throwIfAborted(input.abortSignal);
     try {
-        await sendOutboxItem(input.client, item);
-        input.outbox.acknowledge({ holder: input.holderId, id: item.id, leaseEpoch: item.lease_epoch });
+        const current = input.outbox.renewLease({
+            holder: input.holderId,
+            id: item.id,
+            leaseEpoch: item.lease_epoch,
+            leaseMs: input.policy.leaseMs,
+        });
+        await sendOutboxItem(input.client, current);
+        input.outbox.acknowledge({ holder: input.holderId, id: current.id, leaseEpoch: current.lease_epoch });
         result.sent++;
-        sentByKind[item.kind] = (sentByKind[item.kind] ?? 0) + 1;
+        sentByKind[current.kind] = (sentByKind[current.kind] ?? 0) + 1;
     }
     catch (error) {
         failOutboxItem(input, item, error, result);
     }
 }
 function failOutboxItem(input, item, error, result) {
-    const message = error instanceof Error ? error.message : String(error);
-    if (error instanceof OutboxPayloadShapeError || item.attempt_count + 1 >= input.policy.maxAttempts) {
-        input.outbox.deadLetter({
+    const message = sanitizeCollectorGapDetails(error instanceof Error ? error.message : String(error));
+    try {
+        if (error instanceof OutboxPayloadShapeError || item.attempt_count + 1 >= input.policy.maxAttempts) {
+            input.outbox.deadLetter({
+                error: message,
+                holder: input.holderId,
+                id: item.id,
+                leaseEpoch: item.lease_epoch,
+            });
+            result.deadLettered++;
+            return;
+        }
+        input.outbox.failRetryable({
             error: message,
             holder: input.holderId,
             id: item.id,
             leaseEpoch: item.lease_epoch,
+            retryBackoffMs: input.policy.retryBackoffMs * (item.attempt_count + 1),
         });
-        result.deadLettered++;
-        return;
+        result.failed++;
     }
-    input.outbox.failRetryable({
-        error: message,
-        holder: input.holderId,
-        id: item.id,
-        leaseEpoch: item.lease_epoch,
-        retryBackoffMs: input.policy.retryBackoffMs * (item.attempt_count + 1),
-    });
-    result.failed++;
+    catch (transitionError) {
+        if (isLeaseNotCurrentError(transitionError)) {
+            result.failed++;
+            return;
+        }
+        throw transitionError;
+    }
+}
+function isLeaseNotCurrentError(error) {
+    return error instanceof Error && error.message.startsWith("local outbox lease not current");
 }
 class OutboxPayloadShapeError extends Error {
     constructor(message) {
@@ -937,6 +1131,34 @@ function isUnresolvedScanBudgetGap(item, policy) {
     }
     return policy.maxEnqueuedBatchesPerRun <= Number(match[1]);
 }
+export const LOCAL_COLLECTOR_LIFECYCLE_STATES = [
+    "healthy_idle",
+    "draining",
+    "retryable_backlog",
+    "dead_letter",
+    "stale_lease",
+    "coverage_missing",
+];
+export function deriveLocalCollectorLifecycleState(input) {
+    const { summary } = input;
+    if (summary.deadLetter > 0) {
+        return "dead_letter";
+    }
+    if (summary.staleLeases > 0) {
+        return "stale_lease";
+    }
+    const claimableNow = Math.max(0, summary.ready - summary.retrying);
+    if (summary.leased > 0 || claimableNow > 0) {
+        return "draining";
+    }
+    if (summary.retrying > 0) {
+        return "retryable_backlog";
+    }
+    if (input.coverageObserved === false && input.recordBatchCount > 0) {
+        return "coverage_missing";
+    }
+    return "healthy_idle";
+}
 function heartbeatStatusForSummary(summary, policy) {
     if (summary.deadLetter > 0) {
         return "blocked";
@@ -963,6 +1185,16 @@ export function buildHeartbeatOutboxDiagnostics(summary, options = {}) {
         total: summary.total,
     };
 }
+function buildHeartbeatDeadLetterError(outbox, sourceInstanceId) {
+    const summary = outbox.deadLetterErrorSummary({ sourceInstanceId });
+    if (summary.dead_letter_count === 0) {
+        return null;
+    }
+    return {
+        kind: "dead_letter_backlog",
+        top_dead_letter_classes: summary.top_classes,
+    };
+}
 function countOpenBacklogGaps(outbox, sourceInstanceId) {
     return outbox.countOpenGaps({ sourceInstanceId });
 }

package/dist/polyfill-connectors/src/connector-exit.js

@@ -0,0 +1,62 @@
+const STDOUT_DRAIN_TIMEOUT_MS = 3000;
+const RUNTIME_ACK_TIMEOUT_MS = 30 * 60 * 1000;
+export function flushAndExitAfterRuntimeAck(code, options = {}) {
+    const stdin = options.stdin ?? process.stdin;
+    const stdout = options.stdout ?? process.stdout;
+    const exit = options.exit ?? process.exit;
+    const stdoutDrainTimeoutMs = options.stdoutDrainTimeoutMs ?? STDOUT_DRAIN_TIMEOUT_MS;
+    const runtimeAckTimeoutMs = options.runtimeAckTimeoutMs ?? RUNTIME_ACK_TIMEOUT_MS;
+    let finished = false;
+    let drainTimer = null;
+    let ackTimer = null;
+    const finish = () => {
+        if (finished) {
+            return;
+        }
+        finished = true;
+        if (drainTimer) {
+            clearTimeout(drainTimer);
+        }
+        if (ackTimer) {
+            clearTimeout(ackTimer);
+        }
+        exit(code);
+    };
+    const waitForRuntimeAck = () => {
+        if (drainTimer) {
+            clearTimeout(drainTimer);
+            drainTimer = null;
+        }
+        if (stdin.readableEnded || stdin.destroyed) {
+            finish();
+            return;
+        }
+        const cleanup = () => {
+            stdin.off("close", onRuntimeAck);
+            stdin.off("end", onRuntimeAck);
+            stdin.off("error", onRuntimeAck);
+        };
+        const onRuntimeAck = () => {
+            cleanup();
+            finish();
+        };
+        stdin.once("close", onRuntimeAck);
+        stdin.once("end", onRuntimeAck);
+        stdin.once("error", onRuntimeAck);
+        ackTimer = setTimeout(() => {
+            cleanup();
+            finish();
+        }, runtimeAckTimeoutMs);
+        ackTimer.unref();
+    };
+    if (stdout.writableLength > 0) {
+        stdout.once("drain", waitForRuntimeAck);
+        drainTimer = setTimeout(() => {
+            stdout.off("drain", waitForRuntimeAck);
+            waitForRuntimeAck();
+        }, stdoutDrainTimeoutMs);
+        drainTimer.unref();
+        return;
+    }
+    waitForRuntimeAck();
+}

package/dist/polyfill-connectors/src/connector-runtime-protocol.d.ts

@@ -31,6 +31,19 @@ export interface DetailGapStartEntry {
     status: "pending";
     stream: string;
 }
+export interface DetailGapsPageRequestMessage {
+    max_bytes?: number;
+    reference_only: true;
+    request_id: string;
+    streams?: readonly string[];
+    type: "DETAIL_GAPS_PAGE_REQUEST";
+}
+export interface DetailGapsPageResponse {
+    detail_gaps: readonly DetailGapStartEntry[];
+    reference_only: true;
+    request_id: string;
+    type: "DETAIL_GAPS_PAGE_RESPONSE";
+}
 export interface InteractionResponse {
     data?: Record<string, string>;
     error?: {
@@ -70,20 +83,21 @@ export interface AssistanceCompletion {
     message?: string;
     status: AssistanceCompletionStatus;
 }
+export interface DetailGapNetworkPressure {
+    attempt?: number;
+    endpoint_route: string;
+    error_class: string;
+    max_attempts?: number;
+    method: string;
+    retry_after_ms?: number;
+    safe_headers?: Record<string, string | number>;
+    status?: number;
+}
 export interface DetailGapMessage {
     detail?: {
         class?: string;
         http_status?: number;
-        network_pressure?: {
-            attempt?: number;
-            endpoint_route: string;
-            error_class: string;
-            max_attempts?: number;
-            method: string;
-            retry_after_ms?: number;
-            safe_headers?: Record<string, string | number>;
-            status?: number;
-        };
+        network_pressure?: DetailGapNetworkPressure;
     };
     detail_locator: {
         kind: string;
@@ -93,16 +107,7 @@ export interface DetailGapMessage {
         class?: string;
         http_status?: number;
         message?: string;
-        network_pressure?: {
-            attempt?: number;
-            endpoint_route: string;
-            error_class: string;
-            max_attempts?: number;
-            method: string;
-            retry_after_ms?: number;
-            safe_headers?: Record<string, string | number>;
-            status?: number;
-        };
+        network_pressure?: DetailGapNetworkPressure;
     };
     list_cursor?: unknown;
     parent_stream?: string;
@@ -115,6 +120,8 @@ export interface DetailGapMessage {
     type: "DETAIL_GAP";
 }
 export interface DetailCoverageMessage {
+    considered?: number;
+    covered?: number;
     gap_keys?: Array<string | number>;
     hydrated_keys: Array<string | number>;
     optional_skip_keys?: Array<string | number>;
@@ -131,6 +138,18 @@ export interface DetailGapRecoveredMessage {
     stream: string;
     type: "DETAIL_GAP_RECOVERED";
 }
+export interface ProviderBudgetProgress {
+    circuit: {
+        previous_state: "closed" | "half_open" | "open";
+        reason: "provider_failure" | "provider_throttle" | "reset_timeout" | "success";
+        state: "closed" | "half_open" | "open";
+        trigger: "before_request" | "provider_failure" | "provider_throttle" | "success";
+    };
+    elapsed_ms: number;
+    object: "provider_budget_circuit_transition";
+    request_count: number;
+    retry_tokens_remaining?: number | "unbounded";
+}
 export type EmittedMessage = {
     type: "RECORD";
     stream: string;
@@ -146,6 +165,7 @@ export type EmittedMessage = {
     type: "PROGRESS";
     message: string;
     stream?: string;
+    provider_budget?: ProviderBudgetProgress;
 } | ({
     type: "ASSISTANCE";
 } & AssistanceRequest) | ({
@@ -156,7 +176,7 @@ export type EmittedMessage = {
     reason: string;
     message: string;
     diagnostics?: unknown;
-} | DetailGapMessage | DetailCoverageMessage | DetailGapRecoveredMessage | {
+} | DetailGapMessage | DetailCoverageMessage | DetailGapRecoveredMessage | DetailGapsPageRequestMessage | {
     type: "DONE";
     status: "succeeded" | "failed";
     records_emitted: number;