@botbotgo/agent-harness 0.0.75 → 0.0.77

package/dist/runtime/agent-runtime-adapter.js

@@ -64,6 +64,9 @@ function computeRemainingTimeoutMs(deadlineAt, fallbackTimeoutMs) {
     }
     return fallbackTimeoutMs ? Math.min(fallbackTimeoutMs, remaining) : remaining;
 }
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
 function isPlaceholderApiKey(value) {
     return typeof value === "string" && value.trim().toLowerCase() === "dummy";
 }
@@ -485,6 +488,57 @@ export class AgentRuntimeAdapter {
         }
         return 15_000;
     }
+    resolveProviderRetryPolicy(binding) {
+        const resilience = typeof binding.harnessRuntime.resilience === "object" && binding.harnessRuntime.resilience
+            ? binding.harnessRuntime.resilience
+            : {};
+        const providerRetries = typeof resilience.providerRetries === "object" && resilience.providerRetries
+            ? resilience.providerRetries
+            : {};
+        const maxAttempts = typeof providerRetries.maxAttempts === "number" &&
+            Number.isFinite(providerRetries.maxAttempts) &&
+            providerRetries.maxAttempts > 0
+            ? Math.floor(providerRetries.maxAttempts)
+            : 2;
+        const backoffMs = typeof providerRetries.backoffMs === "number" &&
+            Number.isFinite(providerRetries.backoffMs) &&
+            providerRetries.backoffMs >= 0
+            ? Math.floor(providerRetries.backoffMs)
+            : 1_000;
+        const retryableMessages = Array.isArray(providerRetries.retryableMessages)
+            ? providerRetries.retryableMessages.filter((value) => typeof value === "string" && value.trim().length > 0)
+            : [];
+        return {
+            maxAttempts,
+            backoffMs,
+            retryableMessages,
+        };
+    }
+    isRetryableProviderError(binding, error) {
+        const message = error instanceof Error ? error.message : String(error);
+        const normalized = message.toLowerCase();
+        const { retryableMessages } = this.resolveProviderRetryPolicy(binding);
+        return retryableMessages.some((candidate) => normalized.includes(candidate.toLowerCase()));
+    }
+    async invokeWithProviderRetry(binding, operation) {
+        const retryPolicy = this.resolveProviderRetryPolicy(binding);
+        let lastError;
+        for (let attempt = 1; attempt <= retryPolicy.maxAttempts; attempt += 1) {
+            try {
+                return await operation();
+            }
+            catch (error) {
+                lastError = error;
+                if (attempt >= retryPolicy.maxAttempts || !this.isRetryableProviderError(binding, error)) {
+                    throw error;
+                }
+                if (retryPolicy.backoffMs > 0) {
+                    await sleep(retryPolicy.backoffMs);
+                }
+            }
+        }
+        throw lastError instanceof Error ? lastError : new Error(String(lastError));
+    }
     async withTimeout(producer, timeoutMs, operation, stage = operation.includes("stream") ? "stream" : "invoke") {
         if (!timeoutMs) {
             return Promise.resolve(producer());
@@ -1204,8 +1258,10 @@ export class AgentRuntimeAdapter {
             : new Command({ resume: resumePayload });
         let result;
         const callRuntime = async (activeBinding, activeRequest) => {
-            const runnable = await this.create(activeBinding);
-            return (await this.withTimeout(() => runnable.invoke(activeRequest, { configurable: { thread_id: threadId }, ...(options.context ? { context: options.context } : {}) }), this.resolveBindingTimeout(activeBinding), "agent invoke", "invoke"));
+            return this.invokeWithProviderRetry(activeBinding, async () => {
+                const runnable = await this.create(activeBinding);
+                return (await this.withTimeout(() => runnable.invoke(activeRequest, { configurable: { thread_id: threadId }, ...(options.context ? { context: options.context } : {}) }), this.resolveBindingTimeout(activeBinding), "agent invoke", "invoke"));
+            });
         };
         const callRuntimeWithToolParseRecovery = async (activeRequest) => {
             try {

package/dist/runtime/checkpoint-maintenance.d.ts

@@ -18,18 +18,27 @@ type CheckpointMaintenanceTarget = {
     agentId: string;
     dbPath: string;
 };
+export type MaintenanceLoopStatus = {
+    lastStartedAt?: string;
+    lastCompletedAt?: string;
+    lastFailedAt?: string;
+    consecutiveFailures: number;
+    lastError?: string;
+};
 export declare function readCheckpointMaintenanceConfig(workspace: WorkspaceBundle): CheckpointMaintenanceConfig | null;
 export declare function discoverCheckpointMaintenanceTargets(workspace: WorkspaceBundle): CheckpointMaintenanceTarget[];
-export declare function maintainSqliteCheckpoints(dbPath: string, config: CheckpointMaintenanceConfig, nowMs?: number): {
+export declare function maintainSqliteCheckpoints(dbPath: string, config: CheckpointMaintenanceConfig, nowMs?: number): Promise<{
     deletedCount: number;
-};
+}>;
 export declare class CheckpointMaintenanceLoop {
     private readonly targets;
     private readonly config;
     private timer;
     private running;
+    private status;
     constructor(targets: CheckpointMaintenanceTarget[], config: CheckpointMaintenanceConfig);
     runOnce(nowMs?: number): Promise<void>;
+    getStatus(): MaintenanceLoopStatus;
     start(): Promise<void>;
     stop(): Promise<void>;
 }

package/dist/runtime/checkpoint-maintenance.js

@@ -1,5 +1,7 @@
 import path from "node:path";
 import { SqliteSaver } from "@langchain/langgraph-checkpoint-sqlite";
+import { listProtectedCheckpointThreadIds } from "../persistence/sqlite-store.js";
+import { fileExists } from "../utils/fs.js";
 import { getRuntimeDefaults } from "../workspace/support/workspace-ref-utils.js";
 import { ManagedSqliteSaver } from "./sqlite-maintained-checkpoint-saver.js";
 function asObject(value) {
@@ -135,11 +137,18 @@ function totalCheckpointBytes(db) {
     return Number(checkpointsBytes.total ?? 0) + Number(writesBytes.total ?? 0);
 }
 export function maintainSqliteCheckpoints(dbPath, config, nowMs = Date.now()) {
+    return maintainSqliteCheckpointsInternal(dbPath, config, nowMs);
+}
+async function maintainSqliteCheckpointsInternal(dbPath, config, nowMs) {
+    if (!(await fileExists(dbPath))) {
+        return { deletedCount: 0 };
+    }
     const saver = new ManagedSqliteSaver(SqliteSaver.fromConnString(dbPath).db);
     const db = saver.db;
     try {
         saver.prepareMaintenance();
         backfillCheckpointMetadata(db, nowMs);
+        const protectedThreadIds = await listProtectedCheckpointThreadIds(path.join(path.dirname(dbPath), "runtime.sqlite"));
         let deletedCount = 0;
         if (config.policies.maxAgeSeconds !== undefined) {
             const cutoffMs = nowMs - config.policies.maxAgeSeconds * 1000;
@@ -154,13 +163,13 @@ export function maintainSqliteCheckpoints(dbPath, config, nowMs = Date.now()) {
            WHERE meta.created_at_ms <= ?
            ORDER BY meta.created_at_ms ASC, meta.checkpoint_id ASC
            LIMIT ?`)
-                .all(cutoffMs, config.sqlite.sweepBatchSize);
-            deletedCount += deleteCheckpointRows(db, expired);
+                .all(cutoffMs, config.sqlite.sweepBatchSize * 4);
+            deletedCount += deleteCheckpointRows(db, expired.filter((row) => !protectedThreadIds.has(row.thread_id)).slice(0, config.sqlite.sweepBatchSize));
         }
         if (config.policies.maxBytes !== undefined) {
             let currentBytes = totalCheckpointBytes(db);
             while (currentBytes > config.policies.maxBytes) {
-                const oldest = selectOldestRows(db, config.sqlite.sweepBatchSize);
+                const oldest = selectOldestRows(db, config.sqlite.sweepBatchSize * 4).filter((row) => !protectedThreadIds.has(row.thread_id));
                 if (oldest.length === 0) {
                     break;
                 }
@@ -191,15 +200,42 @@ export class CheckpointMaintenanceLoop {
     config;
     timer = null;
     running = false;
+    status = {
+        consecutiveFailures: 0,
+    };
     constructor(targets, config) {
         this.targets = targets;
         this.config = config;
     }
     async runOnce(nowMs = Date.now()) {
-        for (const target of this.targets) {
-            maintainSqliteCheckpoints(target.dbPath, this.config, nowMs);
+        this.status = {
+            ...this.status,
+            lastStartedAt: new Date(nowMs).toISOString(),
+        };
+        try {
+            for (const target of this.targets) {
+                await maintainSqliteCheckpoints(target.dbPath, this.config, nowMs);
+            }
+            this.status = {
+                ...this.status,
+                lastCompletedAt: new Date(nowMs).toISOString(),
+                consecutiveFailures: 0,
+                lastError: undefined,
+            };
+        }
+        catch (error) {
+            this.status = {
+                ...this.status,
+                lastFailedAt: new Date(nowMs).toISOString(),
+                consecutiveFailures: this.status.consecutiveFailures + 1,
+                lastError: error instanceof Error ? error.message : String(error),
+            };
+            throw error;
         }
     }
+    getStatus() {
+        return { ...this.status };
+    }
     async start() {
         if (this.running) {
             return;

package/dist/runtime/harness.d.ts

@@ -1,4 +1,4 @@
-import type { ApprovalRecord, HarnessEvent, HarnessStreamItem, MessageContent, RunRecord, RunStartOptions, RestartConversationOptions, RuntimeAdapterOptions, ResumeOptions, RunOptions, RunResult, RunSummary, ThreadSummary, ThreadRecord, WorkspaceBundle } from "../contracts/types.js";
+import type { ApprovalRecord, HarnessEvent, HarnessStreamItem, RuntimeHealthSnapshot, MessageContent, RunRecord, RunStartOptions, RestartConversationOptions, RuntimeAdapterOptions, ResumeOptions, RunOptions, RunResult, RunSummary, ThreadSummary, ThreadRecord, WorkspaceBundle } from "../contracts/types.js";
 import { type ToolMcpServerOptions } from "../mcp.js";
 import { type InventoryAgentRecord, type InventorySkillRecord } from "./inventory.js";
 import type { RequirementAssessmentOptions } from "./skill-requirements.js";
@@ -23,10 +23,13 @@ export declare class AgentHarnessRuntime {
     private readonly unregisterThreadMemorySync;
     private readonly resolvedRuntimeAdapterOptions;
     private readonly checkpointMaintenance;
+    private readonly runtimeRecordMaintenance;
+    private readonly healthMonitor;
     private readonly recoveryConfig;
     private readonly concurrencyConfig;
     private activeRunSlots;
     private readonly pendingRunSlots;
+    private runtimeEventSequence;
     private toPublicApprovalRecord;
     private normalizeInvocationEnvelope;
     private isTerminalRunState;
@@ -42,6 +45,7 @@ export declare class AgentHarnessRuntime {
     constructor(workspace: WorkspaceBundle, runtimeAdapterOptions?: RuntimeAdapterOptions);
     initialize(): Promise<void>;
     subscribe(listener: (event: HarnessEvent) => void): () => void;
+    getHealth(): Promise<RuntimeHealthSnapshot>;
     private getBinding;
     private listAgentTools;
     private resolveAgentTools;

package/dist/runtime/harness.js

@@ -1,5 +1,5 @@
 import { AUTO_AGENT_ID } from "../contracts/types.js";
-import { FilePersistence } from "../persistence/file-store.js";
+import { SqlitePersistence } from "../persistence/sqlite-store.js";
 import { createPersistentId } from "../utils/id.js";
 import { AGENT_INTERRUPT_SENTINEL_PREFIX, AgentRuntimeAdapter, RuntimeOperationTimeoutError } from "./agent-runtime-adapter.js";
 import { createResourceBackendResolver, createResourceToolResolver } from "../resource/resource.js";
@@ -13,6 +13,8 @@ import { resolveCompiledVectorStore, resolveCompiledVectorStoreRef } from "./sup
 import { ThreadMemorySync } from "./thread-memory-sync.js";
 import { FileBackedStore } from "./store.js";
 import { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, readCheckpointMaintenanceConfig, } from "./checkpoint-maintenance.js";
+import { RuntimeRecordMaintenanceLoop, discoverRuntimeRecordMaintenanceTargets, readRuntimeRecordMaintenanceConfig, } from "./runtime-record-maintenance.js";
+import { HealthMonitor } from "./health-monitor.js";
 import { extractMessageText, normalizeMessageContent } from "../utils/message-content.js";
 import { createToolMcpServerFromTools, serveToolsOverStdioFromHarness } from "../mcp.js";
 import { getBindingAdapterKind, getBindingPrimaryTools, getBindingStoreConfig, isDeepAgentBinding } from "./support/compiled-binding.js";
@@ -38,10 +40,13 @@ export class AgentHarnessRuntime {
     unregisterThreadMemorySync;
     resolvedRuntimeAdapterOptions;
     checkpointMaintenance;
+    runtimeRecordMaintenance;
+    healthMonitor;
     recoveryConfig;
     concurrencyConfig;
     activeRunSlots = 0;
     pendingRunSlots = [];
+    runtimeEventSequence = 0;
     toPublicApprovalRecord(approval) {
         const { toolCallId: _toolCallId, checkpointRef: _checkpointRef, eventRefs: _eventRefs, ...publicApproval } = approval;
         return publicApproval;
@@ -141,7 +146,7 @@ export class AgentHarnessRuntime {
         this.workspace = workspace;
         this.runtimeAdapterOptions = runtimeAdapterOptions;
         const runRoot = this.defaultRunRoot();
-        this.persistence = new FilePersistence(runRoot);
+        this.persistence = new SqlitePersistence(runRoot);
         const defaultStoreConfig = this.listHostBindings()[0]?.harnessRuntime.store;
         this.defaultStore = this.resolveStoreFromConfig(defaultStoreConfig, runRoot) ?? new FileBackedStore(`${runRoot}/store.json`);
         const runtimeMemoryStoreConfig = typeof this.listHostBindings()[0]?.harnessRuntime.runtimeMemory?.store === "object" &&
@@ -189,17 +194,37 @@ export class AgentHarnessRuntime {
         this.checkpointMaintenance = checkpointMaintenanceConfig
             ? new CheckpointMaintenanceLoop(discoverCheckpointMaintenanceTargets(workspace), checkpointMaintenanceConfig)
             : null;
+        const runtimeRecordMaintenanceConfig = readRuntimeRecordMaintenanceConfig(workspace);
+        this.runtimeRecordMaintenance = runtimeRecordMaintenanceConfig
+            ? new RuntimeRecordMaintenanceLoop(discoverRuntimeRecordMaintenanceTargets(workspace), runtimeRecordMaintenanceConfig)
+            : null;
         this.recoveryConfig = getRecoveryConfig(workspace.refs);
         this.concurrencyConfig = getConcurrencyConfig(workspace.refs);
+        this.healthMonitor = new HealthMonitor({
+            workspace,
+            persistence: this.persistence,
+            getActiveRunSlots: () => this.activeRunSlots,
+            getPendingRunSlots: () => this.pendingRunSlots.length,
+            getCheckpointMaintenanceStatus: () => this.checkpointMaintenance?.getStatus() ?? null,
+            getRuntimeRecordMaintenanceStatus: () => this.runtimeRecordMaintenance?.getStatus() ?? null,
+            publishEvent: async (payload) => {
+                this.eventBus.publish(createHarnessEvent("__runtime__", "__runtime__", ++this.runtimeEventSequence, "runtime.health.changed", payload));
+            },
+        });
     }
     async initialize() {
         await this.persistence.initialize();
         await this.checkpointMaintenance?.start();
+        await this.runtimeRecordMaintenance?.start();
+        await this.healthMonitor.start();
         await this.recoverStartupRuns();
     }
     subscribe(listener) {
         return this.eventBus.subscribe(listener);
     }
+    async getHealth() {
+        return this.healthMonitor.getSnapshot();
+    }
     getBinding(agentId) {
         return this.workspace.bindings.get(agentId);
     }
@@ -457,7 +482,16 @@ export class AgentHarnessRuntime {
     }
     async invokeWithHistory(binding, input, threadId, runId, resumePayload, options = {}) {
         const priorHistory = await this.loadPriorHistory(threadId, runId);
-        return this.runtimeAdapter.invoke(binding, input, threadId, runId, resumePayload, priorHistory, options);
+        const startedAt = Date.now();
+        try {
+            const result = await this.runtimeAdapter.invoke(binding, input, threadId, runId, resumePayload, priorHistory, options);
+            this.healthMonitor.recordLlmSuccess(Date.now() - startedAt);
+            return result;
+        }
+        catch (error) {
+            this.healthMonitor.recordLlmFailure(Date.now() - startedAt);
+            throw error;
+        }
     }
     buildPersistedRunRequest(input, invocation) {
         const envelope = invocation.invocation ?? {
@@ -830,6 +864,7 @@ export class AgentHarnessRuntime {
             return;
         }
         let emitted = false;
+        let streamActivityObserved = false;
         const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
         await this.persistence.saveRunRequest(threadId, runId, this.buildPersistedRunRequest(options.input, invocation));
         yield { type: "event", event: await this.emitRunCreated(threadId, runId, {
@@ -851,6 +886,7 @@ export class AgentHarnessRuntime {
                     files: invocation.files,
                 })) {
                     if (chunk) {
+                        streamActivityObserved = true;
                         const normalizedChunk = typeof chunk === "string"
                             ? chunk.startsWith(AGENT_INTERRUPT_SENTINEL_PREFIX)
                                 ? { kind: "interrupt", content: chunk.slice(AGENT_INTERRUPT_SENTINEL_PREFIX.length) }
@@ -964,11 +1000,30 @@ export class AgentHarnessRuntime {
                 return;
             }
             catch (error) {
-                if (emitted) {
+                if (emitted || streamActivityObserved) {
+                    const runtimeFailure = renderRuntimeFailure(error);
                     yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
                             previousState: "running",
                             error: error instanceof Error ? error.message : String(error),
                         }) };
+                    yield {
+                        type: "content",
+                        threadId,
+                        runId,
+                        agentId: selectedAgentId,
+                        content: runtimeFailure,
+                    };
+                    yield {
+                        type: "result",
+                        result: {
+                            threadId,
+                            runId,
+                            agentId: selectedAgentId,
+                            state: "failed",
+                            output: runtimeFailure,
+                            finalMessageText: runtimeFailure,
+                        },
+                    };
                     return;
                 }
                 if (error instanceof RuntimeOperationTimeoutError && error.stage === "invoke") {
@@ -1097,8 +1152,10 @@ export class AgentHarnessRuntime {
             const history = await this.persistence.listThreadMessages(threadId);
             const priorHistory = history.filter((message) => message.runId !== runId);
             const runInput = await this.loadRunInput(threadId, runId);
+            const startedAt = Date.now();
             try {
                 const actual = await this.runtimeAdapter.invoke(binding, "", threadId, runId, resumePayload, priorHistory);
+                this.healthMonitor.recordLlmSuccess(Date.now() - startedAt);
                 await this.persistence.clearRecoveryIntent(threadId, runId);
                 const finalized = await this.finalizeContinuedRun(threadId, runId, runInput, actual, {
                     previousState: "resuming",
@@ -1112,6 +1169,7 @@ export class AgentHarnessRuntime {
                 };
             }
             catch (error) {
+                this.healthMonitor.recordLlmFailure(Date.now() - startedAt);
                 throw error;
             }
         }
@@ -1182,7 +1240,9 @@ export class AgentHarnessRuntime {
         };
     }
     async close() {
+        await this.healthMonitor.stop();
         await this.checkpointMaintenance?.stop();
+        await this.runtimeRecordMaintenance?.stop();
         this.unregisterThreadMemorySync();
         await this.threadMemorySync.close();
     }
@@ -1283,8 +1343,10 @@ export class AgentHarnessRuntime {
             const history = await this.persistence.listThreadMessages(thread.threadId);
             const priorHistory = history.filter((message) => message.runId !== thread.latestRunId);
             const runInput = await this.loadRunInput(thread.threadId, thread.latestRunId);
+            const startedAt = Date.now();
             try {
                 const actual = await this.runtimeAdapter.invoke(binding, "", thread.threadId, thread.latestRunId, recoveryIntent.resumePayload, priorHistory);
+                this.healthMonitor.recordLlmSuccess(Date.now() - startedAt);
                 await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);
                 await this.finalizeContinuedRun(thread.threadId, thread.latestRunId, runInput, actual, {
                     previousState: "resuming",
@@ -1293,6 +1355,7 @@ export class AgentHarnessRuntime {
                 });
             }
             catch (error) {
+                this.healthMonitor.recordLlmFailure(Date.now() - startedAt);
                 if (recoveryIntent.attempts + 1 >= this.recoveryConfig.maxRecoveryAttempts) {
                     await this.persistence.setRunState(thread.threadId, thread.latestRunId, "failed", recoveryIntent.checkpointRef);
                     await this.persistence.clearRecoveryIntent(thread.threadId, thread.latestRunId);

package/dist/runtime/health-monitor.d.ts

@@ -0,0 +1,81 @@
+import type { RuntimeHealthSnapshot, WorkspaceBundle } from "../contracts/types.js";
+import type { RuntimePersistence } from "../persistence/types.js";
+import type { MaintenanceLoopStatus } from "./checkpoint-maintenance.js";
+type HealthMonitorConfig = {
+    enabled: boolean;
+    evaluateIntervalSeconds: number;
+    emitEvents: boolean;
+    thresholds: {
+        llmErrorRate: {
+            windowSeconds: number;
+            degradedAbove: number;
+            unhealthyAbove: number;
+        };
+        llmP95LatencyMs: {
+            windowSeconds: number;
+            degradedAbove: number;
+            unhealthyAbove: number;
+        };
+        pendingApprovals: {
+            degradedAbove: number;
+            unhealthyAbove: number;
+        };
+        stuckRunSeconds: {
+            degradedAbove: number;
+            unhealthyAbove: number;
+        };
+        checkpointBytes: {
+            degradedAbove: number;
+            unhealthyAbove: number;
+        };
+        runtimeDbBytes: {
+            degradedAbove: number;
+            unhealthyAbove: number;
+        };
+        artifactBytes: {
+            degradedAbove: number;
+            unhealthyAbove: number;
+        };
+    };
+};
+type HealthMonitorOptions = {
+    workspace: WorkspaceBundle;
+    persistence: RuntimePersistence;
+    getActiveRunSlots: () => number;
+    getPendingRunSlots: () => number;
+    getCheckpointMaintenanceStatus: () => MaintenanceLoopStatus | null;
+    getRuntimeRecordMaintenanceStatus: () => MaintenanceLoopStatus | null;
+    publishEvent?: (payload: Record<string, unknown>) => void | Promise<void>;
+};
+export declare function readHealthMonitorConfig(workspace: WorkspaceBundle): HealthMonitorConfig;
+export declare class HealthMonitor {
+    private readonly options;
+    private readonly config;
+    private readonly runRoots;
+    private readonly checkpointDbPaths;
+    private readonly llmSamples;
+    private timer;
+    private latestSnapshot;
+    private runtimeEventSequence;
+    constructor(options: HealthMonitorOptions);
+    recordLlmSuccess(latencyMs: number, nowMs?: number): void;
+    recordLlmFailure(latencyMs: number, nowMs?: number): void;
+    private recordLlmSample;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    getSnapshot(): Promise<RuntimeHealthSnapshot>;
+    evaluate(nowMs?: number): Promise<RuntimeHealthSnapshot>;
+    private llmStats;
+    private evaluateLlmCheck;
+    private llmSymptoms;
+    private evaluatePersistenceCheck;
+    private evaluateCapacityCheck;
+    private capacitySymptoms;
+    private evaluateWorkloadCheck;
+    private workloadSymptoms;
+    private countStuckRuns;
+    private sumRuntimeDbBytes;
+    private sumCheckpointDbBytes;
+    private sumArtifactBytes;
+}
+export {};