@botbotgo/agent-harness 0.0.101 → 0.0.103

package/dist/runtime/harness/run/run-operations.d.ts

@@ -0,0 +1,50 @@
+import type { CancelOptions, CompiledAgentBinding, InternalApprovalRecord, MessageContent, ResumeOptions, RunResult, ThreadSummary, TranscriptMessage } from "../../../contracts/types.js";
+import type { RecoveryIntent } from "../../../persistence/types.js";
+type ResumeOperationRuntime = {
+    getApprovalById: (approvalId: string) => Promise<InternalApprovalRecord | null>;
+    getSession: (threadId: string) => Promise<ThreadSummary | null>;
+    resolveApprovalRecord: (options: ResumeOptions, thread: ThreadSummary) => Promise<InternalApprovalRecord>;
+    getBinding: (agentId: string) => CompiledAgentBinding | undefined;
+    buildResumePayload: (binding: CompiledAgentBinding, approval: InternalApprovalRecord, options: ResumeOptions) => unknown;
+    getRunCancellation: (runId: string) => Promise<{
+        requested: boolean;
+        reason?: string;
+    }>;
+    finalizeCancelledRun: (threadId: string, runId: string, previousState: RunResult["state"] | null, reason?: string) => Promise<RunResult>;
+    setRunState: (threadId: string, runId: string, state: RunResult["state"], checkpointRef: string | null) => Promise<void>;
+    acquireRunSlot: (threadId?: string, runId?: string, activeState?: RunResult["state"], priority?: number) => Promise<() => Promise<void>>;
+    resolvePersistedRunPriority: (threadId: string, runId: string) => Promise<number>;
+    saveRecoveryIntent: (threadId: string, runId: string, payload: RecoveryIntent) => Promise<unknown>;
+    emit: (threadId: string, runId: string, sequence: number, eventType: string, payload: Record<string, unknown>) => Promise<unknown>;
+    resolveApproval: (threadId: string, runId: string, approvalId: string, resolution: "approved" | "edited" | "rejected" | "expired") => Promise<unknown>;
+    listThreadMessages: (threadId: string) => Promise<TranscriptMessage[]>;
+    loadRunInput: (threadId: string, runId: string) => Promise<MessageContent>;
+    invoke: (binding: CompiledAgentBinding, input: MessageContent, threadId: string, runId: string, resumePayload: unknown, priorHistory: TranscriptMessage[]) => Promise<RunResult>;
+    recordLlmSuccess: (startedAt: number) => void;
+    recordLlmFailure: (startedAt: number) => void;
+    clearRecoveryIntent: (threadId: string, runId: string) => Promise<void>;
+    finalizeContinuedRun: (binding: CompiledAgentBinding, threadId: string, runId: string, input: MessageContent, actual: RunResult, options: {
+        previousState: RunResult["state"] | null;
+        stateSequence: number;
+        approvalSequence?: number;
+    }) => Promise<RunResult>;
+};
+type CancelOperationRuntime = {
+    getRun: (runId: string) => Promise<{
+        threadId: string;
+        runId: string;
+        agentId: string;
+        state: RunResult["state"];
+    } | null>;
+    requestRunCancel: (runId: string, reason?: string) => Promise<unknown>;
+    dropPendingRunSlot: (runId: string) => boolean;
+    finalizeCancelledRun: (threadId: string, runId: string, previousState: RunResult["state"] | null, reason?: string) => Promise<RunResult>;
+    setRunStateAndEmit: (threadId: string, runId: string, sequence: number, state: RunResult["state"], options: {
+        previousState: string | null;
+        checkpointRef?: string | null;
+        error?: string;
+    }) => Promise<unknown>;
+};
+export declare function resumeRun(runtime: ResumeOperationRuntime, options: ResumeOptions): Promise<RunResult>;
+export declare function cancelRunOperation(runtime: CancelOperationRuntime, options: CancelOptions): Promise<RunResult>;
+export {};

package/dist/runtime/harness/run/run-operations.js

@@ -0,0 +1,113 @@
+import { isTerminalRunState } from "./helpers.js";
+export async function resumeRun(runtime, options) {
+    const approvalById = options.approvalId ? await runtime.getApprovalById(options.approvalId) : null;
+    const thread = options.threadId
+        ? await runtime.getSession(options.threadId)
+        : approvalById
+            ? await runtime.getSession(approvalById.threadId)
+            : null;
+    if (!thread) {
+        throw new Error("resume requires either threadId or approvalId");
+    }
+    const approval = approvalById ?? await runtime.resolveApprovalRecord(options, thread);
+    const threadId = approval.threadId;
+    const runId = approval.runId;
+    const binding = runtime.getBinding(thread.agentId);
+    if (!binding) {
+        throw new Error(`Unknown agent ${thread.agentId}`);
+    }
+    const resumePayload = runtime.buildResumePayload(binding, approval, options);
+    const cancellation = await runtime.getRunCancellation(runId);
+    if (cancellation.requested) {
+        return runtime.finalizeCancelledRun(threadId, runId, thread.status, cancellation.reason);
+    }
+    const checkpointRef = `checkpoints/${threadId}/${runId}/cp-1`;
+    await runtime.setRunState(threadId, runId, "resuming", checkpointRef);
+    const releaseRunSlot = await runtime.acquireRunSlot(threadId, runId, "resuming", await runtime.resolvePersistedRunPriority(threadId, runId));
+    try {
+        await runtime.saveRecoveryIntent(threadId, runId, {
+            kind: "approval-decision",
+            savedAt: new Date().toISOString(),
+            checkpointRef,
+            resumePayload,
+            attempts: 0,
+        });
+        await runtime.emit(threadId, runId, 5, "run.resumed", {
+            resumeKind: "cross-restart",
+            checkpointRef,
+            state: "resuming",
+            approvalId: approval.approvalId,
+            pendingActionId: approval.pendingActionId,
+        });
+        await runtime.resolveApproval(threadId, runId, approval.approvalId, options.decision === "reject" ? "rejected" : options.decision === "edit" ? "edited" : "approved");
+        await runtime.emit(threadId, runId, 6, "approval.resolved", {
+            approvalId: approval.approvalId,
+            pendingActionId: approval.pendingActionId,
+            decision: options.decision ?? "approve",
+            toolName: approval.toolName,
+        });
+        const history = await runtime.listThreadMessages(threadId);
+        const priorHistory = history.filter((message) => message.runId !== runId);
+        const runInput = await runtime.loadRunInput(threadId, runId);
+        const startedAt = Date.now();
+        try {
+            const actual = await runtime.invoke(binding, "", threadId, runId, resumePayload, priorHistory);
+            runtime.recordLlmSuccess(startedAt);
+            const cancelledAfterInvoke = await runtime.getRunCancellation(runId);
+            if (cancelledAfterInvoke.requested) {
+                return runtime.finalizeCancelledRun(threadId, runId, "resuming", cancelledAfterInvoke.reason);
+            }
+            await runtime.clearRecoveryIntent(threadId, runId);
+            const finalized = await runtime.finalizeContinuedRun(binding, threadId, runId, runInput, actual, {
+                previousState: "resuming",
+                stateSequence: 7,
+                approvalSequence: 8,
+            });
+            return {
+                ...finalized,
+                approvalId: finalized.approvalId ?? approval.approvalId,
+                pendingActionId: finalized.pendingActionId ?? approval.pendingActionId,
+            };
+        }
+        catch (error) {
+            runtime.recordLlmFailure(startedAt);
+            throw error;
+        }
+    }
+    finally {
+        await releaseRunSlot();
+    }
+}
+export async function cancelRunOperation(runtime, options) {
+    const run = await runtime.getRun(options.runId);
+    if (!run) {
+        throw new Error(`Unknown run ${options.runId}`);
+    }
+    if (isTerminalRunState(run.state)) {
+        return {
+            threadId: run.threadId,
+            runId: run.runId,
+            agentId: run.agentId,
+            state: run.state,
+            output: run.state,
+        };
+    }
+    await runtime.requestRunCancel(run.runId, options.reason);
+    if (run.state === "queued" || run.state === "waiting_for_approval" || run.state === "claimed") {
+        if (run.state === "queued") {
+            runtime.dropPendingRunSlot(run.runId);
+        }
+        return runtime.finalizeCancelledRun(run.threadId, run.runId, run.state, options.reason);
+    }
+    await runtime.setRunStateAndEmit(run.threadId, run.runId, 103, "cancelling", {
+        previousState: run.state,
+        ...(options.reason ? { error: options.reason } : {}),
+    });
+    return {
+        threadId: run.threadId,
+        runId: run.runId,
+        agentId: run.agentId,
+        state: "cancelling",
+        output: options.reason ? `cancelling: ${options.reason}` : "cancelling",
+    };
+}

package/dist/runtime/harness/run/run-slot-acquisition.d.ts

@@ -0,0 +1,64 @@
+import type { RunResult } from "../../../contracts/types.js";
+import { type PendingRunSlot } from "./run-queue.js";
+export type RunSlotConcurrencyConfig = {
+    maxConcurrentRuns?: number;
+    leaseMs: number;
+    heartbeatIntervalMs: number;
+};
+type RunClaimPersistence = {
+    claimQueuedRun(payload: {
+        threadId: string;
+        runId: string;
+        workerId: string;
+        claimedAt: string;
+        leaseExpiresAt: string;
+    }): Promise<unknown>;
+    renewRunLease(payload: {
+        runId: string;
+        workerId: string;
+        heartbeatAt: string;
+        leaseExpiresAt: string;
+    }): Promise<unknown>;
+    releaseRunClaim(runId: string): Promise<unknown>;
+    enqueueRun(payload: {
+        threadId: string;
+        runId: string;
+        priority: number;
+    }): Promise<unknown>;
+    getRun(runId: string): Promise<{
+        state: RunResult["state"];
+    } | null>;
+};
+type RunSlotRuntime = {
+    persistence: RunClaimPersistence;
+    workerId: string;
+    concurrencyConfig: RunSlotConcurrencyConfig;
+    pendingRunSlots: PendingRunSlot[];
+    getActiveRunSlots: () => number;
+    setActiveRunSlots: (count: number) => void;
+    enqueuePendingRunSlot: (entry: {
+        threadId?: string;
+        runId?: string;
+        priority: number;
+        activate: () => void | Promise<void>;
+        abort: () => void;
+    }) => Array<{
+        threadId: string;
+        runId: string;
+        priority: number;
+        queuePosition: number;
+    }>;
+    emit: (threadId: string, runId: string, sequence: number, eventType: string, payload: Record<string, unknown>) => Promise<unknown>;
+    setRunStateAndEmit: (threadId: string, runId: string, sequence: number, state: RunResult["state"], options: {
+        previousState: string | null;
+        checkpointRef?: string | null;
+        error?: string;
+    }) => Promise<unknown>;
+};
+export declare function acquireRunSlot(runtime: RunSlotRuntime, options?: {
+    threadId?: string;
+    runId?: string;
+    activeState?: RunResult["state"];
+    priority?: number;
+}): Promise<() => Promise<void>>;
+export {};

package/dist/runtime/harness/run/run-slot-acquisition.js

@@ -0,0 +1,157 @@
+import { shiftNextPendingRunSlot } from "./run-queue.js";
+export async function acquireRunSlot(runtime, options = {}) {
+    const { threadId, runId } = options;
+    const activeState = options.activeState ?? "running";
+    const priority = options.priority ?? 0;
+    let stopHeartbeat = () => undefined;
+    const beginLease = async (mode) => {
+        if (!threadId || !runId) {
+            return;
+        }
+        const claimedAt = new Date().toISOString();
+        const leaseExpiresAt = new Date(Date.now() + runtime.concurrencyConfig.leaseMs).toISOString();
+        if (mode === "queue-claim") {
+            await runtime.persistence.claimQueuedRun({
+                threadId,
+                runId,
+                workerId: runtime.workerId,
+                claimedAt,
+                leaseExpiresAt,
+            });
+        }
+        else {
+            await runtime.persistence.renewRunLease({
+                runId,
+                workerId: runtime.workerId,
+                heartbeatAt: claimedAt,
+                leaseExpiresAt,
+            });
+        }
+        if (runtime.concurrencyConfig.heartbeatIntervalMs <= 0) {
+            return;
+        }
+        const timer = setInterval(() => {
+            void runtime.persistence.renewRunLease({
+                runId,
+                workerId: runtime.workerId,
+                heartbeatAt: new Date().toISOString(),
+                leaseExpiresAt: new Date(Date.now() + runtime.concurrencyConfig.leaseMs).toISOString(),
+            });
+        }, runtime.concurrencyConfig.heartbeatIntervalMs);
+        timer.unref?.();
+        stopHeartbeat = () => {
+            clearInterval(timer);
+        };
+    };
+    const releaseLease = async () => {
+        stopHeartbeat();
+        if (runId) {
+            await runtime.persistence.releaseRunClaim(runId);
+        }
+    };
+    const releaseAndAdvance = async (shiftNext) => {
+        await releaseLease();
+        runtime.setActiveRunSlots(Math.max(0, runtime.getActiveRunSlots() - 1));
+        const next = shiftNext();
+        void next?.activate();
+    };
+    const maxConcurrentRuns = runtime.concurrencyConfig.maxConcurrentRuns;
+    if (!maxConcurrentRuns) {
+        await beginLease("direct-heartbeat");
+        return async () => {
+            await releaseLease();
+        };
+    }
+    const canActivateImmediately = runtime.getActiveRunSlots() < maxConcurrentRuns;
+    const useDirectHeartbeatFastPath = canActivateImmediately && maxConcurrentRuns > 1;
+    if (canActivateImmediately) {
+        runtime.setActiveRunSlots(runtime.getActiveRunSlots() + 1);
+        if (threadId && runId && !useDirectHeartbeatFastPath) {
+            await runtime.persistence.enqueueRun({ threadId, runId, priority });
+        }
+        await beginLease(useDirectHeartbeatFastPath ? "direct-heartbeat" : "queue-claim");
+        let released = false;
+        return async () => {
+            if (released) {
+                return;
+            }
+            released = true;
+            await releaseAndAdvance(() => shiftNextPendingRunSlot(runtime.pendingRunSlots));
+        };
+    }
+    const activateQueuedRun = async () => {
+        const currentRun = runId ? await runtime.persistence.getRun(runId) : null;
+        if (currentRun?.state === "cancelled") {
+            return "abort";
+        }
+        runtime.setActiveRunSlots(runtime.getActiveRunSlots() + 1);
+        if (threadId && runId) {
+            await runtime.emit(threadId, runId, 4, "run.dequeued", {
+                queuePosition: 0,
+                activeRunCount: runtime.getActiveRunSlots(),
+                maxConcurrentRuns,
+                priority,
+            });
+            await runtime.setRunStateAndEmit(threadId, runId, 5, activeState, {
+                previousState: "queued",
+            });
+            await beginLease("queue-claim");
+        }
+        return "activate";
+    };
+    if (threadId && runId) {
+        await runtime.persistence.enqueueRun({ threadId, runId, priority });
+        const slotAcquisition = new Promise((resolve, reject) => {
+            const displacedEntries = runtime.enqueuePendingRunSlot({
+                threadId,
+                runId,
+                priority,
+                activate: async () => {
+                    try {
+                        resolve(await activateQueuedRun());
+                    }
+                    catch (error) {
+                        reject(error);
+                    }
+                },
+                abort: () => resolve("abort"),
+            });
+            void Promise.all(displacedEntries.map((candidate) => runtime.emit(candidate.threadId, candidate.runId, 3, "run.queued", {
+                queuePosition: candidate.queuePosition,
+                activeRunCount: runtime.getActiveRunSlots(),
+                maxConcurrentRuns,
+                priority: candidate.priority,
+            })));
+        });
+        const queuePosition = runtime.pendingRunSlots.findIndex((entry) => entry.runId === runId) + 1;
+        await runtime.setRunStateAndEmit(threadId, runId, 2, "queued", {
+            previousState: activeState,
+        });
+        await runtime.emit(threadId, runId, 3, "run.queued", {
+            queuePosition,
+            activeRunCount: runtime.getActiveRunSlots(),
+            maxConcurrentRuns,
+            priority,
+        });
+        const slotAcquisitionResult = await slotAcquisition;
+        if (slotAcquisitionResult === "abort") {
+            return async () => undefined;
+        }
+        let released = false;
+        return async () => {
+            if (released) {
+                return;
+            }
+            released = true;
+            await releaseAndAdvance(() => runtime.pendingRunSlots.shift());
+        };
+    }
+    let released = false;
+    return async () => {
+        if (released) {
+            return;
+        }
+        released = true;
+        await releaseAndAdvance(() => shiftNextPendingRunSlot(runtime.pendingRunSlots));
+    };
+}

package/dist/runtime/harness/run/startup-runtime.d.ts

@@ -0,0 +1,37 @@
+import type { RunResult, ThreadSummary } from "../../../contracts/types.js";
+import type { RuntimePersistence } from "../../../persistence/types.js";
+import type { ConcurrencyConfig, RecoveryConfig } from "../../../workspace/support/workspace-ref-utils.js";
+import { recoverQueuedStartupRun } from "./recovery.js";
+type Startable = {
+    start(): Promise<void>;
+};
+type StartupRecoveryContext = Parameters<typeof recoverQueuedStartupRun>[0];
+export declare function initializeHarnessRuntime(input: {
+    persistence: RuntimePersistence;
+    checkpointMaintenance: Startable | null;
+    runtimeRecordMaintenance: Startable | null;
+    healthMonitor: Startable | null;
+    recoverStartupRuns: () => Promise<void>;
+}): Promise<void>;
+export declare function recoverStartupRuns(input: {
+    recoveryConfig: RecoveryConfig;
+    persistence: RuntimePersistence;
+    createStartupRecoveryContext: () => StartupRecoveryContext;
+    reclaimExpiredClaimedRuns: (nowIso?: string) => Promise<void>;
+}): Promise<void>;
+export declare function reclaimExpiredClaimedRuns(input: {
+    persistence: RuntimePersistence;
+    setRunStateAndEmit: (threadId: string, runId: string, sequence: number, state: RunResult["state"], options: {
+        previousState: string | null;
+        checkpointRef?: string | null;
+        error?: string;
+    }) => Promise<unknown>;
+    emit: (threadId: string, runId: string, sequence: number, eventType: string, payload: Record<string, unknown>) => Promise<unknown>;
+    concurrencyConfig: ConcurrencyConfig;
+    getActiveRunSlots: () => number;
+}, nowIso?: string): Promise<void>;
+export declare function isStaleRunningRun(input: {
+    persistence: RuntimePersistence;
+    concurrencyConfig: ConcurrencyConfig;
+}, thread: ThreadSummary, nowMs?: number): Promise<boolean>;
+export {};

package/dist/runtime/harness/run/startup-runtime.js

@@ -0,0 +1,68 @@
+import { recoverQueuedStartupRun, recoverResumingStartupRun, recoverRunningStartupRun, } from "./recovery.js";
+export async function initializeHarnessRuntime(input) {
+    await input.persistence.initialize();
+    await input.checkpointMaintenance?.start();
+    await input.runtimeRecordMaintenance?.start();
+    await input.healthMonitor?.start();
+    await input.recoverStartupRuns();
+}
+export async function recoverStartupRuns(input) {
+    if (!input.recoveryConfig.enabled) {
+        return;
+    }
+    await input.reclaimExpiredClaimedRuns();
+    const threads = await input.persistence.listSessions();
+    const recoveryContext = input.createStartupRecoveryContext();
+    for (const thread of threads) {
+        const handled = await recoverQueuedStartupRun(recoveryContext, thread) ||
+            await recoverRunningStartupRun(recoveryContext, thread) ||
+            await recoverResumingStartupRun(recoveryContext, thread);
+        if (handled) {
+            continue;
+        }
+    }
+}
+export async function reclaimExpiredClaimedRuns(input, nowIso = new Date().toISOString()) {
+    const expiredClaims = await input.persistence.listExpiredClaimedRuns(nowIso);
+    for (const claim of expiredClaims) {
+        const thread = await input.persistence.getSession(claim.threadId);
+        if (!thread) {
+            await input.persistence.releaseRunClaim(claim.runId);
+            continue;
+        }
+        const lifecycle = await input.persistence.getRunLifecycle(claim.threadId, claim.runId);
+        if (lifecycle.state === "claimed") {
+            await input.persistence.enqueueRun({
+                threadId: claim.threadId,
+                runId: claim.runId,
+                priority: claim.priority,
+                queueKey: claim.queueKey,
+                availableAt: nowIso,
+            });
+            await input.setRunStateAndEmit(claim.threadId, claim.runId, 99, "queued", {
+                previousState: "claimed",
+            });
+            await input.emit(claim.threadId, claim.runId, 100, "run.queued", {
+                queuePosition: 0,
+                activeRunCount: input.getActiveRunSlots(),
+                maxConcurrentRuns: input.concurrencyConfig.maxConcurrentRuns,
+                recoveredOnStartup: true,
+                reclaimReason: "expired-lease",
+            });
+            continue;
+        }
+        await input.persistence.releaseRunClaim(claim.runId);
+    }
+}
+export async function isStaleRunningRun(input, thread, nowMs = Date.now()) {
+    const control = await input.persistence.getRunControl(thread.latestRunId);
+    const heartbeatAt = control?.heartbeatAt;
+    if (!heartbeatAt) {
+        return true;
+    }
+    const heartbeatAtMs = Date.parse(heartbeatAt);
+    if (!Number.isFinite(heartbeatAtMs)) {
+        return true;
+    }
+    return nowMs - heartbeatAtMs >= input.concurrencyConfig.heartbeatTimeoutMs;
+}

package/dist/runtime/harness/run/stream-run.d.ts

@@ -0,0 +1,53 @@
+import type { CompiledAgentBinding, HarnessEvent, HarnessStreamItem, MessageContent, RunResult, TranscriptMessage } from "../../../contracts/types.js";
+type RuntimeStreamChunk = string | {
+    kind: "content" | "interrupt" | "reasoning" | "step" | "tool-result" | "upstream-event";
+    content?: string;
+    toolName?: string;
+    output?: unknown;
+    isError?: boolean;
+    event?: {
+        format?: string;
+        raw?: unknown;
+    } | Record<string, unknown>;
+};
+type StreamRunOptions = {
+    binding: CompiledAgentBinding;
+    input: MessageContent;
+    invocation: {
+        context?: Record<string, unknown>;
+        state?: Record<string, unknown>;
+        files?: Record<string, unknown>;
+    };
+    threadId: string;
+    runId: string;
+    selectedAgentId: string;
+    isNewThread: boolean;
+    runCreatedEventPromise: Promise<HarnessEvent>;
+    releaseRunSlotPromise: Promise<() => Promise<void>>;
+    loadPriorHistory: (threadId: string, runId: string) => Promise<TranscriptMessage[]>;
+    stream: (binding: CompiledAgentBinding, input: MessageContent, threadId: string, priorHistory: TranscriptMessage[], options: {
+        context?: Record<string, unknown>;
+        state?: Record<string, unknown>;
+        files?: Record<string, unknown>;
+        runId: string;
+    }) => AsyncIterable<RuntimeStreamChunk>;
+    invokeWithHistory: (binding: CompiledAgentBinding, input: MessageContent, threadId: string, runId: string) => Promise<RunResult>;
+    emit: (threadId: string, runId: string, sequence: number, eventType: string, payload: Record<string, unknown>) => Promise<HarnessEvent>;
+    setRunStateAndEmit: (threadId: string, runId: string, sequence: number, state: RunResult["state"], options: {
+        previousState: string | null;
+        checkpointRef?: string | null;
+        error?: string;
+    }) => Promise<HarnessEvent>;
+    requestApprovalAndEmit: (threadId: string, runId: string, input: MessageContent, interruptContent: string | undefined, checkpointRef: string, sequence: number) => Promise<{
+        approval: {
+            approvalId: string;
+            pendingActionId: string;
+        };
+        event: HarnessEvent;
+    }>;
+    appendAssistantMessage: (threadId: string, runId: string, content?: string) => Promise<void>;
+    clearRunRequest: (threadId: string, runId: string) => Promise<void>;
+    emitSyntheticFallback: (threadId: string, runId: string, selectedAgentId: string, error: unknown) => Promise<void>;
+};
+export declare function streamHarnessRun(options: StreamRunOptions): AsyncGenerator<HarnessStreamItem>;
+export {};