@botbotgo/agent-harness 0.0.45 → 0.0.46

package/README.md

@@ -17,7 +17,7 @@ What it provides:
 - YAML-defined runtime assembly for hosts, models, routing, recovery, concurrency, MCP, and maintenance policy
 - backend-adapted execution with a generic runtime contract and current LangChain v1 / DeepAgents adapters
 - local `resources/tools/` and `resources/skills/` loading
-- persisted runs, threads, approvals, events, and resumable checkpoints
+- persisted runs, threads, approvals, events, queued tasks, and resumable checkpoints
 
 ## Quick Start
 
@@ -35,6 +35,8 @@ your-workspace/
     agent-context.md
     workspace.yaml
     models.yaml
+    embedding-models.yaml
+    vector-stores.yaml
     stores.yaml
     tools.yaml
     mcp.yaml
@@ -77,7 +79,7 @@ try {
 - Persisted threads, runs, approvals, and lifecycle events
 - Recovery policy and resumable checkpoints
 - Background checkpoint maintenance
-- Runtime-level concurrency control
+- Runtime-level concurrency control and queued-run persistence
 
 ## How To Use
 
@@ -150,6 +152,16 @@ const result = await run(runtime, {
 
 `subscribe(...)` is a read-only observer surface over stored lifecycle events.
 
+The event stream includes:
+
+- `run.created`
+- `run.queued`
+- `run.dequeued`
+- `run.state.changed`
+- `approval.requested`
+- `approval.resolved`
+- `output.delta`
+
 ### Inspect Threads And Approvals
 
 ```ts
@@ -225,29 +237,105 @@ Use Kubernetes-style YAML:
 
 Use distinct names for named objects such as models, stores, checkpointers, tools, and MCP servers.
 
-### `config/workspace.yaml`
+### Client-Configurable YAML Reference
+
+This section is the client-facing explanation of what can be configured in YAML today and what each field changes at runtime.
+
+There are three layers of client configuration:
+
+- runtime-level policy in `config/workspace.yaml`
+- reusable object catalogs in `config/*.yaml`
+- agent assembly in `config/agents/*.yaml`
 
-Use this file for runtime-level policy:
+### `config/workspace.yaml`
 
-- `runRoot`
-- `routing.rules`
-- `routing.defaultAgentId`
-- `routing.systemPrompt`
-- `routing.modelRouting`
-- `concurrency.maxConcurrentRuns`
-- `recovery.enabled`
-- `recovery.resumeOnStartup`
-- `recovery.maxRecoveryAttempts`
-- `maintenance.checkpoints.*`
+Use this file for runtime-level policy shared by the whole workspace.
+
+Primary fields:
+
+- `runRoot`: root directory where the runtime stores thread indexes, runs, approvals, artifacts, queued requests, and default local persistence
+- `routing.defaultAgentId`: default host selected when no explicit routing rule matches
+- `routing.rules`: ordered YAML routing rules evaluated before backend routing
+- `routing.systemPrompt`: optional model-classifier prompt used only when model routing is enabled
+- `routing.modelRouting`: opt in to model-driven host classification fallback
+- `concurrency.maxConcurrentRuns`: maximum number of active runs; extra runs enter the persistent queue
+- `recovery.enabled`: enables runtime-managed startup recovery
+- `recovery.resumeOnStartup`: compatibility alias for resuming interrupted approval-driven runs on startup
+- `recovery.resumeResumingRunsOnStartup`: explicit control for resuming interrupted approval-driven runs on startup
+- `recovery.maxRecoveryAttempts`: upper bound for startup recovery retries
+- `maintenance.checkpoints.enabled`: turns on background checkpoint cleanup
+- `maintenance.checkpoints.schedule.intervalSeconds`: maintenance loop interval
+- `maintenance.checkpoints.schedule.runOnStartup`: run checkpoint cleanup during startup
+- `maintenance.checkpoints.policies.maxAgeSeconds`: age-based checkpoint cleanup
+- `maintenance.checkpoints.policies.maxBytes`: size-based checkpoint cleanup
+- `maintenance.checkpoints.sqlite.sweepBatchSize`: batch size for SQLite cleanup scans
+- `maintenance.checkpoints.sqlite.vacuum`: vacuum SQLite after deletions
 
 If `runRoot` is omitted, the runtime defaults to `<workspace-root>/run-data`.
 
+Example:
+
+```yaml
+apiVersion: agent-harness/v1alpha1
+kind: Runtime
+metadata:
+  name: default
+spec:
+  runRoot: ./.agent
+  concurrency:
+    maxConcurrentRuns: 3
+  routing:
+    defaultAgentId: orchestra
+    modelRouting: false
+    rules:
+      - agentId: orchestra
+        contains: ["latest", "recent", "today", "news"]
+      - agentId: orchestra
+        regex:
+          - "\\b(create|build|implement|fix|debug|review|inspect)\\b"
+  maintenance:
+    checkpoints:
+      enabled: true
+      schedule:
+        intervalSeconds: 3600
+        runOnStartup: true
+      policies:
+        maxAgeSeconds: 604800
+      sqlite:
+        sweepBatchSize: 200
+        vacuum: false
+  recovery:
+    enabled: true
+    resumeResumingRunsOnStartup: true
+    maxRecoveryAttempts: 3
+```
+
+Notes:
+
+- `routing.rules` only choose the starting host agent; they do not replace backend planning semantics
+- queued runs are persisted under `runRoot` and continue after process restart
+- `running` runs are only replayed on startup when the bound tools are retryable
+
 ### `config/agent-context.md`
 
 Use this file for shared startup context loaded into agents at construction time.
 
 Put stable project context here. Do not use it as mutable long-term memory.
 
+Good uses:
+
+- product positioning
+- codebase conventions
+- stable domain vocabulary
+- organization-specific rules
+
+Bad uses:
+
+- transient scratch notes
+- per-run execution state
+- approval packets
+- long-term memory that should live in the store
+
 ### `config/models.yaml`
 
 Use one file for multiple named models:
@@ -259,6 +347,7 @@ spec:
   - name: default
     provider: openai
     model: gpt-4.1
+    temperature: 0.2
   - name: planner
     provider: openai
     model: gpt-4.1-mini
@@ -266,6 +355,70 @@ spec:
 
 These load as `model/default` and `model/planner`.
 
+Client-configurable model fields:
+
+- `name`: catalog name referenced by `model/<name>`
+- `provider`: provider family such as `openai`, `openai-compatible`, `ollama`, `anthropic`, or `google`
+- `model`: provider model id
+- top-level provider init fields such as `temperature`, `baseUrl`, API-specific settings, and client options
+- `clientRef`: optional external client reference
+- `fallbacks`: optional fallback model refs
+- `metadata`: optional model metadata
+
+### `config/embedding-models.yaml`
+
+Use this file for named embedding model presets used by retrieval-oriented tools.
+
+```yaml
+apiVersion: agent-harness/v1alpha1
+kind: EmbeddingModels
+spec:
+  - name: default
+    provider: ollama
+    model: nomic-embed-text
+    baseUrl: http://localhost:11434
+```
+
+Client-configurable embedding fields:
+
+- `name`
+- `provider`
+- `model`
+- top-level provider init fields such as `baseUrl`
+- `clientRef`
+- `metadata`
+
+These load as `embedding-model/default`.
+
+### `config/vector-stores.yaml`
+
+Use this file for named vector store presets referenced by retrieval tools.
+
+```yaml
+apiVersion: agent-harness/v1alpha1
+kind: VectorStores
+spec:
+  - name: default
+    storeKind: LibSQLVectorStore
+    url: file:.agent/vector-store.db
+    table: rag_chunks
+    column: embedding
+    embeddingModelRef: embedding-model/default
+```
+
+Client-configurable vector store fields:
+
+- `name`
+- `storeKind`
+- `url`
+- `authToken`
+- `table`
+- `column`
+- `embeddingModelRef`
+- `metadata`
+
+These load as `vector-store/default`.
+
 ### `config/stores.yaml`
 
 Use one file for named persistence presets:
@@ -285,6 +438,73 @@ spec:
 
 These load as `store/default` and `checkpointer/default`.
 
+Client-configurable store fields:
+
+- `kind: Store` for backend stores
+- `kind: Checkpointer` for resumable execution state
+- `name` for refs
+- `storeKind` such as `FileStore`, `InMemoryStore`, `RedisStore`, `PostgresStore`
+- `checkpointerKind` such as `MemorySaver`, `FileCheckpointer`, `SqliteSaver`
+- storage-specific fields such as `path`, connection strings, auth, and provider options
+
+### `config/tools.yaml`
+
+Use this file for reusable tool presets and tool bundles.
+
+Minimal collection form:
+
+```yaml
+apiVersion: agent-harness/v1alpha1
+kind: Tools
+spec:
+  - kind: Tool
+    name: fetch_docs
+    type: function
+    description: Fetch a documentation page.
+```
+
+Client-configurable tool fields:
+
+- `name`
+- `type`: `function`, `backend`, `mcp`, or `bundle`
+- `description`
+- `implementationName` for local JS tool modules
+- `inputSchema.ref`
+- `backend.operation`
+- `mcp.ref` or `mcp.tool`
+- `refs` for bundle composition
+- `hitl.enabled` and `hitl.allow` for approval-gated tools
+- `retryable: true` for tools that are safe to replay during startup recovery
+- `config` for tool-specific options
+
+Use `retryable` carefully. Mark a tool retryable only when repeated execution is safe or intentionally idempotent.
+
+### `config/mcp.yaml`
+
+Use this file for reusable MCP server definitions and MCP-backed tool presets.
+
+```yaml
+apiVersion: agent-harness/v1alpha1
+kind: McpServers
+spec:
+  - name: docs
+    transport: http
+    url: https://example.com/mcp
+  - name: local-browser
+    transport: stdio
+    command: node
+    args: ["./mcp-browser-server.mjs"]
+```
+
+Client-configurable MCP fields:
+
+- `name`
+- `transport`: `stdio`, `http`, `sse`, or `websocket`
+- `command`, `args`, `env`, `cwd` for stdio servers
+- `url`, `token`, `headers` for network servers
+
+These load as `mcp/<name>`.
+
 ### `config/agents/*.yaml`
 
 Prefer the generic agent form and declare the current execution backend explicitly:
@@ -303,20 +523,75 @@ spec:
 
 `kind: DeepAgent` and `kind: LangChainAgent` remain supported as compatibility forms, but `kind: Agent` is the recommended product-facing entry point.
 
-Common fields include:
-
-- `modelRef`
-- `execution.backend`
-- `systemPrompt`
-- `tools`
-- `skills`
-- `memory`
-- `checkpointer`
-- `store`
-- `backend`
-- `middleware`
-- `subagents`
-- `mcpServers`
+Common client-configurable agent fields:
+
+- `metadata.name`
+- `metadata.description`
+- `spec.execution.backend`
+- `spec.modelRef`
+- `spec.systemPrompt`
+- `spec.tools`
+- `spec.skills`
+- `spec.memory`
+- `spec.checkpointer`
+- `spec.store`
+- `spec.backend`
+- `spec.middleware`
+- `spec.subagents`
+- `spec.mcpServers`
+- `spec.responseFormat`
+- `spec.contextSchema`
+
+Typical patterns:
+
+- use `direct` as a lightweight host for simple one-turn requests
+- use `orchestra` as the main execution host for tools, multi-step work, and delegation
+- keep routing policy in `config/workspace.yaml`, not buried in prompts
+
+Example direct agent:
+
+```yaml
+apiVersion: agent-harness/v1alpha1
+kind: Agent
+metadata:
+  name: direct
+spec:
+  execution:
+    backend: langchain-v1
+  modelRef: model/default
+  checkpointer:
+    ref: checkpointer/default
+  systemPrompt: |-
+    You are the direct agent.
+    Answer simple requests directly.
+```
+
+Example orchestra agent:
+
+```yaml
+apiVersion: agent-harness/v1alpha1
+kind: Agent
+metadata:
+  name: orchestra
+spec:
+  execution:
+    backend: deepagent
+  modelRef: model/default
+  memory:
+    - path: config/agent-context.md
+  store:
+    ref: store/default
+  checkpointer:
+    ref: checkpointer/default
+  backend:
+    kind: CompositeBackend
+    state:
+      kind: VfsSandbox
+      timeout: 600
+    routes:
+      /memories/:
+        kind: StoreBackend
+```
 
 ### `resources/`
 
@@ -329,6 +604,24 @@ Tool modules are discovered from `resources/tools/*.js`, `resources/tools/*.mjs`
 
 The preferred tool module format is exporting `tool({...})`.
 
+Example:
+
+```js
+import { z } from "zod";
+import { tool } from "@botbotgo/agent-harness/tools";
+
+export const local_lookup = tool({
+  description: "Lookup a ticker from a local tool module.",
+  retryable: true,
+  schema: {
+    ticker: z.string().min(1),
+  },
+  async invoke(input) {
+    return input.ticker.toUpperCase();
+  },
+});
+```
+
 Keep runtime extension source under `resources/`. Keep tests outside the published source tree, for example under repository `test/`.
 
 ## Design Notes
@@ -337,11 +630,12 @@ Keep runtime extension source under `resources/`. Keep tests outside the publish
 - agent-level execution behavior stays upstream
 - application-level orchestration and lifecycle management stays in the harness
 - checkpoint resume is treated as a system-managed runtime behavior, not a primary public abstraction
+- public runtime contract generic does not mean backend-agnostic implementation internals; it means client-facing semantics stay stable even when adapters change
 
 ## API Summary
 
 - `createAgentHarness(...)`
-- `run(...)`
+- `run(runtime, {...})`
 - `subscribe(...)`
 - `listThreads(...)`
 - `getThread(...)`

package/dist/config/workspace.yaml

@@ -22,6 +22,11 @@ spec:
   # Value options: relative workspace path like `./.agent`, or an absolute filesystem path.
   runRoot: ./.agent
 
+  # agent-harness feature: runtime-level task queue and maximum number of concurrent runs.
+  # Additional runs wait in the harness queue until a slot becomes available.
+  concurrency:
+    maxConcurrentRuns: 3
+
   # agent-harness feature: optional host-router prompt override used when the runtime chooses between
   # top-level host agents such as a main execution host and an optional low-latency side host.
   # Use placeholders so the same prompt can survive host renames:
@@ -90,19 +95,17 @@ spec:
   # - oldest-first deletion by time policy and/or size policy
   # - background scheduling inside the harness lifecycle
   #
-  # Example:
-  # maintenance:
-  #   checkpoints:
-  #     enabled: true
-  #     schedule:
-  #       intervalSeconds: 3600
-  #       runOnStartup: true
-  #     policies:
-  #       maxAgeSeconds: 604800
-  #       maxBytes: 1073741824
-  #     sqlite:
-  #       sweepBatchSize: 200
-  #       vacuum: false
+  maintenance:
+    checkpoints:
+      enabled: true
+      schedule:
+        intervalSeconds: 3600
+        runOnStartup: true
+      policies:
+        maxAgeSeconds: 604800
+      sqlite:
+        sweepBatchSize: 200
+        vacuum: false
 
   # agent-harness feature: runtime-managed recovery policy for interrupted runs.
   # This keeps checkpoint resume as an internal lifecycle concern instead of a primary user-facing API concept.
@@ -112,8 +115,7 @@ spec:
   # - persisted approval-decision intent for cross-restart resume continuation
   # - bounded retry attempts to avoid infinite restart loops
   #
-  # Example:
-  # recovery:
-  #   enabled: true
-  #   resumeResumingRunsOnStartup: true
-  #   maxRecoveryAttempts: 3
+  recovery:
+    enabled: true
+    resumeResumingRunsOnStartup: true
+    maxRecoveryAttempts: 3

package/dist/contracts/types.d.ts

@@ -4,7 +4,7 @@ export type RuntimeCapabilities = {
     delegation?: boolean;
     memory?: boolean;
 };
-export type RunState = "running" | "waiting_for_approval" | "resuming" | "completed" | "failed";
+export type RunState = "queued" | "running" | "waiting_for_approval" | "resuming" | "completed" | "failed";
 export type ParsedAgentObject = {
     id: string;
     executionMode: ExecutionMode;
@@ -85,6 +85,7 @@ export type ParsedToolObject = {
         enabled: boolean;
         allow?: Array<"approve" | "edit" | "reject">;
     };
+    retryable?: boolean;
     sourcePath: string;
 };
 export type LangChainAgentParams = {
@@ -172,6 +173,7 @@ export type CompiledTool = {
         enabled: boolean;
         allow: Array<"approve" | "edit" | "reject">;
     };
+    retryable?: boolean;
     runtimeValue: {
         name: string;
         description: string;
@@ -221,7 +223,7 @@ export type ThreadSummary = {
     status: RunState;
 };
 export type SessionRecord = ThreadSummary;
-export type KnownHarnessEventType = "run.created" | "run.state.changed" | "run.resumed" | "approval.requested" | "approval.resolved" | "artifact.created" | "output.delta" | "reasoning.delta" | "runtime.synthetic_fallback";
+export type KnownHarnessEventType = "run.created" | "run.queued" | "run.dequeued" | "run.state.changed" | "run.resumed" | "approval.requested" | "approval.resolved" | "artifact.created" | "output.delta" | "reasoning.delta" | "runtime.synthetic_fallback";
 export type HarnessEventType = KnownHarnessEventType | (string & {});
 export type HarnessEvent = {
     eventId: string;

package/dist/extensions.js

@@ -126,6 +126,7 @@ registerToolKind({
                         allow: tool.hitl.allow ?? ["approve", "edit", "reject"],
                     }
                     : undefined,
+                retryable: tool.retryable,
                 runtimeValue: { name: tool.name, description: tool.description, type: "function" },
             },
         ];
@@ -158,6 +159,7 @@ registerToolKind({
                         allow: tool.hitl.allow ?? ["approve", "edit", "reject"],
                     }
                     : undefined,
+                retryable: tool.retryable,
                 runtimeValue: { name: tool.name, description: tool.description, type: "backend" },
             },
         ];
@@ -190,6 +192,7 @@ registerToolKind({
                         allow: tool.hitl.allow ?? ["approve", "edit", "reject"],
                     }
                     : undefined,
+                retryable: tool.retryable,
                 runtimeValue: { name: tool.name, description: tool.description, type: "mcp" },
             },
         ];

package/dist/package-version.d.ts

@@ -1 +1 @@
-export declare const AGENT_HARNESS_VERSION = "0.0.44";
+export declare const AGENT_HARNESS_VERSION = "0.0.45";

package/dist/package-version.js

@@ -1 +1 @@
-export const AGENT_HARNESS_VERSION = "0.0.44";
+export const AGENT_HARNESS_VERSION = "0.0.45";

package/dist/persistence/file-store.d.ts

@@ -1,4 +1,4 @@
-import type { ArtifactListing, ArtifactRecord, DelegationRecord, HarnessEvent, InternalApprovalRecord, RunState, ThreadSummary, ThreadRunRecord, TranscriptMessage } from "../contracts/types.js";
+import type { ArtifactListing, ArtifactRecord, DelegationRecord, HarnessEvent, InternalApprovalRecord, InvocationEnvelope, MessageContent, RunState, ThreadSummary, ThreadRunRecord, TranscriptMessage } from "../contracts/types.js";
 type ThreadMeta = {
     threadId: string;
     workspaceId: string;
@@ -39,6 +39,11 @@ type RecoveryIntent = {
     resumePayload: unknown;
     attempts: number;
 };
+type PersistedRunRequest = {
+    input: MessageContent;
+    invocation?: InvocationEnvelope;
+    savedAt: string;
+};
 export declare class FilePersistence {
     private readonly runRoot;
     constructor(runRoot: string);
@@ -73,6 +78,9 @@ export declare class FilePersistence {
     getRunApprovals(threadId: string, runId: string): Promise<InternalApprovalRecord[]>;
     getRunMeta(threadId: string, runId: string): Promise<RunMeta>;
     getRunLifecycle(threadId: string, runId: string): Promise<Lifecycle>;
+    saveRunRequest(threadId: string, runId: string, request: PersistedRunRequest): Promise<void>;
+    getRunRequest(threadId: string, runId: string): Promise<PersistedRunRequest | null>;
+    clearRunRequest(threadId: string, runId: string): Promise<void>;
     listDelegations(): Promise<DelegationRecord[]>;
     createApproval(record: InternalApprovalRecord): Promise<void>;
     resolveApproval(threadId: string, runId: string, approvalId: string, status: InternalApprovalRecord["status"]): Promise<InternalApprovalRecord>;

package/dist/persistence/file-store.js

@@ -266,6 +266,22 @@ export class FilePersistence {
     async getRunLifecycle(threadId, runId) {
         return readJson(path.join(this.runDir(threadId, runId), "lifecycle.json"));
     }
+    async saveRunRequest(threadId, runId, request) {
+        await writeJson(path.join(this.runDir(threadId, runId), "request.json"), request);
+    }
+    async getRunRequest(threadId, runId) {
+        const requestPath = path.join(this.runDir(threadId, runId), "request.json");
+        if (!(await fileExists(requestPath))) {
+            return null;
+        }
+        return readJson(requestPath);
+    }
+    async clearRunRequest(threadId, runId) {
+        const requestPath = path.join(this.runDir(threadId, runId), "request.json");
+        if (await fileExists(requestPath)) {
+            await rm(requestPath, { force: true });
+        }
+    }
     async listDelegations() {
         const delegationsDir = path.join(this.runRoot, "indexes", "delegations");
         if (!(await fileExists(delegationsDir))) {

package/dist/resource/resource-impl.d.ts

@@ -6,6 +6,7 @@ export type ResourceToolInfo = {
     backendOperation: string;
     name: string;
     description: string;
+    retryable?: boolean;
     hitl?: {
         enabled: boolean;
         allow: Array<"approve" | "edit" | "reject">;

package/dist/runtime/harness.d.ts

@@ -40,6 +40,7 @@ export declare class AgentHarnessRuntime {
     private getBinding;
     private listAgentTools;
     private resolveAgentTools;
+    private supportsRunningReplay;
     listThreads(filter?: {
         agentId?: string;
     }): Promise<ThreadSummary[]>;
@@ -62,6 +63,8 @@ export declare class AgentHarnessRuntime {
     private loadRunInput;
     private appendAssistantMessage;
     private invokeWithHistory;
+    private buildPersistedRunRequest;
+    private executeQueuedRun;
     private checkpointRefForState;
     private finalizeContinuedRun;
     private emitOutputDeltaAndCreateItem;

package/dist/runtime/harness.js

@@ -213,6 +213,10 @@ export class AgentHarnessRuntime {
             resolvedTool: resolvedTools[index],
         }));
     }
+    supportsRunningReplay(binding) {
+        const tools = getBindingPrimaryTools(binding);
+        return tools.every((tool) => tool.retryable === true);
+    }
     async listThreads(filter) {
         const threadSummaries = await this.persistence.listSessions();
         if (!filter?.agentId) {
@@ -385,6 +389,72 @@ export class AgentHarnessRuntime {
         const priorHistory = await this.loadPriorHistory(threadId, runId);
         return this.runtimeAdapter.invoke(binding, input, threadId, runId, resumePayload, priorHistory, options);
     }
+    buildPersistedRunRequest(input, invocation) {
+        const envelope = invocation.invocation ?? {
+            ...(invocation.context ? { context: invocation.context } : {}),
+            ...(invocation.state ? { inputs: invocation.state } : {}),
+            ...(invocation.files ? { attachments: invocation.files } : {}),
+        };
+        return {
+            input: normalizeMessageContent(input),
+            invocation: envelope && Object.keys(envelope).length > 0
+                ? {
+                    ...(envelope.context ? { context: envelope.context } : {}),
+                    ...(envelope.inputs ? { inputs: envelope.inputs } : {}),
+                    ...(envelope.attachments ? { attachments: envelope.attachments } : {}),
+                    ...(envelope.capabilities ? { capabilities: envelope.capabilities } : {}),
+                }
+                : undefined,
+            savedAt: new Date().toISOString(),
+        };
+    }
+    async executeQueuedRun(binding, input, threadId, runId, agentId, options = {}) {
+        const previousState = options.previousState ?? "running";
+        if (previousState === "queued") {
+            await this.emit(threadId, runId, 101, "run.dequeued", {
+                queuePosition: 0,
+                activeRunCount: this.activeRunSlots,
+                maxConcurrentRuns: this.concurrencyConfig.maxConcurrentRuns,
+                recoveredOnStartup: true,
+            });
+            await this.setRunStateAndEmit(threadId, runId, 102, "running", {
+                previousState: "queued",
+            });
+        }
+        try {
+            const actual = await this.invokeWithHistory(binding, input, threadId, runId, undefined, {
+                context: options.context,
+                state: options.state,
+                files: options.files,
+            });
+            const finalized = await this.finalizeContinuedRun(threadId, runId, input, actual, {
+                previousState: previousState === "queued" ? "running" : previousState,
+                stateSequence: options.stateSequence ?? 103,
+                approvalSequence: options.approvalSequence ?? 104,
+            });
+            return {
+                ...finalized,
+                agentId,
+            };
+        }
+        catch (error) {
+            await this.emitSyntheticFallback(threadId, runId, agentId, error, 103);
+            await this.setRunStateAndEmit(threadId, runId, 104, "failed", {
+                previousState: previousState === "queued" ? "running" : previousState,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return {
+                threadId,
+                runId,
+                agentId,
+                state: "failed",
+                output: renderRuntimeFailure(error),
+            };
+        }
+        finally {
+            await this.persistence.clearRunRequest(threadId, runId);
+        }
+    }
     checkpointRefForState(threadId, runId, state) {
         return state === "waiting_for_approval" ? `checkpoints/${threadId}/${runId}/cp-1` : null;
     }
@@ -492,17 +562,56 @@ export class AgentHarnessRuntime {
         }
         await listener(value);
     }
-    async acquireRunSlot() {
+    async acquireRunSlot(threadId, runId, activeState = "running") {
         const maxConcurrentRuns = this.concurrencyConfig.maxConcurrentRuns;
         if (!maxConcurrentRuns) {
             return () => undefined;
         }
-        if (this.activeRunSlots >= maxConcurrentRuns) {
-            await new Promise((resolve) => {
-                this.pendingRunSlots.push(resolve);
+        if (this.activeRunSlots < maxConcurrentRuns) {
+            this.activeRunSlots += 1;
+            let released = false;
+            return () => {
+                if (released) {
+                    return;
+                }
+                released = true;
+                this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
+                const next = this.pendingRunSlots.shift();
+                void next?.();
+            };
+        }
+        if (threadId && runId) {
+            const queuePosition = this.pendingRunSlots.length + 1;
+            await this.setRunStateAndEmit(threadId, runId, 2, "queued", {
+                previousState: activeState,
+            });
+            await this.emit(threadId, runId, 3, "run.queued", {
+                queuePosition,
+                activeRunCount: this.activeRunSlots,
+                maxConcurrentRuns,
             });
         }
-        this.activeRunSlots += 1;
+        await new Promise((resolve, reject) => {
+            this.pendingRunSlots.push(async () => {
+                try {
+                    this.activeRunSlots += 1;
+                    if (threadId && runId) {
+                        await this.emit(threadId, runId, 4, "run.dequeued", {
+                            queuePosition: 0,
+                            activeRunCount: this.activeRunSlots,
+                            maxConcurrentRuns,
+                        });
+                        await this.setRunStateAndEmit(threadId, runId, 5, activeState, {
+                            previousState: "queued",
+                        });
+                    }
+                    resolve();
+                }
+                catch (error) {
+                    reject(error);
+                }
+            });
+        });
         let released = false;
         return () => {
             if (released) {
@@ -511,7 +620,7 @@ export class AgentHarnessRuntime {
             released = true;
             this.activeRunSlots = Math.max(0, this.activeRunSlots - 1);
             const next = this.pendingRunSlots.shift();
-            next?.();
+            void next?.();
         };
     }
     async dispatchRunListeners(stream, listeners) {
@@ -587,88 +696,68 @@ export class AgentHarnessRuntime {
         if (options.listeners) {
             return this.dispatchRunListeners(this.streamEvents(options), options.listeners);
         }
-        const releaseRunSlot = await this.acquireRunSlot();
+        const invocation = this.normalizeInvocationEnvelope(options);
+        const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
+        const binding = this.workspace.bindings.get(selectedAgentId);
+        if (!binding) {
+            throw new Error(`Unknown agent ${selectedAgentId}`);
+        }
+        const policyDecision = this.policyEngine.evaluate(binding);
+        if (!policyDecision.allowed) {
+            throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
+        }
+        const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
+        await this.persistence.saveRunRequest(threadId, runId, this.buildPersistedRunRequest(options.input, invocation));
+        await this.emitRunCreated(threadId, runId, {
+            agentId: binding.agent.id,
+            requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
+            selectedAgentId,
+            executionMode: getBindingAdapterKind(binding),
+        });
+        const releaseRunSlot = await this.acquireRunSlot(threadId, runId);
         try {
-            const invocation = this.normalizeInvocationEnvelope(options);
-            const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
-            const binding = this.workspace.bindings.get(selectedAgentId);
-            if (!binding) {
-                throw new Error(`Unknown agent ${selectedAgentId}`);
-            }
-            const policyDecision = this.policyEngine.evaluate(binding);
-            if (!policyDecision.allowed) {
-                throw new Error(`Policy evaluation blocked agent ${selectedAgentId}: ${policyDecision.reasons.join(", ")}`);
-            }
-            const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
-            await this.emitRunCreated(threadId, runId, {
-                agentId: binding.agent.id,
-                requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
-                selectedAgentId,
-                executionMode: getBindingAdapterKind(binding),
+            return await this.executeQueuedRun(binding, options.input, threadId, runId, selectedAgentId, {
+                context: invocation.context,
+                state: invocation.state,
+                files: invocation.files,
+                previousState: "running",
+                stateSequence: 6,
+                approvalSequence: 7,
             });
-            try {
-                const actual = await this.invokeWithHistory(binding, options.input, threadId, runId, undefined, {
-                    context: invocation.context,
-                    state: invocation.state,
-                    files: invocation.files,
-                });
-                const finalized = await this.finalizeContinuedRun(threadId, runId, options.input, actual, {
-                    previousState: null,
-                    stateSequence: 3,
-                    approvalSequence: 4,
-                });
-                return {
-                    ...finalized,
-                    agentId: selectedAgentId,
-                };
-            }
-            catch (error) {
-                await this.emitSyntheticFallback(threadId, runId, selectedAgentId, error);
-                await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                    previousState: null,
-                    error: error instanceof Error ? error.message : String(error),
-                });
-                return {
-                    threadId,
-                    runId,
-                    agentId: selectedAgentId,
-                    state: "failed",
-                    output: renderRuntimeFailure(error),
-                };
-            }
         }
         finally {
             releaseRunSlot();
         }
     }
     async *streamEvents(options) {
-        const releaseRunSlot = await this.acquireRunSlot();
-        try {
-            const invocation = this.normalizeInvocationEnvelope(options);
-            const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
-            const binding = this.workspace.bindings.get(selectedAgentId);
-            if (!binding) {
-                const result = await this.run(options);
-                for (const line of result.output.split("\n")) {
-                    yield {
-                        type: "content",
-                        threadId: result.threadId,
-                        runId: result.runId,
-                        agentId: result.agentId ?? selectedAgentId,
-                        content: `${line}\n`,
-                    };
-                }
-                return;
+        const invocation = this.normalizeInvocationEnvelope(options);
+        const selectedAgentId = await this.resolveSelectedAgentId(options.input, options.agentId, options.threadId);
+        const binding = this.workspace.bindings.get(selectedAgentId);
+        if (!binding) {
+            const result = await this.run(options);
+            for (const line of result.output.split("\n")) {
+                yield {
+                    type: "content",
+                    threadId: result.threadId,
+                    runId: result.runId,
+                    agentId: result.agentId ?? selectedAgentId,
+                    content: `${line}\n`,
+                };
             }
-            let emitted = false;
-            const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
-            yield { type: "event", event: await this.emitRunCreated(threadId, runId, {
-                    agentId: selectedAgentId,
-                    requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
-                    selectedAgentId,
-                    input: options.input,
-                    state: "running",
-                }) };
+            return;
+        }
+        let emitted = false;
+        const { threadId, runId } = await this.ensureThreadStarted(selectedAgentId, binding, options.input, options.threadId);
+        await this.persistence.saveRunRequest(threadId, runId, this.buildPersistedRunRequest(options.input, invocation));
+        yield { type: "event", event: await this.emitRunCreated(threadId, runId, {
+                agentId: selectedAgentId,
+                requestedAgentId: options.agentId ?? AUTO_AGENT_ID,
+                selectedAgentId,
+                input: options.input,
+                state: "running",
+            }) };
+        const releaseRunSlot = await this.acquireRunSlot(threadId, runId);
+        try {
             try {
                 const priorHistory = await this.loadPriorHistory(threadId, runId);
                 let assistantOutput = "";
@@ -686,11 +775,11 @@ export class AgentHarnessRuntime {
                             : chunk;
                         if (normalizedChunk.kind === "interrupt") {
                             const checkpointRef = `checkpoints/${threadId}/${runId}/cp-1`;
-                            const waitingEvent = await this.setRunStateAndEmit(threadId, runId, 4, "waiting_for_approval", {
-                                previousState: null,
+                            const waitingEvent = await this.setRunStateAndEmit(threadId, runId, 6, "waiting_for_approval", {
+                                previousState: "running",
                                 checkpointRef,
                             });
-                            const approvalRequest = await this.requestApprovalAndEmit(threadId, runId, options.input, normalizedChunk.content, checkpointRef, 5);
+                            const approvalRequest = await this.requestApprovalAndEmit(threadId, runId, options.input, normalizedChunk.content, checkpointRef, 7);
                             yield {
                                 type: "event",
                                 event: waitingEvent,
@@ -783,22 +872,22 @@ export class AgentHarnessRuntime {
                         finalMessageText: assistantOutput,
                     },
                 };
-                yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "completed", {
-                        previousState: null,
+                yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "completed", {
+                        previousState: "running",
                     }) };
                 return;
             }
             catch (error) {
                 if (emitted) {
-                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                            previousState: null,
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
+                            previousState: "running",
                             error: error instanceof Error ? error.message : String(error),
                         }) };
                     return;
                 }
                 if (error instanceof RuntimeOperationTimeoutError && error.stage === "invoke") {
-                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                            previousState: null,
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
+                            previousState: "running",
                             error: error.message,
                         }) };
                     yield {
@@ -836,15 +925,15 @@ export class AgentHarnessRuntime {
                             agentId: selectedAgentId,
                         },
                     };
-                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, actual.state, {
-                            previousState: null,
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, actual.state, {
+                            previousState: "running",
                         }) };
                     return;
                 }
                 catch (invokeError) {
                     await this.emitSyntheticFallback(threadId, runId, selectedAgentId, invokeError);
-                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 4, "failed", {
-                            previousState: null,
+                    yield { type: "event", event: await this.setRunStateAndEmit(threadId, runId, 6, "failed", {
+                            previousState: "running",
                             error: invokeError instanceof Error ? invokeError.message : String(invokeError),
                         }) };
                     yield {
@@ -870,29 +959,30 @@ export class AgentHarnessRuntime {
             }
         }
         finally {
+            await this.persistence.clearRunRequest(threadId, runId);
             releaseRunSlot();
         }
     }
     async resume(options) {
-        const releaseRunSlot = await this.acquireRunSlot();
+        const approvalById = options.approvalId ? await this.persistence.getApproval(options.approvalId) : null;
+        const thread = options.threadId
+            ? await this.getSession(options.threadId)
+            : approvalById
+                ? await this.getSession(approvalById.threadId)
+                : null;
+        if (!thread) {
+            throw new Error("resume requires either threadId or approvalId");
+        }
+        const approval = approvalById ?? await this.resolveApprovalRecord(options, thread);
+        const threadId = approval.threadId;
+        const runId = approval.runId;
+        const binding = this.workspace.bindings.get(thread.agentId);
+        if (!binding) {
+            throw new Error(`Unknown agent ${thread.agentId}`);
+        }
+        await this.persistence.setRunState(threadId, runId, "resuming", `checkpoints/${threadId}/${runId}/cp-1`);
+        const releaseRunSlot = await this.acquireRunSlot(threadId, runId, "resuming");
         try {
-            const approvalById = options.approvalId ? await this.persistence.getApproval(options.approvalId) : null;
-            const thread = options.threadId
-                ? await this.getSession(options.threadId)
-                : approvalById
-                    ? await this.getSession(approvalById.threadId)
-                    : null;
-            if (!thread) {
-                throw new Error("resume requires either threadId or approvalId");
-            }
-            const approval = approvalById ?? await this.resolveApprovalRecord(options, thread);
-            const threadId = approval.threadId;
-            const runId = approval.runId;
-            const binding = this.workspace.bindings.get(thread.agentId);
-            if (!binding) {
-                throw new Error(`Unknown agent ${thread.agentId}`);
-            }
-            await this.persistence.setRunState(threadId, runId, "resuming", `checkpoints/${threadId}/${runId}/cp-1`);
             await this.persistence.saveRecoveryIntent(threadId, runId, {
                 kind: "approval-decision",
                 savedAt: new Date().toISOString(),
@@ -978,12 +1068,72 @@ export class AgentHarnessRuntime {
         await this.close();
     }
     async recoverStartupRuns() {
-        if (!this.recoveryConfig.enabled || !this.recoveryConfig.resumeResumingRunsOnStartup) {
+        if (!this.recoveryConfig.enabled) {
             return;
         }
         const threads = await this.persistence.listSessions();
         for (const thread of threads) {
-            if (thread.status !== "resuming") {
+            if (thread.status === "queued") {
+                const runMeta = await this.persistence.getRunMeta(thread.threadId, thread.latestRunId);
+                const binding = this.workspace.bindings.get(runMeta.agentId);
+                if (!binding) {
+                    continue;
+                }
+                const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
+                if (!request) {
+                    await this.setRunStateAndEmit(thread.threadId, thread.latestRunId, 100, "failed", {
+                        previousState: "queued",
+                        error: "missing persisted run request for queued run recovery",
+                    });
+                    continue;
+                }
+                const releaseRunSlot = await this.acquireRunSlot();
+                try {
+                    await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
+                        context: request.invocation?.context,
+                        state: request.invocation?.inputs,
+                        files: request.invocation?.attachments,
+                        previousState: "queued",
+                        stateSequence: 103,
+                        approvalSequence: 104,
+                    });
+                }
+                finally {
+                    releaseRunSlot();
+                }
+                continue;
+            }
+            if (thread.status === "running") {
+                const runMeta = await this.persistence.getRunMeta(thread.threadId, thread.latestRunId);
+                const binding = this.workspace.bindings.get(runMeta.agentId);
+                if (!binding || !this.supportsRunningReplay(binding)) {
+                    continue;
+                }
+                const request = await this.persistence.getRunRequest(thread.threadId, thread.latestRunId);
+                if (!request) {
+                    continue;
+                }
+                const releaseRunSlot = await this.acquireRunSlot();
+                try {
+                    await this.emit(thread.threadId, thread.latestRunId, 100, "run.resumed", {
+                        resumeKind: "startup-running-recovery",
+                        state: "running",
+                    });
+                    await this.executeQueuedRun(binding, request.input, thread.threadId, thread.latestRunId, runMeta.agentId, {
+                        context: request.invocation?.context,
+                        state: request.invocation?.inputs,
+                        files: request.invocation?.attachments,
+                        previousState: "running",
+                        stateSequence: 103,
+                        approvalSequence: 104,
+                    });
+                }
+                finally {
+                    releaseRunSlot();
+                }
+                continue;
+            }
+            if (thread.status !== "resuming" || !this.recoveryConfig.resumeResumingRunsOnStartup) {
                 continue;
             }
             const binding = this.workspace.bindings.get(thread.agentId);

package/dist/runtime/thread-memory-sync.js

@@ -44,6 +44,8 @@ function renderOpenApprovalsMarkdown(approvals) {
 }
 const THREAD_MEMORY_EVENT_TYPES = new Set([
     "run.state.changed",
+    "run.queued",
+    "run.dequeued",
     "approval.resolved",
     "approval.requested",
 ]);

package/dist/tool-modules.d.ts

@@ -9,6 +9,7 @@ export type LoadedToolModule = {
     invoke: (input: unknown, context?: Record<string, unknown>) => Promise<unknown> | unknown;
     schema: SchemaLike;
     description: string;
+    retryable?: boolean;
 };
 export declare function isSupportedToolModulePath(filePath: string): boolean;
 export declare function discoverAnnotatedFunctionNames(sourceText: string): string[];

package/dist/tool-modules.js

@@ -76,6 +76,7 @@ function loadToolObjectDefinition(imported, exportName) {
         invoke: definition.invoke,
         schema: normalizeToolSchema(definition.schema),
         description: definition.description.trim(),
+        retryable: definition.retryable === true,
     };
 }
 export function isSupportedToolModulePath(filePath) {
@@ -113,6 +114,11 @@ export function discoverToolModuleDefinitions(sourceText, imported) {
             invoke: invoke,
             schema,
             description: readToolDescription(imported, implementationName, schema),
+            retryable: typeof imported[`${implementationName}Retryable`] === "boolean"
+                ? imported[`${implementationName}Retryable`] === true
+                : typeof imported.retryable === "boolean"
+                    ? imported.retryable === true
+                    : undefined,
         });
     }
     return discovered;
@@ -139,5 +145,10 @@ export function loadToolModuleDefinition(imported, implementationName) {
         invoke: invoke,
         schema,
         description: readToolDescription(imported, implementationName, schema),
+        retryable: typeof imported[`${implementationName}Retryable`] === "boolean"
+            ? imported[`${implementationName}Retryable`] === true
+            : typeof imported.retryable === "boolean"
+                ? imported.retryable === true
+                : undefined,
     };
 }

package/dist/tools.d.ts

@@ -5,6 +5,7 @@ export type ToolDefinitionObject = {
     name?: string;
     description: string;
     schema: SchemaInput;
+    retryable?: boolean;
     invoke: (input: unknown, context?: Record<string, unknown>) => Promise<unknown> | unknown;
     [TOOL_DEFINITION_MARKER]: true;
 };
@@ -15,6 +16,7 @@ export declare function tool(definition: {
     name?: string;
     description: string;
     schema: SchemaInput;
+    retryable?: boolean;
     invoke: (input: unknown, context?: Record<string, unknown>) => Promise<unknown> | unknown;
 }): ToolDefinitionObject;
 export {};

package/dist/workspace/object-loader.js

@@ -566,6 +566,7 @@ export async function readToolModuleItems(root) {
                     name: definition.implementationName,
                     description: definition.description,
                     implementationName: definition.implementationName,
+                    ...(definition.retryable !== undefined ? { retryable: definition.retryable } : {}),
                 },
                 sourcePath: filePath,
             });

package/dist/workspace/resource-compilers.js

@@ -254,6 +254,7 @@ export function parseToolObject(object) {
                 : undefined,
         bundleRefs,
         hitl: parseHitlPolicy(value.hitl),
+        retryable: value.retryable === true,
         sourcePath: object.sourcePath,
     };
 }

package/dist/workspace/support/workspace-ref-utils.d.ts

@@ -22,7 +22,7 @@ export type RecoveryConfig = {
     maxRecoveryAttempts: number;
 };
 export type ConcurrencyConfig = {
-    maxConcurrentRuns?: number;
+    maxConcurrentRuns: number;
 };
 export declare function getWorkspaceObject(refs: Map<string, WorkspaceObject | ParsedAgentObject>, ref: string | undefined): WorkspaceObject | undefined;
 export declare function getRuntimeDefaults(refs: Map<string, WorkspaceObject | ParsedAgentObject>): Record<string, unknown> | undefined;

package/dist/workspace/support/workspace-ref-utils.js

@@ -62,7 +62,7 @@ export function getConcurrencyConfig(refs) {
         Number.isFinite(concurrency.maxConcurrentRuns) &&
         concurrency.maxConcurrentRuns > 0
         ? Math.floor(concurrency.maxConcurrentRuns)
-        : undefined;
+        : 3;
     return { maxConcurrentRuns };
 }
 export function getRoutingSystemPrompt(refs) {

package/dist/workspace/tool-hydration.js

@@ -105,6 +105,7 @@ export async function hydrateResourceAndExternalTools(tools, toolSourceRefs, wor
             backendOperation: existing?.backendOperation ?? resourceTool.backendOperation,
             bundleRefs: existing?.bundleRefs ?? [],
             hitl: existing?.hitl ?? resourceTool.hitl,
+            retryable: existing?.retryable ?? resourceTool.retryable,
             sourcePath: existing?.sourcePath ?? resourceTool.toolPath,
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botbotgo/agent-harness",
-  "version": "0.0.45",
+  "version": "0.0.46",
   "description": "Workspace runtime for multi-agent applications",
   "type": "module",
   "packageManager": "npm@10.9.2",