@salesforce/sfdx-agent-harness-mastra 0.8.0 → 0.10.0

package/README.md

@@ -24,6 +24,24 @@ const manager = await createAgentManager('/path/to/storage', new MastraHarnessFa
 See the [`@salesforce/sfdx-agent-sdk` README](../sfdx-agent-sdk/README.md) for the full consumer-facing API. For
 internal architecture, see [ARCHITECTURE.md](ARCHITECTURE.md).
 
+### Removing the storage directory on Windows
+
+`manager.shutdown()` closes the libsql client (forcedotcom/agentic-dx#429), but the underlying SQLite/WAL handle release
+on Windows is asynchronous at the OS level. After `shutdown()` returns, Windows can take anywhere from ~60 ms to several
+seconds to actually release the file lock — so a naïve `fs.rm(storageRoot)` immediately afterward will fail with
+`EBUSY`. Pass `maxRetries` and `retryDelay` so Node retries until the OS lets go:
+
+```ts
+import { rm } from 'node:fs/promises';
+
+await manager.shutdown();
+await rm(storageRoot, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
+```
+
+Linux and macOS release the handle eagerly; the retry options are no-ops on those platforms. Tracking upstream at
+[mastra-ai/mastra#17285](https://github.com/mastra-ai/mastra/issues/17285) — when `LibSQLStore` exposes a public
+`close()` that includes a WAL checkpoint, this delay should shrink or disappear.
+
 ## Public API
 
 ### Core

package/dist/mastra-harness.d.ts

@@ -2,7 +2,7 @@ import { Agent } from '@mastra/core/agent';
 import { SkillSearchProcessor, ToolSearchProcessor } from '@mastra/core/processors';
 import type { ChunkType } from '@mastra/core/stream';
 import { MCPClient } from '@mastra/mcp';
-import { type AgentHarness, type WithAgentConfig, type ChatStreamResult, type HarnessAgentConfig, type MCPConfiguration, type McpServerInfo, type Message, type StreamOptions, type TelemetryEventCallback, type ToolResultInfo } from '@salesforce/sfdx-agent-sdk';
+import { McpServerStatus, type AgentHarness, type WithAgentConfig, type ChatStreamResult, type HarnessAgentConfig, type MCPConfiguration, type McpServerErrorDetail, type McpServerInfo, type Message, type StreamOptions, type TelemetryEventCallback, type ToolResultInfo } from '@salesforce/sfdx-agent-sdk';
 import type { MastraModelConfigFactory } from './model-config-factory.js';
 import { type MastraAgentConfig, type SkillSearchHandle, type ToolSearchHandle } from './tool-search.js';
 import type { LLMGatewayClient } from '@salesforce/llm-gateway-sdk';
@@ -20,10 +20,19 @@ import { Workspace } from '@mastra/core/workspace';
  * if the consumer breaks out of the event stream early (common in
  * approval UIs), we must finish reading the stream to ensure the
  * snapshot is persisted before we attempt to resume.
+ *
+ * `disposeAbortListener` is invoked whenever the entry is replaced or
+ * removed so the per-run abort listener (installed by `stream()` against
+ * the caller's `streamOptions.abortSignal`) cannot fire on a run that is
+ * no longer the active one for the thread. Mirrors the SDK invariant
+ * "every chat turn ends with exactly one terminal `FinishEvent`" by
+ * ensuring we don't accidentally abort a continuation run that
+ * `approveToolCall` / `declineToolCall` has already authorized.
  */
 interface ActiveRun {
     runId: string;
     fullStream: ReadableStream<ChunkType>;
+    disposeAbortListener?: () => void;
 }
 /**
  * Per-server MCP state. Each enabled server gets its own MCPClient and
@@ -37,8 +46,27 @@ interface ActiveRun {
  */
 interface McpServerState {
     client: MCPClient;
+    /**
+     * Last `McpServerStatus` value emitted (or seeded to `Connecting` at
+     * construction). Updated synchronously by `emitMcpStatusChange()` so a
+     * follow-up `getMcpServerInfo()` call observes the world the most recent
+     * `mcp-server-status-changed` event described.
+     */
+    currentStatus: McpServerStatus;
     epoch: number;
+    /**
+     * Sanitized human-readable error message. V8 stack frames and file paths
+     * have been stripped — safe for UI banners and telemetry tags. The full
+     * error (with `.stack`) is preserved on the `mcp-server-discovery-failed`
+     * event's `error` field.
+     */
     error?: string;
+    /**
+     * Structured failure projection. Mirrors the SDK's `McpServerErrorDetail`
+     * shape so consumers can route on category / code / retriable. Populated
+     * alongside `error`.
+     */
+    errorDetail?: McpServerErrorDetail;
     ready: Promise<void>;
     settled: boolean;
     tools: ToolsInput;
@@ -119,10 +147,18 @@ export declare class MastraHarness implements MastraAgentHarness {
      * would mix process state.
      */
     protected readonly activeAgents: Map<string, MastraAgentState>;
+    private readonly libsqlClient;
     private readonly storage;
     private readonly memory;
     private readonly mastra;
     private readonly buses;
+    /**
+     * Tracks the in-flight storage init so {@link shutdown} can settle it
+     * before closing the libsql client. Without this, Mastra's lazy storage
+     * init can fire after `client.close()` and surface as an unhandled
+     * `CLIENT_CLOSED` rejection (#429 follow-up).
+     */
+    private readonly storageReady;
     /**
      * Mastra-specific extension surface, typed off `MastraAgentHarness`. The
      * SDK never reads this; consumers reach through `manager.extensions.mastra`.
@@ -134,6 +170,9 @@ export declare class MastraHarness implements MastraAgentHarness {
      * - MUST disconnect the mcpClient for every active agent before clearing state.
      * - MUST clear the `activeAgents` map.
      * - MUST shut down the global `this.mastra` orchestrator.
+     * - MUST close the libsql client so the underlying SQLite handle is released
+     *   (Windows blocks fs.rm of the storage dir while the WAL/shm files are open;
+     *   see forcedotcom/agentic-dx#429).
      */
     shutdown(): Promise<void>;
     onTelemetry(callback: TelemetryEventCallback): Unsubscribe;
@@ -250,6 +289,10 @@ export declare class MastraHarness implements MastraAgentHarness {
      * - MUST return a `ChatStreamResult` with `textStream` and `eventStream`.
      * - On pre-stream rejection from `agent.stream()`, MUST re-throw without emitting telemetry —
      *   the SDK's chat-session boundary catches the rejection and emits `chat-stream-error`.
+     * - MUST install a per-run abort listener on `options.abortSignal` so that aborting
+     *   while a tool-approval is pending tears down the suspended Mastra run and clears
+     *   the thread's `ActiveRun` entry. See ARCHITECTURE.md → "Tool Approval Flow → Abort
+     *   during pending approval (#409)".
      */
     stream(agentId: string, threadId: string, message: string, options?: StreamOptions): Promise<ChatStreamResult>;
     /**
@@ -343,6 +386,25 @@ export declare class MastraHarness implements MastraAgentHarness {
      */
     private splitIntoChunks;
     private getAgentStateOrThrow;
+    /**
+     * Synchronously flips `serverState.currentStatus` (no-op when the next
+     * status equals the current) and returns a closure that, when invoked,
+     * emits the corresponding `mcp-server-status-changed` event. Returns
+     * `undefined` when the status didn't change — the caller has nothing
+     * to emit.
+     *
+     * Splitting the state flip from the emit lets callers update internal
+     * state synchronously alongside `settled` (so a synchronous listener
+     * reading `getMcpServerInfo()` from inside the discovery telemetry
+     * callback observes the post-settle world per issue #373) while still
+     * deferring the status-change emit until after the discovery telemetry
+     * has fired.
+     *
+     * The returned closure swallows `DISPOSED` errors the same way the
+     * discovery emit sites do: background discovery callbacks can resolve
+     * after `shutdown()` has disposed the buses.
+     */
+    private prepareMcpStatusChange;
     /**
      * Runs MCP discovery for one server: emits the
      * `mcp-server-discovery-started` / `…-completed` / `…-failed` telemetry
@@ -358,11 +420,43 @@ export declare class MastraHarness implements MastraAgentHarness {
      * Errors observed during discovery are recorded on the entry's `error`
      * field; the returned promise resolves either way (it only rejects when
      * an aborted operation throws after we have already cleaned up). The
-     * `settled` flag is pinned by `.finally()` so `getMcpServerInfo()` can
-     * tell `Connecting` from `Connected`/`Error`.
+     * `settled` flag is pinned BEFORE the terminal telemetry emit so a
+     * synchronous listener calling `getMcpServerInfo()` from inside the
+     * `mcp-server-discovery-{completed,failed}` callback observes the same
+     * world the event described — see issue #373.
      */
     private runMcpDiscovery;
     private toChatStreamResult;
+    /**
+     * Replace (or set) the active run for a thread, disposing any prior
+     * entry's per-run abort listener so the prior signal cannot fire on the
+     * new run.
+     */
+    private replaceActiveRun;
+    /**
+     * Remove the active run for a thread, disposing its per-run abort listener
+     * if present. Safe to call when no entry exists.
+     */
+    private removeActiveRun;
+    /**
+     * Wire `abortSignal.abort()` to tear down the suspended Mastra run for
+     * `threadId`. Returns a disposer that removes the listener; callers must
+     * invoke it whenever the entry is replaced or removed so a stale signal
+     * cannot abort an unrelated continuation run.
+     *
+     * Identity guard: the listener checks `agentState.activeRuns.get(threadId) === run`
+     * before calling `abortRunStream`, so a defensive race with another
+     * `stream()` call on the same thread cannot double-abort.
+     *
+     * Closes #409 — without this listener, aborting while a tool-approval is
+     * pending leaves Mastra's prepared run alive and `activeRuns[threadId]`
+     * pointing at a runId the consumer can no longer settle, so subsequent
+     * `approveToolCall` reaches Mastra and surfaces a confusing
+     * "could not find a suspended run" error instead of the SDK-level
+     * "no active run found" guard.
+     */
+    private installPerRunAbortListener;
+    private handlePerRunAbort;
     private getMemoryOrThrow;
     private agentResourceId;
     /**

package/dist/mastra-harness.js

@@ -10,6 +10,7 @@ import { Memory } from '@mastra/memory';
 import { SkillSearchProcessor, ToolSearchProcessor } from '@mastra/core/processors';
 import { MASTRA_RESOURCE_ID_KEY, MASTRA_THREAD_ID_KEY, RequestContext } from '@mastra/core/request-context';
 import { LibSQLStore } from '@mastra/libsql';
+import { createClient } from '@libsql/client';
 import { MCPClient } from '@mastra/mcp';
 import { AgentSDKError, AgentSDKErrorType, DEFAULT_MAX_STEPS, HarnessBusOwner, McpServerStatus, } from '@salesforce/sfdx-agent-sdk';
 import { MastraEventAdapter } from './mastra-event-adapter.js';
@@ -17,6 +18,7 @@ import { mapMastraToMessage, mapMessageToMastra, partToTranscriptText } from './
 import { isStandardSchemaWithJSON, standardSchemaToJSONSchema } from '@mastra/schema-compat';
 import { mapToolDefinitionsToMastraTools } from './mastra-tool-mapper.js';
 import { hasEnabledServers, mapToMastraMCPServers } from './mcp-config-mapper.js';
+import { buildMcpErrorDetail, sanitizeMcpErrorMessage } from './mcp-error-classifier.js';
 import { DefaultMastraModelConfigFactory } from './default-model-config-factory.js';
 import { partitionPool, } from './tool-search.js';
 import { buildSummaryPrompt, Clock, getErrorMessage, RealClock, splitFrontmatterAndBody, UUIDGenerator, } from '@salesforce/agentic-common';
@@ -311,10 +313,18 @@ export class MastraHarness {
      * would mix process state.
      */
     activeAgents = new Map();
+    libsqlClient;
     storage;
     memory;
     mastra;
     buses = new HarnessBusOwner();
+    /**
+     * Tracks the in-flight storage init so {@link shutdown} can settle it
+     * before closing the libsql client. Without this, Mastra's lazy storage
+     * init can fire after `client.close()` and surface as an unhandled
+     * `CLIENT_CLOSED` rejection (#429 follow-up).
+     */
+    storageReady;
     /**
      * Mastra-specific extension surface, typed off `MastraAgentHarness`. The
      * SDK never reads this; consumers reach through `manager.extensions.mastra`.
@@ -338,9 +348,24 @@ export class MastraHarness {
         this.modelConfigFactory = modelConfigFactory;
         this.clock = clock;
         this.idGenerator = idGenerator;
+        // Construct the libsql Client ourselves and inject it into LibSQLStore
+        // via the `{ client }` constructor form. We keep our own reference so
+        // shutdown() can call client.close() — Mastra's shutdown() does not
+        // close the storage, and LibSQLStore declares `client` as private, so
+        // we cannot reach in. Without an explicit close, the SQLite WAL/shm
+        // handles stay open on Windows and block fs.rm of the storage dir
+        // (forcedotcom/agentic-dx#429).
+        //
+        // Upstream tracking: https://github.com/mastra-ai/mastra/issues/17285
+        // — when LibSQLStore exposes a public close(), revert to the simpler
+        // `new LibSQLStore({ id, url })` form and call store.close() in
+        // shutdown().
+        this.libsqlClient = createClient({
+            url: `file:${this.storageRootFolder}/mastra.db`,
+        });
         this.storage = new LibSQLStore({
             id: 'mastra-storage',
-            url: `file:${this.storageRootFolder}/mastra.db`,
+            client: this.libsqlClient,
         });
         this.memory = new Memory({
             storage: this.storage,
@@ -357,15 +382,30 @@ export class MastraHarness {
             storage: this.storage,
             memory: { shared: this.memory },
         });
+        // Eagerly run storage init so Mastra's augment-with-init proxy caches
+        // the result. Without this, any lazy storage call deferred until after
+        // shutdown() (e.g. background tool/agent registration paths in Mastra)
+        // races our libsqlClient.close() and surfaces as an unhandled
+        // CLIENT_CLOSED rejection. We retain the promise so shutdown() can
+        // settle it before closing the client (a swallowed rejection here is
+        // fine — Mastra retains its own init proxy and surfaces real init
+        // errors at the next storage call site).
+        this.storageReady = this.storage.init().catch(() => undefined);
     }
     /**
      * @requirements
      * - MUST disconnect the mcpClient for every active agent before clearing state.
      * - MUST clear the `activeAgents` map.
      * - MUST shut down the global `this.mastra` orchestrator.
+     * - MUST close the libsql client so the underlying SQLite handle is released
+     *   (Windows blocks fs.rm of the storage dir while the WAL/shm files are open;
+     *   see forcedotcom/agentic-dx#429).
      */
     async shutdown() {
         await Promise.all([...this.activeAgents.values()].map(async (state) => {
+            for (const run of state.activeRuns.values())
+                run.disposeAbortListener?.();
+            state.activeRuns.clear();
             await Promise.allSettled(Array.from(state.mcpServers.values()).map(async (s) => {
                 await s.ready.catch(() => undefined);
                 await s.client.disconnect();
@@ -374,6 +414,29 @@ export class MastraHarness {
         }));
         this.activeAgents.clear();
         await this.mastra?.shutdown();
+        // Settle the constructor's eager init before closing — see the
+        // `storageReady` field comment for the why.
+        await this.storageReady;
+        // Fold the WAL into the main DB and switch journal_mode=DELETE
+        // so SQLite unlinks the -wal/-shm sidecars on close. Without this,
+        // Windows holds the sidecars open well past `client.close()` and
+        // racing `fs.rm` calls (e.g. vitest's per-test cleanup) crash the
+        // worker. Mirrors the fix Mastra shipped on `LibSQLStore.close()`
+        // in `@mastra/libsql@1.12.0-alpha.0`. Replace this block with a
+        // call to `storage.close()` once that release ships stable — see
+        // https://github.com/forcedotcom/agentic-dx/issues/475.
+        try {
+            await this.libsqlClient.execute('PRAGMA wal_checkpoint(TRUNCATE);');
+            await this.libsqlClient.execute('PRAGMA journal_mode=DELETE;');
+        }
+        catch (err) {
+            // Best-effort: still close even if checkpoint fails (e.g. the
+            // client was already closed by a prior shutdown path).
+            this.buses.getLogBus()?.warn('libsql WAL checkpoint failed before close', {
+                error: getErrorMessage(err),
+            });
+        }
+        this.libsqlClient.close();
         this.buses.dispose();
     }
     onTelemetry(callback) {
@@ -424,6 +487,7 @@ export class MastraHarness {
                 // after the synchronous `set` and `ready` assignment complete.
                 const entry = {
                     client,
+                    currentStatus: McpServerStatus.Connecting,
                     epoch: 0,
                     ready: Promise.resolve(),
                     settled: false,
@@ -556,6 +620,9 @@ export class MastraHarness {
      */
     async destroyAgent(agentId) {
         const state = this.getAgentStateOrThrow(agentId);
+        for (const run of state.activeRuns.values())
+            run.disposeAbortListener?.();
+        state.activeRuns.clear();
         await Promise.allSettled(Array.from(state.mcpServers.values()).map(async (s) => {
             await s.ready.catch(() => undefined);
             await s.client.disconnect();
@@ -576,19 +643,35 @@ export class MastraHarness {
         const state = this.getAgentStateOrThrow(agentId);
         const results = [];
         for (const [name, serverState] of state.mcpServers) {
-            if (!serverState.settled) {
-                results.push({ name, status: McpServerStatus.Connecting, tools: [] });
-            }
-            else if (serverState.error) {
-                results.push({ error: serverState.error, name, status: McpServerStatus.Error, tools: [] });
-            }
-            else {
+            // `currentStatus` is the source of truth: it tracks `Reconnecting`
+            // (transient state during a reconnect on a previously-Connected
+            // server) which the `settled`/`error` pair cannot represent.
+            // `settled` still gates whether tools are visible — Mastra's
+            // discovery has resolved at most once per epoch.
+            const status = serverState.currentStatus;
+            if (status === McpServerStatus.Connected && serverState.settled) {
                 results.push({
                     name,
                     status: McpServerStatus.Connected,
                     tools: mapMastraToolsToMcpToolInfo(serverState.tools, this.buses.getLogBus()),
                 });
             }
+            else if (status === McpServerStatus.Error && serverState.settled) {
+                const info = {
+                    error: serverState.error,
+                    name,
+                    status: McpServerStatus.Error,
+                    tools: [],
+                };
+                if (serverState.errorDetail)
+                    info.errorDetail = serverState.errorDetail;
+                results.push(info);
+            }
+            else {
+                // Connecting (initial), Reconnecting (mid-cycle), or any
+                // status whose discovery has not yet settled.
+                results.push({ name, status, tools: [] });
+            }
         }
         if (state.mcpConfig) {
             for (const [name, serverConfig] of Object.entries(state.mcpConfig)) {
@@ -627,6 +710,17 @@ export class MastraHarness {
         serverState.error = undefined;
         serverState.tools = {};
         serverState.settled = false;
+        // Status transition before transport work begins. A reconnect on a
+        // previously-`Connected` server lands in `Reconnecting` (transport is
+        // being cycled); reconnects from `Error` are a recovery path and use
+        // the regular `Connecting` placeholder so the (prev,next) pair on
+        // success reads `Connecting → Connected` like an initial discovery.
+        // Concurrent reconnects (already `Reconnecting`) coalesce via the
+        // emit-helper's same-status no-op.
+        const transientStatus = serverState.currentStatus === McpServerStatus.Connected
+            ? McpServerStatus.Reconnecting
+            : McpServerStatus.Connecting;
+        this.prepareMcpStatusChange(agentId, serverName, serverState, transientStatus)?.();
         // Emit `started` BEFORE awaiting the transport reconnect so subscribers
         // see started → (completed | failed) timestamps in chronological order.
         // The previous shape had `started` emitted from the catch block AFTER
@@ -650,9 +744,12 @@ export class MastraHarness {
             // Drop telemetry + state writes if a newer epoch has superseded us.
             if (state.mcpServers.get(serverName)?.epoch !== epoch)
                 return;
-            const message = err instanceof Error ? err.message : String(err);
+            const message = sanitizeMcpErrorMessage(err);
+            const errorDetail = buildMcpErrorDetail(err);
             serverState.error = message;
+            serverState.errorDetail = errorDetail;
             serverState.settled = true;
+            const emitStatusChange = this.prepareMcpStatusChange(agentId, serverName, serverState, McpServerStatus.Error, message);
             this.buses.getLogBus()?.warn('MCP server reconnect failed', {
                 serverName,
                 error: message,
@@ -665,12 +762,14 @@ export class MastraHarness {
                     serverName,
                     durationMs: this.clock.now().getTime() - startTime,
                     error: err instanceof Error ? err : new Error(message),
+                    errorDetail,
                 });
             }
             catch (emitErr) {
                 if (!isDisposedError(emitErr))
                     throw emitErr;
             }
+            emitStatusChange?.();
             return;
         }
         // Pass the existing startTime so runMcpDiscovery does NOT emit its own
@@ -703,7 +802,7 @@ export class MastraHarness {
      * - MUST remove any active run tracked for this thread to prevent stale references.
      */
     async destroyThread(agentId, threadId) {
-        this.getAgentStateOrThrow(agentId).activeRuns.delete(threadId);
+        this.removeActiveRun(this.getAgentStateOrThrow(agentId), threadId);
         const memory = await this.getMemoryOrThrow(agentId);
         await memory.deleteThread(threadId);
     }
@@ -830,6 +929,10 @@ export class MastraHarness {
      * - MUST return a `ChatStreamResult` with `textStream` and `eventStream`.
      * - On pre-stream rejection from `agent.stream()`, MUST re-throw without emitting telemetry —
      *   the SDK's chat-session boundary catches the rejection and emits `chat-stream-error`.
+     * - MUST install a per-run abort listener on `options.abortSignal` so that aborting
+     *   while a tool-approval is pending tears down the suspended Mastra run and clears
+     *   the thread's `ActiveRun` entry. See ARCHITECTURE.md → "Tool Approval Flow → Abort
+     *   during pending approval (#409)".
      */
     async stream(agentId, threadId, message, options) {
         const agentState = this.getAgentStateOrThrow(agentId);
@@ -843,7 +946,7 @@ export class MastraHarness {
             abortSignal: options?.abortSignal,
             requireToolApproval: options?.requireToolApproval,
         });
-        return this.toChatStreamResult(agentState, threadId, output);
+        return this.toChatStreamResult(agentState, threadId, output, options?.abortSignal);
     }
     // ── Tool Execution ───────────────────────────────────────────────
     /**
@@ -1080,6 +1183,47 @@ export class MastraHarness {
         }
         return state;
     }
+    /**
+     * Synchronously flips `serverState.currentStatus` (no-op when the next
+     * status equals the current) and returns a closure that, when invoked,
+     * emits the corresponding `mcp-server-status-changed` event. Returns
+     * `undefined` when the status didn't change — the caller has nothing
+     * to emit.
+     *
+     * Splitting the state flip from the emit lets callers update internal
+     * state synchronously alongside `settled` (so a synchronous listener
+     * reading `getMcpServerInfo()` from inside the discovery telemetry
+     * callback observes the post-settle world per issue #373) while still
+     * deferring the status-change emit until after the discovery telemetry
+     * has fired.
+     *
+     * The returned closure swallows `DISPOSED` errors the same way the
+     * discovery emit sites do: background discovery callbacks can resolve
+     * after `shutdown()` has disposed the buses.
+     */
+    prepareMcpStatusChange(agentId, serverName, serverState, nextStatus, error) {
+        const previousStatus = serverState.currentStatus;
+        if (previousStatus === nextStatus)
+            return undefined;
+        serverState.currentStatus = nextStatus;
+        return () => {
+            try {
+                this.buses.emitTelemetry({
+                    type: 'mcp-server-status-changed',
+                    timestamp: this.clock.now(),
+                    agentId,
+                    serverName,
+                    previousStatus,
+                    nextStatus,
+                    ...(error !== undefined ? { error } : {}),
+                });
+            }
+            catch (emitErr) {
+                if (!isDisposedError(emitErr))
+                    throw emitErr;
+            }
+        };
+    }
     /**
      * Runs MCP discovery for one server: emits the
      * `mcp-server-discovery-started` / `…-completed` / `…-failed` telemetry
@@ -1095,8 +1239,10 @@ export class MastraHarness {
      * Errors observed during discovery are recorded on the entry's `error`
      * field; the returned promise resolves either way (it only rejects when
      * an aborted operation throws after we have already cleaned up). The
-     * `settled` flag is pinned by `.finally()` so `getMcpServerInfo()` can
-     * tell `Connecting` from `Connected`/`Error`.
+     * `settled` flag is pinned BEFORE the terminal telemetry emit so a
+     * synchronous listener calling `getMcpServerInfo()` from inside the
+     * `mcp-server-discovery-{completed,failed}` callback observes the same
+     * world the event described — see issue #373.
      */
     runMcpDiscovery(agentId, serverName, client, serverStates, epoch, abortSignal, 
     // When set, reuse the caller's startTime and skip emitting `started` —
@@ -1135,7 +1281,13 @@ export class MastraHarness {
             const s = currentEntry();
             if (!s)
                 return; // superseded — drop write + telemetry
+            // Flip `settled` AND `currentStatus` BEFORE emitting any
+            // terminal telemetry so a synchronous listener calling
+            // getMcpServerInfo() from inside the `completed` callback
+            // observes the same world the event described (issue #373).
             s.tools = namespacedTools;
+            s.settled = true;
+            const emitStatusChange = this.prepareMcpStatusChange(agentId, serverName, s, McpServerStatus.Connected);
             // listToolsetsWithErrors() runs in the background and can resolve
             // after `shutdown()` has disposed the buses. Swallow only `DISPOSED`
             // so a real listener exception still surfaces.
@@ -1153,6 +1305,7 @@ export class MastraHarness {
                 if (!isDisposedError(emitErr))
                     throw emitErr;
             }
+            emitStatusChange?.();
         })
             .catch(async (err) => {
             if (abortSignal?.aborted) {
@@ -1164,10 +1317,18 @@ export class MastraHarness {
                 const s = currentEntry();
                 if (!s)
                     return; // superseded — drop write + telemetry
-                s.error = err instanceof Error ? err.message : String(err);
+                // Flip `settled` AND `currentStatus` BEFORE emitting any
+                // terminal telemetry — see the matching comment in the
+                // success path above (issue #373).
+                const errorMessage = sanitizeMcpErrorMessage(err);
+                const errorDetail = buildMcpErrorDetail(err);
+                s.error = errorMessage;
+                s.errorDetail = errorDetail;
+                s.settled = true;
+                const emitStatusChange = this.prepareMcpStatusChange(agentId, serverName, s, McpServerStatus.Error, errorMessage);
                 this.buses.getLogBus()?.warn('MCP server discovery failed', {
                     serverName,
-                    error: err instanceof Error ? err.message : String(err),
+                    error: errorMessage,
                 });
                 try {
                     this.buses.emitTelemetry({
@@ -1177,33 +1338,117 @@ export class MastraHarness {
                         serverName,
                         durationMs: this.clock.now().getTime() - startTime,
                         error: err instanceof Error ? err : new Error(String(err)),
+                        errorDetail,
                     });
                 }
                 catch (emitErr) {
                     if (!isDisposedError(emitErr))
                         throw emitErr;
                 }
+                emitStatusChange?.();
             }
-        })
-            .finally(() => {
-            const s = currentEntry();
-            if (s)
-                s.settled = true;
         });
     }
-    toChatStreamResult(agentState, threadId, output) {
+    toChatStreamResult(agentState, threadId, output, abortSignal) {
         // Entries are keyed by threadId so each thread holds at most one entry.
         // Stale entries are overwritten here on the next stream; explicit cleanup
         // happens via destroyThread() / destroyAgent() / shutdown(). Proactive
         // removal on stream completion is not safe because the suspend/resume
         // flow (submitToolResult, approveToolCall) needs the entry after the stream closes.
-        agentState.activeRuns.set(threadId, { runId: output.runId, fullStream: output.fullStream });
+        const newRun = { runId: output.runId, fullStream: output.fullStream };
+        if (abortSignal) {
+            newRun.disposeAbortListener = this.installPerRunAbortListener(agentState, threadId, newRun, abortSignal);
+        }
+        this.replaceActiveRun(agentState, threadId, newRun);
         const adapter = new MastraEventAdapter(this.buses.getLogBus());
         return {
-            eventStream: this.createEventStream(output.fullStream, adapter, agentState),
+            eventStream: this.createEventStream(output.fullStream, adapter, agentState, abortSignal),
             textStream: this.readableStreamToAsyncGenerator(output.textStream),
         };
     }
+    /**
+     * Replace (or set) the active run for a thread, disposing any prior
+     * entry's per-run abort listener so the prior signal cannot fire on the
+     * new run.
+     */
+    replaceActiveRun(agentState, threadId, run) {
+        const prior = agentState.activeRuns.get(threadId);
+        prior?.disposeAbortListener?.();
+        agentState.activeRuns.set(threadId, run);
+    }
+    /**
+     * Remove the active run for a thread, disposing its per-run abort listener
+     * if present. Safe to call when no entry exists.
+     */
+    removeActiveRun(agentState, threadId) {
+        const prior = agentState.activeRuns.get(threadId);
+        prior?.disposeAbortListener?.();
+        agentState.activeRuns.delete(threadId);
+    }
+    /**
+     * Wire `abortSignal.abort()` to tear down the suspended Mastra run for
+     * `threadId`. Returns a disposer that removes the listener; callers must
+     * invoke it whenever the entry is replaced or removed so a stale signal
+     * cannot abort an unrelated continuation run.
+     *
+     * Identity guard: the listener checks `agentState.activeRuns.get(threadId) === run`
+     * before calling `abortRunStream`, so a defensive race with another
+     * `stream()` call on the same thread cannot double-abort.
+     *
+     * Closes #409 — without this listener, aborting while a tool-approval is
+     * pending leaves Mastra's prepared run alive and `activeRuns[threadId]`
+     * pointing at a runId the consumer can no longer settle, so subsequent
+     * `approveToolCall` reaches Mastra and surfaces a confusing
+     * "could not find a suspended run" error instead of the SDK-level
+     * "no active run found" guard.
+     */
+    installPerRunAbortListener(agentState, threadId, run, abortSignal) {
+        if (abortSignal.aborted) {
+            // Listener attaches *after* agent.stream() has resolved — by which
+            // point Mastra has already observed the same signal and started
+            // its own abort flow. We still need to clear the activeRuns entry
+            // so the SDK's "no active run" guard fires on subsequent
+            // approve/decline calls.
+            queueMicrotask(() => this.handlePerRunAbort(agentState, threadId, run));
+            return () => {
+                /* nothing to remove */
+            };
+        }
+        const onAbort = () => this.handlePerRunAbort(agentState, threadId, run);
+        abortSignal.addEventListener('abort', onAbort, { once: true });
+        return () => abortSignal.removeEventListener('abort', onAbort);
+    }
+    handlePerRunAbort(agentState, threadId, run) {
+        // The body runs synchronously inside `controller.abort()` (via
+        // EventTarget.dispatchEvent), so a thrown error here would surface
+        // as a synchronous throw at the consumer's `controller.abort()`
+        // call site — wrapping the whole body in try/catch keeps that path
+        // contained even if `LogBus.warn` itself throws via a misbehaving
+        // subscriber.
+        try {
+            // Identity guard: only abort if this is still the active run
+            // for the thread. A subsequent stream() / approveToolCall()
+            // call may have replaced the entry; that path's own listener
+            // is responsible for its run.
+            if (agentState.activeRuns.get(threadId) !== run)
+                return;
+            agentState.activeRuns.delete(threadId);
+            agentState.agent.abortRunStream(run.runId);
+        }
+        catch (err) {
+            // Mastra's abortRunStream returns a boolean today; defend
+            // against a future shape change so a regression doesn't crash
+            // the consumer's abort dispatch.
+            try {
+                this.buses
+                    .getLogBus()
+                    ?.warn(`Failed to abort suspended run "${run.runId}" on thread "${threadId}": ${getErrorMessage(err)}`, { threadId, runId: run.runId }, err instanceof Error ? err : new Error(String(err)));
+            }
+            catch {
+                /* swallow — synchronous abort dispatch must not throw */
+            }
+        }
+    }
     async getMemoryOrThrow(agentId) {
         const agentState = this.getAgentStateOrThrow(agentId);
         const memory = await agentState.agent.getMemory();
@@ -1233,11 +1478,22 @@ export class MastraHarness {
         ctx.set(MASTRA_RESOURCE_ID_KEY, this.agentResourceId(agentId));
         return ctx;
     }
-    async *createEventStream(fullStream, adapter, agentState) {
+    async *createEventStream(fullStream, adapter, agentState, abortSignal) {
+        // Tracks whether the consumer has observed a terminal event already.
+        // Either `finish` (natural completion) or any `error` event counts —
+        // the SDK's `wrapEventStream` treats any error as terminal and
+        // appends its own synthetic finish, so synthesizing an abort
+        // terminal on top would duplicate the SDK invariant of one terminal
+        // per turn. The narrower "only abort-coded errors count" predicate
+        // would mis-fire on tripwire/regular errors that race the abort.
+        let sawTerminal = false;
         for await (const chunk of this.readableStreamToAsyncGenerator(fullStream)) {
             const event = adapter.toChatEvent(chunk);
             if (!event)
                 continue;
+            if (event.type === 'finish' || event.type === 'error') {
+                sawTerminal = true;
+            }
             if (isEnrichableToolEvent(event)) {
                 const toolName = event.type === 'tool-approval-request' ? event.toolCall.toolName : event.toolName;
                 const metadata = lookupMcpToolMetadata(agentState, toolName);
@@ -1252,6 +1508,29 @@ export class MastraHarness {
             }
             yield event;
         }
+        // The fullStream closed. If abort fired during a suspended-run path
+        // (consumer aborted while a tool-approval was pending), Mastra
+        // releases the prepared run silently — the fullStream just ends with
+        // no `abort` chunk — so synthesize the terminal pair the SDK
+        // invariant requires. See ARCHITECTURE.md → "Tool Approval Flow →
+        // Abort during pending approval (#409)".
+        //
+        // Adapter-coupling note: this gate depends on `MastraEventAdapter`
+        // not yielding a `finish` chunk between `tool-call-approval` and
+        // the consumer's resume. True today (the adapter only emits
+        // `finish` for the terminal Mastra `finish` chunk). A future
+        // Mastra version that injects a step-level `finish` mid-suspend
+        // would set `sawTerminal` early and silently suppress the synth.
+        // If that lands, switch the predicate to track only the terminal
+        // `finish` (e.g. via a per-stream "suspended" flag).
+        if (!sawTerminal && abortSignal?.aborted) {
+            yield {
+                type: 'error',
+                error: new DOMException('The stream was aborted.', 'AbortError'),
+                code: 'abort',
+            };
+            yield { type: 'finish', finishReason: 'error' };
+        }
     }
     async *readableStreamToAsyncGenerator(stream) {
         const reader = stream.getReader();

package/dist/mcp-error-classifier.d.ts

@@ -0,0 +1,34 @@
+import type { McpServerErrorCategory, McpServerErrorDetail } from '@salesforce/sfdx-agent-sdk';
+/**
+ * Strip V8 stack frames ("    at <frame>") and any framing that arrived inside
+ * the message string. The result is a single, human-readable summary line safe
+ * for UI banners, telemetry tags, and Sentry fingerprints. The full stack is
+ * still available on the underlying `Error` instance (`error.stack`) for
+ * debug-mode log surfaces.
+ *
+ * Inputs are loose on purpose: a string passes through, a non-Error object's
+ * `String(...)` form is sanitized the same way, and `undefined`/`null` round-trip
+ * through `String(...)`.
+ */
+export declare function sanitizeMcpErrorMessage(err: unknown): string;
+/**
+ * Map a caught error to the SDK's stable {@link McpServerErrorCategory}. The
+ * lookup is layered:
+ *
+ *  1. `DOMException` `AbortError` and Node's `code === 'ABORT_ERR'` map to
+ *     `'aborted'` regardless of the `.message`.
+ *  2. MCP-shaped errors (an `Error` carrying a numeric `code`, see
+ *     {@link isMcpErrorLike}) match by JSON-RPC 2.0 code (`-32001` etc.).
+ *  3. Free-form `.message` text is matched against substrings the underlying
+ *     transports surface (`401`, `403`, `5xx`, `ECONNREFUSED`, etc.). This is
+ *     intentionally a small lookup — categories are stable, message shapes
+ *     are not, so the heuristic stays narrow and falls back to `'unknown'`.
+ */
+export declare function classifyMcpError(err: unknown): McpServerErrorCategory;
+/**
+ * Build the SDK-side {@link McpServerErrorDetail} from a caught error. The
+ * underlying JSON-RPC `code` (when the error is MCP-shaped, see
+ * {@link isMcpErrorLike}) is preserved so consumers can route on the code
+ * without having to introspect `cause`.
+ */
+export declare function buildMcpErrorDetail(err: unknown): McpServerErrorDetail;

package/dist/mcp-error-classifier.js

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2026, Salesforce, Inc. All rights reserved.
+ * See LICENSE.txt for license terms.
+ */
+import { isAbortError } from '@salesforce/agentic-common';
+/**
+ * Structural duck-type for MCP-shaped errors. Matches the upstream
+ * `@modelcontextprotocol/sdk` `McpError` class (`extends Error`, numeric
+ * `code`) without an `instanceof` check. We avoid the import for two reasons:
+ *
+ * 1. **Dual-package hazard.** If `@mastra/mcp` ever pins
+ *    `@modelcontextprotocol/sdk` to a version that diverges from one this
+ *    package would pin, npm hoists two copies and `instanceof` returns false
+ *    across realms. Structural matching survives that case.
+ * 2. **JSON-RPC codes are the contract, not the class.** The numeric codes
+ *    (`-32001 RequestTimeout`, `-32603 InternalError`, …) are stable per the
+ *    JSON-RPC 2.0 spec. Any error with `code: number` originating from
+ *    MCP-shaped infrastructure will match — which is what consumers want.
+ *
+ * Node's `SystemError` (e.g. `ECONNREFUSED`) also has a `.code` field but as
+ * a `string`, so the `typeof === 'number'` check disambiguates.
+ */
+function isMcpErrorLike(err) {
+    return err instanceof Error && typeof err.code === 'number';
+}
+/**
+ * Strip V8 stack frames ("    at <frame>") and any framing that arrived inside
+ * the message string. The result is a single, human-readable summary line safe
+ * for UI banners, telemetry tags, and Sentry fingerprints. The full stack is
+ * still available on the underlying `Error` instance (`error.stack`) for
+ * debug-mode log surfaces.
+ *
+ * Inputs are loose on purpose: a string passes through, a non-Error object's
+ * `String(...)` form is sanitized the same way, and `undefined`/`null` round-trip
+ * through `String(...)`.
+ */
+export function sanitizeMcpErrorMessage(err) {
+    const raw = err instanceof Error ? err.message : String(err);
+    const lines = raw.split(/\r?\n/);
+    // Keep the first line that is NOT a V8 stack frame. V8 frames start with
+    // arbitrary leading whitespace followed by "at " (matched at line start).
+    for (const line of lines) {
+        if (!/^\s*at\s/.test(line)) {
+            return line.trim();
+        }
+    }
+    // All lines were stack frames — fall back to the raw single-line input.
+    return lines[0]?.trim() ?? '';
+}
+/**
+ * Map a caught error to the SDK's stable {@link McpServerErrorCategory}. The
+ * lookup is layered:
+ *
+ *  1. `DOMException` `AbortError` and Node's `code === 'ABORT_ERR'` map to
+ *     `'aborted'` regardless of the `.message`.
+ *  2. MCP-shaped errors (an `Error` carrying a numeric `code`, see
+ *     {@link isMcpErrorLike}) match by JSON-RPC 2.0 code (`-32001` etc.).
+ *  3. Free-form `.message` text is matched against substrings the underlying
+ *     transports surface (`401`, `403`, `5xx`, `ECONNREFUSED`, etc.). This is
+ *     intentionally a small lookup — categories are stable, message shapes
+ *     are not, so the heuristic stays narrow and falls back to `'unknown'`.
+ */
+export function classifyMcpError(err) {
+    if (isAbortError(err))
+        return 'aborted';
+    if (isMcpErrorLike(err)) {
+        switch (err.code) {
+            case -32001: // RequestTimeout
+                return 'connect-timeout';
+            case -32700: // ParseError
+            case -32600: // InvalidRequest
+            case -32601: // MethodNotFound
+            case -32602: // InvalidParams
+            case -32603: // InternalError
+                return 'protocol-error';
+            case -32000: // ConnectionClosed
+                return 'transport-eof';
+            default:
+                return 'unknown';
+        }
+    }
+    const message = err instanceof Error ? err.message : typeof err === 'string' ? err : '';
+    if (/\b401\b/.test(message))
+        return 'http-401';
+    if (/\b403\b/.test(message))
+        return 'http-403';
+    if (/\b4\d\d\b/.test(message))
+        return 'http-4xx';
+    if (/\b5\d\d\b/.test(message))
+        return 'http-5xx';
+    if (/ECONNREFUSED|ECONNRESET|EPIPE|socket hang up/i.test(message))
+        return 'transport-eof';
+    if (/timed out|timeout/i.test(message))
+        return 'connect-timeout';
+    return 'unknown';
+}
+/**
+ * Whether the SDK considers the failure transient (worth a retry via
+ * `Agent.reconnectMcpServer` / `Agent.refreshMcpAuth`). Aborted operations and
+ * permanent protocol violations are non-retriable; everything else is retriable
+ * by default — failing-open here is correct because the consumer always has
+ * the option to ignore the hint.
+ */
+function isCategoryRetriable(category) {
+    switch (category) {
+        case 'aborted':
+        case 'protocol-error':
+        case 'config-error':
+            return false;
+        default:
+            return true;
+    }
+}
+/**
+ * Build the SDK-side {@link McpServerErrorDetail} from a caught error. The
+ * underlying JSON-RPC `code` (when the error is MCP-shaped, see
+ * {@link isMcpErrorLike}) is preserved so consumers can route on the code
+ * without having to introspect `cause`.
+ */
+export function buildMcpErrorDetail(err) {
+    const category = classifyMcpError(err);
+    const detail = {
+        category,
+        retriable: isCategoryRetriable(category),
+    };
+    if (isMcpErrorLike(err)) {
+        detail.code = err.code;
+    }
+    return detail;
+}
+//# sourceMappingURL=mcp-error-classifier.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@salesforce/sfdx-agent-harness-mastra",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "description": "Mastra-backed AgentHarness implementation for @salesforce/sfdx-agent-sdk",
   "type": "module",
   "main": "dist/index.js",
@@ -28,20 +28,21 @@
   ],
   "dependencies": {
     "@ai-sdk/provider": "^3.0.10",
+    "@libsql/client": "^0.15.15",
     "@mastra/core": "^1.36.0",
     "@mastra/libsql": "^1.11.1",
     "@mastra/mcp": "^1.8.0",
     "@mastra/memory": "^1.19.0",
     "@mastra/schema-compat": "^1.2.10",
-    "@salesforce/agentic-common": "0.5.0",
-    "@salesforce/llm-gateway-sdk": "0.6.0"
+    "@salesforce/agentic-common": "0.6.0",
+    "@salesforce/llm-gateway-sdk": "0.8.0"
   },
   "peerDependencies": {
-    "@salesforce/sfdx-agent-sdk": "0.9.0"
+    "@salesforce/sfdx-agent-sdk": "0.11.0"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
-    "@salesforce/sfdx-agent-sdk": "0.9.0",
+    "@salesforce/sfdx-agent-sdk": "0.11.0",
     "@types/node": "^22.19.17",
     "@vitest/coverage-istanbul": "^4.1.7",
     "@vitest/eslint-plugin": "^1.6.17",