@gaunt-sloth/agent 2.0.0-alpha.23 → 2.0.0-alpha.24

package/dist/middleware/frontendImageInjectionMiddleware.js

@@ -0,0 +1,146 @@
+/**
+ * @packageDocumentation
+ * Opt-in middleware that turns a frontend "capture image" tool result into a vision message the
+ * model can actually see.
+ *
+ * A browser/frontend tool such as CopilotKit's `capture_image` fulfils client-side and posts its
+ * result back to gth's AG-UI server as a trailing `tool`-role message whose content is a JSON
+ * string `{"mimeType":"image/...","data":"<base64>"}`. gth hands that to the model as a plain-string
+ * ToolMessage — no vision block — so the model literally cannot see the photo. This middleware
+ * detects that ToolMessage in `beforeModel`, parses the envelope, and injects a `HumanMessage`
+ * carrying a provider-appropriate vision block before the next model call.
+ *
+ * Strictly opt-in: it fires ONLY when referenced by name in `config.middleware`
+ * (`"middleware": ["frontend-image-injection"]`), never auto-injected. Promoted from
+ * pukeko-robot-controller's proven middleware (RC-21), minus the robot-specific motion-tool coupling.
+ */
+import { createMiddleware } from 'langchain';
+import { HumanMessage, isToolMessage } from '@langchain/core/messages';
+/** Default frontend capture tool name. Overridable via the `toolName` middleware setting. */
+export const DEFAULT_CAPTURE_TOOL_NAME = 'capture_image';
+/**
+ * A vision content block the target provider's `@langchain` converter actually decodes. Verified
+ * against the installed converters (RC-21):
+ *   - **ollama** → `{ type:'image_url', image_url:'<data-URL string>' }`. ChatOllama's
+ *     `convertToOllamaMessages` only handles `image_url` blocks (extractBase64FromDataUrl); the
+ *     LangChain standard `source_type` block throws "Unsupported content type: image".
+ *   - **OpenAI-compatible** (`openai`, `openrouter`, `deepseek`, `xai`, `groq` — all extend
+ *     `ChatOpenAI` / report `_llmType()==='openai'`) → `{ type:'image_url', image_url:{ url:'<data-URL>' } }`.
+ *     This native OpenAI shape is correct on BOTH the Completions API AND the Responses API (GS2-74
+ *     flips reasoning-capable openai models to Responses). A raw `source_type` standard block
+ *     serialises to an *invalid* image part on the Responses path, so we emit the provider-native
+ *     shape rather than lean on `@langchain/core`'s (deprecated, internal) auto-conversion.
+ *   - **anthropic / google-genai / vertexai** (and any unknown/default) → the LangChain standard
+ *     base64 data content block `{ type:'image', source_type:'base64', mime_type, data }`, which
+ *     those native converters decode directly and which is the most broadly decodable fallback.
+ *
+ * Pure and exported so each provider branch can be unit-tested directly.
+ */
+export function imageBlockFor(provider, mimeType, data) {
+    const dataUrl = `data:${mimeType};base64,${data}`;
+    switch (provider) {
+        case 'ollama':
+            return { type: 'image_url', image_url: dataUrl };
+        case 'openai':
+        case 'openrouter':
+        case 'deepseek':
+        case 'xai':
+        case 'groq':
+            return { type: 'image_url', image_url: { url: dataUrl } };
+        case 'anthropic':
+        case 'google-genai':
+        case 'vertexai':
+        default:
+            return {
+                type: 'image',
+                source_type: 'base64',
+                mime_type: mimeType,
+                data,
+            };
+    }
+}
+/**
+ * Create the frontend-image-injection middleware.
+ *
+ * @param opts.provider - provider string selecting the vision-block shape (see {@link imageBlockFor}).
+ * @param opts.toolName - capture tool name (default {@link DEFAULT_CAPTURE_TOOL_NAME}).
+ */
+export function createFrontendImageInjectionMiddleware(opts) {
+    const toolName = opts.toolName ?? DEFAULT_CAPTURE_TOOL_NAME;
+    // thread_id → set of tool_call_ids whose image (or error note) has already been injected. Without
+    // this, a summarizer/replayed history that retains the capture ToolMessage would re-inject its
+    // frame on the next turn, appended after the newest turn's content and mispairing the assistant
+    // message with a stale frame (the RC-21 idempotency guard).
+    //
+    // Closure-scoped (one Map per middleware instance), NOT a module global as the robot had it. The
+    // AG-UI server caches one agent per client-toolset signature (getAgentForTools), so a thread's
+    // turns share this instance and idempotency-across-turns is preserved — while unrelated agents (and
+    // tests) get independent state instead of a process-lifetime Map that accumulates every thread_id.
+    const injectedByThread = new Map();
+    return createMiddleware({
+        name: 'frontend-image-injection',
+        beforeModel: async (state, runtime) => {
+            const messages = state.messages || [];
+            // The AG-UI server sets runConfig.configurable.thread_id (= the run's threadId) and threads it
+            // to the graph, so per-session idempotency keys correctly. '__default__' is only reached off
+            // that path (e.g. a bare invoke with no thread_id), which still behaves correctly per-instance.
+            const threadId = runtime?.configurable?.thread_id ?? '__default__';
+            let injectedIds = injectedByThread.get(threadId);
+            if (!injectedIds) {
+                injectedIds = new Set();
+                injectedByThread.set(threadId, injectedIds);
+            }
+            // Scan forward so injected frames stay in chronological order; skip any tool_call_id already
+            // injected on this thread (idempotent across a retained/replayed tail).
+            const injected = [];
+            for (let i = 0; i < messages.length; i++) {
+                const msg = messages[i];
+                if (
+                // RC-21 (golden fix): use the duck-typed `isToolMessage`, NEVER `msg instanceof
+                // ToolMessage`. A consumer (galvanized/robot) importing this middleware across a `file:`-dep
+                // boundary resolves a SECOND @langchain/core copy, and a capture ToolMessage constructed by
+                // the AG-UI pipeline's core copy is not an instance of the `ToolMessage` class we import —
+                // `instanceof` silently returns false across copies and no frame is ever injected.
+                isToolMessage(msg) &&
+                    typeof msg.content === 'string' &&
+                    msg.name === toolName) {
+                    const id = msg.tool_call_id;
+                    if (!id || injectedIds.has(id))
+                        continue;
+                    try {
+                        injected.push({ payload: JSON.parse(msg.content), id });
+                    }
+                    catch {
+                        // Non-JSON tool result — skip injection (leave the guard clean).
+                    }
+                }
+            }
+            if (injected.length === 0)
+                return undefined;
+            const newMessages = [...messages];
+            for (const { payload, id } of injected) {
+                if (payload.error) {
+                    // Mark injected so the error note isn't re-emitted on a later turn.
+                    injectedIds.add(id);
+                    newMessages.push(new HumanMessage({ content: `Camera unavailable: ${payload.error}` }));
+                    continue;
+                }
+                if (payload.mimeType && payload.data) {
+                    // RC-21: mark injected ONLY on a successful frame emission. A capture result that arrived
+                    // WITHOUT its base64 `data` (dropped upstream) injects nothing and must NOT poison the
+                    // guard — a later data-bearing result for the same tool_call_id can still recover.
+                    injectedIds.add(id);
+                    const block = imageBlockFor(opts.provider, payload.mimeType, payload.data);
+                    newMessages.push(new HumanMessage({
+                        content: [{ type: 'text', text: 'Camera frame captured:' }, block],
+                    }));
+                }
+                // else: a capture result whose `data` is absent — inject nothing, leave the guard clean.
+            }
+            // If every candidate was data-less (nothing appended), return no state update rather than an
+            // identical copy — keeps beforeModel a true no-op and leaves the guard clean for recovery.
+            return newMessages.length > messages.length ? { messages: newMessages } : undefined;
+        },
+    });
+}
+//# sourceMappingURL=frontendImageInjectionMiddleware.js.map

package/dist/middleware/frontendImageInjectionMiddleware.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontendImageInjectionMiddleware.js","sourceRoot":"","sources":["../../src/middleware/frontendImageInjectionMiddleware.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,gBAAgB,EAAwB,MAAM,WAAW,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAavE,6FAA6F;AAC7F,MAAM,CAAC,MAAM,yBAAyB,GAAG,eAAe,CAAC;AAkBzD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB,EAAE,QAAgB,EAAE,IAAY;IAC5E,MAAM,OAAO,GAAG,QAAQ,QAAQ,WAAW,IAAI,EAAE,CAAC;IAClD,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO,EAAE,IAAI,EAAE,WAAoB,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;QAC5D,KAAK,QAAQ,CAAC;QACd,KAAK,YAAY,CAAC;QAClB,KAAK,UAAU,CAAC;QAChB,KAAK,KAAK,CAAC;QACX,KAAK,MAAM;YACT,OAAO,EAAE,IAAI,EAAE,WAAoB,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC;QACrE,KAAK,WAAW,CAAC;QACjB,KAAK,cAAc,CAAC;QACpB,KAAK,UAAU,CAAC;QAChB;YACE,OAAO;gBACL,IAAI,EAAE,OAAgB;gBACtB,WAAW,EAAE,QAAiB;gBAC9B,SAAS,EAAE,QAAQ;gBACnB,IAAI;aACL,CAAC;IACN,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sCAAsC,CACpD,IAAmC;IAEnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,yBAAyB,CAAC;IAE5D,kGAAkG;IAClG,+FAA+F;IAC/F,gGAAgG;IAChG,4DAA4D;IAC5D,EAAE;IACF,iGAAiG;IACjG,+FAA+F;IAC/F,oGAAoG;IACpG,mGAAmG;IACnG,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAuB,CAAC;IAExD,OAAO,gBAAgB,CAAC;QACtB,IAAI,EAAE,0BAA0B;QAEhC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;YACtC,+FAA+F;YAC/F,6FAA6F;YAC7F,gGAAgG;YAChG,MAAM,QAAQ,GAAG,OAAO,EAAE,YAAY,EAAE,SAAS,IAAI,aAAa,CAAC;YACnE,IAAI,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;gBAChC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;YAC9C,CAAC;YAED,6FAA6F;YAC7F,wEAAwE;YACxE,MAAM,QAAQ,GAAiD,EAAE,CAAC;YAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACxB;gBACE,gFAAgF;gBAChF,6FAA6F;gBAC7F,4FAA4F;gBAC5F,2FAA2F;gBAC3F,mFAAmF;gBACnF,aAAa,CAAC,GAAG,CAAC;oBAClB,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;oBAC/B,GAAG,CAAC,IAAI,KAAK,QAAQ,EACrB,CAAC;oBACD,MAAM,EAAE,GAAG,GAAG,CAAC,YAAY,CAAC;oBAC5B,IAAI,CAAC,EAAE,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;wBAAE,SAAS;oBACzC,IAAI,CAAC;wBACH,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;oBAC1E,CAAC;oBAAC,MAAM,CAAC;wBACP,iEAAiE;oBACnE,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAE5C,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;YAClC,KAAK,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,QAAQ,EAAE,CAAC;gBACvC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAClB,oEAAoE;oBACpE,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACpB,WAAW,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,uBAAuB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;oBACxF,SAAS;gBACX,CAAC;gBACD,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;oBACrC,0FAA0F;oBAC1F,uFAAuF;oBACvF,mFAAmF;oBACnF,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACpB,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC3E,WAAW,CAAC,IAAI,CACd,IAAI,YAAY,CAAC;wBACf,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,EAAE,EAAE,KAAK,CAAmB;qBACrF,CAAC,CACH,CAAC;gBACJ,CAAC;gBACD,yFAAyF;YAC3F,CAAC;YAED,6FAA6F;YAC7F,2FAA2F;YAC3F,OAAO,WAAW,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACtF,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/middleware/registry.js

@@ -9,6 +9,28 @@ import { displayWarning } from '@gaunt-sloth/core/utils/consoleUtils.js';
 import { debugLog } from '@gaunt-sloth/core/utils/debugUtils.js';
 import { anthropicPromptCachingMiddleware, summarizationMiddleware, } from 'langchain';
 import { createBinaryContentInjectionMiddleware, } from '#src/middleware/binaryContentInjectionMiddleware.js';
+import { createFrontendImageInjectionMiddleware } from '#src/middleware/frontendImageInjectionMiddleware.js';
+/**
+ * Derive the provider string the vision middleware maps to a per-provider block shape. Prefers the
+ * loader-stashed raw `llm.type` ({@link GthConfig.modelProviderType}) — the exact gth provider
+ * namespace (`anthropic`/`openrouter`/`deepseek`/`xai`/`groq`/`ollama`/`google-genai`/`vertexai`/…)
+ * — and falls back to the live model's `_llmType()` only when it is absent (module configs). The
+ * OpenAI-compatible shims (openrouter/deepseek/xai/groq) all report `_llmType() === 'openai'`, which
+ * maps to the same `image_url:{url}` block, so the fallback stays shape-correct.
+ */
+function resolveVisionProvider(gthConfig) {
+    if (gthConfig.modelProviderType)
+        return gthConfig.modelProviderType;
+    const llm = gthConfig.llm;
+    try {
+        if (typeof llm?._llmType === 'function')
+            return llm._llmType();
+    }
+    catch {
+        // A misbehaving _llmType must not break middleware resolution.
+    }
+    return '';
+}
 const predefinedMiddlewareFactories = {
     /**
      * Anthropic prompt caching middleware. see https://docs.langchain.com/oss/javascript/langchain/middleware#anthropic-prompt-caching
@@ -24,7 +46,20 @@ const predefinedMiddlewareFactories = {
      * as HumanMessage content blocks before the next model call.
      * This works around LangChain's limitation where ToolMessage doesn't support binary content.
      */
-    'binary-content-injection': (settings, gthConfig) => createBinaryContentInjectionMiddleware(settings, gthConfig),
+    'binary-content-injection': (settings, gthConfig) => createBinaryContentInjectionMiddleware({
+        ...settings,
+        provider: resolveVisionProvider(gthConfig),
+    }, gthConfig),
+    /**
+     * Frontend image injection middleware (RC-22). Converts a frontend capture tool's
+     * `{mimeType,data}` ToolMessage (default tool name `capture_image`) into a provider-appropriate
+     * vision HumanMessage the model can see. Strictly opt-in — resolved ONLY when named in
+     * `config.middleware` (no auto-inject branch). Optional `toolName` setting overrides the tool name.
+     */
+    'frontend-image-injection': (settings, gthConfig) => Promise.resolve(createFrontendImageInjectionMiddleware({
+        provider: resolveVisionProvider(gthConfig),
+        toolName: typeof settings.toolName === 'string' ? settings.toolName : undefined,
+    })),
 };
 function isPredefinedMiddlewareName(name) {
     return name in predefinedMiddlewareFactories;

package/dist/middleware/registry.js.map

@@ -1 +1 @@
-{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/middleware/registry.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AASH,OAAO,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,uCAAuC,CAAC;AACjE,OAAO,EACL,gCAAgC,EAChC,uBAAuB,GAExB,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,sCAAsC,GAEvC,MAAM,qDAAqD,CAAC;AAO7D,MAAM,6BAA6B,GAAG;IACpC;;OAEG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,sCAAsC,CAAC,QAAwC,EAAE,SAAS,CAAC;IAC7F;;OAEG;IACH,aAAa,EAAE,CACb,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,6BAA6B,CAAC,QAA+B,EAAE,SAAS,CAAC;IAC3E;;;;;OAKG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,sCAAsC,CACpC,QAAoD,EACpD,SAAS,CACV;CACkD,CAAC;AAExD,SAAS,0BAA0B,CACjC,IAAY;IAEZ,OAAO,IAAI,IAAI,6BAA6B,CAAC;AAC/C,CAAC;AAED,SAAS,4BAA4B,CACnC,MAAwB;IAExB,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,MAAM,IAAI,MAAM;QAChB,OAAQ,MAA4B,CAAC,IAAI,KAAK,QAAQ;QACtD,0BAA0B,CAAE,MAA2B,CAAC,IAAI,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sCAAsC,CAC1D,MAAoC,EACpC,CAAY;IAEZ,QAAQ,CAAC,0DAA0D,MAAM,CAAC,GAAG,IAAI,SAAS,EAAE,CAAC,CAAC;IAE9F,0CAA0C;IAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,gCAAgC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAChF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,MAA2B,EAC3B,SAAoB;IAEpB,QAAQ,CAAC,mCAAmC,CAAC,CAAC;IAE9C,OAAO,OAAO,CAAC,OAAO,CACpB,uBAAuB,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,GAAG;QACpC,GAAG,MAAM;KACV,CAAC,CACH,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAuC,EACvC,SAAoB;IAEpB,MAAM,gBAAgB,GAAG,OAAO,IAAI,EAAE,CAAC;IACvC,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,iFAAiF;IACjF,wDAAwD;IACxD,MAAM,gBAAgB,GACpB,SAAS,CAAC,aAAa,KAAK,SAAS;QACrC,SAAS,CAAC,aAAa,KAAK,KAAK;QACjC,SAAS,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;IACrC,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,IAAI,CAC/C,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,KAAK,0BAA0B;QAChC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,0BAA0B,CAAC,CAClF,CAAC;IAEF,IAAI,gBAAgB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7C,QAAQ,CAAC,+EAA+E,CAAC,CAAC;QAC1F,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,0BAA0B,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,oEAAoE;YACpE,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;YAC3E,CAAC;YACD,oDAAoD;iBAC/C,IAAI,4BAA4B,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,MAAM,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;gBACrC,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YAC/E,CAAC;YACD,mDAAmD;iBAC9C,CAAC;gBACJ,QAAQ,CAAC,0BAA0B,CAAC,CAAC;gBACrC,UAAU,CAAC,IAAI,CAAC,MAAyB,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CACZ,gCAAgC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACzF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,0BAA0B,CACvC,IAAY,EACZ,QAAiC,EACjC,SAAoB;IAEpB,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;IACpD,OAAO,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AACtC,CAAC"}
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/middleware/registry.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AASH,OAAO,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,uCAAuC,CAAC;AACjE,OAAO,EACL,gCAAgC,EAChC,uBAAuB,GAExB,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,sCAAsC,GAEvC,MAAM,qDAAqD,CAAC;AAC7D,OAAO,EAAE,sCAAsC,EAAE,MAAM,qDAAqD,CAAC;AAO7G;;;;;;;GAOG;AACH,SAAS,qBAAqB,CAAC,SAAoB;IACjD,IAAI,SAAS,CAAC,iBAAiB;QAAE,OAAO,SAAS,CAAC,iBAAiB,CAAC;IACpE,MAAM,GAAG,GAAG,SAAS,CAAC,GAA8C,CAAC;IACrE,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,EAAE,QAAQ,KAAK,UAAU;YAAE,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,+DAA+D;IACjE,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,6BAA6B,GAAG;IACpC;;OAEG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,sCAAsC,CAAC,QAAwC,EAAE,SAAS,CAAC;IAC7F;;OAEG;IACH,aAAa,EAAE,CACb,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,6BAA6B,CAAC,QAA+B,EAAE,SAAS,CAAC;IAC3E;;;;;OAKG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,sCAAsC,CACpC;QACE,GAAI,QAAqD;QACzD,QAAQ,EAAE,qBAAqB,CAAC,SAAS,CAAC;KAC3C,EACD,SAAS,CACV;IACH;;;;;OAKG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,OAAO,CAAC,OAAO,CACb,sCAAsC,CAAC;QACrC,QAAQ,EAAE,qBAAqB,CAAC,SAAS,CAAC;QAC1C,QAAQ,EAAE,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;KAChF,CAAC,CACH;CACkD,CAAC;AAExD,SAAS,0BAA0B,CACjC,IAAY;IAEZ,OAAO,IAAI,IAAI,6BAA6B,CAAC;AAC/C,CAAC;AAED,SAAS,4BAA4B,CACnC,MAAwB;IAExB,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,MAAM,IAAI,MAAM;QAChB,OAAQ,MAA4B,CAAC,IAAI,KAAK,QAAQ;QACtD,0BAA0B,CAAE,MAA2B,CAAC,IAAI,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sCAAsC,CAC1D,MAAoC,EACpC,CAAY;IAEZ,QAAQ,CAAC,0DAA0D,MAAM,CAAC,GAAG,IAAI,SAAS,EAAE,CAAC,CAAC;IAE9F,0CAA0C;IAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,gCAAgC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAChF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,MAA2B,EAC3B,SAAoB;IAEpB,QAAQ,CAAC,mCAAmC,CAAC,CAAC;IAE9C,OAAO,OAAO,CAAC,OAAO,CACpB,uBAAuB,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,GAAG;QACpC,GAAG,MAAM;KACV,CAAC,CACH,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAuC,EACvC,SAAoB;IAEpB,MAAM,gBAAgB,GAAG,OAAO,IAAI,EAAE,CAAC;IACvC,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,iFAAiF;IACjF,wDAAwD;IACxD,MAAM,gBAAgB,GACpB,SAAS,CAAC,aAAa,KAAK,SAAS;QACrC,SAAS,CAAC,aAAa,KAAK,KAAK;QACjC,SAAS,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;IACrC,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,IAAI,CAC/C,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,KAAK,0BAA0B;QAChC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,0BAA0B,CAAC,CAClF,CAAC;IAEF,IAAI,gBAAgB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7C,QAAQ,CAAC,+EAA+E,CAAC,CAAC;QAC1F,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,0BAA0B,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,oEAAoE;YACpE,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;YAC3E,CAAC;YACD,oDAAoD;iBAC/C,IAAI,4BAA4B,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,MAAM,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;gBACrC,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YAC/E,CAAC;YACD,mDAAmD;iBAC9C,CAAC;gBACJ,QAAQ,CAAC,0BAA0B,CAAC,CAAC;gBACrC,UAAU,CAAC,IAAI,CAAC,MAAyB,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CACZ,gCAAgC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACzF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,0BAA0B,CACvC,IAAY,EACZ,QAAiC,EACjC,SAAoB;IAEpB,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;IACpD,OAAO,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AACtC,CAAC"}

package/dist/middleware/types.d.ts

@@ -10,7 +10,7 @@ import { AgentMiddleware } from 'langchain';
 /**
  * Predefined middleware types that can be configured via JSON config.
  */
-export type PredefinedMiddlewareName = 'anthropic-prompt-caching' | 'summarization' | 'binary-content-injection';
+export type PredefinedMiddlewareName = 'anthropic-prompt-caching' | 'summarization' | 'binary-content-injection' | 'frontend-image-injection';
 /**
  * Configuration for Anthropic prompt caching middleware.
  */
@@ -70,6 +70,18 @@ export interface ImageFormatTransformConfig {
  */
 export interface BinaryContentInjectionConfig {
 }
+/**
+ * Configuration for frontend image injection middleware (RC-22).
+ * Converts a frontend capture tool's `{mimeType,data}` tool result into a vision HumanMessage the
+ * model can see. Opt-in — resolved only when named in `config.middleware`.
+ */
+export interface FrontendImageInjectionConfig {
+    /**
+     * Name of the frontend capture tool whose result is converted into a vision message.
+     * Defaults to `capture_image`.
+     */
+    toolName?: string;
+}
 /**
  * Union type of all predefined middleware configurations.
  */
@@ -79,7 +91,9 @@ export type PredefinedMiddlewareConfig = ({
     name: 'summarization';
 } & SummarizationConfig) | ({
     name: 'binary-content-injection';
-} & BinaryContentInjectionConfig);
+} & BinaryContentInjectionConfig) | ({
+    name: 'frontend-image-injection';
+} & FrontendImageInjectionConfig);
 /**
  * Middleware configuration that can be specified in JSON or JS config.
  * - String: Name of predefined middleware with default settings

package/dist/modules/a2a/A2AClientWrapper.d.ts

@@ -31,7 +31,7 @@ export interface A2ASendContext {
  * @experimental
  */
 export declare class A2AClientWrapper {
-    private client;
+    private clientPromise;
     private config;
     constructor(config: A2AClientConfig);
     /**

package/dist/modules/a2a/A2AClientWrapper.js

@@ -6,17 +6,29 @@
 import { A2AClient } from '@a2a-js/sdk/client';
 import { debugLog, debugLogError } from '@gaunt-sloth/core/utils/debugUtils.js';
 import { v4 as uuidv4 } from 'uuid';
+/** The A2A spec's well-known agent-card path. `A2AClient.fromCardUrl` needs the full CARD url,
+ * whereas {@link A2AClientConfig.agentUrl} is the agent's BASE url — so we append this, exactly as
+ * the deprecated `new A2AClient(url)` string constructor did internally (its `resolveAgentCardUrl`
+ * used the same default). Keeps the fetched card URL byte-identical across the migration. */
+const AGENT_CARD_PATH = '.well-known/agent-card.json';
 /**
  * Wrapper around the A2A SDK client for communicating with external agents.
  * @experimental
  */
 export class A2AClientWrapper {
-    client;
+    clientPromise;
     config;
     constructor(config) {
         this.config = config;
-        // Initialize with agent URL. The SDK will fetch the agent card.
-        this.client = new A2AClient(config.agentUrl);
+        // Construct via the non-deprecated `A2AClient.fromCardUrl(cardUrl)` (the string `new
+        // A2AClient(url)` constructor is deprecated and console.warns). `fromCardUrl` is async, so we
+        // hold the promise and await it per send. The agent card is fetched from the well-known path
+        // appended to the base agent URL — reproducing the old constructor's fetch target exactly.
+        const cardUrl = `${config.agentUrl.replace(/\/+$/, '')}/${AGENT_CARD_PATH}`;
+        this.clientPromise = A2AClient.fromCardUrl(cardUrl);
+        // Guard against an unhandled rejection if the wrapper is constructed but never used; real
+        // callers still observe the rejection when they await `clientPromise` inside a send method.
+        void this.clientPromise.catch(() => undefined);
     }
     /**
      * Sends a message to the A2A agent and returns the response.
@@ -26,8 +38,9 @@ export class A2AClientWrapper {
     async sendMessage(messageText) {
         debugLog(`Sending message to A2A agent ${this.config.agentId} at ${this.config.agentUrl}: ${messageText}`);
         try {
+            const client = await this.clientPromise;
             const messageId = uuidv4();
-            const response = await this.client.sendMessage({
+            const response = await client.sendMessage({
                 message: {
                     kind: 'message',
                     messageId: messageId,
@@ -82,7 +95,8 @@ export class A2AClientWrapper {
             `${this.config.agentUrl}: ${messageText}` +
             (context?.contextId ? ` [contextId=${context.contextId}]` : ''));
         try {
-            const response = await this.client.sendMessage({
+            const client = await this.clientPromise;
+            const response = await client.sendMessage({
                 message: {
                     kind: 'message',
                     messageId: uuidv4(),

package/dist/modules/a2a/A2AClientWrapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"A2AClientWrapper.js","sourceRoot":"","sources":["../../../src/modules/a2a/A2AClientWrapper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAChF,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AAiCpC;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IACnB,MAAM,CAAY;IAClB,MAAM,CAAkB;IAEhC,YAAY,MAAuB;QACjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,gEAAgE;QAChE,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,WAAmB;QACnC,QAAQ,CACN,gCAAgC,IAAI,CAAC,MAAM,CAAC,OAAO,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,WAAW,EAAE,CACjG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;gBAC7C,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS;oBACf,SAAS,EAAE,SAAS;oBACpB,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;iBAC7C;aACF,CAAC,CAAC;YAEH,QAAQ,CAAC,qCAAqC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAE1E,2BAA2B;YAC3B,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,8DAA8D;gBAC9D,MAAM,IAAI,KAAK,CAAC,cAAc,IAAI,CAAC,SAAS,CAAE,QAAgB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3E,CAAC;YAED,6BAA6B;YAC7B,6DAA6D;YAC7D,8DAA8D;YAC9D,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAa,CAAC;YAEtC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9E,0CAA0C;gBAC1C,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC9E,CAAC;iBAAM,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACxB,OAAO,eAAe,MAAM,CAAC,KAAK,EAAE,CAAC;YACvC,CAAC;YAED,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;YAC3D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,sBAAsB,CAC1B,WAAmB,EACnB,OAAwB;QAExB,QAAQ,CACN,+CAA+C,IAAI,CAAC,MAAM,CAAC,OAAO,MAAM;YACtE,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,WAAW,EAAE;YACzC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,eAAe,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAClE,CAAC;QACF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;gBAC7C,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS;oBACf,SAAS,EAAE,MAAM,EAAE;oBACnB,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;oBAC5C,kFAAkF;oBAClF,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/D,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvD;aACF,CAAC,CAAC;YAEH,QAAQ,CAAC,qCAAqC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAE1E,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,8DAA8D;gBAC9D,MAAM,IAAI,KAAK,CAAC,cAAc,IAAI,CAAC,SAAS,CAAE,QAAgB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3E,CAAC;YAED,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAwB,CAAC;YACjD,OAAO,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,mDAAmD,EAAE,KAAK,CAAC,CAAC;YAC1E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;2FACuF;IAC/E,MAAM,CAAC,aAAa,CAAC,MAAsB;QACjD,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAc,CAAC;YAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC;YAChD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC;YACzE,MAAM,IAAI,GACR,gBAAgB,CAAC,WAAW,CAAC,WAAW,IAAI,aAAa,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;QAC9D,CAAC;QACD,MAAM,OAAO,GAAG,MAAiB,CAAC;QAClC,MAAM,IAAI,GAAG,gBAAgB,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACpF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC1E,CAAC;IAED;yEACqE;IAC7D,MAAM,CAAC,WAAW,CAAC,KAAyB;QAClD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC5C,OAAO,KAAK;aACT,MAAM,CAAC,CAAC,IAAI,EAA2C,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;aAChF,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;aACxB,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"A2AClientWrapper.js","sourceRoot":"","sources":["../../../src/modules/a2a/A2AClientWrapper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAChF,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AAEpC;;;6FAG6F;AAC7F,MAAM,eAAe,GAAG,6BAA6B,CAAC;AAiCtD;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IACnB,aAAa,CAAqB;IAClC,MAAM,CAAkB;IAEhC,YAAY,MAAuB;QACjC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,qFAAqF;QACrF,8FAA8F;QAC9F,6FAA6F;QAC7F,2FAA2F;QAC3F,MAAM,OAAO,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC;QAC5E,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACpD,0FAA0F;QAC1F,4FAA4F;QAC5F,KAAK,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,WAAmB;QACnC,QAAQ,CACN,gCAAgC,IAAI,CAAC,MAAM,CAAC,OAAO,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,WAAW,EAAE,CACjG,CAAC;QACF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC;YACxC,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC;gBACxC,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS;oBACf,SAAS,EAAE,SAAS;oBACpB,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;iBAC7C;aACF,CAAC,CAAC;YAEH,QAAQ,CAAC,qCAAqC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAE1E,2BAA2B;YAC3B,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,8DAA8D;gBAC9D,MAAM,IAAI,KAAK,CAAC,cAAc,IAAI,CAAC,SAAS,CAAE,QAAgB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3E,CAAC;YAED,6BAA6B;YAC7B,6DAA6D;YAC7D,8DAA8D;YAC9D,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAa,CAAC;YAEtC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9E,0CAA0C;gBAC1C,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC9E,CAAC;iBAAM,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACxB,OAAO,eAAe,MAAM,CAAC,KAAK,EAAE,CAAC;YACvC,CAAC;YAED,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;YAC3D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,KAAK,CAAC,sBAAsB,CAC1B,WAAmB,EACnB,OAAwB;QAExB,QAAQ,CACN,+CAA+C,IAAI,CAAC,MAAM,CAAC,OAAO,MAAM;YACtE,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,WAAW,EAAE;YACzC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,eAAe,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAClE,CAAC;QACF,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC;YACxC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC;gBACxC,OAAO,EAAE;oBACP,IAAI,EAAE,SAAS;oBACf,SAAS,EAAE,MAAM,EAAE;oBACnB,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;oBAC5C,kFAAkF;oBAClF,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/D,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvD;aACF,CAAC,CAAC;YAEH,QAAQ,CAAC,qCAAqC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAE1E,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,8DAA8D;gBAC9D,MAAM,IAAI,KAAK,CAAC,cAAc,IAAI,CAAC,SAAS,CAAE,QAAgB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3E,CAAC;YAED,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAwB,CAAC;YACjD,OAAO,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,mDAAmD,EAAE,KAAK,CAAC,CAAC;YAC1E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;2FACuF;IAC/E,MAAM,CAAC,aAAa,CAAC,MAAsB;QACjD,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAc,CAAC;YAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC;YAChD,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC;YACzE,MAAM,IAAI,GACR,gBAAgB,CAAC,WAAW,CAAC,WAAW,IAAI,aAAa,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;QAC9D,CAAC;QACD,MAAM,OAAO,GAAG,MAAiB,CAAC;QAClC,MAAM,IAAI,GAAG,gBAAgB,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACpF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC1E,CAAC;IAED;yEACqE;IAC7D,MAAM,CAAC,WAAW,CAAC,KAAyB;QAClD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC5C,OAAO,KAAK;aACT,MAAM,CAAC,CAAC,IAAI,EAA2C,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC;aAChF,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;aACxB,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;CACF"}

package/dist/modules/apiAgUiModule.d.ts

@@ -1,2 +1,70 @@
 import { GthConfig } from '@gaunt-sloth/core/config.js';
+import type { BaseMessage } from '@langchain/core/messages';
+/** An AG-UI wire message as received on the run input (the shape {@link convertMessage} accepts). */
+type AgUiWireMessage = {
+    role: string;
+    content?: string;
+    id: string;
+    toolCalls?: Array<{
+        id: string;
+        type: string;
+        function: {
+            name: string;
+            arguments: string;
+        };
+    }>;
+    toolCallId?: string;
+};
+/** Per-message options for {@link convertMessage}. */
+interface ConvertMessageOptions {
+    /**
+     * Whether an assistant text-emitted tool call may be PROMOTED to a native tool_call for this
+     * message. Defaults to `true` (the EXT-35 behaviour). {@link convertMessages} sets this to `false`
+     * for a DANGLING history call (one not followed by its tool result) so a stalled replayed call
+     * stays plain text — see EXT-43 and that function's doc.
+     */
+    allowTextCallPromotion?: boolean;
+}
+/**
+ * Convert AG-UI message format to LangChain BaseMessage.
+ *
+ * `allowedToolNames` is the set of tool names bound to this run (config.tools + any run-input
+ * client tools). It gates EXT-35 plain-text tool-call repair on the assistant branch: an incoming
+ * assistant message with NO native `toolCalls` whose content is a STANDALONE text-emitted call
+ * (bracket / `<function=…>` / Harmony — the dialects small/local models produce) is promoted to a
+ * native tool_call so a replayed history turn is a real tool call rather than inert prose. An empty
+ * (or absent) allow-list promotes nothing — the prose-safe default. This runs alongside
+ * {@link parseToolArguments} (which rescues malformed args on an ALREADY-native tool_call).
+ *
+ * EXT-43: `options.allowTextCallPromotion` (default `true`) lets a caller suppress promotion for a
+ * single message; {@link convertMessages} uses it to leave a DANGLING history call as text.
+ */
+export declare function convertMessage(msg: AgUiWireMessage, allowedToolNames?: Set<string>, options?: ConvertMessageOptions): BaseMessage;
+/**
+ * Convert a whole AG-UI history array to LangChain messages, applying TWO symmetric replay guards
+ * so a poisoned history can never abort every subsequent turn on the thread.
+ *
+ * EXT-43 (forward, dangling-CALL): EXT-35's per-message promotion is unconditional, which is correct
+ * for a call that WILL be executed this turn. But when replaying HISTORY, promoting a STALLED text
+ * call (one the client recorded but that never ran) yields an `AIMessage` with `tool_calls` and NO
+ * following `tool_result` — a shape a strict provider (Anthropic) 400s on, where the pre-EXT-35
+ * plain text was valid. So promotion is allowed ONLY when the assistant message is immediately
+ * followed by a `tool` result message; a dangling call stays plain text (`allowTextCallPromotion:
+ * false`).
+ *
+ * RC-18 (backward, orphan-RESULT): the mirror image. A replayed `role:'tool'` message whose matching
+ * `tool_call` id is absent from EVERY PRECEDING assistant message is an ORPHAN — converting it to a
+ * `ToolMessage` yields a tool result with no preceding `AIMessage.tool_calls`, which the same strict
+ * provider 400s on (`Invalid parameter: messages with role 'tool' must be a response to a preceding
+ * message with 'tool_calls'`, INVALID_TOOL_RESULTS). Such orphans arise when a terminal
+ * (`returnDirect`) tool call's result is reconstructed by the client without its parenting assistant
+ * `tool_call`. We DROP the orphan (match on tool_call_id, NOT adjacency; keep genuine pairs; do NOT
+ * fabricate a synthetic call — mirroring EXT-43's demote-don't-invent spirit). Ids are accumulated
+ * in iteration order, so a result whose matching call appears only LATER is still an orphan.
+ *
+ * The live middleware path (`GthLangChainAgent`, fixing the CURRENT turn) is unaffected — both
+ * guards are history-replay only.
+ */
+export declare function convertMessages(messages: AgUiWireMessage[], allowedToolNames?: Set<string>): BaseMessage[];
 export declare function startAgUiServer(config: GthConfig, port: number): Promise<void>;
+export {};

package/dist/modules/apiAgUiModule.js

@@ -6,6 +6,7 @@ import { GthDeepAgent } from '#src/core/GthDeepAgent.js';
 import { GthLangChainAgent } from '@gaunt-sloth/core/core/GthLangChainAgent.js';
 import { defaultStatusCallback, displayInfo, displayWarning, } from '@gaunt-sloth/core/utils/consoleUtils.js';
 import { getNewRunnableConfig } from '@gaunt-sloth/core/utils/llmUtils.js';
+import { textToNativeToolCalls } from '@gaunt-sloth/core/core/toolCallRepair/index.js';
 import { HumanMessage, AIMessage, SystemMessage, ToolMessage } from '@langchain/core/messages';
 import { MemorySaver } from '@langchain/langgraph';
 import { tool } from '@langchain/core/tools';
@@ -97,9 +98,20 @@ function parseToolArguments(raw, toolName) {
     }
 }
 /**
- * Convert AG-UI message format to LangChain BaseMessage
+ * Convert AG-UI message format to LangChain BaseMessage.
+ *
+ * `allowedToolNames` is the set of tool names bound to this run (config.tools + any run-input
+ * client tools). It gates EXT-35 plain-text tool-call repair on the assistant branch: an incoming
+ * assistant message with NO native `toolCalls` whose content is a STANDALONE text-emitted call
+ * (bracket / `<function=…>` / Harmony — the dialects small/local models produce) is promoted to a
+ * native tool_call so a replayed history turn is a real tool call rather than inert prose. An empty
+ * (or absent) allow-list promotes nothing — the prose-safe default. This runs alongside
+ * {@link parseToolArguments} (which rescues malformed args on an ALREADY-native tool_call).
+ *
+ * EXT-43: `options.allowTextCallPromotion` (default `true`) lets a caller suppress promotion for a
+ * single message; {@link convertMessages} uses it to leave a DANGLING history call as text.
  */
-function convertMessage(msg) {
+export function convertMessage(msg, allowedToolNames, options) {
     const content = typeof msg.content === 'string' ? msg.content : '';
     switch (msg.role) {
         case 'user':
@@ -116,6 +128,17 @@ function convertMessage(msg) {
                     })),
                 });
             }
+            // EXT-35: no native tool_calls — a small/local model may have emitted the call as assistant
+            // TEXT. Promote a standalone text-emitted call (gated by the bound-tool allow-list + payload
+            // cap + standalone-only) to a native tool_call; otherwise fall through to plain text.
+            // EXT-43: `allowTextCallPromotion === false` (a dangling history call) short-circuits the
+            // promotion so the message stays plain text.
+            const repairedToolCalls = options?.allowTextCallPromotion !== false && allowedToolNames && allowedToolNames.size > 0
+                ? textToNativeToolCalls(content, { allowedToolNames })
+                : undefined;
+            if (repairedToolCalls) {
+                return new AIMessage({ content: '', tool_calls: repairedToolCalls });
+            }
             return new AIMessage(content);
         }
         case 'system':
@@ -127,6 +150,65 @@ function convertMessage(msg) {
             return new HumanMessage(content);
     }
 }
+/**
+ * Convert a whole AG-UI history array to LangChain messages, applying TWO symmetric replay guards
+ * so a poisoned history can never abort every subsequent turn on the thread.
+ *
+ * EXT-43 (forward, dangling-CALL): EXT-35's per-message promotion is unconditional, which is correct
+ * for a call that WILL be executed this turn. But when replaying HISTORY, promoting a STALLED text
+ * call (one the client recorded but that never ran) yields an `AIMessage` with `tool_calls` and NO
+ * following `tool_result` — a shape a strict provider (Anthropic) 400s on, where the pre-EXT-35
+ * plain text was valid. So promotion is allowed ONLY when the assistant message is immediately
+ * followed by a `tool` result message; a dangling call stays plain text (`allowTextCallPromotion:
+ * false`).
+ *
+ * RC-18 (backward, orphan-RESULT): the mirror image. A replayed `role:'tool'` message whose matching
+ * `tool_call` id is absent from EVERY PRECEDING assistant message is an ORPHAN — converting it to a
+ * `ToolMessage` yields a tool result with no preceding `AIMessage.tool_calls`, which the same strict
+ * provider 400s on (`Invalid parameter: messages with role 'tool' must be a response to a preceding
+ * message with 'tool_calls'`, INVALID_TOOL_RESULTS). Such orphans arise when a terminal
+ * (`returnDirect`) tool call's result is reconstructed by the client without its parenting assistant
+ * `tool_call`. We DROP the orphan (match on tool_call_id, NOT adjacency; keep genuine pairs; do NOT
+ * fabricate a synthetic call — mirroring EXT-43's demote-don't-invent spirit). Ids are accumulated
+ * in iteration order, so a result whose matching call appears only LATER is still an orphan.
+ *
+ * The live middleware path (`GthLangChainAgent`, fixing the CURRENT turn) is unaffected — both
+ * guards are history-replay only.
+ */
+export function convertMessages(messages, allowedToolNames) {
+    // Ids of tool calls emitted by PRECEDING assistant messages, accumulated as we iterate in order.
+    // A `tool` result whose tool_call_id is not yet in this set has no preceding assistant tool_call.
+    const seenToolCallIds = new Set();
+    const converted = [];
+    messages.forEach((msg, index) => {
+        // RC-18 backward orphan-RESULT guard.
+        if (msg.role === 'tool') {
+            const toolCallId = msg.toolCallId || msg.id;
+            if (!seenToolCallIds.has(toolCallId)) {
+                displayWarning(`Dropping orphan tool result (tool_call_id ${JSON.stringify(toolCallId)}) with no ` +
+                    'preceding assistant tool_call in the replayed history; converting it would 400 the ' +
+                    'provider (INVALID_TOOL_RESULTS).');
+                return; // drop — do not convert to a ToolMessage
+            }
+            converted.push(convertMessage(msg, allowedToolNames));
+            return;
+        }
+        // Record this assistant's native tool_call ids BEFORE any following tool result is checked, so
+        // a genuine call→result pair (id present on a preceding assistant) survives the guard above.
+        if (msg.role === 'assistant' && msg.toolCalls) {
+            for (const tc of msg.toolCalls) {
+                if (tc?.id)
+                    seenToolCallIds.add(tc.id);
+            }
+        }
+        // EXT-43 forward dangling-CALL guard (PRESERVED, unchanged).
+        const followedByToolResult = messages[index + 1]?.role === 'tool';
+        converted.push(convertMessage(msg, allowedToolNames, {
+            allowTextCallPromotion: followedByToolResult,
+        }));
+    });
+    return converted;
+}
 /**
  * Construct the AG-UI agent for the configured backend (B5).
  * - `agent.backend: 'deep'` → {@link GthDeepAgent} (deepagents runtime, experimental).
@@ -215,6 +297,13 @@ export async function startAgUiServer(config, port) {
         // the server's statically-configured agent.
         const hasClientTools = Array.isArray(tools) && tools.length > 0;
         const activeAgent = hasClientTools ? await getAgentForTools(tools) : agent;
+        // EXT-35: the names of the tools bound to THIS run (server config.tools + any run-input client
+        // tools) — the allow-list for plain-text tool-call repair in convertMessage. Only a text-emitted
+        // call naming one of these is promoted to a native tool_call; an empty set promotes nothing.
+        const allowedToolNames = new Set([
+            ...(config.tools ?? []).map((t) => t?.name),
+            ...(hasClientTools ? tools.map((t) => t.name) : []),
+        ].filter((name) => Boolean(name)));
         const encoder = new EventEncoder({ accept: req.headers.accept });
         res.setHeader('Content-Type', encoder.getContentType());
         res.setHeader('Cache-Control', 'no-cache');
@@ -290,7 +379,7 @@ export async function startAgUiServer(config, port) {
                     ? queued
                         .map((s) => typeof s === 'string'
                         ? new HumanMessage(s)
-                        : convertMessage(s))
+                        : convertMessage(s, allowedToolNames))
                         .filter((m) => Boolean(m))
                     : [];
                 eventStream = activeAgent.streamWithEventsResume(forwardedProps.command.resume, runConfig, queuedMessages, ac.signal);
@@ -299,7 +388,9 @@ export async function startAgUiServer(config, port) {
                 // The system prompt (backstory + guidelines + mode prompt + identity) lives in the
                 // deep-agent graph via createDeepAgent({ systemPrompt }) — see GthDeepAgent — so it is no
                 // longer prepended here. A separate, non-first SystemMessage would be rejected by Anthropic.
-                const langChainMessages = (messages || []).map(convertMessage);
+                // EXT-43: convertMessages (not a bare map) applies the dangling-call guard so a stalled
+                // text call replayed in history is not promoted to a native tool_call with no result.
+                const langChainMessages = convertMessages((messages || []), allowedToolNames);
                 eventStream = activeAgent.streamWithEvents(langChainMessages, runConfig, ac.signal);
             }
             for await (const event of eventStream) {