@gaunt-sloth/agent 2.0.0-alpha.9 → 2.0.0-beta.1

package/dist/mcp/tlsTrust.d.ts

@@ -0,0 +1,61 @@
+/**
+ * @packageDocumentation
+ * Process-global TLS trust for outbound HTTPS, driven by the `tls` config block.
+ *
+ * Motivation: an `http`-transport MCP server behind a private/corporate CA (e.g. a dev monolith
+ * with a self-signed cert) makes Node's global `fetch` reject the connection with
+ * `TypeError: fetch failed` (`SELF_SIGNED_CERT_IN_CHAIN`). The only pre-existing workaround was to
+ * prepend `NODE_EXTRA_CA_CERTS=<ca.crt>` on every invocation — that env var is read ONCE at Node
+ * startup, so it can't be set from config after launch.
+ *
+ * Node's `fetch` runs on undici and ignores `https.globalAgent`, so the CA must be installed via an
+ * undici dispatcher. `@langchain/mcp-adapters` forwards only `serverName`/`headers`/`authProvider`
+ * to the SDK transport (it drops the SDK's `requestInit`/`fetch` hooks), so a per-server CA is not
+ * reachable through the adapter. The trust is therefore installed **process-globally** via
+ * `setGlobalDispatcher`. That is acceptable for adding a CA (additive trust weakens nothing) — but
+ * note the insecure latch below is global too, and so affects LLM provider calls, not only MCP.
+ */
+import type { GthConfig } from '@gaunt-sloth/core/config.js';
+type TlsConfig = GthConfig['tls'];
+/** A cert path that could not be read, with the reason (surfaced by the installer). */
+export interface TlsCertFailure {
+    path: string;
+    message: string;
+}
+/** The undici `connect` options computed from a `tls` config, plus what happened while reading. */
+export interface BuiltDispatcher {
+    /** Node's default roots FIRST, then successfully-read user certs. `connect.ca` REPLACES the */
+    /** default store, so the roots must be included or every public-CA fetch breaks. */
+    ca: string[];
+    /** `false` disables verification entirely (process-global). Defaults to `true`. */
+    rejectUnauthorized: boolean;
+    /** How many configured certs were read successfully. */
+    loadedCount: number;
+    /** Configured certs that could not be read (fail-soft: skipped, not fatal). */
+    failures: TlsCertFailure[];
+}
+/**
+ * Compute the undici dispatcher options from a `tls` config. Pure aside from the injected
+ * `readCert` (so unit tests stub file reads): resolves nothing about the environment itself.
+ *
+ * Returns `null` only when NOTHING is configured — no `tls` block, or an empty `tls` with no cert
+ * entries and verification left at its secure default. When certs ARE configured it always returns
+ * the object (even if every read failed), so the caller can surface those `failures` rather than
+ * silently swallowing a bad cert path. Whether to actually install is the caller's call:
+ * `loadedCount > 0 || !rejectUnauthorized`.
+ *
+ * `readCert` receives each configured path verbatim and must return the PEM contents or throw.
+ */
+export declare function buildDispatcherOptions(tls: TlsConfig, readCert: (path: string) => string, baseRoots?: readonly string[]): BuiltDispatcher | null;
+/**
+ * Install the process-global undici dispatcher from `config.tls`, if any. Idempotent (installs at
+ * most once per process). Fail-soft: an unreadable cert is warned about and skipped, never fatal.
+ * Emits a loud security warning when verification is disabled.
+ *
+ * Wired into `getMcpClient` before the MCP client is created; because it is global it also covers
+ * LLM/tool fetches — which is desired for the CA case and is the danger to flag for the latch.
+ */
+export declare function installMcpTlsTrust(config: GthConfig): void;
+/** Test-only: reset the install-once guard between cases. */
+export declare function resetMcpTlsTrustForTests(): void;
+export {};

package/dist/mcp/tlsTrust.js

@@ -0,0 +1,109 @@
+/**
+ * @packageDocumentation
+ * Process-global TLS trust for outbound HTTPS, driven by the `tls` config block.
+ *
+ * Motivation: an `http`-transport MCP server behind a private/corporate CA (e.g. a dev monolith
+ * with a self-signed cert) makes Node's global `fetch` reject the connection with
+ * `TypeError: fetch failed` (`SELF_SIGNED_CERT_IN_CHAIN`). The only pre-existing workaround was to
+ * prepend `NODE_EXTRA_CA_CERTS=<ca.crt>` on every invocation — that env var is read ONCE at Node
+ * startup, so it can't be set from config after launch.
+ *
+ * Node's `fetch` runs on undici and ignores `https.globalAgent`, so the CA must be installed via an
+ * undici dispatcher. `@langchain/mcp-adapters` forwards only `serverName`/`headers`/`authProvider`
+ * to the SDK transport (it drops the SDK's `requestInit`/`fetch` hooks), so a per-server CA is not
+ * reachable through the adapter. The trust is therefore installed **process-globally** via
+ * `setGlobalDispatcher`. That is acceptable for adding a CA (additive trust weakens nothing) — but
+ * note the insecure latch below is global too, and so affects LLM provider calls, not only MCP.
+ */
+import { rootCertificates } from 'node:tls';
+import { readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { resolve } from 'node:path';
+import { Agent, setGlobalDispatcher } from 'undici';
+import { getProjectDir } from '@gaunt-sloth/core/utils/systemUtils.js';
+import { displayWarning } from '@gaunt-sloth/core/utils/consoleUtils.js';
+import { debugLog } from '@gaunt-sloth/core/utils/debugUtils.js';
+/**
+ * Compute the undici dispatcher options from a `tls` config. Pure aside from the injected
+ * `readCert` (so unit tests stub file reads): resolves nothing about the environment itself.
+ *
+ * Returns `null` only when NOTHING is configured — no `tls` block, or an empty `tls` with no cert
+ * entries and verification left at its secure default. When certs ARE configured it always returns
+ * the object (even if every read failed), so the caller can surface those `failures` rather than
+ * silently swallowing a bad cert path. Whether to actually install is the caller's call:
+ * `loadedCount > 0 || !rejectUnauthorized`.
+ *
+ * `readCert` receives each configured path verbatim and must return the PEM contents or throw.
+ */
+export function buildDispatcherOptions(tls, readCert, baseRoots = rootCertificates) {
+    const rejectUnauthorized = tls?.rejectUnauthorized ?? true;
+    const certPaths = tls?.extraCaCerts ?? [];
+    // Nothing configured at all: no cert entries and verification not being disabled.
+    if (certPaths.length === 0 && rejectUnauthorized)
+        return null;
+    const userCerts = [];
+    const failures = [];
+    for (const path of certPaths) {
+        try {
+            userCerts.push(readCert(path));
+        }
+        catch (error) {
+            failures.push({ path, message: error instanceof Error ? error.message : String(error) });
+        }
+    }
+    return {
+        // ca REPLACES the trust store — keep the default roots or public-CA fetches (incl. the LLM
+        // provider) silently break.
+        ca: [...baseRoots, ...userCerts],
+        rejectUnauthorized,
+        loadedCount: userCerts.length,
+        failures,
+    };
+}
+// Install-once guard: the trust is process-global and set-once, so re-running resolveTools (re-init)
+// must not re-replace the dispatcher nor re-emit the security warning every time.
+let installed = false;
+/**
+ * Install the process-global undici dispatcher from `config.tls`, if any. Idempotent (installs at
+ * most once per process). Fail-soft: an unreadable cert is warned about and skipped, never fatal.
+ * Emits a loud security warning when verification is disabled.
+ *
+ * Wired into `getMcpClient` before the MCP client is created; because it is global it also covers
+ * LLM/tool fetches — which is desired for the CA case and is the danger to flag for the latch.
+ */
+export function installMcpTlsTrust(config) {
+    if (installed) {
+        debugLog('MCP TLS trust already installed this process; skipping.');
+        return;
+    }
+    const built = buildDispatcherOptions(config.tls, resolveAndReadCert);
+    if (!built)
+        return;
+    // Surface unreadable certs even if we end up installing nothing — a bad cert path must never be
+    // swallowed silently. Warn BEFORE the "nothing actionable" bail below.
+    for (const failure of built.failures) {
+        displayWarning(`TLS: could not read CA certificate '${failure.path}' (${failure.message}); skipping it.`);
+    }
+    // Nothing left to install: every configured cert failed to read and verification is still on.
+    // (Failures were already surfaced above.) Leave Node's default dispatcher untouched.
+    if (built.loadedCount === 0 && built.rejectUnauthorized)
+        return;
+    installed = true;
+    if (!built.rejectUnauthorized) {
+        displayWarning('TLS: certificate verification is DISABLED (tls.rejectUnauthorized: false). This is INSECURE ' +
+            'and applies to ALL outbound HTTPS this process makes — MCP servers AND LLM provider calls. ' +
+            'Use only against trusted dev endpoints; prefer tls.extraCaCerts to trust a specific CA instead.');
+    }
+    setGlobalDispatcher(new Agent({ connect: { ca: built.ca, rejectUnauthorized: built.rejectUnauthorized } }));
+    debugLog(`MCP TLS trust installed: ${built.loadedCount} extra CA cert(s), rejectUnauthorized=${built.rejectUnauthorized}.`);
+}
+/** Resolve a configured cert path (`~` / relative-to-project / absolute) and read it as UTF-8. */
+function resolveAndReadCert(path) {
+    const expanded = path === '~' || path.startsWith('~/') ? homedir() + path.slice(1) : path;
+    return readFileSync(resolve(getProjectDir(), expanded), 'utf8');
+}
+/** Test-only: reset the install-once guard between cases. */
+export function resetMcpTlsTrustForTests() {
+    installed = false;
+}
+//# sourceMappingURL=tlsTrust.js.map

package/dist/mcp/tlsTrust.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tlsTrust.js","sourceRoot":"","sources":["../../src/mcp/tlsTrust.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,QAAQ,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,MAAM,wCAAwC,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,uCAAuC,CAAC;AAuBjE;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,sBAAsB,CACpC,GAAc,EACd,QAAkC,EAClC,SAAS,GAAsB,gBAAgB;IAE/C,MAAM,kBAAkB,GAAG,GAAG,EAAE,kBAAkB,IAAI,IAAI,CAAC;IAC3D,MAAM,SAAS,GAAG,GAAG,EAAE,YAAY,IAAI,EAAE,CAAC;IAE1C,kFAAkF;IAClF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAE9D,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAqB,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,OAAO;QACL,2FAA2F;QAC3F,4BAA4B;QAC5B,EAAE,EAAE,CAAC,GAAG,SAAS,EAAE,GAAG,SAAS,CAAC;QAChC,kBAAkB;QAClB,WAAW,EAAE,SAAS,CAAC,MAAM;QAC7B,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,qGAAqG;AACrG,kFAAkF;AAClF,IAAI,SAAS,GAAG,KAAK,CAAC;AAEtB;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAiB;IAClD,IAAI,SAAS,EAAE,CAAC;QACd,QAAQ,CAAC,yDAAyD,CAAC,CAAC;QACpE,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG,sBAAsB,CAAC,MAAM,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;IACrE,IAAI,CAAC,KAAK;QAAE,OAAO;IAEnB,gGAAgG;IAChG,uEAAuE;IACvE,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACrC,cAAc,CACZ,uCAAuC,OAAO,CAAC,IAAI,MAAM,OAAO,CAAC,OAAO,iBAAiB,CAC1F,CAAC;IACJ,CAAC;IAED,8FAA8F;IAC9F,qFAAqF;IACrF,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC,IAAI,KAAK,CAAC,kBAAkB;QAAE,OAAO;IAEhE,SAAS,GAAG,IAAI,CAAC;IAEjB,IAAI,CAAC,KAAK,CAAC,kBAAkB,EAAE,CAAC;QAC9B,cAAc,CACZ,8FAA8F;YAC5F,6FAA6F;YAC7F,iGAAiG,CACpG,CAAC;IACJ,CAAC;IAED,mBAAmB,CACjB,IAAI,KAAK,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,EAAE,EAAE,CAAC,CACvF,CAAC;IACF,QAAQ,CACN,4BAA4B,KAAK,CAAC,WAAW,yCAAyC,KAAK,CAAC,kBAAkB,GAAG,CAClH,CAAC;AACJ,CAAC;AAED,kGAAkG;AAClG,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,QAAQ,GAAG,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1F,OAAO,YAAY,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC;AAClE,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,wBAAwB;IACtC,SAAS,GAAG,KAAK,CAAC;AACpB,CAAC"}

package/dist/middleware/binaryContentInjectionMiddleware.d.ts

@@ -18,5 +18,12 @@ import { type AgentMiddleware } from 'langchain';
 import type { GthConfig } from '@gaunt-sloth/core/config.js';
 export interface BinaryContentInjectionMiddlewareSettings {
     name?: 'binary-content-injection';
+    /**
+     * The gth provider string selecting the per-provider image-block shape (see
+     * {@link imageBlockFor}). Derived by the registry factory via `resolveVisionProvider(gthConfig)`;
+     * may be `''`/absent, in which case the standard base64 image block is emitted. Only affects the
+     * `image` format — file/video/audio blocks are provider-agnostic.
+     */
+    provider?: string;
 }
-export declare function createBinaryContentInjectionMiddleware(_settings: BinaryContentInjectionMiddlewareSettings, _gthConfig: GthConfig): Promise<AgentMiddleware>;
+export declare function createBinaryContentInjectionMiddleware(settings: BinaryContentInjectionMiddlewareSettings, _gthConfig: GthConfig): Promise<AgentMiddleware>;

package/dist/middleware/binaryContentInjectionMiddleware.js

@@ -18,6 +18,7 @@ import { createMiddleware } from 'langchain';
 import path from 'node:path';
 import { debugLog } from '@gaunt-sloth/core/utils/debugUtils.js';
 import { ToolMessage, HumanMessage } from '@langchain/core/messages';
+import { imageBlockFor } from '#src/middleware/frontendImageInjectionMiddleware.js';
 /**
  * Parse the special binary format string returned by gth_read_binary tool.
  * Format: gth_read_binary;type:${type};path:${encodedPath};data:${media_type};base64,${data}
@@ -73,8 +74,10 @@ function getFormatLabel(formatType) {
     };
     return labels[formatType] || 'file';
 }
-export function createBinaryContentInjectionMiddleware(_settings, _gthConfig) {
+export function createBinaryContentInjectionMiddleware(settings, _gthConfig) {
     debugLog('Creating binary content injection middleware');
+    // The provider selecting the image-block shape; '' falls to the standard base64 block.
+    const provider = settings.provider ?? '';
     return Promise.resolve(createMiddleware({
         name: 'binary-content-injection',
         // Before next model call, detect and inject HumanMessage with binary content
@@ -100,7 +103,13 @@ export function createBinaryContentInjectionMiddleware(_settings, _gthConfig) {
                 const newMessages = [...messages];
                 for (const { binaryData } of binaryMessages) {
                     const formatLabel = getFormatLabel(binaryData.formatType);
-                    const contentBlock = createContentBlock(binaryData);
+                    // Images go through the per-provider builder so OpenAI reasoning models (Responses API,
+                    // GS2-74) get a valid `image_url` block instead of the standard `source_type` data block,
+                    // which @langchain/openai mis-serialises to an invalid Responses image part (GS2-75).
+                    // file/video/audio keep the standard block (they already convert correctly).
+                    const contentBlock = binaryData.formatType === 'image'
+                        ? imageBlockFor(provider, binaryData.media_type, binaryData.data)
+                        : createContentBlock(binaryData);
                     const humanMessage = new HumanMessage({
                         content: [
                             {

package/dist/middleware/binaryContentInjectionMiddleware.js.map

@@ -1 +1 @@
-{"version":3,"file":"binaryContentInjectionMiddleware.js","sourceRoot":"","sources":["../../src/middleware/binaryContentInjectionMiddleware.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,gBAAgB,EAAwB,MAAM,WAAW,CAAC;AACnE,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAE,MAAM,uCAAuC,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAcrE;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,OAAe;IACzC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEnD,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,8BAA8B;QAC9B,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAErD,OAAO;YACL,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC;YACxB,IAAI,EAAE,WAAW;YACjB,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC;YACxB,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;SACrB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,UAA+B;IACzD,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC;IAEpD,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,QAAQ;QACrB,SAAS,EAAE,UAAU;QACrB,IAAI;QACJ,QAAQ,EAAE;YACR,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;SACzC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,UAAkB;IACxC,MAAM,MAAM,GAA2B;QACrC,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,MAAM;KACb,CAAC;IACF,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,SAAmD,EACnD,UAAqB;IAErB,QAAQ,CAAC,8CAA8C,CAAC,CAAC;IAEzD,OAAO,OAAO,CAAC,OAAO,CACpB,gBAAgB,CAAC;QACf,IAAI,EAAE,0BAA0B;QAEhC,6EAA6E;QAC7E,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;YAEtC,gDAAgD;YAChD,MAAM,cAAc,GAAqE,EAAE,CAAC;YAE5F,uEAAuE;YACvE,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7E,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACxB,IACE,GAAG,YAAY,WAAW;oBAC1B,GAAG,CAAC,IAAI,KAAK,iBAAiB;oBAC9B,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAC/B,CAAC;oBACD,MAAM,aAAa,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtD,IAAI,aAAa,EAAE,CAAC;wBAClB,cAAc,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,CAAC;oBACnE,CAAC;gBACH,CAAC;YACH,CAAC;YAED,qDAAqD;YACrD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,QAAQ,CAAC,aAAa,cAAc,CAAC,MAAM,sCAAsC,CAAC,CAAC;gBAEnF,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAElC,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,cAAc,EAAE,CAAC;oBAC5C,MAAM,WAAW,GAAG,cAAc,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;oBAC1D,MAAM,YAAY,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;oBAEpD,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC;wBACpC,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,eAAe,WAAW,yBAAyB;6BAC1D;4BACD,YAAY;yBACK;qBACpB,CAAC,CAAC;oBAEH,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBACjC,CAAC;gBAED,8CAA8C;gBAC9C,OAAO;oBACL,QAAQ,EAAE,WAAW;iBACtB,CAAC;YACJ,CAAC;YAED,kCAAkC;YAClC,OAAO,SAAS,CAAC;QACnB,CAAC;KACF,CAAC,CACH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"binaryContentInjectionMiddleware.js","sourceRoot":"","sources":["../../src/middleware/binaryContentInjectionMiddleware.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,gBAAgB,EAAwB,MAAM,WAAW,CAAC;AACnE,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,QAAQ,EAAE,MAAM,uCAAuC,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAErE,OAAO,EAAE,aAAa,EAAE,MAAM,qDAAqD,CAAC;AAoBpF;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,OAAe;IACzC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAEnD,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,8BAA8B;QAC9B,MAAM,WAAW,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAErD,OAAO;YACL,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC;YACxB,IAAI,EAAE,WAAW;YACjB,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC;YACxB,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;SACrB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,UAA+B;IACzD,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC;IAEpD,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,QAAQ;QACrB,SAAS,EAAE,UAAU;QACrB,IAAI;QACJ,QAAQ,EAAE;YACR,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;SACzC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,UAAkB;IACxC,MAAM,MAAM,GAA2B;QACrC,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,MAAM;KACb,CAAC;IACF,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,QAAkD,EAClD,UAAqB;IAErB,QAAQ,CAAC,8CAA8C,CAAC,CAAC;IAEzD,uFAAuF;IACvF,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;IAEzC,OAAO,OAAO,CAAC,OAAO,CACpB,gBAAgB,CAAC;QACf,IAAI,EAAE,0BAA0B;QAEhC,6EAA6E;QAC7E,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;YAEtC,gDAAgD;YAChD,MAAM,cAAc,GAAqE,EAAE,CAAC;YAE5F,uEAAuE;YACvE,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7E,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACxB,IACE,GAAG,YAAY,WAAW;oBAC1B,GAAG,CAAC,IAAI,KAAK,iBAAiB;oBAC9B,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAC/B,CAAC;oBACD,MAAM,aAAa,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtD,IAAI,aAAa,EAAE,CAAC;wBAClB,cAAc,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,CAAC;oBACnE,CAAC;gBACH,CAAC;YACH,CAAC;YAED,qDAAqD;YACrD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,QAAQ,CAAC,aAAa,cAAc,CAAC,MAAM,sCAAsC,CAAC,CAAC;gBAEnF,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAElC,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,cAAc,EAAE,CAAC;oBAC5C,MAAM,WAAW,GAAG,cAAc,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;oBAC1D,wFAAwF;oBACxF,0FAA0F;oBAC1F,sFAAsF;oBACtF,6EAA6E;oBAC7E,MAAM,YAAY,GAChB,UAAU,CAAC,UAAU,KAAK,OAAO;wBAC/B,CAAC,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC;wBACjE,CAAC,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;oBAErC,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC;wBACpC,OAAO,EAAE;4BACP;gCACE,IAAI,EAAE,MAAM;gCACZ,IAAI,EAAE,eAAe,WAAW,yBAAyB;6BAC1D;4BACD,YAAY;yBACK;qBACpB,CAAC,CAAC;oBAEH,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBACjC,CAAC;gBAED,8CAA8C;gBAC9C,OAAO;oBACL,QAAQ,EAAE,WAAW;iBACtB,CAAC;YACJ,CAAC;YAED,kCAAkC;YAClC,OAAO,SAAS,CAAC;QACnB,CAAC;KACF,CAAC,CACH,CAAC;AACJ,CAAC"}

package/dist/middleware/frontendImageInjectionMiddleware.d.ts

@@ -0,0 +1,107 @@
+/**
+ * @packageDocumentation
+ * Opt-in middleware that turns a frontend "capture image" tool result into a vision message the
+ * model can actually see.
+ *
+ * A browser/frontend tool such as CopilotKit's `capture_image` fulfils client-side and posts its
+ * result back to gth's AG-UI server as a trailing `tool`-role message whose content is a JSON
+ * string `{"mimeType":"image/...","data":"<base64>"}`. gth hands that to the model as a plain-string
+ * ToolMessage — no vision block — so the model literally cannot see the photo. This middleware
+ * detects that ToolMessage in `beforeModel`, parses the envelope, and injects a `HumanMessage`
+ * carrying a provider-appropriate vision block before the next model call.
+ *
+ * Strictly opt-in: it fires ONLY when referenced by name in `config.middleware`
+ * (`"middleware": ["frontend-image-injection"]`), never auto-injected. Promoted from
+ * pukeko-robot-controller's proven middleware (RC-21), minus the robot-specific motion-tool coupling.
+ */
+import { type AgentMiddleware } from 'langchain';
+/** Default frontend capture tool name. Overridable via the `toolName` middleware setting. */
+export declare const DEFAULT_CAPTURE_TOOL_NAME = "capture_image";
+export interface FrontendImageInjectionOptions {
+    /**
+     * Provider selecting the vision-block shape. Derived by the registry factory from
+     * `gthConfig.modelProviderType` (the raw `llm.type`), falling back to `llm._llmType()`. One of
+     * gth's provider strings (`anthropic`, `openai`, `openrouter`, `deepseek`, `xai`, `groq`,
+     * `ollama`, `google-genai`, `vertexai`, `huggingface`, …); unknown values fall to the standard
+     * base64 block.
+     */
+    provider: string;
+    /**
+     * Tool name whose `{mimeType,data}` result is converted into a vision message. Defaults to
+     * {@link DEFAULT_CAPTURE_TOOL_NAME} (`capture_image`).
+     */
+    toolName?: string;
+}
+/**
+ * A vision content block the target provider's `@langchain` converter actually decodes. Verified
+ * against the installed converters (RC-21):
+ *   - **ollama** → `{ type:'image_url', image_url:'<data-URL string>' }`. ChatOllama's
+ *     `convertToOllamaMessages` only handles `image_url` blocks (extractBase64FromDataUrl); the
+ *     LangChain standard `source_type` block throws "Unsupported content type: image".
+ *   - **OpenAI-compatible** (`openai`, `openrouter`, `deepseek`, `xai`, `groq` — served by an
+ *     OpenAI-compatible Chat Completions API, whichever LangChain class fronts it) →
+ *     `{ type:'image_url', image_url:{ url:'<data-URL>' } }`.
+ *     This native OpenAI shape is correct on BOTH the Completions API AND the Responses API (GS2-74
+ *     flips reasoning-capable openai models to Responses). A raw `source_type` standard block
+ *     serialises to an *invalid* image part on the Responses path, so we emit the provider-native
+ *     shape rather than lean on `@langchain/core`'s (deprecated, internal) auto-conversion.
+ *   - **anthropic** → the provider-native block `{ type:'image', source:{ type:'base64',
+ *     media_type, data } }`. The LangChain standard block is NOT usable here (RC-32): in
+ *     `@langchain/anthropic`'s `_formatContentBlocks`, the `isDataContentBlock` branch yields its
+ *     conversion and then FALLS THROUGH — no `continue` — into the chain below, where
+ *     `type === 'image'` matches the very same block and yields a SECOND one whose `media_type` is
+ *     read from camelCase `mimeType` (a key the snake_case standard block never has) and so
+ *     defaults to the literal `image/jpeg`. Every frame is therefore sent twice, and a non-JPEG
+ *     capture 400s outright on the mislabelled copy. The native block is recognised earlier by
+ *     `_isAnthropicImageBlockParam` and passed through untouched, exactly once.
+ *   - **google-genai / vertexai** (and any unknown/default) → the LangChain standard base64 data
+ *     content block `{ type:'image', source_type:'base64', mime_type, data }`, which those native
+ *     converters decode directly and which is the most broadly decodable fallback.
+ *
+ * The through-line: emit what the target provider's converter consumes natively rather than lean on
+ * a generic auto-conversion — the same lesson as GS2-75 on the OpenAI Responses path.
+ *
+ * Pure and exported so each provider branch can be unit-tested directly.
+ */
+export declare function imageBlockFor(provider: string, mimeType: string, data: string): {
+    type: 'image_url';
+    image_url: string;
+    source?: undefined;
+    source_type?: undefined;
+    mime_type?: undefined;
+    data?: undefined;
+} | {
+    type: 'image_url';
+    image_url: {
+        url: string;
+    };
+    source?: undefined;
+    source_type?: undefined;
+    mime_type?: undefined;
+    data?: undefined;
+} | {
+    image_url?: undefined;
+    type: 'image';
+    source: {
+        type: 'base64';
+        media_type: string;
+        data: string;
+    };
+    source_type?: undefined;
+    mime_type?: undefined;
+    data?: undefined;
+} | {
+    image_url?: undefined;
+    source?: undefined;
+    type: 'image';
+    source_type: 'base64';
+    mime_type: string;
+    data: string;
+};
+/**
+ * Create the frontend-image-injection middleware.
+ *
+ * @param opts.provider - provider string selecting the vision-block shape (see {@link imageBlockFor}).
+ * @param opts.toolName - capture tool name (default {@link DEFAULT_CAPTURE_TOOL_NAME}).
+ */
+export declare function createFrontendImageInjectionMiddleware(opts: FrontendImageInjectionOptions): AgentMiddleware;

package/dist/middleware/frontendImageInjectionMiddleware.js

@@ -0,0 +1,167 @@
+/**
+ * @packageDocumentation
+ * Opt-in middleware that turns a frontend "capture image" tool result into a vision message the
+ * model can actually see.
+ *
+ * A browser/frontend tool such as CopilotKit's `capture_image` fulfils client-side and posts its
+ * result back to gth's AG-UI server as a trailing `tool`-role message whose content is a JSON
+ * string `{"mimeType":"image/...","data":"<base64>"}`. gth hands that to the model as a plain-string
+ * ToolMessage — no vision block — so the model literally cannot see the photo. This middleware
+ * detects that ToolMessage in `beforeModel`, parses the envelope, and injects a `HumanMessage`
+ * carrying a provider-appropriate vision block before the next model call.
+ *
+ * Strictly opt-in: it fires ONLY when referenced by name in `config.middleware`
+ * (`"middleware": ["frontend-image-injection"]`), never auto-injected. Promoted from
+ * pukeko-robot-controller's proven middleware (RC-21), minus the robot-specific motion-tool coupling.
+ */
+import { createMiddleware } from 'langchain';
+import { HumanMessage, isToolMessage } from '@langchain/core/messages';
+/** Default frontend capture tool name. Overridable via the `toolName` middleware setting. */
+export const DEFAULT_CAPTURE_TOOL_NAME = 'capture_image';
+/**
+ * A vision content block the target provider's `@langchain` converter actually decodes. Verified
+ * against the installed converters (RC-21):
+ *   - **ollama** → `{ type:'image_url', image_url:'<data-URL string>' }`. ChatOllama's
+ *     `convertToOllamaMessages` only handles `image_url` blocks (extractBase64FromDataUrl); the
+ *     LangChain standard `source_type` block throws "Unsupported content type: image".
+ *   - **OpenAI-compatible** (`openai`, `openrouter`, `deepseek`, `xai`, `groq` — served by an
+ *     OpenAI-compatible Chat Completions API, whichever LangChain class fronts it) →
+ *     `{ type:'image_url', image_url:{ url:'<data-URL>' } }`.
+ *     This native OpenAI shape is correct on BOTH the Completions API AND the Responses API (GS2-74
+ *     flips reasoning-capable openai models to Responses). A raw `source_type` standard block
+ *     serialises to an *invalid* image part on the Responses path, so we emit the provider-native
+ *     shape rather than lean on `@langchain/core`'s (deprecated, internal) auto-conversion.
+ *   - **anthropic** → the provider-native block `{ type:'image', source:{ type:'base64',
+ *     media_type, data } }`. The LangChain standard block is NOT usable here (RC-32): in
+ *     `@langchain/anthropic`'s `_formatContentBlocks`, the `isDataContentBlock` branch yields its
+ *     conversion and then FALLS THROUGH — no `continue` — into the chain below, where
+ *     `type === 'image'` matches the very same block and yields a SECOND one whose `media_type` is
+ *     read from camelCase `mimeType` (a key the snake_case standard block never has) and so
+ *     defaults to the literal `image/jpeg`. Every frame is therefore sent twice, and a non-JPEG
+ *     capture 400s outright on the mislabelled copy. The native block is recognised earlier by
+ *     `_isAnthropicImageBlockParam` and passed through untouched, exactly once.
+ *   - **google-genai / vertexai** (and any unknown/default) → the LangChain standard base64 data
+ *     content block `{ type:'image', source_type:'base64', mime_type, data }`, which those native
+ *     converters decode directly and which is the most broadly decodable fallback.
+ *
+ * The through-line: emit what the target provider's converter consumes natively rather than lean on
+ * a generic auto-conversion — the same lesson as GS2-75 on the OpenAI Responses path.
+ *
+ * Pure and exported so each provider branch can be unit-tested directly.
+ */
+export function imageBlockFor(provider, mimeType, data) {
+    const dataUrl = `data:${mimeType};base64,${data}`;
+    switch (provider) {
+        case 'ollama':
+            return { type: 'image_url', image_url: dataUrl };
+        case 'openai':
+        case 'openrouter':
+        case 'deepseek':
+        case 'xai':
+        case 'groq':
+            return { type: 'image_url', image_url: { url: dataUrl } };
+        case 'anthropic':
+            return {
+                type: 'image',
+                source: { type: 'base64', media_type: mimeType, data },
+            };
+        case 'google-genai':
+        case 'vertexai':
+        default:
+            return {
+                type: 'image',
+                source_type: 'base64',
+                mime_type: mimeType,
+                data,
+            };
+    }
+}
+/**
+ * Create the frontend-image-injection middleware.
+ *
+ * @param opts.provider - provider string selecting the vision-block shape (see {@link imageBlockFor}).
+ * @param opts.toolName - capture tool name (default {@link DEFAULT_CAPTURE_TOOL_NAME}).
+ */
+export function createFrontendImageInjectionMiddleware(opts) {
+    const toolName = opts.toolName ?? DEFAULT_CAPTURE_TOOL_NAME;
+    // thread_id → set of tool_call_ids whose image (or error note) has already been injected. It
+    // scopes injection to ONE frame per capture per graph thread: `beforeModel` runs before every
+    // model call, so a multi-step turn would otherwise stack a copy of the same frame at each step,
+    // each one appended after the newest content and mispairing the assistant message with a stale
+    // frame (the RC-21 idempotency guard). It is deliberately keyed on the GRAPH thread rather than
+    // the client session — the AG-UI server gives each fresh run its own checkpoint thread (RC-31),
+    // and that run replays the whole history into an empty graph, so it must re-inject the frame or
+    // the model would lose the photo the moment the capture turn ended.
+    //
+    // Closure-scoped (one Map per middleware instance), NOT a module global as the robot had it. The
+    // AG-UI server caches one agent per client-toolset signature (getAgentForTools), so a thread's
+    // turns share this instance and idempotency-across-turns is preserved — while unrelated agents (and
+    // tests) get independent state instead of a process-lifetime Map that accumulates every thread_id.
+    const injectedByThread = new Map();
+    return createMiddleware({
+        name: 'frontend-image-injection',
+        beforeModel: async (state, runtime) => {
+            const messages = state.messages || [];
+            // The AG-UI server sets runConfig.configurable.thread_id (= the run's threadId) and threads it
+            // to the graph, so per-session idempotency keys correctly. '__default__' is only reached off
+            // that path (e.g. a bare invoke with no thread_id), which still behaves correctly per-instance.
+            const threadId = runtime?.configurable?.thread_id ?? '__default__';
+            let injectedIds = injectedByThread.get(threadId);
+            if (!injectedIds) {
+                injectedIds = new Set();
+                injectedByThread.set(threadId, injectedIds);
+            }
+            // Scan forward so injected frames stay in chronological order; skip any tool_call_id already
+            // injected on this thread (idempotent across a retained/replayed tail).
+            const injected = [];
+            for (let i = 0; i < messages.length; i++) {
+                const msg = messages[i];
+                if (
+                // RC-21 (golden fix): use the duck-typed `isToolMessage`, NEVER `msg instanceof
+                // ToolMessage`. A consumer (galvanized/robot) importing this middleware across a `file:`-dep
+                // boundary resolves a SECOND @langchain/core copy, and a capture ToolMessage constructed by
+                // the AG-UI pipeline's core copy is not an instance of the `ToolMessage` class we import —
+                // `instanceof` silently returns false across copies and no frame is ever injected.
+                isToolMessage(msg) &&
+                    typeof msg.content === 'string' &&
+                    msg.name === toolName) {
+                    const id = msg.tool_call_id;
+                    if (!id || injectedIds.has(id))
+                        continue;
+                    try {
+                        injected.push({ payload: JSON.parse(msg.content), id });
+                    }
+                    catch {
+                        // Non-JSON tool result — skip injection (leave the guard clean).
+                    }
+                }
+            }
+            if (injected.length === 0)
+                return undefined;
+            const newMessages = [...messages];
+            for (const { payload, id } of injected) {
+                if (payload.error) {
+                    // Mark injected so the error note isn't re-emitted on a later turn.
+                    injectedIds.add(id);
+                    newMessages.push(new HumanMessage({ content: `Camera unavailable: ${payload.error}` }));
+                    continue;
+                }
+                if (payload.mimeType && payload.data) {
+                    // RC-21: mark injected ONLY on a successful frame emission. A capture result that arrived
+                    // WITHOUT its base64 `data` (dropped upstream) injects nothing and must NOT poison the
+                    // guard — a later data-bearing result for the same tool_call_id can still recover.
+                    injectedIds.add(id);
+                    const block = imageBlockFor(opts.provider, payload.mimeType, payload.data);
+                    newMessages.push(new HumanMessage({
+                        content: [{ type: 'text', text: 'Camera frame captured:' }, block],
+                    }));
+                }
+                // else: a capture result whose `data` is absent — inject nothing, leave the guard clean.
+            }
+            // If every candidate was data-less (nothing appended), return no state update rather than an
+            // identical copy — keeps beforeModel a true no-op and leaves the guard clean for recovery.
+            return newMessages.length > messages.length ? { messages: newMessages } : undefined;
+        },
+    });
+}
+//# sourceMappingURL=frontendImageInjectionMiddleware.js.map

package/dist/middleware/frontendImageInjectionMiddleware.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontendImageInjectionMiddleware.js","sourceRoot":"","sources":["../../src/middleware/frontendImageInjectionMiddleware.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,gBAAgB,EAAwB,MAAM,WAAW,CAAC;AACnE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAavE,6FAA6F;AAC7F,MAAM,CAAC,MAAM,yBAAyB,GAAG,eAAe,CAAC;AAkBzD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB,EAAE,QAAgB,EAAE,IAAY;IAC5E,MAAM,OAAO,GAAG,QAAQ,QAAQ,WAAW,IAAI,EAAE,CAAC;IAClD,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO,EAAE,IAAI,EAAE,WAAoB,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;QAC5D,KAAK,QAAQ,CAAC;QACd,KAAK,YAAY,CAAC;QAClB,KAAK,UAAU,CAAC;QAChB,KAAK,KAAK,CAAC;QACX,KAAK,MAAM;YACT,OAAO,EAAE,IAAI,EAAE,WAAoB,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC;QACrE,KAAK,WAAW;YACd,OAAO;gBACL,IAAI,EAAE,OAAgB;gBACtB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAiB,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE;aAChE,CAAC;QACJ,KAAK,cAAc,CAAC;QACpB,KAAK,UAAU,CAAC;QAChB;YACE,OAAO;gBACL,IAAI,EAAE,OAAgB;gBACtB,WAAW,EAAE,QAAiB;gBAC9B,SAAS,EAAE,QAAQ;gBACnB,IAAI;aACL,CAAC;IACN,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sCAAsC,CACpD,IAAmC;IAEnC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,yBAAyB,CAAC;IAE5D,6FAA6F;IAC7F,8FAA8F;IAC9F,gGAAgG;IAChG,+FAA+F;IAC/F,gGAAgG;IAChG,gGAAgG;IAChG,gGAAgG;IAChG,oEAAoE;IACpE,EAAE;IACF,iGAAiG;IACjG,+FAA+F;IAC/F,oGAAoG;IACpG,mGAAmG;IACnG,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAuB,CAAC;IAExD,OAAO,gBAAgB,CAAC;QACtB,IAAI,EAAE,0BAA0B;QAEhC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;YACtC,+FAA+F;YAC/F,6FAA6F;YAC7F,gGAAgG;YAChG,MAAM,QAAQ,GAAG,OAAO,EAAE,YAAY,EAAE,SAAS,IAAI,aAAa,CAAC;YACnE,IAAI,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;gBAChC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;YAC9C,CAAC;YAED,6FAA6F;YAC7F,wEAAwE;YACxE,MAAM,QAAQ,GAAiD,EAAE,CAAC;YAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACxB;gBACE,gFAAgF;gBAChF,6FAA6F;gBAC7F,4FAA4F;gBAC5F,2FAA2F;gBAC3F,mFAAmF;gBACnF,aAAa,CAAC,GAAG,CAAC;oBAClB,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;oBAC/B,GAAG,CAAC,IAAI,KAAK,QAAQ,EACrB,CAAC;oBACD,MAAM,EAAE,GAAG,GAAG,CAAC,YAAY,CAAC;oBAC5B,IAAI,CAAC,EAAE,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;wBAAE,SAAS;oBACzC,IAAI,CAAC;wBACH,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;oBAC1E,CAAC;oBAAC,MAAM,CAAC;wBACP,iEAAiE;oBACnE,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAE5C,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;YAClC,KAAK,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,QAAQ,EAAE,CAAC;gBACvC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBAClB,oEAAoE;oBACpE,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACpB,WAAW,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,uBAAuB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;oBACxF,SAAS;gBACX,CAAC;gBACD,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;oBACrC,0FAA0F;oBAC1F,uFAAuF;oBACvF,mFAAmF;oBACnF,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACpB,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;oBAC3E,WAAW,CAAC,IAAI,CACd,IAAI,YAAY,CAAC;wBACf,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,EAAE,EAAE,KAAK,CAAmB;qBACrF,CAAC,CACH,CAAC;gBACJ,CAAC;gBACD,yFAAyF;YAC3F,CAAC;YAED,6FAA6F;YAC7F,2FAA2F;YAC3F,OAAO,WAAW,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACtF,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/middleware/registry.js

@@ -9,6 +9,41 @@ import { displayWarning } from '@gaunt-sloth/core/utils/consoleUtils.js';
 import { debugLog } from '@gaunt-sloth/core/utils/debugUtils.js';
 import { anthropicPromptCachingMiddleware, summarizationMiddleware, } from 'langchain';
 import { createBinaryContentInjectionMiddleware, } from '#src/middleware/binaryContentInjectionMiddleware.js';
+import { createFrontendImageInjectionMiddleware } from '#src/middleware/frontendImageInjectionMiddleware.js';
+/**
+ * Derive the provider string the vision middleware maps to a per-provider block shape. Prefers the
+ * loader-stashed raw `llm.type` ({@link GthConfig.modelProviderType}) — the exact gth provider
+ * namespace (`anthropic`/`openrouter`/`deepseek`/`xai`/`groq`/`ollama`/`google-genai`/`vertexai`/…)
+ * — and falls back to the live model's `_llmType()` only when it is absent (module configs).
+ * `_llmType()` is the model class's own label rather than the gth provider namespace, so the two
+ * only mostly agree. Where they do: `openrouter`/`deepseek`/`xai`/`groq`/`ollama`/`anthropic` each
+ * report their own name, which `imageBlockFor` matches directly, and both Gemini providers report
+ * `google`, which lands on the default branch — the same standard block its `google-genai`/
+ * `vertexai` cases return.
+ *
+ * Two labels do NOT agree, and this fallback is the only path either can arrive on:
+ *   - `huggingface` reports `openai`, so the fallback emits `image_url:{url}` where the configured
+ *     `type` would take the default standard block. Which of the two the HF router actually accepts
+ *     is unmeasured — see [[CFG-45]]. Do not add a branch by analogy.
+ *   - `ChatXAIResponses` reports `xai-responses`, which matches no case and so takes the default
+ *     standard block — while `imageBlockFor`'s own docstring records that a standard block
+ *     serialises to an INVALID image part on the Responses path. gth itself only ever constructs
+ *     `ChatXAI`, so this is reachable only from a module config that builds the Responses class
+ *     itself.
+ */
+function resolveVisionProvider(gthConfig) {
+    if (gthConfig.modelProviderType)
+        return gthConfig.modelProviderType;
+    const llm = gthConfig.llm;
+    try {
+        if (typeof llm?._llmType === 'function')
+            return llm._llmType();
+    }
+    catch {
+        // A misbehaving _llmType must not break middleware resolution.
+    }
+    return '';
+}
 const predefinedMiddlewareFactories = {
     /**
      * Anthropic prompt caching middleware. see https://docs.langchain.com/oss/javascript/langchain/middleware#anthropic-prompt-caching
@@ -24,7 +59,20 @@ const predefinedMiddlewareFactories = {
      * as HumanMessage content blocks before the next model call.
      * This works around LangChain's limitation where ToolMessage doesn't support binary content.
      */
-    'binary-content-injection': (settings, gthConfig) => createBinaryContentInjectionMiddleware(settings, gthConfig),
+    'binary-content-injection': (settings, gthConfig) => createBinaryContentInjectionMiddleware({
+        ...settings,
+        provider: resolveVisionProvider(gthConfig),
+    }, gthConfig),
+    /**
+     * Frontend image injection middleware (RC-22). Converts a frontend capture tool's
+     * `{mimeType,data}` ToolMessage (default tool name `capture_image`) into a provider-appropriate
+     * vision HumanMessage the model can see. Strictly opt-in — resolved ONLY when named in
+     * `config.middleware` (no auto-inject branch). Optional `toolName` setting overrides the tool name.
+     */
+    'frontend-image-injection': (settings, gthConfig) => Promise.resolve(createFrontendImageInjectionMiddleware({
+        provider: resolveVisionProvider(gthConfig),
+        toolName: typeof settings.toolName === 'string' ? settings.toolName : undefined,
+    })),
 };
 function isPredefinedMiddlewareName(name) {
     return name in predefinedMiddlewareFactories;

package/dist/middleware/registry.js.map

@@ -1 +1 @@
-{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/middleware/registry.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AASH,OAAO,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,uCAAuC,CAAC;AACjE,OAAO,EACL,gCAAgC,EAChC,uBAAuB,GAExB,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,sCAAsC,GAEvC,MAAM,qDAAqD,CAAC;AAO7D,MAAM,6BAA6B,GAAG;IACpC;;OAEG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,sCAAsC,CAAC,QAAwC,EAAE,SAAS,CAAC;IAC7F;;OAEG;IACH,aAAa,EAAE,CACb,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,6BAA6B,CAAC,QAA+B,EAAE,SAAS,CAAC;IAC3E;;;;;OAKG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,sCAAsC,CACpC,QAAoD,EACpD,SAAS,CACV;CACkD,CAAC;AAExD,SAAS,0BAA0B,CACjC,IAAY;IAEZ,OAAO,IAAI,IAAI,6BAA6B,CAAC;AAC/C,CAAC;AAED,SAAS,4BAA4B,CACnC,MAAwB;IAExB,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,MAAM,IAAI,MAAM;QAChB,OAAQ,MAA4B,CAAC,IAAI,KAAK,QAAQ;QACtD,0BAA0B,CAAE,MAA2B,CAAC,IAAI,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sCAAsC,CAC1D,MAAoC,EACpC,CAAY;IAEZ,QAAQ,CAAC,0DAA0D,MAAM,CAAC,GAAG,IAAI,SAAS,EAAE,CAAC,CAAC;IAE9F,0CAA0C;IAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,gCAAgC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAChF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,MAA2B,EAC3B,SAAoB;IAEpB,QAAQ,CAAC,mCAAmC,CAAC,CAAC;IAE9C,OAAO,OAAO,CAAC,OAAO,CACpB,uBAAuB,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,GAAG;QACpC,GAAG,MAAM;KACV,CAAC,CACH,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAuC,EACvC,SAAoB;IAEpB,MAAM,gBAAgB,GAAG,OAAO,IAAI,EAAE,CAAC;IACvC,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,iFAAiF;IACjF,wDAAwD;IACxD,MAAM,gBAAgB,GACpB,SAAS,CAAC,aAAa,KAAK,SAAS;QACrC,SAAS,CAAC,aAAa,KAAK,KAAK;QACjC,SAAS,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;IACrC,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,IAAI,CAC/C,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,KAAK,0BAA0B;QAChC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,0BAA0B,CAAC,CAClF,CAAC;IAEF,IAAI,gBAAgB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7C,QAAQ,CAAC,+EAA+E,CAAC,CAAC;QAC1F,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,0BAA0B,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,oEAAoE;YACpE,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;YAC3E,CAAC;YACD,oDAAoD;iBAC/C,IAAI,4BAA4B,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,MAAM,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;gBACrC,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YAC/E,CAAC;YACD,mDAAmD;iBAC9C,CAAC;gBACJ,QAAQ,CAAC,0BAA0B,CAAC,CAAC;gBACrC,UAAU,CAAC,IAAI,CAAC,MAAyB,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CACZ,gCAAgC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACzF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,0BAA0B,CACvC,IAAY,EACZ,QAAiC,EACjC,SAAoB;IAEpB,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;IACpD,OAAO,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AACtC,CAAC"}
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/middleware/registry.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AASH,OAAO,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,MAAM,uCAAuC,CAAC;AACjE,OAAO,EACL,gCAAgC,EAChC,uBAAuB,GAExB,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,sCAAsC,GAEvC,MAAM,qDAAqD,CAAC;AAC7D,OAAO,EAAE,sCAAsC,EAAE,MAAM,qDAAqD,CAAC;AAO7G;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,SAAS,qBAAqB,CAAC,SAAoB;IACjD,IAAI,SAAS,CAAC,iBAAiB;QAAE,OAAO,SAAS,CAAC,iBAAiB,CAAC;IACpE,MAAM,GAAG,GAAG,SAAS,CAAC,GAA8C,CAAC;IACrE,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,EAAE,QAAQ,KAAK,UAAU;YAAE,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,+DAA+D;IACjE,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,6BAA6B,GAAG;IACpC;;OAEG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,sCAAsC,CAAC,QAAwC,EAAE,SAAS,CAAC;IAC7F;;OAEG;IACH,aAAa,EAAE,CACb,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,6BAA6B,CAAC,QAA+B,EAAE,SAAS,CAAC;IAC3E;;;;;OAKG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,sCAAsC,CACpC;QACE,GAAI,QAAqD;QACzD,QAAQ,EAAE,qBAAqB,CAAC,SAAS,CAAC;KAC3C,EACD,SAAS,CACV;IACH;;;;;OAKG;IACH,0BAA0B,EAAE,CAC1B,QAAiC,EACjC,SAAoB,EACM,EAAE,CAC5B,OAAO,CAAC,OAAO,CACb,sCAAsC,CAAC;QACrC,QAAQ,EAAE,qBAAqB,CAAC,SAAS,CAAC;QAC1C,QAAQ,EAAE,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;KAChF,CAAC,CACH;CACkD,CAAC;AAExD,SAAS,0BAA0B,CACjC,IAAY;IAEZ,OAAO,IAAI,IAAI,6BAA6B,CAAC;AAC/C,CAAC;AAED,SAAS,4BAA4B,CACnC,MAAwB;IAExB,OAAO,CACL,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,MAAM,IAAI,MAAM;QAChB,OAAQ,MAA4B,CAAC,IAAI,KAAK,QAAQ;QACtD,0BAA0B,CAAE,MAA2B,CAAC,IAAI,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sCAAsC,CAC1D,MAAoC,EACpC,CAAY;IAEZ,QAAQ,CAAC,0DAA0D,MAAM,CAAC,GAAG,IAAI,SAAS,EAAE,CAAC,CAAC;IAE9F,0CAA0C;IAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,gCAAgC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAChF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,MAA2B,EAC3B,SAAoB;IAEpB,QAAQ,CAAC,mCAAmC,CAAC,CAAC;IAE9C,OAAO,OAAO,CAAC,OAAO,CACpB,uBAAuB,CAAC;QACtB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,GAAG;QACpC,GAAG,MAAM;KACV,CAAC,CACH,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAAuC,EACvC,SAAoB;IAEpB,MAAM,gBAAgB,GAAG,OAAO,IAAI,EAAE,CAAC;IACvC,MAAM,UAAU,GAAsB,EAAE,CAAC;IAEzC,iFAAiF;IACjF,wDAAwD;IACxD,MAAM,gBAAgB,GACpB,SAAS,CAAC,aAAa,KAAK,SAAS;QACrC,SAAS,CAAC,aAAa,KAAK,KAAK;QACjC,SAAS,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;IACrC,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,IAAI,CAC/C,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,KAAK,0BAA0B;QAChC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,0BAA0B,CAAC,CAClF,CAAC;IAEF,IAAI,gBAAgB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7C,QAAQ,CAAC,+EAA+E,CAAC,CAAC;QAC1F,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,0BAA0B,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,oEAAoE;YACpE,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,MAAM,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;YAC3E,CAAC;YACD,oDAAoD;iBAC/C,IAAI,4BAA4B,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,MAAM,EAAE,IAAI,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;gBACrC,UAAU,CAAC,IAAI,CAAC,MAAM,0BAA0B,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YAC/E,CAAC;YACD,mDAAmD;iBAC9C,CAAC;gBACJ,QAAQ,CAAC,0BAA0B,CAAC,CAAC;gBACrC,UAAU,CAAC,IAAI,CAAC,MAAyB,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CACZ,gCAAgC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACzF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,0BAA0B,CACvC,IAAY,EACZ,QAAiC,EACjC,SAAoB;IAEpB,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;IACpD,OAAO,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AACtC,CAAC"}