@gaunt-sloth/tools 0.0.1 → 0.1.1

package/README.md

@@ -23,3 +23,11 @@ Tools and middleware for Gaunt Sloth.
 import { builtInToolsConfig } from '@gaunt-sloth/tools/builtInToolsConfig.js';
 import { resolveMiddleware } from '@gaunt-sloth/tools/middleware/registry.js';
 ```
+
+## Related packages
+
+- [`@gaunt-sloth/core`](../core) — Core utilities, config, and agent infrastructure
+- [`@gaunt-sloth/tools`](../tools) — Built-in tools, filesystem toolkit, and middleware registry (this package)
+- [`@gaunt-sloth/api`](../api) — API server, AG-UI, MCP, and A2A integration
+- [`@gaunt-sloth/review`](../review) — Review and Q&A modules with standalone CLI
+- [`gaunt-sloth-assistant`](../assistant) — Main CLI application

package/dist/middleware/frontendImageInjectionMiddleware.d.ts

@@ -0,0 +1,15 @@
+/**
+ * @packageDocumentation
+ * Middleware to inject camera frames (images) as HumanMessage.
+ *
+ * This middleware:
+ * 1. Detects the capture_image tool calls
+ * 2. Parses the JSON payload from ToolMessage content
+ * 3. Injects HumanMessage with image blocks before next model call
+ */
+import { type AgentMiddleware } from 'langchain';
+import type { GthConfig } from '@gaunt-sloth/core/config.js';
+export interface FrontendImageInjectionMiddlewareSettings {
+    name?: 'frontend-image-injection';
+}
+export declare function createFrontendImageInjectionMiddleware(_settings: FrontendImageInjectionMiddlewareSettings, _gthConfig: GthConfig): Promise<AgentMiddleware>;

package/dist/middleware/frontendImageInjectionMiddleware.js

@@ -0,0 +1,72 @@
+/**
+ * @packageDocumentation
+ * Middleware to inject camera frames (images) as HumanMessage.
+ *
+ * This middleware:
+ * 1. Detects the capture_image tool calls
+ * 2. Parses the JSON payload from ToolMessage content
+ * 3. Injects HumanMessage with image blocks before next model call
+ */
+import { createMiddleware } from 'langchain';
+import { debugLog } from '@gaunt-sloth/core/utils/debugUtils.js';
+import { ToolMessage, HumanMessage } from '@langchain/core/messages';
+export function createFrontendImageInjectionMiddleware(_settings, _gthConfig) {
+    debugLog('Creating frontend image injection middleware');
+    return Promise.resolve(createMiddleware({
+        name: 'frontend-image-injection',
+        beforeModel: async (state) => {
+            const messages = state.messages || [];
+            // Find recent ToolMessages from capture_image
+            const imageMessages = [];
+            for (let i = messages.length - 1; i >= Math.max(0, messages.length - 5); i--) {
+                const msg = messages[i];
+                if (msg instanceof ToolMessage &&
+                    msg.name === 'capture_image' &&
+                    typeof msg.content === 'string') {
+                    try {
+                        const payload = JSON.parse(msg.content);
+                        imageMessages.push({ message: msg, payload });
+                    }
+                    catch {
+                        // Ignore invalid JSON
+                    }
+                }
+            }
+            // If we found image content, inject HumanMessage(s)
+            if (imageMessages.length > 0) {
+                debugLog(`Injecting ${imageMessages.length} HumanMessage(s) from capture_image`);
+                const newMessages = [...messages];
+                for (const { payload } of imageMessages) {
+                    if (payload.error) {
+                        newMessages.push(new HumanMessage({
+                            content: `Camera unavailable: ${payload.error}`
+                        }));
+                    }
+                    else if (payload.mimeType && payload.data) {
+                        const contentBlock = {
+                            type: 'image',
+                            source_type: 'base64',
+                            mime_type: payload.mimeType,
+                            data: payload.data,
+                        };
+                        const humanMessage = new HumanMessage({
+                            content: [
+                                {
+                                    type: 'text',
+                                    text: `Camera frame captured:`,
+                                },
+                                contentBlock,
+                            ],
+                        });
+                        newMessages.push(humanMessage);
+                    }
+                }
+                if (newMessages.length > messages.length) {
+                    return { messages: newMessages };
+                }
+            }
+            return undefined;
+        },
+    }));
+}
+//# sourceMappingURL=frontendImageInjectionMiddleware.js.map

package/dist/middleware/frontendImageInjectionMiddleware.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontendImageInjectionMiddleware.js","sourceRoot":"","sources":["../../src/middleware/frontendImageInjectionMiddleware.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,gBAAgB,EAAwB,MAAM,WAAW,CAAC;AAEnE,OAAO,EAAE,QAAQ,EAAE,MAAM,uCAAuC,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAOrE,MAAM,UAAU,sCAAsC,CACpD,SAAmD,EACnD,UAAqB;IAErB,QAAQ,CAAC,8CAA8C,CAAC,CAAC;IAEzD,OAAO,OAAO,CAAC,OAAO,CACpB,gBAAgB,CAAC;QACf,IAAI,EAAE,0BAA0B;QAEhC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;YAEtC,8CAA8C;YAC9C,MAAM,aAAa,GAAkD,EAAE,CAAC;YAExE,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7E,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACxB,IACE,GAAG,YAAY,WAAW;oBAC1B,GAAG,CAAC,IAAI,KAAK,eAAe;oBAC5B,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAC/B,CAAC;oBACD,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;wBACxC,aAAa,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;oBAChD,CAAC;oBAAC,MAAM,CAAC;wBACP,sBAAsB;oBACxB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,oDAAoD;YACpD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,QAAQ,CAAC,aAAa,aAAa,CAAC,MAAM,qCAAqC,CAAC,CAAC;gBAEjF,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAElC,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,aAAa,EAAE,CAAC;oBACxC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;wBAClB,WAAW,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC;4BAChC,OAAO,EAAE,uBAAuB,OAAO,CAAC,KAAK,EAAE;yBAChD,CAAC,CAAC,CAAC;oBACN,CAAC;yBAAM,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;wBAC5C,MAAM,YAAY,GAAG;4BACnB,IAAI,EAAE,OAAO;4BACb,WAAW,EAAE,QAAQ;4BACrB,SAAS,EAAE,OAAO,CAAC,QAAQ;4BAC3B,IAAI,EAAE,OAAO,CAAC,IAAI;yBACnB,CAAC;wBAEF,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC;4BACpC,OAAO,EAAE;gCACP;oCACE,IAAI,EAAE,MAAM;oCACZ,IAAI,EAAE,wBAAwB;iCAC/B;gCACD,YAAY;6BACK;yBACpB,CAAC,CAAC;wBAEH,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACjC,CAAC;gBACH,CAAC;gBAED,IAAI,WAAW,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACzC,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;gBACnC,CAAC;YACH,CAAC;YAED,OAAO,SAAS,CAAC;QACnB,CAAC;KACF,CAAC,CACH,CAAC;AACJ,CAAC"}

package/dist/tools/captureImageTool.d.ts

@@ -0,0 +1,2 @@
+import { z } from 'zod';
+export declare function get(): import("@langchain/core/tools").DynamicStructuredTool<z.ZodObject<{}, z.core.$strip>, Record<string, never>, Record<string, never>, string, unknown, "capture_image">;

package/dist/tools/captureImageTool.js

@@ -0,0 +1,16 @@
+import { tool } from '@langchain/core/tools';
+import { z } from 'zod';
+export function get() {
+    const t = tool(async () => {
+        // This is a stub that will be overridden by GthLangChainAgent for client: true tools
+        return 'Client tool stub executed on server';
+    }, {
+        name: 'capture_image',
+        description: 'Capture a photo from the robot webcam. Returns a base64 encoded image frame.',
+        schema: z.object({}),
+    });
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    t.metadata = { client: true };
+    return t;
+}
+//# sourceMappingURL=captureImageTool.js.map

package/dist/tools/captureImageTool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"captureImageTool.js","sourceRoot":"","sources":["../../src/tools/captureImageTool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,uBAAuB,CAAC;AAC7C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,UAAU,GAAG;IACjB,MAAM,CAAC,GAAG,IAAI,CACZ,KAAK,IAAI,EAAE;QACT,qFAAqF;QACrF,OAAO,qCAAqC,CAAC;IAC/C,CAAC,EACD;QACE,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE,8EAA8E;QAC3F,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;KACrB,CACF,CAAC;IACF,8DAA8D;IAC7D,CAAS,CAAC,QAAQ,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACvC,OAAO,CAAC,CAAC;AACX,CAAC"}

package/package.json

@@ -1,9 +1,25 @@
 {
   "name": "@gaunt-sloth/tools",
-  "version": "0.0.1",
+  "version": "0.1.1",
   "description": "Tools and middleware for Gaunt Sloth",
   "license": "MIT",
   "author": "Andrew Kondratev",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Galvanized-Pukeko/gaunt-sloth-assistant.git",
+    "directory": "packages/tools"
+  },
+  "bugs": {
+    "url": "https://github.com/Galvanized-Pukeko/gaunt-sloth-assistant/issues"
+  },
+  "homepage": "https://github.com/Galvanized-Pukeko/gaunt-sloth-assistant/tree/main/packages/tools#readme",
+  "keywords": [
+    "ai",
+    "agent",
+    "llm",
+    "tools",
+    "middleware"
+  ],
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -18,9 +34,9 @@
     "#src/*.js": "./dist/*.js"
   },
   "dependencies": {
-    "@gaunt-sloth/core": "^0.0.1",
-    "diff": "^8.0.3",
-    "langchain": "^1.2.36"
+    "@gaunt-sloth/core": "0.1.1",
+    "diff": "^9.0.0",
+    "langchain": "^1.3.5"
   },
   "files": [
     "./dist/*"