@hybridaione/hybridclaw 0.2.1 → 0.2.2

package/CHANGELOG.md

@@ -8,6 +8,29 @@
 
 ### Fixed
 
+## [0.2.2](https://github.com/HybridAIOne/hybridclaw/tree/v0.2.2)
+
+### Added
+
+- **Discord image attachment ingest/cache**: Added receive-time image ingest with local cache under `data/discord-media-cache`, preserving attachment order and carrying `path`, `mimeType`, `sizeBytes`, and `originalUrl` per media item.
+- **Structured media context pipeline**: Added typed media payload (`MediaPaths`/`MediaUrls`/`MediaTypes` equivalents) from Discord runtime through gateway/container request boundaries.
+- **Attachment vision tools**: Added `vision_analyze` (and `image` alias) for Discord-uploaded image analysis using local cached paths first, with Discord CDN URL fallback.
+- **Native multimodal injection**: Added direct image-part injection for vision-capable models, with automatic retry without image parts if the model rejects multimodal payloads.
+- **Scoped Vitest test configs**: Added dedicated `vitest.{unit,integration,e2e,live}.config.ts` files and matching npm scripts (`test:unit`, `test:integration`, `test:e2e`, `test:live`, `test:watch`) for explicit suite boundaries.
+
+### Changed
+
+- **Discord channel module layout**: Completed migration of Discord runtime internals into `src/channels/discord/*`, including `runtime.ts` and `stream.ts`, and removed legacy root-level `src/discord.ts` shim.
+- **Image-question tool routing**: Discord image questions now prioritize attachment vision (`vision_analyze`) and block `browser_vision` unless the user explicitly asks about the active browser tab/page.
+- **Browser vision scope guidance**: Updated `browser_vision` tool description to clarify it is for browser-page tasks only, not Discord-uploaded files.
+- **Test runner strategy**: Switched from compiled test artifacts (`dist-tests` + `tsconfig.tests.json`) to direct TypeScript execution via Vitest.
+- **Test file location and conventions**: Moved basic test files from `src/*.test.ts` to `tests/` and aligned naming/scoping conventions for unit/integration/e2e/live suites.
+
+### Fixed
+
+- **Discord image analysis fallback behavior**: Added safer cache/CDN fallback handling and guardrails (Discord CDN allowlist, size/type limits, per-image success/failure logging) to avoid brittle image-analysis failures.
+- **Regression coverage for wrong vision tool selection**: Added basic regression test coverage that Discord image questions should not route to browser screenshot vision.
+
 ## [0.2.1](https://github.com/HybridAIOne/hybridclaw/tree/v0.2.1)
 
 ### Added

package/README.md

@@ -11,15 +11,17 @@ npm install -g @hybridaione/hybridclaw
 hybridclaw onboarding
 ```
 
-Latest release: [v0.2.1](https://github.com/HybridAIOne/hybridclaw/releases/tag/v0.2.1)
+Latest release: [v0.2.2](https://github.com/HybridAIOne/hybridclaw/releases/tag/v0.2.2)
 
-## What's new in v0.2.1
+## What's new in v0.2.2
 
-- Added OpenClaw-style Discord `message` tool actions (`read`, `member-info`, `channel-info`) to the container runtime
-- Added gateway endpoint `POST /api/discord/action` for Discord context lookups from tools
-- Replaced prompt-time Discord presence snapshots with cache-backed `member-info` presence fields (`status`, `activities`)
-- Routed Discord context lookups through gateway API from container with host remapping and token propagation
-- Enabled `message` tool in heartbeat and base subagent allowlists
+- Added Discord attachment ingest/cache with structured media context (`path`, `mime`, `size`, `original_url`) passed into the agent pipeline
+- Added `vision_analyze`/`image` tools for Discord-uploaded image analysis (local cached path first, Discord CDN fallback)
+- Added native model vision image-part injection for vision-capable models, with safe fallback if multimodal input is rejected
+- Routed Discord image questions away from `browser_vision` (unless explicitly about the active browser tab/page)
+- Completed Discord runtime migration into `src/channels/discord/*` and removed the legacy root-level `src/discord.ts` shim
+- Switched tests from compiled `dist-tests` artifacts to direct TypeScript execution via Vitest
+- Moved basic tests to `tests/` with explicit scope naming conventions
 
 ## HybridAI Advantage
 
@@ -349,6 +351,34 @@ System prompt assembly is handled by a formal hook pipeline:
 
 Hook toggles live in `config.json` under `promptHooks`.
 
+## Testing
+
+Run checks locally:
+
+```bash
+# Typecheck only (no emit)
+npm run typecheck
+
+# Strict TS lint gate (unused locals/params)
+npm run lint
+
+# Unit tests (default `npm test`)
+npm run test:unit
+
+# Scoped suites (ready for dedicated tests)
+npm run test:integration
+npm run test:e2e
+npm run test:live
+```
+
+Test layout and scopes:
+
+- tests live under `tests/` (not `src/`)
+- unit tests: `tests/**/*.test.ts` (excluding `*.integration|*.e2e|*.live`)
+- integration tests: `tests/**/*.integration.test.ts`
+- e2e tests: `tests/**/*.e2e.test.ts`
+- live tests: `tests/**/*.live.test.ts`
+
 ## Commands
 
 CLI runtime commands:
@@ -379,12 +409,14 @@ In Discord, use `!claw help` to see all commands. Key ones:
 ## Project structure
 
 ```
-src/gateway.ts          Core runtime entrypoint (DB, scheduler, heartbeat, HTTP API)
-src/tui.ts              Terminal adapter (thin client to gateway)
-src/discord.ts          Discord integration and message transport
-src/gateway-service.ts  Core shared agent/session logic used by gateway API
-src/gateway-client.ts   HTTP client used by thin clients (e.g. TUI)
-container/src/          Agent code (tools, HybridAI client, IPC)
-templates/              Workspace bootstrap files
-data/                   Runtime data (gitignored): SQLite DB, sessions, agent workspaces
+src/gateway.ts                    Core runtime entrypoint (DB, scheduler, heartbeat, HTTP API)
+src/tui.ts                        Terminal adapter (thin client to gateway)
+src/channels/discord/runtime.ts   Discord runtime integration and message transport
+src/channels/discord/*.ts         Discord responsibility modules (inbound, delivery, mentions, attachments, tools, stream)
+src/gateway-service.ts            Core shared agent/session logic used by gateway API
+src/gateway-client.ts             HTTP client used by thin clients (e.g. TUI)
+tests/                            Vitest suites (unit/integration/e2e/live scopes)
+container/src/                    Agent code (tools, HybridAI client, IPC)
+templates/                        Workspace bootstrap files
+data/                             Runtime data (gitignored): SQLite DB, sessions, agent workspaces
 ```

package/container/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "hybridclaw-agent",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "hybridclaw-agent",
-      "version": "0.2.1",
+      "version": "0.2.2",
       "dependencies": {
         "@mozilla/readability": "^0.6.0",
         "agent-browser": "^0.15.1",

package/container/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hybridclaw-agent",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "type": "module",
   "scripts": {
     "build": "tsc",

package/container/src/browser-tools.ts

@@ -1199,7 +1199,7 @@ export const BROWSER_TOOL_DEFINITIONS: ToolDefinition[] = [
     function: {
       name: 'browser_vision',
       description:
-        'Capture a screenshot and analyze it with a vision model using the provided question.',
+        'Capture the current browser page screenshot and analyze it with a vision model. Use only for active browser-tab/page tasks, not for Discord-uploaded files.',
       parameters: {
         type: 'object',
         properties: {

package/container/src/index.ts

@@ -1,4 +1,6 @@
 import path from 'path';
+import fs from 'fs';
+import { URL } from 'url';
 
 import { emitRuntimeEvent, runAfterToolHooks, runBeforeToolHooks } from './extensions.js';
 import { callHybridAI, callHybridAIStream, HybridAIRequestError } from './hybridai-client.js';
@@ -15,12 +17,23 @@ import {
   getPendingSideEffects,
   resetSideEffects,
   setGatewayContext,
+  setMediaContext,
   setModelContext,
   setScheduledTasks,
   setSessionContext,
   TOOL_DEFINITIONS,
 } from './tools.js';
-import type { ArtifactMetadata, ChatMessage, ContainerInput, ContainerOutput, ToolDefinition, ToolExecution } from './types.js';
+import type {
+  ArtifactMetadata,
+  ChatContentPart,
+  ChatMessage,
+  ChatMessageContent,
+  ContainerInput,
+  ContainerOutput,
+  MediaContextItem,
+  ToolDefinition,
+  ToolExecution,
+} from './types.js';
 
 const MAX_ITERATIONS = 20;
 const IDLE_TIMEOUT_MS = parseInt(process.env.CONTAINER_IDLE_TIMEOUT || '300000', 10); // 5 min
@@ -45,10 +58,159 @@ const ARTIFACT_MIME_TYPES: Record<string, string> = {
   '.svg': 'image/svg+xml',
   '.webp': 'image/webp',
 };
+const DISCORD_MEDIA_CACHE_ROOT = '/discord-media-cache';
+const NATIVE_VISION_MAX_IMAGES = 8;
+const NATIVE_VISION_MAX_IMAGE_BYTES = 10 * 1024 * 1024;
+const DISCORD_CDN_HOST_PATTERNS: RegExp[] = [
+  /^cdn\.discordapp\.com$/i,
+  /^media\.discordapp\.net$/i,
+  /^cdn\.discordapp\.net$/i,
+  /^images-ext-\d+\.discordapp\.net$/i,
+];
 
 /** API key received once via stdin, held in memory for the container lifetime. */
 let storedApiKey = '';
 
+function normalizeMessageContentToText(content: ChatMessageContent): string {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+  const chunks: string[] = [];
+  for (const part of content) {
+    if (!part || typeof part !== 'object') continue;
+    if (part.type !== 'text') continue;
+    if (typeof part.text !== 'string') continue;
+    if (part.text.trim()) chunks.push(part.text.trim());
+  }
+  return chunks.join('\n').trim();
+}
+
+function normalizePathSlashes(raw: string): string {
+  return raw.replace(/\\/g, '/');
+}
+
+function normalizeAllowedLocalImagePath(rawPath: string): string | null {
+  const trimmed = rawPath.trim();
+  if (!trimmed) return null;
+
+  const workspace = path.posix.normalize(WORKSPACE_ROOT);
+  const mediaRoot = path.posix.normalize(DISCORD_MEDIA_CACHE_ROOT);
+
+  const candidate = trimmed.startsWith('/')
+    ? path.posix.normalize(normalizePathSlashes(trimmed))
+    : path.posix.normalize(path.posix.join(workspace, normalizePathSlashes(trimmed)));
+
+  const underWorkspace = candidate === workspace || candidate.startsWith(`${workspace}/`);
+  const underMediaRoot = candidate === mediaRoot || candidate.startsWith(`${mediaRoot}/`);
+  if (!underWorkspace && !underMediaRoot) return null;
+  return candidate;
+}
+
+function inferImageMimeType(filePath: string, fallbackMime: string | null | undefined): string {
+  const normalizedFallback = String(fallbackMime || '').trim().toLowerCase();
+  if (normalizedFallback.startsWith('image/')) return normalizedFallback;
+  const ext = path.posix.extname(filePath).toLowerCase();
+  return ARTIFACT_MIME_TYPES[ext] || 'image/png';
+}
+
+function isSafeDiscordCdnUrl(raw: string): boolean {
+  let parsed: URL;
+  try {
+    parsed = new URL(raw);
+  } catch {
+    return false;
+  }
+  if (parsed.protocol !== 'https:') return false;
+  return DISCORD_CDN_HOST_PATTERNS.some((pattern) => pattern.test(parsed.hostname));
+}
+
+function modelSupportsNativeVision(model: string): boolean {
+  const normalized = model.toLowerCase();
+  if (!normalized) return false;
+  if (
+    normalized.includes('gpt-5')
+    || normalized.includes('gpt-4o')
+    || normalized.includes('gpt-4.1')
+    || normalized.includes('o1')
+    || normalized.includes('o3')
+    || normalized.includes('vision')
+    || normalized.includes('multimodal')
+    || normalized.includes('gemini')
+    || normalized.includes('claude-3')
+  ) {
+    return true;
+  }
+  return false;
+}
+
+async function resolveMediaImagePartUrl(item: MediaContextItem): Promise<string | null> {
+  const localPath = item.path ? normalizeAllowedLocalImagePath(item.path) : null;
+  if (localPath) {
+    try {
+      const image = await fs.promises.readFile(localPath);
+      if (image.length > NATIVE_VISION_MAX_IMAGE_BYTES) {
+        console.error(`[media] skipping ${localPath}: ${image.length}B exceeds native vision max`);
+      } else {
+        const mimeType = inferImageMimeType(localPath, item.mimeType);
+        const base64 = image.toString('base64');
+        return `data:${mimeType};base64,${base64}`;
+      }
+    } catch (err) {
+      console.error(`[media] failed to read local media ${localPath}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+
+  const fallbackCandidates = [item.url, item.originalUrl].map((value) => String(value || '').trim()).filter(Boolean);
+  for (const candidate of fallbackCandidates) {
+    if (!isSafeDiscordCdnUrl(candidate)) continue;
+    return candidate;
+  }
+  return null;
+}
+
+async function injectNativeVisionContent(
+  messages: ChatMessage[],
+  model: string,
+  media: MediaContextItem[] | undefined,
+): Promise<ChatMessage[]> {
+  if (!Array.isArray(media) || media.length === 0) return messages;
+  if (!modelSupportsNativeVision(model)) return messages;
+
+  const mediaSlice = media.slice(0, NATIVE_VISION_MAX_IMAGES);
+  const imageParts: ChatContentPart[] = [];
+  for (const item of mediaSlice) {
+    const url = await resolveMediaImagePartUrl(item);
+    if (!url) continue;
+    imageParts.push({ type: 'image_url', image_url: { url } });
+  }
+  if (imageParts.length === 0) return messages;
+
+  const latestUserIndex = (() => {
+    for (let i = messages.length - 1; i >= 0; i -= 1) {
+      if (messages[i].role === 'user') return i;
+    }
+    return -1;
+  })();
+  if (latestUserIndex < 0) return messages;
+
+  const cloned = messages.map((msg) => ({ ...msg }));
+  const existingText = normalizeMessageContentToText(cloned[latestUserIndex].content);
+  const contentParts: ChatContentPart[] = [];
+  const nativeVisionHint =
+    '[NativeVision] Image parts are attached in this message. Analyze them directly and skip extra vision tool pre-analysis unless explicitly required.';
+  if (existingText) {
+    contentParts.push({ type: 'text', text: `${existingText}\n\n${nativeVisionHint}` });
+  } else {
+    contentParts.push({ type: 'text', text: nativeVisionHint });
+  }
+  contentParts.push(...imageParts);
+  cloned[latestUserIndex] = {
+    ...cloned[latestUserIndex],
+    content: contentParts,
+  };
+  console.error(`[media] injected ${imageParts.length} native vision image part(s) for model ${model}`);
+  return cloned;
+}
+
 /**
  * Read a single line from stdin (the initial request JSON containing secrets).
  * Resolves on the first newline — does not consume the entire stream, so docker -i
@@ -104,20 +266,21 @@ function latestUserPrompt(messages: ChatMessage[]): string {
   for (let i = messages.length - 1; i >= 0; i--) {
     const message = messages[i];
     if (message.role !== 'user') continue;
-    const text = String(message.content || '').replace(/\s+/g, ' ').trim();
+    const text = normalizeMessageContentToText(message.content).replace(/\s+/g, ' ').trim();
     if (!text) continue;
     return text.slice(0, 1_200);
   }
   return 'Continue the task';
 }
 
-function parseRalphChoice(content: string | null): 'CONTINUE' | 'STOP' | null {
-  if (!content) return null;
+function parseRalphChoice(content: ChatMessageContent): 'CONTINUE' | 'STOP' | null {
+  const normalizedContent = normalizeMessageContentToText(content);
+  if (!normalizedContent) return null;
   const re = /<choice>\s*([^<]*)\s*<\/choice>/gi;
   let match: RegExpExecArray | null = null;
   let lastChoice: string | null = null;
   while (true) {
-    match = re.exec(content);
+    match = re.exec(normalizedContent);
     if (!match) break;
     lastChoice = (match[1] || '').trim().toUpperCase();
   }
@@ -125,13 +288,14 @@ function parseRalphChoice(content: string | null): 'CONTINUE' | 'STOP' | null {
   return null;
 }
 
-function stripRalphChoiceTags(content: string | null): string | null {
-  if (content == null) return content;
-  const stripped = content
+function stripRalphChoiceTags(content: ChatMessageContent): string | null {
+  const normalizedContent = normalizeMessageContentToText(content);
+  if (!normalizedContent) return null;
+  const stripped = normalizedContent
     .replace(/<choice>\s*[^<]*\s*<\/choice>/gi, '')
     .replace(/\n{3,}/g, '\n\n')
     .trim();
-  return stripped || content;
+  return stripped || normalizedContent;
 }
 
 function buildRalphPrompt(taskPrompt: string, missingChoice: boolean): string {
@@ -487,14 +651,35 @@ async function processRequest(
  * Main loop: read first request from stdin (with secrets), then poll IPC for follow-ups.
  */
 function resolveTools(input: ContainerInput): ToolDefinition[] {
-  const tools = input.allowedTools
+  let tools = input.allowedTools
     ? TOOL_DEFINITIONS.filter((t) => input.allowedTools!.includes(t.function.name))
     : [...TOOL_DEFINITIONS];
+  if (Array.isArray(input.blockedTools) && input.blockedTools.length > 0) {
+    const blocked = new Set(
+      input.blockedTools
+        .map((name) => String(name || '').trim())
+        .filter(Boolean),
+    );
+    tools = tools.filter((tool) => !blocked.has(tool.function.name));
+  }
   // Sort alphabetically for deterministic system-prompt ordering (KV cache stability)
   tools.sort((a, b) => a.function.name.localeCompare(b.function.name));
   return tools;
 }
 
+function shouldRetryWithoutNativeVision(error: string | undefined): boolean {
+  const normalized = String(error || '').toLowerCase();
+  if (!normalized) return false;
+  return (
+    normalized.includes('image_url')
+    || normalized.includes('unsupported image')
+    || normalized.includes('unsupported content')
+    || normalized.includes('vision')
+    || normalized.includes('multimodal')
+    || normalized.includes('content part')
+  );
+}
+
 async function main(): Promise<void> {
   console.error(`[hybridclaw-agent] started, idle timeout ${IDLE_TIMEOUT_MS}ms`);
 
@@ -510,9 +695,15 @@ async function main(): Promise<void> {
   setSessionContext(firstInput.sessionId);
   setGatewayContext(firstInput.gatewayBaseUrl, firstInput.gatewayApiToken, firstInput.channelId);
   setModelContext(firstInput.baseUrl, storedApiKey, firstInput.model, firstInput.chatbotId);
-
-  const firstOutput = await processRequest(
+  setMediaContext(firstInput.media);
+  const firstMessages = await injectNativeVisionContent(
     firstInput.messages,
+    firstInput.model,
+    firstInput.media,
+  );
+
+  let firstOutput = await processRequest(
+    firstMessages,
     storedApiKey,
     firstInput.baseUrl,
     firstInput.model,
@@ -520,6 +711,22 @@ async function main(): Promise<void> {
     firstInput.enableRag,
     resolveTools(firstInput),
   );
+  if (
+    firstMessages !== firstInput.messages
+    && firstOutput.status === 'error'
+    && shouldRetryWithoutNativeVision(firstOutput.error)
+  ) {
+    console.error('[media] native vision injection rejected by model; retrying without image parts');
+    firstOutput = await processRequest(
+      firstInput.messages,
+      storedApiKey,
+      firstInput.baseUrl,
+      firstInput.model,
+      firstInput.chatbotId,
+      firstInput.enableRag,
+      resolveTools(firstInput),
+    );
+  }
 
   firstOutput.sideEffects = getPendingSideEffects();
   writeOutput(firstOutput);
@@ -544,9 +751,15 @@ async function main(): Promise<void> {
     setSessionContext(input.sessionId);
     setGatewayContext(input.gatewayBaseUrl, input.gatewayApiToken, input.channelId);
     setModelContext(input.baseUrl, apiKey, input.model, input.chatbotId);
-
-    const output = await processRequest(
+    setMediaContext(input.media);
+    const preparedMessages = await injectNativeVisionContent(
       input.messages,
+      input.model,
+      input.media,
+    );
+
+    let output = await processRequest(
+      preparedMessages,
       apiKey,
       input.baseUrl,
       input.model,
@@ -554,6 +767,22 @@ async function main(): Promise<void> {
       input.enableRag,
       resolveTools(input),
     );
+    if (
+      preparedMessages !== input.messages
+      && output.status === 'error'
+      && shouldRetryWithoutNativeVision(output.error)
+    ) {
+      console.error('[media] native vision injection rejected by model; retrying without image parts');
+      output = await processRequest(
+        input.messages,
+        apiKey,
+        input.baseUrl,
+        input.model,
+        input.chatbotId,
+        input.enableRag,
+        resolveTools(input),
+      );
+    }
 
     output.sideEffects = getPendingSideEffects();
     writeOutput(output);

package/container/src/token-usage.ts

@@ -26,12 +26,28 @@ export function createTokenUsageStats(): TokenUsageStats {
   };
 }
 
-export function estimateTextTokens(text: string | null | undefined): number {
+export function estimateTextTokens(text: unknown): number {
   const normalized = typeof text === 'string' ? text : '';
   if (!normalized) return 0;
   return Math.max(1, Math.ceil(normalized.length / CHARS_PER_TOKEN));
 }
 
+function normalizeContentText(content: ChatMessage['content']): string {
+  if (typeof content === 'string') return content;
+  if (!Array.isArray(content)) return '';
+  const chunks: string[] = [];
+  for (const part of content) {
+    if (part?.type === 'text' && typeof part.text === 'string') {
+      chunks.push(part.text);
+      continue;
+    }
+    if (part?.type === 'image_url' && part.image_url?.url) {
+      chunks.push('[image]');
+    }
+  }
+  return chunks.join('\n');
+}
+
 export function estimateMessageTokens(messages: ChatMessage[]): number {
   if (!Array.isArray(messages) || messages.length === 0) return 0;
 
@@ -39,7 +55,7 @@ export function estimateMessageTokens(messages: ChatMessage[]): number {
   for (const message of messages) {
     total += 4;
     total += estimateTextTokens(message.role);
-    total += estimateTextTokens(message.content);
+    total += estimateTextTokens(normalizeContentText(message.content));
     if (message.tool_calls) total += estimateTextTokens(JSON.stringify(message.tool_calls));
     if (message.tool_call_id) total += estimateTextTokens(message.tool_call_id);
   }