@dotsetlabs/dotclaw 2.4.0 → 2.6.0

package/container/agent-runner/src/openrouter-input.ts

@@ -0,0 +1,159 @@
+import fs from 'fs';
+import path from 'path';
+
+import type { ContainerInput } from './container-protocol.js';
+
+const MAX_IMAGE_BYTES = 10 * 1024 * 1024; // 10MB per image
+const MAX_TOTAL_IMAGE_BYTES = 20 * 1024 * 1024; // 20MB total across all images
+const IMAGE_MIME_TYPES = new Set(['image/jpeg', 'image/png', 'image/gif', 'image/webp']);
+
+export interface OpenRouterInputTextPart {
+  type: 'input_text';
+  text: string;
+}
+
+export interface OpenRouterInputImagePart {
+  type: 'input_image';
+  detail: 'auto';
+  imageUrl: string;
+}
+
+export type OpenRouterUserContentPart = OpenRouterInputTextPart | OpenRouterInputImagePart;
+
+export interface OpenRouterInputMessage {
+  role: 'user' | 'assistant';
+  content: string | OpenRouterUserContentPart[];
+}
+
+function inferImageMimeFromName(fileName?: string): string | null {
+  if (!fileName || typeof fileName !== 'string') return null;
+  const extension = path.extname(fileName).toLowerCase();
+  if (!extension) return null;
+  if (extension === '.jpg' || extension === '.jpeg') return 'image/jpeg';
+  if (extension === '.png') return 'image/png';
+  if (extension === '.gif') return 'image/gif';
+  if (extension === '.webp') return 'image/webp';
+  return null;
+}
+
+function normalizeMimeType(input: unknown, fallbackName?: string): string | null {
+  const fromInput = typeof input === 'string'
+    ? input.toLowerCase().split(';')[0].trim()
+    : '';
+  const candidate = fromInput || inferImageMimeFromName(fallbackName);
+  if (!candidate || !IMAGE_MIME_TYPES.has(candidate)) return null;
+  return candidate;
+}
+
+function jsonStringifySafe(value: unknown): string {
+  try {
+    const serialized = JSON.stringify(value);
+    return typeof serialized === 'string' ? serialized : String(value);
+  } catch {
+    return String(value);
+  }
+}
+
+export function coerceInputContentToText(content: unknown): string {
+  if (typeof content === 'string') return content;
+  if (content == null) return '';
+
+  const collectFromRecord = (record: Record<string, unknown>): string | null => {
+    if (typeof record.text === 'string' && record.text.trim()) return record.text;
+    if (typeof record.content === 'string' && record.content.trim()) return record.content;
+    if (typeof record.output === 'string' && record.output.trim()) return record.output;
+    if (typeof record.refusal === 'string' && record.refusal.trim()) return record.refusal;
+    return null;
+  };
+
+  if (Array.isArray(content)) {
+    const parts = content
+      .map((part) => {
+        if (!part || typeof part !== 'object') return null;
+        const record = part as Record<string, unknown>;
+        return collectFromRecord(record);
+      })
+      .filter((part): part is string => typeof part === 'string');
+    if (parts.length > 0) return parts.join('\n');
+    return jsonStringifySafe(content);
+  }
+
+  if (typeof content === 'object') {
+    const extracted = collectFromRecord(content as Record<string, unknown>);
+    if (extracted) return extracted;
+    return jsonStringifySafe(content);
+  }
+
+  return String(content);
+}
+
+export function messagesToOpenRouterInput(
+  messages: Array<{ role: 'user' | 'assistant'; content: unknown }>
+): OpenRouterInputMessage[] {
+  return messages.map(message => ({
+    role: message.role,
+    content: coerceInputContentToText(message.content)
+  }));
+}
+
+export function loadImageAttachmentsForInput(
+  attachments?: ContainerInput['attachments'],
+  options?: { log?: (message: string) => void }
+): OpenRouterInputImagePart[] {
+  if (!Array.isArray(attachments) || attachments.length === 0) return [];
+
+  const log = options?.log;
+  const images: OpenRouterInputImagePart[] = [];
+  let totalBytes = 0;
+
+  for (const attachment of attachments) {
+    if (!attachment || attachment.type !== 'photo' || typeof attachment.path !== 'string' || !attachment.path) {
+      continue;
+    }
+
+    const mime = normalizeMimeType(attachment.mime_type, attachment.file_name);
+    if (!mime) continue;
+
+    try {
+      const stat = fs.statSync(attachment.path);
+      if (stat.size > MAX_IMAGE_BYTES) {
+        log?.(`Skipping image ${attachment.path}: ${stat.size} bytes exceeds ${MAX_IMAGE_BYTES}`);
+        continue;
+      }
+      if (totalBytes + stat.size > MAX_TOTAL_IMAGE_BYTES) {
+        log?.(`Skipping image ${attachment.path}: cumulative size would exceed ${MAX_TOTAL_IMAGE_BYTES}`);
+        break;
+      }
+
+      const data = fs.readFileSync(attachment.path);
+      totalBytes += data.length;
+      images.push({
+        type: 'input_image',
+        detail: 'auto',
+        imageUrl: `data:${mime};base64,${data.toString('base64')}`
+      });
+    } catch (err) {
+      const detail = err instanceof Error ? err.message : String(err);
+      log?.(`Failed to load image ${attachment.path}: ${detail}`);
+    }
+  }
+
+  return images;
+}
+
+export function injectImagesIntoContextInput(
+  contextInput: OpenRouterInputMessage[],
+  imageParts: OpenRouterInputImagePart[]
+): void {
+  if (!Array.isArray(contextInput) || contextInput.length === 0 || imageParts.length === 0) return;
+  const lastMessage = contextInput[contextInput.length - 1];
+  if (!lastMessage || lastMessage.role !== 'user') return;
+
+  lastMessage.content = [
+    {
+      type: 'input_text',
+      text: coerceInputContentToText(lastMessage.content)
+    },
+    ...imageParts
+  ];
+}

package/container/agent-runner/src/system-prompt.ts

@@ -110,7 +110,9 @@ function buildScheduledSection(params: SystemPromptParams): string {
 
 function buildResponseGuidanceSection(): string {
   return [
-    '- Always answer the user\'s question directly before reaching for tools.',
+    '- Answer directly when the request can be completed from conversation context without external state.',
+    '- When the request requires file/system/network actions or fresh state, execute tools first before finalizing.',
+    '- Never claim an action happened unless corresponding tool calls succeeded in this turn.',
     '- If the user asks about your previous actions (e.g., "did you use X tool?"), reflect on the conversation history — do not re-execute the task.',
     '- If the user asks a simple factual question, answer from your knowledge — do not call tools unless you need to verify or act.',
     '- When you have genuinely nothing to say, respond with ONLY: NO_REPLY (your entire message must be just this token, nothing else).'
@@ -128,6 +130,8 @@ function buildToolCallStyleSection(): string {
 function buildToolGuidanceSection(params: SystemPromptParams): string {
   const lines = [
     'Key tool rules:',
+    '- Never claim file/system/web actions succeeded unless tool calls in this turn confirm them.',
+    '- For large file edits, avoid one giant Write payload; split into smaller Write/Edit steps and verify with Read.',
     '- User attachments arrive in /workspace/group/inbox/ (see <attachment> tags). Process with Read/Bash/Python.',
     '- To send media from the web: download_url → send_photo/send_file/send_audio.',
     '- Charts/plots: matplotlib → savefig → send_photo. Graphviz → dot -Tpng → send_photo.',
@@ -285,8 +289,14 @@ export function buildSystemPrompt(params: SystemPromptParams): string {
   const toolReliability = trimLevel >= 2 ? '' : (
     params.toolReliability && params.toolReliability.length > 0
       ? params.toolReliability
-        .sort((a, b) => a.success_rate - b.success_rate)
-        .slice(0, 20)
+        .filter(t => t.count >= 5 && (t.success_rate < 0.98 || (Number.isFinite(t.avg_duration_ms) && (t.avg_duration_ms || 0) > 2500)))
+        .sort((a, b) => {
+          if (a.success_rate !== b.success_rate) return a.success_rate - b.success_rate;
+          const aDur = Number.isFinite(a.avg_duration_ms) ? (a.avg_duration_ms || 0) : 0;
+          const bDur = Number.isFinite(b.avg_duration_ms) ? (b.avg_duration_ms || 0) : 0;
+          return bDur - aDur;
+        })
+        .slice(0, 8)
         .map(t => {
           const pct = `${Math.round(t.success_rate * 100)}%`;
           const avg = Number.isFinite(t.avg_duration_ms) ? `${Math.round(t.avg_duration_ms!)}ms` : 'n/a';