@phuetz/code-buddy 1.1.0 → 1.2.0

package/dist/tools/registry/vision-tools.js

@@ -123,12 +123,7 @@ export class BrowserVisionTool {
             const screenshotDir = path.join(rootDir, '.codebuddy', 'browser-vision');
             await fs.mkdir(screenshotDir, { recursive: true });
             const screenshotPath = path.join(screenshotDir, `browser-vision-${sanitizeFilename(this.options.createId?.() ?? String(Date.now()))}.png`);
-            const screenshot = await this.browser.execute({
-                action: 'screenshot',
-                fullPage: input.full_page === true,
-                format: 'png',
-                outputPath: screenshotPath,
-            });
+            const screenshot = await this.captureScreenshotWithRetry(input, screenshotPath);
             if (!screenshot.success)
                 return screenshot;
             const analysis = await analyzeVisionImage(screenshotPath, {
@@ -247,6 +242,33 @@ export class BrowserVisionTool {
         await this.browser.execute({ action: 'close' }).catch(() => undefined);
         await this.browser.dispose?.();
     }
+    async captureScreenshotWithRetry(input, screenshotPath) {
+        const fullPage = input.full_page === true;
+        const baseInput = {
+            action: 'screenshot',
+            fullPage,
+            format: 'png',
+            outputPath: screenshotPath,
+        };
+        const first = await this.browser.execute(baseInput);
+        if (first.success)
+            return first;
+        await delay(150);
+        const second = await this.browser.execute(baseInput);
+        if (second.success)
+            return second;
+        if (!fullPage) {
+            await delay(150);
+            const fullPageRetry = await this.browser.execute({
+                ...baseInput,
+                fullPage: true,
+            });
+            if (fullPageRetry.success)
+                return fullPageRetry;
+            return combineScreenshotFailures(first, second, fullPageRetry);
+        }
+        return combineScreenshotFailures(first, second);
+    }
 }
 // ============================================================================
 // OcrExtractTool
@@ -444,6 +466,234 @@ export class CameraSnapshotTool {
     }
     isAvailable() { return true; }
 }
+const DEFAULT_CAMERA_VISION_MODEL = 'gemma4:12b';
+const CAMERA_VISION_TIMEOUT_MS = 180_000;
+export class CameraAnalyzeTool {
+    options;
+    runtime;
+    name = 'camera_analyze';
+    description = 'Capture one local webcam frame and return a natural-language description from a local multimodal vision model (default Ollama gemma4:12b). Requires ffmpeg, OS camera permission, and a reachable local vision model.';
+    constructor(options = {}, runtime = {}) {
+        this.options = options;
+        this.runtime = runtime;
+    }
+    async execute(input, context) {
+        const cwd = this.options.rootDir ?? context?.cwd;
+        const prompt = optionalString(input, 'prompt') ?? 'Describe what you see.';
+        const model = optionalString(input, 'model') ?? DEFAULT_CAMERA_VISION_MODEL;
+        const device = optionalString(input, 'device');
+        const outputPath = optionalString(input, 'output_path');
+        const includeOcr = input.include_ocr === true;
+        // (1) Capture a frame from the local webcam.
+        const captureSnapshot = this.runtime.captureSnapshot ?? captureCameraSnapshot;
+        const snapshot = await captureSnapshot({
+            cwd,
+            ...(outputPath ? { outputPath } : {}),
+            ...(device ? { device } : {}),
+            timeoutMs: typeof input.timeout_ms === 'number' ? input.timeout_ms : undefined,
+        });
+        if (!snapshot.success || !snapshot.path) {
+            return {
+                success: false,
+                error: snapshot.error || 'Camera snapshot failed before analysis',
+                output: snapshot.command,
+            };
+        }
+        // (2) base64 the captured PNG into a data URL.
+        let dataUrl;
+        try {
+            const bytes = await fs.readFile(snapshot.path);
+            dataUrl = `data:image/png;base64,${bytes.toString('base64')}`;
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Failed to read captured frame ${snapshot.path}: ${error instanceof Error ? error.message : String(error)}`,
+            };
+        }
+        // (3) Make a REAL vision completion against the local Ollama /v1 endpoint.
+        let description;
+        try {
+            description = await this.describeImage(model, prompt, dataUrl);
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: error instanceof Error ? error.message : String(error),
+                output: `Captured frame saved to ${snapshot.path}, but the vision model could not describe it.`,
+            };
+        }
+        // (4) Optionally add local metadata/OCR evidence.
+        let ocr;
+        if (includeOcr) {
+            try {
+                const analysis = await analyzeVisionImage(snapshot.path, {
+                    ...this.options,
+                    rootDir: cwd,
+                    includeOcr: true,
+                    ocrLanguage: optionalString(input, 'ocr_language') ?? 'eng',
+                    source: 'image',
+                });
+                ocr = analysis.ocr;
+            }
+            catch {
+                // OCR is best-effort evidence; never fail the description on OCR errors.
+            }
+        }
+        // (5) Return the description text as the primary output.
+        const data = {
+            imagePath: snapshot.path,
+            description,
+            model,
+            ...(ocr ? { ocr } : {}),
+        };
+        return {
+            success: true,
+            output: description,
+            data,
+        };
+    }
+    async describeImage(model, prompt, dataUrl) {
+        const fetchImpl = this.runtime.fetch ?? fetch;
+        const endpoint = resolveOllamaChatEndpoint(this.runtime.env ?? process.env);
+        const controller = new AbortController();
+        let timedOut = false;
+        const timer = setTimeout(() => {
+            timedOut = true;
+            controller.abort();
+        }, CAMERA_VISION_TIMEOUT_MS);
+        let response;
+        let text;
+        try {
+            response = await fetchImpl(endpoint, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                signal: controller.signal,
+                body: JSON.stringify({
+                    model,
+                    temperature: 0.2,
+                    messages: [
+                        {
+                            role: 'user',
+                            content: [
+                                { type: 'text', text: prompt },
+                                { type: 'image_url', image_url: { url: dataUrl } },
+                            ],
+                        },
+                    ],
+                }),
+            });
+            // Read the body INSIDE the try so the abort timeout also covers a server
+            // that sends headers then stalls the body (otherwise response.text() hangs).
+            text = await response.text();
+        }
+        catch (error) {
+            const reason = error instanceof Error ? error.message : String(error);
+            if (timedOut) {
+                throw new Error(`Vision model ${model} timed out after ${CAMERA_VISION_TIMEOUT_MS}ms at ${endpoint}. ` +
+                    'A cold local model load can be slow; retry once the model is warm.');
+            }
+            throw new Error(`Vision model unreachable at ${endpoint} (model ${model}): ${reason}. ` +
+                'Start Ollama (ollama serve) and pull a multimodal model, or set OLLAMA_HOST.');
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        if (!response.ok) {
+            throw new Error(`Vision model returned HTTP ${response.status} from ${endpoint}: ${text.slice(0, 500)}`);
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(text);
+        }
+        catch {
+            throw new Error(`Vision model returned non-JSON response from ${endpoint}: ${text.slice(0, 300)}`);
+        }
+        const description = extractCompletionText(parsed);
+        if (!description) {
+            throw new Error(`Vision model ${model} returned an empty description.`);
+        }
+        return description;
+    }
+    getSchema() {
+        return {
+            name: this.name,
+            description: this.description,
+            parameters: {
+                type: 'object',
+                properties: {
+                    prompt: {
+                        type: 'string',
+                        description: 'What to ask the vision model about the frame. Default "Describe what you see."',
+                    },
+                    device: {
+                        type: 'string',
+                        description: 'Optional ffmpeg camera device. Linux example: /dev/video0; Windows: video=Integrated Camera; macOS: 0.',
+                    },
+                    model: {
+                        type: 'string',
+                        description: 'Local multimodal model id served by Ollama. Default gemma4:12b.',
+                    },
+                    include_ocr: {
+                        type: 'boolean',
+                        description: 'Also attach local OCR text evidence from the captured frame. Default false.',
+                    },
+                    ocr_language: {
+                        type: 'string',
+                        description: 'OCR language code when include_ocr is true. Default eng.',
+                    },
+                    output_path: {
+                        type: 'string',
+                        description: 'Optional output image path. Defaults to .codebuddy/camera/camera-<timestamp>.png in the active workspace.',
+                    },
+                    timeout_ms: {
+                        type: 'number',
+                        description: 'Capture timeout in milliseconds (default: 10000).',
+                        minimum: 1000,
+                        maximum: 60000,
+                    },
+                },
+                required: [],
+            },
+        };
+    }
+    validate(input) {
+        if (input === undefined || input === null)
+            return { valid: true };
+        if (typeof input !== 'object')
+            return { valid: false, errors: ['Input must be an object'] };
+        const data = input;
+        if (data.prompt !== undefined && typeof data.prompt !== 'string') {
+            return { valid: false, errors: ['prompt must be a string'] };
+        }
+        if (data.device !== undefined && typeof data.device !== 'string') {
+            return { valid: false, errors: ['device must be a string'] };
+        }
+        if (data.model !== undefined && typeof data.model !== 'string') {
+            return { valid: false, errors: ['model must be a string'] };
+        }
+        if (data.output_path !== undefined && typeof data.output_path !== 'string') {
+            return { valid: false, errors: ['output_path must be a string'] };
+        }
+        if (data.timeout_ms !== undefined && typeof data.timeout_ms !== 'number') {
+            return { valid: false, errors: ['timeout_ms must be a number'] };
+        }
+        return { valid: true };
+    }
+    getMetadata() {
+        return {
+            name: this.name,
+            description: this.description,
+            category: 'media',
+            keywords: ['camera', 'webcam', 'see', 'vision', 'describe', 'look', 'photo', 'companion', 'eyes', 'analyze'],
+            priority: 7,
+            modifiesFiles: true,
+            requiresConfirmation: true,
+            makesNetworkRequests: true,
+        };
+    }
+    isAvailable() { return true; }
+}
 // ============================================================================
 // Factory Function
 // ============================================================================
@@ -454,8 +704,48 @@ export function createVisionTools(options = {}) {
         new OcrExtractTool(),
         new ImageAnalyzeTool(),
         new CameraSnapshotTool(),
+        new CameraAnalyzeTool(options),
     ];
 }
+/**
+ * Resolve the Ollama OpenAI-compatible chat endpoint, honoring OLLAMA_HOST.
+ * OLLAMA_HOST is typically `host:port` with no scheme and no /v1 path.
+ */
+export function resolveOllamaChatEndpoint(env) {
+    const raw = (env.OLLAMA_HOST ?? '').trim();
+    if (!raw)
+        return 'http://localhost:11434/v1/chat/completions';
+    const withScheme = /^https?:\/\//i.test(raw) ? raw : `http://${raw}`;
+    const base = withScheme.replace(/\/+$/, '').replace(/\/v1$/i, '');
+    return `${base}/v1/chat/completions`;
+}
+/**
+ * Extract assistant text from an OpenAI-compatible chat completion. The
+ * `content` field may be a plain string or an array of typed parts.
+ */
+export function extractCompletionText(response) {
+    if (!response || typeof response !== 'object')
+        return undefined;
+    const choices = response.choices;
+    if (!Array.isArray(choices) || choices.length === 0)
+        return undefined;
+    const first = choices[0];
+    if (!first || typeof first !== 'object')
+        return undefined;
+    const message = first.message;
+    if (!message || typeof message !== 'object')
+        return undefined;
+    const content = message.content;
+    if (typeof content === 'string')
+        return content.trim() || undefined;
+    if (Array.isArray(content)) {
+        const parts = content
+            .map(part => (part && typeof part === 'object' ? part.text : undefined))
+            .filter((part) => typeof part === 'string' && part.trim().length > 0);
+        return parts.join('\n').trim() || undefined;
+    }
+    return undefined;
+}
 function requiredString(data, key) {
     const value = optionalString(data, key);
     if (!value) {
@@ -470,4 +760,16 @@ function optionalString(data, key) {
 function sanitizeFilename(id) {
     return id.trim().replace(/[^A-Za-z0-9_.-]+/g, '-').replace(/^-+|-+$/g, '') || String(Date.now());
 }
+async function delay(ms) {
+    await new Promise(resolve => setTimeout(resolve, ms));
+}
+function combineScreenshotFailures(...results) {
+    const errors = results
+        .map(result => result.error || result.output)
+        .filter((message) => typeof message === 'string' && message.trim().length > 0);
+    return {
+        success: false,
+        error: `Browser screenshot failed after retries: ${errors.join(' | ') || 'unknown error'}`,
+    };
+}
 //# sourceMappingURL=vision-tools.js.map

package/dist/tools/registry/windows-tools.d.ts

@@ -0,0 +1,2 @@
+import type { ITool } from './types.js';
+export declare function createWindowsTools(): ITool[];

package/dist/tools/registry/windows-tools.js

@@ -0,0 +1,47 @@
+import { OfficeMacroTool } from '../office-macro-tool.js';
+class OfficeMacroExecuteAdapter {
+    name = 'office_macro_execute';
+    description = 'Executes VBA or PowerShell macros directly in Windows Microsoft Office applications.';
+    impl = new OfficeMacroTool();
+    schema = {
+        name: 'office_macro_execute',
+        description: 'Executes VBA or PowerShell macros directly in Windows Microsoft Office applications.',
+        parameters: {
+            type: "object",
+            properties: {
+                application: { type: "string" },
+                macroCode: { type: "string" },
+                type: { type: "string" },
+                runHeadless: { type: "boolean" },
+            },
+            required: ["application", "macroCode", "type"],
+        },
+    };
+    metadata = {
+        name: 'office_macro_execute',
+        category: 'system',
+        priority: 1,
+        description: 'Executes macros in Microsoft Office (Windows only).',
+        keywords: ['office', 'excel', 'word', 'powerpoint', 'vba', 'macro', 'windows', 'com'],
+    };
+    getSchema() {
+        return this.schema;
+    }
+    async execute(args, context) {
+        const input = args;
+        return this.impl.execute(input);
+    }
+    validate(args) {
+        if (!args.application)
+            return { valid: false, errors: ['Missing application'] };
+        if (!args.macroCode)
+            return { valid: false, errors: ['Missing macroCode'] };
+        if (!args.type)
+            return { valid: false, errors: ['Missing type'] };
+        return { valid: true };
+    }
+}
+export function createWindowsTools() {
+    return [new OfficeMacroExecuteAdapter()];
+}
+//# sourceMappingURL=windows-tools.js.map

package/dist/tools/route-peer-tool.js

@@ -10,6 +10,7 @@ import { DEFAULT_DISPATCH_POLICY_PREVIEW_TOOLS, FLEET_DISPATCH_PROFILES, buildHe
 import { resolveActiveCustomAgentDispatchProfile, shouldPropagateResolvedDispatchProfile, } from '../agent/custom/custom-agent-runtime.js';
 import { getFleetRegistry } from '../fleet/fleet-registry.js';
 import { classifyTaskComplexity } from '../optimization/model-routing.js';
+import { getGlobalEventBus } from '../events/event-bus.js';
 const DEFAULT_DESCRIBE_TIMEOUT_MS = 5_000;
 const MAX_CHAIN_ROLES = 6;
 export async function executeRoutePeer(params) {
@@ -153,6 +154,19 @@ export async function executeRoutePeer(params) {
             },
             ...(chainNextCalls ? { nextCalls: chainNextCalls } : {}),
         };
+        try {
+            if (output.recommendation && output.recommendation.peer) {
+                getGlobalEventBus().emit('fleet:activity', {
+                    activityType: 'fleet.route',
+                    title: 'Fleet Route Planned',
+                    description: `Task routed to peer ${output.recommendation.peer}`,
+                    metadata: { peer: output.recommendation.peer, model: output.recommendation.model, prompt: params.prompt }
+                });
+            }
+        }
+        catch (err) {
+            // ignore
+        }
         return {
             success: true,
             output: JSON.stringify(output, null, 2),

package/dist/tools/screenshot-tool.js

@@ -167,8 +167,20 @@ export class ScreenshotTool {
                 args.push('-f', outputPath);
             }
             else if (availableTool === 'import') {
-                // ImageMagick import
-                if (options.window) {
+                // ImageMagick import. Without an explicit window/region, `import` enters
+                // interactive crosshair mode and blocks forever waiting for a click — so
+                // a headless fullscreen capture MUST target the root window. Only an
+                // explicit region (-crop) leaves the root grab intact.
+                if (options.region) {
+                    args.push('-window', 'root');
+                    args.push('-crop', `${options.region.width}x${options.region.height}+${options.region.x}+${options.region.y}`, 
+                    // Reset the virtual canvas so the saved PNG is a clean WxH+0+0, not the
+                    // full-screen page geometry with a crop offset.
+                    '+repage');
+                }
+                else {
+                    // fullscreen or active-window: capture the whole root (import's -window
+                    // can't reliably target a foreign window by title on X11 without an id).
                     args.push('-window', 'root');
                 }
                 args.push(outputPath);

package/dist/tools/text-to-speech-tool.d.ts

@@ -1,5 +1,5 @@
 import { spawn } from 'child_process';
-export type TextToSpeechProvider = 'auto' | 'system' | 'edge-tts' | 'espeak' | 'say' | 'kokoro' | 'audioreader';
+export type TextToSpeechProvider = 'auto' | 'system' | 'edge-tts' | 'espeak' | 'say' | 'audioreader';
 export interface TextToSpeechOptions {
     rootDir?: string;
     now?: () => Date;

package/dist/tools/text-to-speech-tool.js

@@ -100,7 +100,7 @@ async function resolveProvider(provider, options) {
     if (await commandExists('espeak', { platform })) {
         return 'espeak';
     }
-    throw new Error('No local TTS provider available. Install edge-tts/espeak, use macOS say, Windows PowerShell SAPI, or choose kokoro/audioreader when configured.');
+    throw new Error('No local TTS provider available. Install edge-tts/espeak, use macOS say, Windows PowerShell SAPI, or choose audioreader when configured.');
 }
 function resolveOutputFormat(requested, provider, outputPath) {
     if (outputPath) {
@@ -126,7 +126,6 @@ function validateProviderFormat(provider, format) {
         'edge-tts': ['mp3'],
         espeak: ['wav'],
         say: ['aiff'],
-        kokoro: ['wav'],
         audioreader: ['wav'],
     };
     if (!supported[provider].includes(format)) {
@@ -180,15 +179,6 @@ function buildProviderCommand(provider, input, options) {
                     input.text,
                 ],
             };
-        case 'kokoro':
-            return {
-                kind: 'node',
-                run: async () => {
-                    const { kokoroTtsService } = await import('../utils/kokoro-tts.js');
-                    const buffer = await kokoroTtsService.generateSpeech(input.text, input.voice ?? 'af_bella');
-                    await fs.writeFile(input.outputPath, buffer);
-                },
-            };
         case 'audioreader':
             return {
                 kind: 'node',
@@ -197,7 +187,7 @@ function buildProviderCommand(provider, input, options) {
                         method: 'POST',
                         headers: { 'Content-Type': 'application/json' },
                         body: JSON.stringify({
-                            model: 'kokoro',
+                            model: 'audioreader',
                             input: input.text,
                             voice: input.voice ?? 'ff_siwis',
                             speed: 1.0,

package/dist/tools/tool-selector.js

@@ -14,63 +14,7 @@
  * - Adaptive thresholds based on success metrics
  * - LRU cache for repeated queries
  */
-/**
- * Simple LRU Cache implementation
- */
-class LRUCache {
-    cache;
-    maxSize;
-    ttlMs;
-    constructor(maxSize = 100, ttlMs = 5 * 60 * 1000) {
-        this.cache = new Map();
-        this.maxSize = maxSize;
-        this.ttlMs = ttlMs;
-    }
-    get(key) {
-        const entry = this.cache.get(key);
-        if (!entry)
-            return undefined;
-        // Check TTL
-        if (Date.now() - entry.timestamp > this.ttlMs) {
-            this.cache.delete(key);
-            return undefined;
-        }
-        // Update access count and move to end (most recently used)
-        entry.accessCount++;
-        this.cache.delete(key);
-        this.cache.set(key, entry);
-        return entry.value;
-    }
-    set(key, value) {
-        // Remove oldest if at capacity
-        if (this.cache.size >= this.maxSize) {
-            const firstKey = this.cache.keys().next().value;
-            if (firstKey !== undefined) {
-                this.cache.delete(firstKey);
-            }
-        }
-        this.cache.set(key, {
-            value,
-            timestamp: Date.now(),
-            accessCount: 1
-        });
-    }
-    has(key) {
-        return this.get(key) !== undefined;
-    }
-    clear() {
-        this.cache.clear();
-    }
-    get size() {
-        return this.cache.size;
-    }
-    getStats() {
-        return {
-            size: this.cache.size,
-            hitRate: 0 // Would need to track hits/misses
-        };
-    }
-}
+import { LRUCache } from '../utils/lru-cache.js';
 import { TOOL_METADATA, CATEGORY_KEYWORDS } from "./metadata.js";
 /**
  * RAG-based Tool Selector.
@@ -116,8 +60,8 @@ export class ToolSelector {
             lastUpdated: new Date()
         };
         // Initialize caches
-        this.classificationCache = new LRUCache(100, 5 * 60 * 1000);
-        this.selectionCache = new LRUCache(50, 2 * 60 * 1000);
+        this.classificationCache = new LRUCache({ maxSize: 100, ttlMs: 5 * 60 * 1000 });
+        this.selectionCache = new LRUCache({ maxSize: 50, ttlMs: 2 * 60 * 1000 });
         this.buildIndex();
     }
     /**