@phuetz/code-buddy 1.1.0 → 1.2.0

package/dist/desktop/codebuddy-engine-adapter.js

@@ -9,6 +9,10 @@
  */
 import { logger } from '../utils/logger.js';
 import { createHash } from 'node:crypto';
+import { maybeContinueGoalAfterTurn } from '../goals/goal-loop.js';
+import { getGoalManager } from '../goals/goal-manager.js';
+const COWORK_GOAL_SESSION_PREFIX = 'cowork:';
+const GOAL_LOOP_HARD_BACKSTOP = 100;
 /**
  * Concrete implementation of EngineAdapter that wraps CodeBuddyAgent.
  *
@@ -40,6 +44,12 @@ export class CodeBuddyEngineAdapter {
      * effect on the next turn (Phase 8).
      */
     agentIdentities = new Map();
+    /**
+     * Configured reasoning/thinking level (`off | minimal | … | xhigh`), set via
+     * {@link setThinkingLevel}. Re-applied to each freshly-created agent so a
+     * model swap / new session keeps the user's chosen level.
+     */
+    thinkingLevel;
     /**
      * Hard cap on cached agents — matches the pi-runner's
      * `MAX_CACHED_SESSIONS` (50) so memory pressure is comparable
@@ -102,6 +112,13 @@ export class CodeBuddyEngineAdapter {
                 if (typeof agent.setVisionGroundingModel === 'function') {
                     agent.setVisionGroundingModel(this.config.visionGroundingModel);
                 }
+                // Re-apply the configured thinking level so a freshly-created agent
+                // (new session or post model-swap) keeps the user's chosen level on the
+                // Gemini-native path. The OpenAI-compat / Grok / Ollama path reads the
+                // global extended-thinking budget per turn, so it needs no per-agent step.
+                if (this.thinkingLevel) {
+                    this.applyGeminiThinkingLevel(agent, this.thinkingLevel);
+                }
                 // Phase 9 — enforce LRU before insertion so we never exceed
                 // the cap. Evict the least-recently-used (head of the
                 // insertion-ordered Map) until there's room.
@@ -165,92 +182,166 @@ export class CodeBuddyEngineAdapter {
                     toolCallCount,
                 };
             }
-            // Stream the response
-            const stream = agent.processUserMessageStream(lastMessage.content);
-            for await (const chunk of stream) {
-                // Check for abort
-                if (abortController.signal.aborted) {
-                    break;
-                }
-                switch (chunk.type) {
-                    case 'content':
-                        if (chunk.content) {
-                            fullContent += chunk.content;
-                            onEvent({ type: 'content', content: chunk.content });
-                        }
-                        break;
-                    case 'reasoning':
-                        if (chunk.reasoning) {
-                            onEvent({ type: 'thinking', thinking: chunk.reasoning });
-                        }
-                        break;
-                    case 'tool_calls':
-                        if (chunk.toolCalls) {
-                            for (const tc of chunk.toolCalls) {
-                                toolCallCount++;
+            const streamingAgent = agent;
+            const goalSessionKey = buildCoworkGoalSessionKey(sessionId);
+            const runPromptTurn = async (prompt) => {
+                let turnContent = '';
+                const toolEvidence = [];
+                // Tracks FAILED tool actions this turn so the goal judge can't be fooled
+                // into a premature "done" by an assistant that narrates success after a
+                // write/patch/command actually failed (Hermes-style mutation verifier).
+                const toolFailures = [];
+                const stream = streamingAgent.processUserMessageStream(prompt);
+                for await (const chunk of stream) {
+                    // Check for abort
+                    if (abortController.signal.aborted) {
+                        return { interrupted: true, judgeResponse: buildGoalJudgeResponse(turnContent, toolEvidence, toolFailures) };
+                    }
+                    switch (chunk.type) {
+                        case 'content':
+                            if (chunk.content) {
+                                turnContent += chunk.content;
+                                fullContent += chunk.content;
+                                onEvent({ type: 'content', content: chunk.content });
+                            }
+                            break;
+                        case 'reasoning':
+                            if (chunk.reasoning) {
+                                onEvent({ type: 'thinking', thinking: chunk.reasoning });
+                            }
+                            break;
+                        case 'tool_calls':
+                            if (chunk.toolCalls) {
+                                for (const tc of chunk.toolCalls) {
+                                    toolCallCount++;
+                                    onEvent({
+                                        type: 'tool_start',
+                                        tool: {
+                                            id: tc.id,
+                                            name: tc.function.name,
+                                            input: tc.function.arguments,
+                                        },
+                                    });
+                                }
+                            }
+                            break;
+                        case 'tool_result':
+                            if (chunk.toolCall && chunk.toolResult) {
+                                const finalOutput = chunk.toolResult.output || chunk.toolResult.error;
+                                const toolStatus = chunk.toolResult.success ? 'success' : 'error';
+                                if (finalOutput) {
+                                    toolEvidence.push(`[tool:${chunk.toolCall.function.name} ${toolStatus}]\n${String(finalOutput)}`);
+                                }
+                                // Record failures INDEPENDENTLY of finalOutput — a tool that
+                                // fails with no output/error would otherwise be invisible to the
+                                // judge (the silent-failure hole).
+                                if (!chunk.toolResult.success) {
+                                    const detail = finalOutput ? `: ${truncateFailureDetail(String(finalOutput))}` : '';
+                                    toolFailures.push(`${chunk.toolCall.function.name}${detail}`);
+                                }
                                 onEvent({
-                                    type: 'tool_start',
+                                    type: 'tool_end',
                                     tool: {
-                                        id: tc.id,
-                                        name: tc.function.name,
-                                        input: tc.function.arguments,
+                                        id: chunk.toolCall.id,
+                                        name: chunk.toolCall.function.name,
+                                        input: chunk.toolCall.function.arguments,
+                                        output: finalOutput,
+                                        isError: !chunk.toolResult.success,
+                                        data: chunk.toolResult.data,
                                     },
                                 });
                             }
-                        }
-                        break;
-                    case 'tool_result':
-                        if (chunk.toolCall && chunk.toolResult) {
-                            const finalOutput = chunk.toolResult.output || chunk.toolResult.error;
-                            onEvent({
-                                type: 'tool_end',
-                                tool: {
-                                    id: chunk.toolCall.id,
-                                    name: chunk.toolCall.function.name,
-                                    output: finalOutput,
-                                    isError: !chunk.toolResult.success,
-                                    data: chunk.toolResult.data,
-                                },
-                            });
-                        }
-                        break;
-                    case 'tool_stream':
-                        if (chunk.toolStreamData) {
-                            onEvent({
-                                type: 'tool_stream',
-                                tool: {
-                                    id: chunk.toolStreamData.toolCallId,
-                                    name: chunk.toolStreamData.toolName,
-                                    delta: chunk.toolStreamData.delta,
-                                },
-                            });
-                        }
-                        break;
-                    case 'token_count':
-                        if (chunk.tokenCount !== undefined) {
-                            totalTokens = chunk.tokenCount;
-                            onEvent({ type: 'token_count', tokenCount: chunk.tokenCount });
-                        }
-                        break;
-                    case 'ask_user':
-                        if (chunk.askUser) {
-                            onEvent({ type: 'ask_user', askUser: chunk.askUser });
-                        }
-                        break;
-                    case 'plan_progress':
-                        if (chunk.planProgress) {
-                            onEvent({ type: 'plan_progress', planProgress: chunk.planProgress });
-                        }
-                        break;
-                    case 'diff_preview':
-                        if (chunk.diffPreview) {
-                            onEvent({ type: 'diff_preview', diffPreview: chunk.diffPreview });
-                        }
-                        break;
-                    case 'done':
-                        onEvent({ type: 'done' });
-                        break;
+                            break;
+                        case 'tool_stream':
+                            if (chunk.toolStreamData) {
+                                onEvent({
+                                    type: 'tool_stream',
+                                    tool: {
+                                        id: chunk.toolStreamData.toolCallId,
+                                        name: chunk.toolStreamData.toolName,
+                                        delta: chunk.toolStreamData.delta,
+                                    },
+                                });
+                            }
+                            break;
+                        case 'token_count':
+                            if (chunk.tokenCount !== undefined) {
+                                totalTokens = chunk.tokenCount;
+                                onEvent({ type: 'token_count', tokenCount: chunk.tokenCount });
+                            }
+                            break;
+                        case 'ask_user':
+                            if (chunk.askUser) {
+                                onEvent({ type: 'ask_user', askUser: chunk.askUser });
+                            }
+                            break;
+                        case 'plan_progress':
+                            if (chunk.planProgress) {
+                                onEvent({ type: 'plan_progress', planProgress: chunk.planProgress });
+                            }
+                            break;
+                        case 'steer':
+                            if (chunk.steer) {
+                                onEvent({ type: 'steer', steer: chunk.steer });
+                            }
+                            break;
+                        case 'diff_preview':
+                            if (chunk.diffPreview) {
+                                onEvent({ type: 'diff_preview', diffPreview: chunk.diffPreview });
+                            }
+                            break;
+                        case 'done':
+                            onEvent({ type: 'done' });
+                            break;
+                    }
+                }
+                return { interrupted: false, judgeResponse: buildGoalJudgeResponse(turnContent, toolEvidence, toolFailures) };
+            };
+            const emitGoalStatus = (message) => {
+                const content = `${fullContent ? '\n\n' : ''}${message}\n\n`;
+                fullContent += content;
+                onEvent({ type: 'content', content });
+            };
+            // Structured goal-status event for host UIs (Cowork goal banner). Emits the
+            // current GoalState snapshot so the renderer can show turn progress without
+            // re-reading goal storage. Safe no-op when no goal is set.
+            const emitGoalSnapshot = () => {
+                const s = getGoalManager(goalSessionKey).state;
+                if (!s || s.status === 'cleared')
+                    return;
+                onEvent({
+                    type: 'goal_status',
+                    goalStatus: {
+                        goal: s.goal,
+                        status: s.status,
+                        turnsUsed: s.turnsUsed,
+                        maxTurns: s.maxTurns,
+                        ...(s.lastVerdict ? { lastVerdict: s.lastVerdict } : {}),
+                        ...(s.lastReason ? { lastReason: s.lastReason } : {}),
+                    },
+                });
+            };
+            // Emit once up-front so the banner appears the instant a goal turn starts
+            // (before the first judge verdict), then again after each judged turn.
+            emitGoalSnapshot();
+            let turn = await runPromptTurn(lastMessage.content);
+            for (let i = 0; i < GOAL_LOOP_HARD_BACKSTOP; i++) {
+                const outcome = await maybeContinueGoalAfterTurn({
+                    client: streamingAgent.getClient?.() ?? null,
+                    lastResponse: turn.judgeResponse,
+                    interrupted: turn.interrupted,
+                    sessionKey: goalSessionKey,
+                });
+                if (outcome?.message) {
+                    emitGoalStatus(outcome.message);
                 }
+                if (outcome) {
+                    emitGoalSnapshot();
+                }
+                if (turn.interrupted || !outcome?.continuationPrompt) {
+                    break;
+                }
+                turn = await runPromptTurn(outcome.continuationPrompt);
             }
             return {
                 content: fullContent,
@@ -278,6 +369,18 @@ export class CodeBuddyEngineAdapter {
             logger.info('[CodeBuddyEngineAdapter] cancelled session', { sessionId });
         }
     }
+    steer(sessionId, prompt) {
+        const controller = this.abortControllers.get(sessionId);
+        const agent = this.agents.get(sessionId);
+        const queue = agent?.getMessageQueue?.();
+        if (!controller || controller.signal.aborted || !queue) {
+            return false;
+        }
+        queue.setMode('steer');
+        queue.enqueue({ content: prompt, source: 'cowork', timestamp: new Date() });
+        logger.info('[CodeBuddyEngineAdapter] steer delivered', { sessionId });
+        return true;
+    }
     clearSession(sessionId) {
         const agent = this.agents.get(sessionId);
         if (agent && typeof agent.dispose === 'function') {
@@ -352,6 +455,51 @@ export class CodeBuddyEngineAdapter {
             logger.warn('[CodeBuddyEngineAdapter] reloadSkills failed', { err });
         }
     }
+    /**
+     * Hot-swap the reasoning/thinking level for live sessions. Updates the global
+     * extended-thinking budget (read per-turn by the OpenAI-compat / Grok / Ollama
+     * providers → effective next turn, no rebuild) and the Gemini-native default on
+     * every cached agent. The level is remembered so future agents inherit it.
+     */
+    async setThinkingLevel(level) {
+        this.thinkingLevel = level;
+        try {
+            const { getExtendedThinking } = await import('../agent/extended-thinking.js');
+            getExtendedThinking().applyThinkingLevel(level);
+        }
+        catch (err) {
+            logger.warn('[CodeBuddyEngineAdapter] applyThinkingLevel failed', { err });
+        }
+        for (const agent of this.agents.values()) {
+            this.applyGeminiThinkingLevel(agent, level);
+        }
+        logger.info('[CodeBuddyEngineAdapter] thinkingLevel set', { level });
+    }
+    /**
+     * Map a UI level to the Gemini-native default thinking level on a single
+     * agent's client. `GeminiThinkingLevel` is `minimal|low|medium|high`, so
+     * `xhigh` clamps to `high` and `off`/unknown leaves the Gemini default
+     * untouched (the global extended-thinking switch already disables it for the
+     * OpenAI-compat path). Best-effort; never throws.
+     */
+    applyGeminiThinkingLevel(agent, level) {
+        const geminiLevel = level === 'minimal' || level === 'low' || level === 'medium'
+            ? level
+            : level === 'high' || level === 'xhigh'
+                ? 'high'
+                : null;
+        if (!geminiLevel)
+            return;
+        try {
+            const getClient = agent.getClient;
+            if (typeof getClient === 'function') {
+                getClient.call(agent)?.setDefaultThinkingLevel(geminiLevel);
+            }
+        }
+        catch {
+            /* best effort — Gemini path only */
+        }
+    }
     /**
      * Synchronise the core MCPManager singleton with the host's view of
      * the MCP servers. Called by Cowork at boot and after any
@@ -481,4 +629,33 @@ export class CodeBuddyEngineAdapter {
         });
     }
 }
+function buildCoworkGoalSessionKey(sessionId) {
+    return `${COWORK_GOAL_SESSION_PREFIX}${sessionId}`;
+}
+/** Cap a single failure detail so the footer stays compact for the judge. */
+export function truncateFailureDetail(detail, max = 160) {
+    const flat = detail.replace(/\s+/g, ' ').trim();
+    return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
+}
+/**
+ * Build a prominent, NON-bracketed footer listing failed tool actions. The goal
+ * judge prompt (goal-state.ts) treats `[tool:…]` bracketed lines as ignorable
+ * metadata, so failures must be surfaced as a plain instruction line or the
+ * judge discounts them. Returns '' when there are no failures.
+ */
+export function buildToolFailureFooter(failures) {
+    if (!failures.length)
+        return '';
+    return (`⚠️ ${failures.length} tool action(s) failed this turn: ${failures.join('; ')}. ` +
+        `Do NOT treat the goal as done if these failures block it — fix or work around them first.`);
+}
+export function buildGoalJudgeResponse(content, toolEvidence, toolFailures = []) {
+    const footer = buildToolFailureFooter(toolFailures);
+    const parts = [
+        content.trim(),
+        ...toolEvidence.map((part) => part.trim()),
+        footer, // last → most prominent for the LLM judge
+    ].filter(Boolean);
+    return parts.join('\n\n');
+}
 //# sourceMappingURL=codebuddy-engine-adapter.js.map

package/dist/desktop/engine-adapter.d.ts

@@ -40,6 +40,11 @@ export interface EngineAdapter {
      * Cancel a running session.
      */
     cancel(sessionId: string): void;
+    /**
+     * Deliver user guidance into an active run when the underlying agent
+     * supports steer mode. Returns false when there is no active run/agent.
+     */
+    steer?(sessionId: string, prompt: string): boolean | Promise<boolean>;
     /**
      * Clear internal state for a session (free memory, close resources).
      */
@@ -80,6 +85,15 @@ export interface EngineAdapter {
      * Optional — adapters without a skills system can omit this.
      */
     reloadSkills?(): Promise<void>;
+    /**
+     * Hot-swap the reasoning/thinking level for live sessions —
+     * `off | minimal | low | medium | high | xhigh`. Updates the global
+     * extended-thinking budget (read per-turn by the OpenAI-compat / Grok /
+     * Ollama providers) and the Gemini-native default on cached agents, so the
+     * Cowork ReasoningLevelPicker takes effect on the next turn without a session
+     * restart. Optional — adapters without a thinking system can omit this.
+     */
+    setThinkingLevel?(level: string): Promise<void>;
     /**
      * Set the default visual grounding fallback configuration.
      */

package/dist/desktop-automation/automation-manager.js

@@ -320,6 +320,13 @@ export class DesktopAutomationManager extends EventEmitter {
             const nativeProvider = await this.createNativeProvider();
             if (nativeProvider) {
                 this.registerProvider(nativeProvider);
+                // The created provider registers under its own name (e.g. 'nutjs' on
+                // Linux when xdotool/xclip are unavailable). Also expose it under the
+                // requested 'native' key so the preferred-provider lookup resolves it
+                // instead of silently falling through to the mock provider.
+                if (!this.providers.has('native')) {
+                    this.providers.set('native', nativeProvider);
+                }
             }
         }
         // Try preferred provider first
@@ -689,6 +696,15 @@ export class DesktopAutomationManager extends EventEmitter {
                     return new WindowsNativeProvider({ wsl: false });
                 }
                 case 'linux': {
+                    // Prefer nut-js (libnut): it provides mouse/keyboard/screen control
+                    // without xdotool/xclip/wmctrl, which are frequently absent on minimal
+                    // or remote Linux desktops. Fall back to the xdotool-based provider
+                    // (which also does window management) only when libnut is unavailable.
+                    const { NutJsProvider } = await import('./nutjs-provider.js');
+                    const nut = new NutJsProvider();
+                    if (await nut.isAvailable()) {
+                        return nut;
+                    }
                     const { LinuxNativeProvider } = await import('./linux-native-provider.js');
                     return new LinuxNativeProvider();
                 }

package/dist/desktop-automation/omniparser-runner.d.ts

@@ -0,0 +1,63 @@
+export interface OmniParserElement {
+    /** Index in the server's parsed_content_list — this is the number drawn on the annotated (SOM) image. */
+    id: number;
+    /** 'text' | 'icon' (whatever the server reports). */
+    type: string;
+    /** OCR text or icon caption (the server's `content`). */
+    content: string;
+    /** Whether OmniParser flagged the element as interactive/clickable (`interactivity`). */
+    interactable: boolean;
+    /** Bounding box [x1, y1, x2, y2]. Pixels when `normalized` is false, else 0-1 ratios. */
+    bbox: [number, number, number, number];
+    /** Box center [x, y] — pixels when `normalized` is false. Handy for a follow-up click. */
+    center: [number, number];
+    /** True when bbox/center are 0-1 ratios (image dimensions were unknown). */
+    normalized: boolean;
+}
+export interface OmniParserResult {
+    elements: OmniParserElement[];
+    /** Base64 of the Set-of-Marks (numbered) image returned by the server, or the original on failure. */
+    annotatedImageBase64: string;
+}
+export interface ParseScreenOptions {
+    /** Screenshot pixel width — when provided, normalized boxes are scaled to pixels. */
+    width?: number;
+    /** Screenshot pixel height. */
+    height?: number;
+}
+/**
+ * Runner for OmniParser (Microsoft's screen-parsing model — github.com/microsoft/OmniParser).
+ *
+ * Talks to the OmniParser v2 `omniparserserver` FastAPI service (omnitool/omniparserserver):
+ *   - POST {base}/parse/  body {"base64_image": "<b64>"}
+ *       -> {"som_image_base64": "<b64>", "parsed_content_list": [{type, content, interactivity, bbox}], "latency"}
+ *   - GET  {base}/probe/  -> health check
+ *
+ * `bbox` from the server is normalized to 0-1; we scale it to pixels when the
+ * screenshot dimensions are known so the coordinates are directly clickable.
+ *
+ * The model is GPU/Python and must be self-hosted; configure the base URL via
+ * OMNIPARSER_API_URL (default http://localhost:8000). Off-server, every call
+ * degrades gracefully to empty elements + the original screenshot.
+ */
+export declare class OmniParserRunner {
+    private baseUrl;
+    private apiKey;
+    constructor();
+    private get parseUrl();
+    private get probeUrl();
+    /**
+     * Send a base64-encoded screenshot to the OmniParser server for analysis.
+     *
+     * @param imageBase64 The raw screenshot in base64 (no data: prefix).
+     * @param opts Optional screenshot pixel dimensions used to scale normalized boxes.
+     * @returns Parsed elements and the annotated (Set-of-Marks) image.
+     */
+    parseScreen(imageBase64: string, opts?: ParseScreenOptions): Promise<OmniParserResult>;
+    /** Map OmniParser's parsed_content_list into our element shape, scaling boxes to pixels when possible. */
+    private mapElements;
+    /**
+     * Check if the OmniParser server is reachable via its /probe/ health endpoint.
+     */
+    isAvailable(): Promise<boolean>;
+}

package/dist/desktop-automation/omniparser-runner.js

@@ -0,0 +1,115 @@
+import { logger } from '../utils/logger.js';
+/**
+ * Runner for OmniParser (Microsoft's screen-parsing model — github.com/microsoft/OmniParser).
+ *
+ * Talks to the OmniParser v2 `omniparserserver` FastAPI service (omnitool/omniparserserver):
+ *   - POST {base}/parse/  body {"base64_image": "<b64>"}
+ *       -> {"som_image_base64": "<b64>", "parsed_content_list": [{type, content, interactivity, bbox}], "latency"}
+ *   - GET  {base}/probe/  -> health check
+ *
+ * `bbox` from the server is normalized to 0-1; we scale it to pixels when the
+ * screenshot dimensions are known so the coordinates are directly clickable.
+ *
+ * The model is GPU/Python and must be self-hosted; configure the base URL via
+ * OMNIPARSER_API_URL (default http://localhost:8000). Off-server, every call
+ * degrades gracefully to empty elements + the original screenshot.
+ */
+export class OmniParserRunner {
+    baseUrl;
+    apiKey;
+    constructor() {
+        // OMNIPARSER_API_URL is the server base (scheme://host:port); endpoints are derived.
+        const raw = process.env.OMNIPARSER_API_URL || 'http://localhost:8000';
+        this.baseUrl = raw.replace(/\/+$/, '');
+        this.apiKey = process.env.OMNIPARSER_API_KEY;
+    }
+    get parseUrl() {
+        return `${this.baseUrl}/parse/`;
+    }
+    get probeUrl() {
+        return `${this.baseUrl}/probe/`;
+    }
+    /**
+     * Send a base64-encoded screenshot to the OmniParser server for analysis.
+     *
+     * @param imageBase64 The raw screenshot in base64 (no data: prefix).
+     * @param opts Optional screenshot pixel dimensions used to scale normalized boxes.
+     * @returns Parsed elements and the annotated (Set-of-Marks) image.
+     */
+    async parseScreen(imageBase64, opts = {}) {
+        try {
+            logger.debug('Sending screenshot to OmniParser API', { url: this.parseUrl });
+            const headers = {
+                'Content-Type': 'application/json',
+            };
+            if (this.apiKey) {
+                headers['Authorization'] = `Bearer ${this.apiKey}`;
+            }
+            const response = await fetch(this.parseUrl, {
+                method: 'POST',
+                headers,
+                // OmniParser v2 ParseRequest expects the field `base64_image`.
+                body: JSON.stringify({ base64_image: imageBase64 }),
+            });
+            if (!response.ok) {
+                throw new Error(`OmniParser API returned status: ${response.status} ${response.statusText}`);
+            }
+            const data = (await response.json());
+            const elements = this.mapElements(data.parsed_content_list ?? [], opts);
+            return {
+                elements,
+                annotatedImageBase64: data.som_image_base64 || imageBase64,
+            };
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            logger.error('OmniParser API call failed', { error: msg, url: this.parseUrl });
+            // Graceful fallback if the server is down/misconfigured: original image, no elements.
+            return {
+                elements: [],
+                annotatedImageBase64: imageBase64,
+            };
+        }
+    }
+    /** Map OmniParser's parsed_content_list into our element shape, scaling boxes to pixels when possible. */
+    mapElements(raw, opts) {
+        const { width, height } = opts;
+        const scalable = typeof width === 'number' && width > 0 && typeof height === 'number' && height > 0;
+        return raw.map((item, index) => {
+            const b = Array.isArray(item.bbox) && item.bbox.length === 4 ? item.bbox : [0, 0, 0, 0];
+            let [x1, y1, x2, y2] = [Number(b[0]) || 0, Number(b[1]) || 0, Number(b[2]) || 0, Number(b[3]) || 0];
+            if (scalable) {
+                x1 = Math.round(x1 * width);
+                y1 = Math.round(y1 * height);
+                x2 = Math.round(x2 * width);
+                y2 = Math.round(y2 * height);
+            }
+            return {
+                id: index,
+                type: item.type ?? 'unknown',
+                content: item.content ?? '',
+                interactable: item.interactivity ?? false,
+                bbox: [x1, y1, x2, y2],
+                center: [Math.round((x1 + x2) / 2), Math.round((y1 + y2) / 2)],
+                normalized: !scalable,
+            };
+        });
+    }
+    /**
+     * Check if the OmniParser server is reachable via its /probe/ health endpoint.
+     */
+    async isAvailable() {
+        try {
+            const headers = {};
+            if (this.apiKey) {
+                headers['Authorization'] = `Bearer ${this.apiKey}`;
+            }
+            const response = await fetch(this.probeUrl, { method: 'GET', headers });
+            return response.ok;
+        }
+        catch {
+            return false;
+        }
+    }
+}
+//# sourceMappingURL=omniparser-runner.js.map

package/dist/desktop-automation/smart-snapshot.d.ts

@@ -132,6 +132,8 @@ export declare class SmartSnapshotManager extends EventEmitter {
     private snapshotCache;
     private static readonly MAX_CACHE_SIZE;
     private nextRef;
+    /** Cached python interpreter that has the Atspi GI binding (`''` = none found, `null` = not yet probed). */
+    private atspiPython;
     constructor(config?: Partial<SmartSnapshotConfig>);
     private runPowerShellEncoded;
     /**
@@ -206,6 +208,15 @@ export declare class SmartSnapshotManager extends EventEmitter {
      * Detect elements on WSL2 via PowerShell UIAutomation (same script as detectWindowsElements)
      */
     private detectWSLElements;
+    /**
+     * Resolve a python3 interpreter that actually has the Atspi GObject-Introspection
+     * binding. `python3` on PATH is frequently a conda/pyenv interpreter that lacks the
+     * system `gi`/Atspi typelib (PyGObject is installed against the system python), so a
+     * bare `python3 -c "import gi"` silently fails and the snapshot falls back to mock
+     * elements. We probe a small candidate list once and cache the winner.
+     * Returns '' when no interpreter has the binding.
+     */
+    private resolveAtspiPython;
     /**
      * Detect elements on native Linux via AT-SPI
      */