onbuzz 4.7.2 → 4.8.0

package/src/tools/desktop/__tests__/DesktopTool.e2e.test.js

@@ -0,0 +1,306 @@
+/**
+ * DesktopTool — end-to-end contract tests.
+ *
+ * These tests don't hit the network or a real OS. They wire the tool
+ * to a stubbed grounding adapter that returns Foundry-shaped responses
+ * AND a stubbed osController, then assert the full pipeline executes:
+ *
+ *   intent string  →  screenshot  →  ground  →  parseCoords  →  click
+ *
+ * The point is to verify the seams between layers (KimiAdapter,
+ * groundingMetrics, DesktopTool) actually compose correctly. Unit
+ * tests above this layer mock one side; these mock the wire and
+ * exercise the rest end-to-end.
+ *
+ * Why not run against a real Foundry deployment in CI: that requires
+ * a live FOUNDRY_KEY which we don't ship to GitHub. The bench harness
+ * (scripts/grounding-bench/) is the real-world counterpart — operators
+ * run it locally before flipping the kill switch off in prod.
+ */
+
+import { jest, describe, test, expect } from '@jest/globals';
+import { DesktopTool } from '../DesktopTool.js';
+import { KimiAdapter } from '../../../services/grounding/KimiAdapter.js';
+
+/** A fetch double that returns a realistic Foundry chat-completion body. */
+function fetchReturning(content, usage = { prompt_tokens: 800, completion_tokens: 50 }) {
+  return jest.fn(async () => ({
+    ok: true,
+    status: 200,
+    text: async () => JSON.stringify({
+      id: 'resp-e2e',
+      model: 'Kimi-K2.6',
+      choices: [{
+        message: {
+          role: 'assistant',
+          content,
+          reasoning_content: 'I see the target in the screenshot.',
+        },
+        finish_reason: 'stop',
+      }],
+      usage,
+    }),
+  }));
+}
+
+function fakeOsc() {
+  const calls = [];
+  return {
+    screenshot:  jest.fn(async () => Buffer.from([0x89, 0x50, 0x4E, 0x47])),
+    screenSize:  jest.fn(async () => ({ width: 1920, height: 1080, dpr: 1 })),
+    mouseClick:  jest.fn(async (...args) => calls.push({ name: 'mouseClick', args })),
+    mouseScroll: jest.fn(async (...args) => calls.push({ name: 'mouseScroll', args })),
+    mouseDrag:   jest.fn(async (...args) => calls.push({ name: 'mouseDrag', args })),
+    typeText:    jest.fn(async () => {}),
+    keyPress:    jest.fn(async () => {}),
+    listWindows: jest.fn(async () => []),
+    describeCapabilities: jest.fn(async () => ({ os: 'win32', mouseInput: true })),
+    _calls: calls,
+  };
+}
+
+function buildToolWithRealAdapter({ content, fetchImpl }) {
+  const adapter = new KimiAdapter({
+    mode:     'direct',
+    endpoint: 'https://test.foundry.example/',
+    apiKey:   'e2e-key',
+    fetchImpl: fetchImpl || fetchReturning(content),
+  });
+  const osc = fakeOsc();
+  const tool = DesktopTool.withDependencies({
+    aiService: { baseUrl: 'https://api', apiKeyManager: null, config: { apiKey: 'k' } },
+    osController: osc,
+    config: {},
+    logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() },
+  });
+  // Inject the real adapter so the FULL parse path runs against a
+  // realistic Foundry-shaped response.
+  tool._adapterCache = adapter;
+  return { tool, osc, adapter };
+}
+
+const ALLOW = { allowedActions: ['screenshot', 'click', 'scroll', 'drag', 'type', 'key_press', 'list_windows', 'focus_window', 'describe_capabilities'] };
+
+describe('DesktopTool E2E — intent grounding pipeline', () => {
+  test('click(intent="X") drives screenshot → KimiAdapter → parseCoords → mouseClick', async () => {
+    const { tool, osc } = buildToolWithRealAdapter({ content: 'x=512, y=384' });
+    const r = await tool.execute(
+      { action: 'click', intent: 'the centre dot' },
+      { toolConfig: ALLOW, sessionId: 'e2e' },
+    );
+
+    expect(r.success).toBe(true);
+    expect(r.coords).toEqual({ x: 512, y: 384 });
+    expect(r.groundedFrom).toBe('the centre dot');
+    expect(osc.screenshot).toHaveBeenCalledTimes(1);
+    expect(osc.mouseClick).toHaveBeenCalledWith(512, 384, expect.any(Object));
+  });
+
+  test('handles Foundry response with no parseable coords → GROUNDING_FAILED', async () => {
+    const { tool } = buildToolWithRealAdapter({ content: "I can't see that button on the screen." });
+    const r = await tool.execute(
+      { action: 'click', intent: 'the unicorn icon' },
+      { toolConfig: ALLOW },
+    );
+    expect(r.success).toBe(false);
+    expect(r.code).toBe('GROUNDING_FAILED');
+    expect(r.error).toMatch(/unicorn|can't see/i);
+  });
+
+  test('handles Foundry response with off-screen coords → COORDS_OUT_OF_BOUNDS', async () => {
+    const { tool, osc } = buildToolWithRealAdapter({ content: 'x=5000, y=5000' });
+    const r = await tool.execute(
+      { action: 'click', intent: 'somewhere bad' },
+      { toolConfig: ALLOW },
+    );
+    expect(r.success).toBe(false);
+    expect(r.code).toBe('COORDS_OUT_OF_BOUNDS');
+    expect(osc.mouseClick).not.toHaveBeenCalled();
+  });
+
+  test('coord-margin rejects coords too close to edge', async () => {
+    const { tool, osc } = buildToolWithRealAdapter({ content: 'x=5, y=5' });
+    // Margin of 50px means (5,5) is too close to the (0,0) corner.
+    const r = await tool.execute(
+      { action: 'click', intent: 'top-left' },
+      { toolConfig: { ...ALLOW, coordMarginPx: 50 } },
+    );
+    expect(r.success).toBe(false);
+    expect(r.code).toBe('COORDS_OUT_OF_BOUNDS');
+    expect(osc.mouseClick).not.toHaveBeenCalled();
+  });
+
+  test('upstream Foundry 500 → bubbles as PROVIDER_ERROR (tool wraps as OP_FAILED)', async () => {
+    const failingFetch = jest.fn(async () => ({
+      ok: false,
+      status: 500,
+      text: async () => JSON.stringify({ error: { message: 'Foundry exploded' } }),
+    }));
+    const { tool, osc } = buildToolWithRealAdapter({
+      content: 'unused',
+      fetchImpl: failingFetch,
+    });
+    const r = await tool.execute(
+      { action: 'click', intent: 'anywhere' },
+      { toolConfig: ALLOW },
+    );
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/Foundry/);
+    expect(osc.mouseClick).not.toHaveBeenCalled();
+  });
+
+  test('drag with two intents grounds twice from one screenshot', async () => {
+    // The adapter's fetch returns different coords on consecutive calls
+    // so we can verify both ends ground independently.
+    let call = 0;
+    const fetchImpl = jest.fn(async () => ({
+      ok: true,
+      status: 200,
+      text: async () => JSON.stringify({
+        choices: [{
+          message: {
+            content: call++ === 0 ? 'x=100, y=100' : 'x=900, y=900',
+          },
+        }],
+        usage: { prompt_tokens: 100, completion_tokens: 5 },
+      }),
+    }));
+    const { tool, osc } = buildToolWithRealAdapter({ content: 'unused', fetchImpl });
+    const r = await tool.execute(
+      { action: 'drag', fromIntent: 'A', toIntent: 'B' },
+      { toolConfig: ALLOW },
+    );
+
+    expect(r.success).toBe(true);
+    expect(r.from).toEqual({ x: 100, y: 100 });
+    expect(r.to).toEqual({ x: 900, y: 900 });
+    expect(osc.screenshot).toHaveBeenCalledTimes(1); // reused
+    expect(fetchImpl).toHaveBeenCalledTimes(2);      // grounded twice
+    expect(osc.mouseDrag).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('DesktopTool E2E — proxy→direct fallback on 404', () => {
+  const originalEndpoint = process.env.FOUNDRY_ENDPOINT;
+  const originalKey      = process.env.FOUNDRY_KEY;
+  afterEach(() => {
+    process.env.FOUNDRY_ENDPOINT = originalEndpoint || '';
+    process.env.FOUNDRY_KEY      = originalKey || '';
+    if (!originalEndpoint) delete process.env.FOUNDRY_ENDPOINT;
+    if (!originalKey)      delete process.env.FOUNDRY_KEY;
+  });
+
+  test('proxy 404 → automatic fallback to direct when env creds exist', async () => {
+    process.env.FOUNDRY_ENDPOINT = 'https://fallback.foundry.example/';
+    process.env.FOUNDRY_KEY      = 'fb-key';
+
+    // First fetch (proxy) returns 404; second fetch (direct) returns OK.
+    let call = 0;
+    const fetchImpl = jest.fn(async (url) => {
+      call++;
+      if (url.includes('/llm/grounding')) {
+        return { ok: false, status: 404, text: async () => JSON.stringify({ error: { message: 'Not Found' } }) };
+      }
+      // Direct path
+      return {
+        ok: true, status: 200,
+        text: async () => JSON.stringify({
+          choices: [{ message: { content: 'x=42, y=84' } }],
+          usage: { prompt_tokens: 5, completion_tokens: 2 },
+        }),
+      };
+    });
+
+    // Build via the REAL _buildAdapter path: instantiate two adapters
+    // (proxy first, then the fallback creates a direct one). We supply
+    // the proxy adapter with our fetchImpl; the fallback creates its
+    // own KimiAdapter which uses globalThis.fetch — patch globalThis.
+    const originalFetch = globalThis.fetch;
+    globalThis.fetch = fetchImpl;
+    try {
+      const { KimiAdapter } = await import('../../../services/grounding/KimiAdapter.js');
+      const proxyAdapter = new KimiAdapter({
+        mode: 'proxy',
+        backendUrl: 'https://api.broken.example',
+        userApiKey: 'u',
+        fetchImpl,
+      });
+      const osc = fakeOsc();
+      const tool = DesktopTool.withDependencies({
+        aiService: { baseUrl: 'https://api.broken.example', apiKeyManager: null, config: { apiKey: 'u' } },
+        osController: osc,
+        logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() },
+      });
+      tool._adapterCache = proxyAdapter;
+
+      const r = await tool.execute(
+        { action: 'click', intent: 'the target' },
+        { toolConfig: ALLOW },
+      );
+      expect(r.success).toBe(true);
+      expect(r.coords).toEqual({ x: 42, y: 84 });
+      expect(call).toBeGreaterThanOrEqual(2);  // proxy + direct
+      expect(osc.mouseClick).toHaveBeenCalledWith(42, 84, expect.any(Object));
+      // Adapter cache should now point at direct mode for subsequent calls.
+      expect(tool._adapterCache.mode).toBe('direct');
+    } finally {
+      globalThis.fetch = originalFetch;
+    }
+  });
+
+  test('proxy 404 with no env creds → surfaces a clear "deploy or set env" error', async () => {
+    delete process.env.FOUNDRY_ENDPOINT;
+    delete process.env.FOUNDRY_KEY;
+
+    const fetchImpl = jest.fn(async () => ({
+      ok: false, status: 404,
+      text: async () => JSON.stringify({ error: { message: 'Not Found' } }),
+    }));
+    const { KimiAdapter } = await import('../../../services/grounding/KimiAdapter.js');
+    const proxyAdapter = new KimiAdapter({
+      mode: 'proxy', backendUrl: 'https://api.broken.example',
+      userApiKey: 'u', fetchImpl,
+    });
+    const tool = DesktopTool.withDependencies({
+      aiService: { baseUrl: 'https://api.broken.example', apiKeyManager: null, config: { apiKey: 'u' } },
+      osController: fakeOsc(),
+      logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() },
+    });
+    tool._adapterCache = proxyAdapter;
+
+    const r = await tool.execute(
+      { action: 'click', intent: 'the target' },
+      { toolConfig: ALLOW },
+    );
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/Backend 404/);
+    expect(r.error).toMatch(/FOUNDRY_ENDPOINT|deployed/);
+  });
+});
+
+describe('DesktopTool E2E — knob propagation', () => {
+  test('reasoning_effort + image-size hint reach the Foundry body', async () => {
+    let capturedBody;
+    const fetchImpl = jest.fn(async (_url, opts) => {
+      capturedBody = JSON.parse(opts.body);
+      return {
+        ok: true, status: 200,
+        text: async () => JSON.stringify({
+          choices: [{ message: { content: 'x=10, y=10' } }],
+          usage: {},
+        }),
+      };
+    });
+    const { tool } = buildToolWithRealAdapter({ content: 'unused', fetchImpl });
+    await tool.execute(
+      { action: 'click', intent: 'the OK button' },
+      { toolConfig: { ...ALLOW, groundingEffort: 'medium' } },
+    );
+
+    expect(capturedBody.reasoning_effort).toBe('medium');
+    // Image-dimension hint must be prepended — empirical 666→41px win.
+    const userMessage = capturedBody.messages[0].content[0].text;
+    expect(userMessage).toMatch(/1920x1080/);
+    expect(userMessage).toMatch(/the OK button/);
+  });
+});