@restregular/dsh-visual-browser 0.0.0-stage → 0.1.0

package/lib/routes.js

@@ -0,0 +1,201 @@
+/**
+ * Same-origin HTTP API behind the Visual Browser panel.
+ *
+ * The panel polls `/state` for the timeline and pulls `/frame?seq=` for the
+ * picture; user clicks, wheel events and keys go back through the same
+ * BrowserManager methods the agent tools use. Frames stay on the host, so a
+ * 1 Hz picture never costs a session token.
+ */
+
+import { BrowserError } from './browser.js';
+
+const BODY_LIMIT = 64 * 1024;
+
+class HttpError extends Error {
+  constructor(status, code, message) {
+    super(message);
+    this.status = status;
+    this.code = code;
+  }
+}
+
+function sendJson(res, status, payload) {
+  const body = Buffer.from(JSON.stringify(payload), 'utf8');
+  res.statusCode = status;
+  res.setHeader('content-type', 'application/json; charset=utf-8');
+  res.setHeader('cache-control', 'no-store');
+  res.setHeader('content-length', String(body.length));
+  res.end(body);
+}
+
+function sendFrame(res, frame) {
+  if (frame === null || frame === undefined) {
+    res.statusCode = 204;
+    res.end();
+    return;
+  }
+  res.statusCode = 200;
+  res.setHeader('content-type', 'image/jpeg');
+  res.setHeader('cache-control', 'no-store');
+  res.setHeader('content-length', String(frame.jpeg.length));
+  res.end(frame.jpeg);
+}
+
+async function readJsonBody(req) {
+  const chunks = [];
+  let size = 0;
+  for await (const chunk of req) {
+    size += chunk.length;
+    if (size > BODY_LIMIT) throw new HttpError(413, 'BODY_TOO_LARGE', 'Request body is too large');
+    chunks.push(chunk);
+  }
+  if (size === 0) return {};
+  const text = Buffer.concat(chunks).toString('utf8');
+  try {
+    const parsed = JSON.parse(text);
+    return parsed && typeof parsed === 'object' ? parsed : {};
+  } catch {
+    throw new HttpError(400, 'BAD_JSON', 'Request body must be JSON');
+  }
+}
+
+function failure(res, error) {
+  const status =
+    error instanceof HttpError
+      ? error.status
+      : error instanceof BrowserError
+        ? error.code === 'BAD_ACTION' || error.code === 'BAD_URL' || error.code === 'BAD_KEY'
+          ? 400
+          : error.code === 'TARGET_NOT_FOUND' || error.code === 'STALE_REF'
+            ? 409
+            : 500
+        : 500;
+  const code = error instanceof HttpError ? error.code : error instanceof BrowserError ? error.code : 'INTERNAL_ERROR';
+  const message = error instanceof Error ? error.message : String(error);
+  sendJson(res, status, { ok: false, error: { code, message } });
+}
+
+/**
+ * Handle one request under `prefix`. Exported so a smoke test can drive it with
+ * a plain request/response pair and a stub manager.
+ */
+export async function handle(manager, prefix, req, res) {
+  const method = String(req.method ?? 'GET').toUpperCase();
+  let url;
+  try {
+    url = new URL(String(req.url ?? '/'), 'http://127.0.0.1');
+  } catch {
+    sendJson(res, 400, { ok: false, error: { code: 'BAD_URL', message: 'Malformed request URL' } });
+    return;
+  }
+  let route = url.pathname;
+  if (route.startsWith(prefix)) route = route.slice(prefix.length);
+  if (route === '' || route === '/') route = '/';
+
+  try {
+    if (method === 'GET' && route === '/') {
+      sendJson(res, 200, { ok: true, name: 'dsh-visual-browser', state: manager.state() });
+      return;
+    }
+    if (method === 'GET' && route === '/state') {
+      sendJson(res, 200, { ok: true, state: manager.state() });
+      return;
+    }
+    if (method === 'GET' && route === '/frame') {
+      const seq = Number(url.searchParams.get('seq') ?? '0');
+      sendFrame(res, manager.frameFor(seq));
+      return;
+    }
+    if (method === 'GET' && route === '/live') {
+      const state = await manager.refreshFrame();
+      const frame = manager.frameFor(state.frameSeq);
+      sendFrame(res, frame);
+      return;
+    }
+    if (method === 'POST' && route === '/start') {
+      const body = await readJsonBody(req);
+      const target = typeof body.url === 'string' ? body.url.trim() : '';
+      if (target !== '') await manager.open(target);
+      else await manager.ensureStarted();
+      sendJson(res, 200, { ok: true, state: manager.state() });
+      return;
+    }
+    if (method === 'POST' && route === '/stop') {
+      await manager.close();
+      sendJson(res, 200, { ok: true, state: manager.state() });
+      return;
+    }
+    if (method === 'POST' && route === '/capture') {
+      const state = await manager.refreshFrame();
+      sendJson(res, 200, { ok: true, state });
+      return;
+    }
+    if (method === 'POST' && route === '/click') {
+      const body = await readJsonBody(req);
+      await manager.clickAt({ x: body.x, y: body.y });
+      sendJson(res, 200, { ok: true, state: manager.state() });
+      return;
+    }
+    if (method === 'POST' && route === '/wheel') {
+      const body = await readJsonBody(req);
+      await manager.wheelAt({ deltaY: body.deltaY, deltaX: body.deltaX });
+      sendJson(res, 200, { ok: true, state: manager.state() });
+      return;
+    }
+    if (method === 'POST' && route === '/key') {
+      const body = await readJsonBody(req);
+      await manager.keyFromUser(String(body.key ?? 'Enter'));
+      sendJson(res, 200, { ok: true, state: manager.state() });
+      return;
+    }
+    if (method === 'POST' && route === '/nav') {
+      const body = await readJsonBody(req);
+      const action = String(body.action ?? '');
+      if (!['back', 'forward', 'reload'].includes(action)) throw new HttpError(400, 'BAD_ACTION', 'action must be back, forward, or reload');
+      await manager.act({ action });
+      sendJson(res, 200, { ok: true, state: manager.state() });
+      return;
+    }
+    sendJson(res, 404, { ok: false, error: { code: 'NOT_FOUND', message: `No route for ${method} ${route}` } });
+  } catch (error) {
+    manager.noteError?.(error);
+    failure(res, error);
+  }
+}
+
+/** Mount the API on the host web server; a deployment without it keeps the tools. */
+export function mountRoutes(ctx, manager, options = {}) {
+  const prefix = typeof options.prefix === 'string' && options.prefix !== '' ? options.prefix : '/visual-browser/api';
+  try {
+    ctx.inject(['webServer'], (scoped) => {
+      const webServer = scoped.get('webServer');
+      if (webServer === undefined || typeof webServer.register !== 'function') {
+        ctx.logger?.info?.('[visual-browser] webServer unavailable; panel API not mounted');
+        return;
+      }
+      try {
+        scoped.effect(
+          () =>
+            webServer.register({
+              kind: 'prefix',
+              path: prefix,
+              handler: (req, res) =>
+                handle(manager, prefix, req, res).catch((error) => {
+                  try {
+                    failure(res, error);
+                  } catch {
+                    /* response already sent */
+                  }
+                }),
+            }),
+          'dsh-visual-browser: api route',
+        );
+        ctx.logger?.info?.(`[visual-browser] panel API mounted at ${prefix}`);
+      } catch (error) {
+        ctx.logger?.warn?.(`[visual-browser] could not mount ${prefix}: ${error?.message ?? error}`);
+      }
+    });
+  } catch (error) {
+    ctx.logger?.warn?.(`[visual-browser] route mount failed: ${error?.message ?? error}`);
+  }
+}

package/lib/tools.js

@@ -0,0 +1,168 @@
+/**
+ * Agent-facing tools.
+ *
+ * Five tools cover the loop: open, look, act, read, screenshot. Each one
+ * returns the compact numbered view again, so a model never has to re-read a
+ * page and never sees the DOM. Every action is also mirrored into the panel
+ * timeline by the manager itself.
+ */
+
+import { BrowserError } from './browser.js';
+import { formatSnapshot } from './format.js';
+
+const TEXT_OUTPUT = {
+  schema: { type: 'string' },
+  render(_args, value) {
+    return [{ type: 'text', text: String(value ?? '') }];
+  },
+};
+
+function hintFor(error) {
+  if (!(error instanceof BrowserError)) return error instanceof Error ? error.message : String(error);
+  switch (error.code) {
+    case 'BROWSER_NOT_FOUND':
+      return `${error.message} Install Google Chrome or Microsoft Edge, or set \`executablePath\` in the dsh-visual-browser config in cordis.patch.yml.`;
+    case 'STALE_REF':
+      return error.message;
+    case 'NOT_RUNNING':
+      return `${error.message} Call browser_open first.`;
+    default:
+      return error.message;
+  }
+}
+
+function snapshotText(snapshot, manager) {
+  return formatSnapshot(snapshot, { maxElements: manager.config.maxElements });
+}
+
+export function browserTools(manager) {
+  return [
+    {
+      name: 'browser_open',
+      description:
+        'Open a URL in the visual browser: a real Chrome that the user can watch in the Visual Browser panel. Launches the browser on first use and returns the page\'s numbered view (interactive elements, headings, a short text excerpt) instead of the DOM, so the result stays small. Use the [n] numbers with browser_act.',
+      parameters: {
+        type: 'object',
+        properties: {
+          url: { type: 'string', description: 'Full URL, or a bare domain such as example.com (https is assumed).' },
+        },
+        required: ['url'],
+        additionalProperties: false,
+      },
+      output: TEXT_OUTPUT,
+      async execute(args) {
+        try {
+          const snapshot = await manager.open(String(args?.url ?? ''));
+          return `OK: opened ${snapshot.url}\n${snapshotText(snapshot, manager)}`;
+        } catch (error) {
+          throw new Error(hintFor(error));
+        }
+      },
+    },
+    {
+      name: 'browser_see',
+      description:
+        'Take a fresh numbered view of the current page: visible interactive elements with [n] numbers, headings and a short text excerpt. Element numbers are reassigned every time, so use numbers from the newest view only. Call this after the page changed on its own, or when an action result looked wrong.',
+      parameters: { type: 'object', properties: {}, additionalProperties: false },
+      output: TEXT_OUTPUT,
+      async execute() {
+        try {
+          const snapshot = await manager.snapshot();
+          return snapshotText(snapshot, manager);
+        } catch (error) {
+          throw new Error(hintFor(error));
+        }
+      },
+    },
+    {
+      name: 'browser_act',
+      description:
+        'Act on the current page, then return the new numbered view. Target an element with a [n] number from the newest view (preferred) or a CSS selector. Supported actions: click, type (ref + text, optional submit to press Enter), key, hover, scroll, select, back, forward, reload, wait, close. Every action is mirrored into the Visual Browser panel for the user.',
+      parameters: {
+        type: 'object',
+        properties: {
+          action: {
+            type: 'string',
+            enum: ['click', 'type', 'key', 'hover', 'scroll', 'select', 'back', 'forward', 'reload', 'wait', 'close'],
+            description: 'What to do.',
+          },
+          ref: { type: 'number', description: 'Element number [n] from the newest view.' },
+          selector: { type: 'string', description: 'CSS selector, for when no numbered view is available.' },
+          text: { type: 'string', description: 'Text to type (type) or the option label/value (select).' },
+          key: { type: 'string', description: 'Key for action=key, such as Enter, Tab, Escape, ArrowDown, Control+a.' },
+          direction: { type: 'string', enum: ['up', 'down', 'left', 'right'], description: 'Scroll direction (default down).' },
+          amount: { type: 'number', description: 'Scroll amount in viewport heights (default 1).' },
+          value: { type: 'string', description: 'Option value or label for action=select.' },
+          ms: { type: 'number', description: 'Milliseconds for action=wait (default 800).' },
+          clear: { type: 'boolean', description: 'type: replace the field content (default true); false appends.' },
+          submit: { type: 'boolean', description: 'type: press Enter afterwards.' },
+        },
+        required: ['action'],
+        additionalProperties: false,
+      },
+      output: TEXT_OUTPUT,
+      async execute(args) {
+        try {
+          const result = await manager.act(args ?? {});
+          if (!result.snapshot) return `OK: ${result.summary}`;
+          return `OK: ${result.summary}\n${snapshotText(result.snapshot, manager)}`;
+        } catch (error) {
+          throw new Error(hintFor(error));
+        }
+      },
+    },
+    {
+      name: 'browser_text',
+      description:
+        'Read text instead of guessing: omit ref/selector for the whole visible page text, or pass a [n] number to read one element (for a link it also returns the href). Use max_chars to raise the cap when the default excerpt is cut.',
+      parameters: {
+        type: 'object',
+        properties: {
+          ref: { type: 'number', description: 'Element number [n] from the newest view.' },
+          selector: { type: 'string', description: 'CSS selector instead of a number.' },
+          max_chars: { type: 'number', description: 'Character cap (default 4000).' },
+        },
+        additionalProperties: false,
+      },
+      output: TEXT_OUTPUT,
+      async execute(args) {
+        try {
+          const value = await manager.readText({
+            ref: args?.ref,
+            selector: args?.selector,
+            maxChars: args?.max_chars,
+          });
+          const header = `URL: ${value.url}${value.title ? `\nTITLE: ${value.title}` : ''}\nKIND: ${value.kind} · ${value.length} chars${value.truncated ? ' (truncated)' : ''}`;
+          const extra = [];
+          if (value.href) extra.push(`HREF: ${value.href}`);
+          if (value.value) extra.push(`VALUE: ${value.value}`);
+          return [header, ...extra, '', value.text || '(empty)'].join('\n');
+        } catch (error) {
+          throw new Error(hintFor(error));
+        }
+      },
+    },
+    {
+      name: 'browser_screenshot',
+      description:
+        'Save a screenshot of the current page to a file and return its absolute path. Use it when pixel detail matters (layout, images, visual verification) and then open the file with read_image. Defaults to a temp directory and png.',
+      parameters: {
+        type: 'object',
+        properties: {
+          path: { type: 'string', description: 'Absolute path to write; defaults to a temp file.' },
+          format: { type: 'string', enum: ['png', 'jpeg'], description: 'Image format (default png).' },
+        },
+        additionalProperties: false,
+      },
+      output: TEXT_OUTPUT,
+      async execute(args) {
+        try {
+          const saved = await manager.screenshot({ path: args?.path, format: args?.format });
+          return `Saved ${saved.format.toUpperCase()} screenshot (${Math.round(saved.bytes / 1024)} KiB): ${saved.path}\nOpen it with read_image when the visual detail matters.`;
+        } catch (error) {
+          throw new Error(hintFor(error));
+        }
+      },
+    },
+  ];
+}

package/locale/en.json

@@ -0,0 +1,6 @@
+{
+  "meta": {
+    "title": "Visual Browser",
+    "description": "A real Chrome for the Agent plus a live view for you: the Agent reads a numbered, token-light list of interactive elements instead of the raw DOM, and every action lands in a panel with the page frame, a replayable timeline, and click-through control."
+  }
+}

package/locale/zh.json

@@ -0,0 +1,6 @@
+{
+  "meta": {
+    "title": "可视化浏览器",
+    "description": "给 Agent 一个真实的 Chrome，也给你一块实时画面：Agent 只读「可交互元素编号列表」这种精简视图而不是整页 DOM，每次操作都会在侧边栏留下画面、可回放的操作时间线，你也可以直接点画面接管浏览器。"
+  }
+}

package/package.json

@@ -1,6 +1,71 @@
 {
   "name": "@restregular/dsh-visual-browser",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Visual browser for DeepSeek Harness: the Agent drives a real Chrome through a token-light numbered view of interactive elements (never the raw DOM), while the Web UI streams the live page frame, the click point of every action, and a replayable operation timeline the user can also take over by clicking the picture.",
+  "icon": "./icon.svg",
+  "type": "module",
+  "main": "index.js",
+  "exports": {
+    ".": "./index.js",
+    "./client": "./client.js",
+    "./locale/*.json": "./locale/*.json",
+    "./icon": "./icon.svg",
+    "./cordis.patch.yml": "./cordis.patch.yml",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "index.js",
+    "client.js",
+    "lib/",
+    "locale/*.json",
+    "cordis.patch.yml",
+    "icon.svg",
+    "README.md",
+    "LICENSE"
+  ],
+  "dsh": {
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    },
+    "client": {
+      "platform": "web",
+      "immediately": true,
+      "inject": [
+        "@deepseek-ai/dsh-client-locale",
+        "@deepseek-ai/dsh-client-ui-layout",
+        "@deepseek-ai/dsh-client-ui-sidebar"
+      ]
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/RestRegular/dsh-visual-browser.git"
+  },
+  "homepage": "https://github.com/RestRegular/dsh-visual-browser#readme",
+  "bugs": {
+    "url": "https://github.com/RestRegular/dsh-visual-browser/issues"
+  },
+  "keywords": [
+    "dsh-plugin",
+    "deepseek-harness",
+    "deepseek",
+    "browser",
+    "browser-automation",
+    "cdp",
+    "chrome-devtools-protocol",
+    "screenshot",
+    "visualization"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "engines": {
+    "node": ">=22.13.0"
+  },
+  "license": "MIT",
+  "author": "RestRegular",
+  "scripts": {
+    "test": "node --check index.js && node --check client.js && node --check lib/browser.js && node test/snapshot.test.mjs && node test/scripts.test.mjs && node test/package.test.mjs"
+  }
+}