@logictan/dsh-browser-agent 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Logic Tan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,60 @@
+# @logictan/dsh-browser-agent
+
+A DSH browser agent that drives **your own Chrome** over CDP. Each page
+observation builds an indexed table of the visible controls; one TypeSafe
+`/v1/systemone` request decides both the **operation** (CLICK / TYPE_TEXT /
+SELECT / SCROLL_UP / SCROLL_DOWN / WAIT / DONE / BLOCKED) and the **target**
+element index; only `TYPE_TEXT` asks a second model for the field value, and
+that model comes from DSH's own `ctx.llm` service rather than a second key.
+
+## Why it connects instead of launching
+
+Launching a browser cannot carry your login state. The `browser-use` provider's
+launch mode hard-codes `--isolated`, whose documented meaning is *keep the
+browser profile in memory, do not save it to disk*. Attaching to an
+already-running Chrome is the only shape that uses a profile that persists.
+
+macOS Chrome refuses `--remote-debugging-port` on the **default** profile
+(`DevTools remote debugging requires a non-default data directory`), so the
+supported shape is a dedicated `--user-data-dir` profile:
+
+```bash
+"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \
+  --remote-debugging-port=9222 \
+  --user-data-dir="$HOME/.dsh/chrome-agent-profile"
+```
+
+Log in once in that window; the login survives restarts because the profile is
+on disk. The profile does **not** travel with config sync — logging in is a
+per-device, one-time step.
+
+The plugin never closes that browser. It disconnects the CDP session when a run
+ends, so your window and tabs are left exactly as they were.
+
+## Settings
+
+`Settings → Plugins → browser-agent → configuration`:
+
+| Field | Meaning |
+| --- | --- |
+| TypeSafe key | `role('secret')`; stored locally, never synced, never read back |
+| TypeSafe endpoint | defaults to `https://api.typesafe.ai/v1/systemone` |
+| TypeSafe model | defaults to `jev-latest` |
+| CDP endpoint | defaults to `http://127.0.0.1:9222` |
+| Max steps | browser actions per run; defaults to 60 |
+| Text provider / model / reasoning effort | route for the `TYPE_TEXT` field-value call, chosen from the live model catalog. All empty = the session's own current route. |
+
+The TypeSafe key is a `role('secret')` field: it must be entered once per
+device (DSH's config sync deliberately strips secret values), and it is never
+read back into the card.
+
+## Tool
+
+`browser_agent({ url, goal })` navigates to `url`, runs the decision loop until
+`DONE`/`BLOCKED` or a cap is reached, and returns a structured trace.
+
+## Scope
+
+v1 covers common HTML/ARIA controls, single-tab, single-frame pages. Iframes,
+shadow roots, canvas, file uploads, nested scrolling, pop-ups and screenshots
+are out of scope.

package/cordis.patch.yml

@@ -0,0 +1,18 @@
+# dsh-browser-agent bundle patch layer.
+#
+# One bare plugin row. The package ships two halves — the host half
+# (exports ".") registers the `browser-agent` settings namespace and the
+# `browser_agent` model tool, the browser half (exports "./client") registers
+# the Plugins-page configuration card.
+#
+# The row must stay a BARE package name: the web plugin table locates each
+# package's `dsh.client` manifest from the specifier of the loader row that
+# mounts it, and that lookup accepts only a bare name — a subpath row resolves
+# to no manifest, so the browser half would silently never load.
+#
+# Row id `browser-agent` equals the host half's `export const name` in
+# src/index.js. It is checked by hand against every other row id in this
+# repository; `aggregate.mjs --check` does not validate id uniqueness.
+- insert:
+    - id: browser-agent
+      name: '@logictan/dsh-browser-agent'

package/lib/actions.js

@@ -0,0 +1,155 @@
+/**
+ * The dynamic action space: observation -> indexed element table.
+ *
+ * One observed node receives one index even when it supports several
+ * operations, and each operation gets its OWN target map. Both matter. A shared
+ * target pool would let a `CLICK` answer name a target only `SELECT` can
+ * execute, and the upstream design rejects that class of answer by
+ * construction rather than by post-hoc checking; a per-node index is what makes
+ * the model's target unambiguous when a control is both clickable and fillable.
+ *
+ * A dropdown option is its own target, addressed `<element>:<option>`, because
+ * "select France" is a different action from "select the Country dropdown". The
+ * option target carries BOTH the value to set and the dropdown's current
+ * selection, since the model needs the latter to decide and the executor needs
+ * the former to act.
+ *
+ * Pure and browser-free, so the numbering rules are testable offline.
+ *
+ * @module @logictan/dsh-browser-agent/actions
+ */
+
+/** Instructions attached to the operation question. */
+export const OPERATION_LABELS = Object.freeze({
+  CLICK: 'Click an element, button, menu option, autocomplete suggestion, or calendar day.',
+  TYPE_TEXT: 'Enter or replace text in an editable field. A small LLM will supply the value from the goal.',
+  SELECT: 'Select an observed dropdown value.',
+});
+
+/**
+ * Labels for the operations that need no observed element.
+ *
+ * `DONE` and `BLOCKED` are not controls — they are terminal answers — but they
+ * belong to the same choice criteria, which is why they share one table.
+ */
+export const CONTROL_LABELS = Object.freeze({
+  SCROLL_UP: 'Scroll the page up.',
+  SCROLL_DOWN: 'Scroll the page down.',
+  WAIT: 'Wait for the page to update.',
+  DONE: 'Every requirement is visibly satisfied.',
+  BLOCKED: 'No supported operation can progress.',
+});
+
+/** Observed operations that address a specific element, keyed by observer kind. */
+const OPERATION_FOR_KIND = Object.freeze({ click: 'CLICK', fill: 'TYPE_TEXT', select: 'SELECT' });
+
+/**
+ * The observer kind that executes one wire operation.
+ *
+ * The executor switches on the observer's kind, not on the wire name: the
+ * operation is spelled `TYPE_TEXT` on the wire and `fill` in the observer. It
+ * is derived from {@link OPERATION_FOR_KIND} so the two directions cannot drift
+ * — deriving it by lowercasing the operation instead would produce `type_text`,
+ * which no executor branch matches.
+ */
+export const KIND_FOR_OPERATION = Object.freeze(
+  Object.fromEntries(Object.entries(OPERATION_FOR_KIND).map(([kind, operation]) => [operation, kind])),
+);
+
+/** Observed state keys copied onto both the element row and its target descriptor. */
+const STATE_KEYS = Object.freeze(['checked', 'selected', 'expanded']);
+
+/** Page-scroll step in CSS pixels, matching the upstream observer. */
+const SCROLL_STEP = 560;
+
+/**
+ * Build the indexed element table and the per-operation target maps.
+ *
+ * @param observed - one entry per observed control ACTION, not per node: a text
+ *   field the observer marks fillable produces a `fill` entry and a `click`
+ *   entry sharing one `node`, and a dropdown produces one `select` entry per
+ *   selectable option. Every entry carries `label`/`value` for the ELEMENT and
+ *   whatever of `checked`/`selected`/`expanded` the observer read; a `select`
+ *   entry additionally carries `optionLabel`/`optionValue` for the option it
+ *   offers, plus `optionDomIndex` — the option's 1-based position in the live
+ *   `<select>`. That position is NOT the entry's position in `observed`: the
+ *   observer skips options that are already selected or disabled, so the offered
+ *   list is a subset.
+ * @param [options] - `controls` lists the target-less operations the observer
+ *   offers. Omit it to offer all of them.
+ * @returns `elements` (the table sent as state), `targets` (per-operation
+ *   index -> descriptor), and `controls` (target-less operations).
+ */
+export function actionSpace(observed, options = {}) {
+  const elements = [];
+  const indices = new Map();
+  const targets = {};
+
+  for (const action of observed) {
+    const operation = OPERATION_FOR_KIND[action.kind];
+    if (operation === undefined) continue;
+
+    if (!indices.has(action.node)) {
+      const element = {
+        index: String(elements.length + 1),
+        label: action.label,
+        role: action.role,
+        operations: [],
+        value: action.value,
+      };
+      for (const key of STATE_KEYS) {
+        if (action[key] !== undefined) element[key] = action[key];
+      }
+      if (operation === 'SELECT') element.options = [];
+      indices.set(action.node, element.index);
+      elements.push(element);
+    }
+
+    const index = indices.get(action.node);
+    const element = elements[Number(index) - 1];
+    if (!element.operations.includes(operation)) element.operations.push(operation);
+
+    const group = (targets[operation] ??= {});
+    const descriptor = {
+      node: action.node,
+      operation,
+      role: action.role,
+      label: action.label,
+      value: action.value,
+    };
+    for (const key of STATE_KEYS) {
+      if (action[key] !== undefined) descriptor[key] = action[key];
+    }
+
+    if (operation === 'SELECT') {
+      const option = {
+        index: `${index}:${element.options.length + 1}`,
+        label: action.optionLabel,
+        value: action.optionValue,
+      };
+      element.options.push(option);
+      group[option.index] = {
+        ...descriptor,
+        label: option.label,
+        value: option.value,
+        // The dropdown's own current selection travels with the option: the
+        // model needs it to decide, and the element row already carries it.
+        currentValue: action.value,
+        element: Number(index),
+        optionIndex: action.optionDomIndex,
+      };
+    } else {
+      group[index] = descriptor;
+    }
+  }
+
+  const offered = options.controls ?? ['SCROLL_UP', 'SCROLL_DOWN', 'WAIT'];
+  const controls = {};
+  for (const name of offered) {
+    if (name === 'SCROLL_UP') controls.SCROLL_UP = { kind: 'scroll', label: CONTROL_LABELS.SCROLL_UP, delta: -SCROLL_STEP };
+    else if (name === 'SCROLL_DOWN') controls.SCROLL_DOWN = { kind: 'scroll', label: CONTROL_LABELS.SCROLL_DOWN, delta: SCROLL_STEP };
+    else if (name === 'WAIT') controls.WAIT = { kind: 'wait', label: CONTROL_LABELS.WAIT };
+  }
+
+  return { elements, targets, controls };
+}

package/lib/cdp.js

@@ -0,0 +1,73 @@
+/**
+ * CDP attachment to the user's own Chrome.
+ *
+ * The plugin attaches to a browser the user started with
+ * `--remote-debugging-port`; it never launches one and never closes one. That
+ * distinction is load-bearing twice over: a launched browser cannot carry login
+ * state (launch mode hard-codes `--isolated`, whose documented meaning is
+ * "do not save the profile to disk"), and closing the connection at the end of
+ * a run must leave the user's windows and tabs untouched.
+ *
+ * @module @logictan/dsh-browser-agent/cdp
+ */
+import { chromium } from 'playwright-core';
+
+/** Default CDP endpoint; the dedicated-profile Chrome listens here. */
+export const DEFAULT_ENDPOINT = 'http://127.0.0.1:9222';
+
+/** How long a connection attempt may take before it is reported as unreachable. */
+const CONNECT_TIMEOUT_MS = 10_000;
+
+/**
+ * Instructions for making the endpoint reachable.
+ *
+ * Included in every connection failure because the cause is almost always
+ * "Chrome is not listening" rather than a plugin fault, and the exact
+ * `--user-data-dir` requirement is the part users get wrong: Chrome refuses
+ * the debugging port on its default profile.
+ */
+export const START_INSTRUCTIONS = [
+  'Start Chrome with a dedicated profile and a debugging port:',
+  '  "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" \\',
+  '    --remote-debugging-port=9222 \\',
+  '    --user-data-dir="$HOME/.dsh/chrome-agent-profile"',
+  'Chrome refuses --remote-debugging-port on its DEFAULT profile, so --user-data-dir is required.',
+  'Log in to the sites you need once; that dedicated profile persists across restarts.',
+].join('\n');
+
+/**
+ * Attach to the browser behind `endpoint`.
+ *
+ * @param endpoint - CDP HTTP endpoint.
+ * @returns the browser connection; the caller disconnects it, never closes it.
+ * @throws {Error} an actionable message naming the endpoint and how to start it.
+ */
+export async function attach(endpoint = DEFAULT_ENDPOINT) {
+  try {
+    return await chromium.connectOverCDP(endpoint, { timeout: CONNECT_TIMEOUT_MS });
+  } catch (cause) {
+    throw new Error(
+      `browser_agent: could not attach to Chrome at ${endpoint} (${cause instanceof Error ? cause.message : String(cause)}).\n${START_INSTRUCTIONS}`,
+    );
+  }
+}
+
+/**
+ * The page a run should drive.
+ *
+ * The user's own tab is reused when the context already has one, so a run does
+ * not silently spawn windows; a new page is opened only when the context has
+ * none. The first context is the browser's default one, which is where a
+ * CDP-attached Chrome puts the profile's tabs.
+ *
+ * @param browser - the connection from {@link attach}.
+ * @returns the page to drive.
+ */
+export async function activePage(browser) {
+  const context = browser.contexts()[0];
+  if (context === undefined) {
+    throw new Error('browser_agent: the attached Chrome exposes no browser context.');
+  }
+  const pages = context.pages();
+  return pages.length > 0 ? pages[0] : await context.newPage();
+}