@logictan/dsh-browser-agent 0.1.0

package/lib/config.js

@@ -0,0 +1,82 @@
+/**
+ * Configuration surface of the browser agent.
+ *
+ * The settings namespace is the one contract three surfaces share: the host
+ * half reads it when a run starts, the Plugins-page card edits it, and the
+ * client half's field list mirrors it. The card ships as plain browser code and
+ * cannot import this module, so the two field lists are kept in step by hand —
+ * changing a key here means changing `src/client.js` too.
+ *
+ * @module @logictan/dsh-browser-agent/config
+ */
+import z from '@deepseek-ai/schemastery';
+import { DEFAULT_ENDPOINT as DEFAULT_CDP_ENDPOINT } from './cdp.js';
+import { DEFAULT_MAX_STEPS } from './loop.js';
+import { DEFAULT_ENDPOINT as DEFAULT_TYPESAFE_ENDPOINT, DEFAULT_MODEL } from './typesafe.js';
+
+/** Settings namespace the host half registers and the card binds. */
+export const SETTINGS_NAMESPACE = 'browser-agent';
+
+/**
+ * Schema of the `browser-agent` settings section.
+ *
+ * The TypeSafe key is `role('secret')`: it is stored in the local settings
+ * document and stripped by every redacting surface, including config sync. That
+ * is a deliberate consequence — a secret never leaves the machine that entered
+ * it — so the key must be typed once per device.
+ *
+ * It deliberately carries NO default. The redaction sidecar reports
+ * `set: value !== undefined` against the RESOLVED value, so `.default('')`
+ * would make the key look permanently present: the card could never tell an
+ * unset key from a set one, and its reset would clear a key it wrongly believed
+ * was there. Leaving the field optional is what keeps that flag truthful.
+ *
+ * The TYPE_TEXT route is three separate fields rather than one. Empty means
+ * "use the session's own current route", which is the only default that stays
+ * correct after config sync moves the file to a machine with different
+ * providers; a hard-coded model would point at a model the target may not have.
+ */
+export const Config = z.object({
+  typesafeApiKey: z.string().role('secret'),
+  typesafeEndpoint: z.string().default(DEFAULT_TYPESAFE_ENDPOINT),
+  typesafeModel: z.string().default(DEFAULT_MODEL),
+  cdpEndpoint: z.string().default(DEFAULT_CDP_ENDPOINT),
+  maxSteps: z.number().step(1).min(1).default(DEFAULT_MAX_STEPS),
+  textProvider: z.string().default(''),
+  textModel: z.string().default(''),
+  textReasoningEffort: z.string().default(''),
+});
+
+/**
+ * The route used for the TYPE_TEXT helper call.
+ *
+ * An explicitly configured pair wins. Otherwise the session's own route is
+ * used, mirroring how the host resolves an auxiliary model call — the same
+ * three-step fallback (configured pair, then the session's routed request, then
+ * its options) that `ctx-mem` uses for its fill call.
+ *
+ * @param config - the resolved settings section.
+ * @param agent - the calling agent, when the tool was invoked from a turn.
+ * @returns the route to call; empty provider/model means "unconfigured".
+ */
+export function resolveTextRoute(config, agent) {
+  if (config.textProvider !== '' && config.textModel !== '') {
+    return {
+      provider: config.textProvider,
+      model: config.textModel,
+      reasoningEffort: config.textReasoningEffort,
+    };
+  }
+
+  const routed = agent?.session?.requestHeader?.()?.config;
+  if (routed?.provider && routed?.model) {
+    return { provider: routed.provider, model: routed.model, reasoningEffort: '' };
+  }
+
+  const options = agent?.options;
+  if (options?.provider && options?.model) {
+    return { provider: options.provider, model: options.model, reasoningEffort: '' };
+  }
+
+  return { provider: '', model: '', reasoningEffort: '' };
+}

package/lib/execute.js

@@ -0,0 +1,106 @@
+/**
+ * Action execution against the attached page.
+ *
+ * Every target is resolved from the observation's own identity map, and its
+ * geometry is re-read immediately before input: the page may have moved since
+ * the observation, and clicking stale coordinates would hit whatever now
+ * occupies them. A target that is gone is reported as a failed step rather than
+ * acted on blind, and the loop re-observes.
+ *
+ * The identity map is `window.__dshBrowserAgent`, written by `./observe.js`.
+ * The page-side functions below therefore read that one shape and nothing else;
+ * if the observer's cache is renamed, both modules must change together.
+ *
+ * @module @logictan/dsh-browser-agent/execute
+ */
+
+/** How long to let the page settle after an interaction, in milliseconds. */
+const SETTLE_MS = 120;
+
+/** A dropdown option gets longer, because its list may still be rendering. */
+const OPTION_SETTLE_MS = 200;
+
+/**
+ * Resolve one code-owned identity to its live node inside the page.
+ *
+ * Must stay self-contained: it is serialized and evaluated in the page.
+ * @param node - the identity from the observation.
+ * @returns whether the node is still actionable, after scrolling it into view.
+ */
+function resolveAndScroll(node) {
+  const element = window.__dshBrowserAgent?.nodes?.get(node);
+  if (!element || !element.isConnected) return false;
+  element.scrollIntoView({ block: 'center', inline: 'center' });
+  return true;
+}
+
+/**
+ * Execute one decision against the page.
+ *
+ * @param input - the page, the resolved decision, and the TYPE_TEXT value
+ *   provider (called only for `TYPE_TEXT`).
+ * @returns a short human-readable description of what was done.
+ * @throws {Error} when the target is no longer actionable, or the value
+ *   provider could not produce a value.
+ */
+export async function execute(input) {
+  const { page, decision, valueProvider } = input;
+
+  if (decision.kind === 'done' || decision.kind === 'blocked') return decision.kind;
+  if (decision.kind === 'wait') {
+    await page.waitForTimeout(SETTLE_MS);
+    return 'waited for the page';
+  }
+  if (decision.kind === 'scroll') {
+    await page.evaluate((delta) => window.scrollBy(0, delta), decision.delta);
+    await page.waitForTimeout(SETTLE_MS);
+    return `scrolled by ${decision.delta}px`;
+  }
+
+  const descriptor = decision.descriptor;
+  const live = await page.evaluate(resolveAndScroll, descriptor.node);
+  if (!live) throw new Error(`target [${decision.target}] is no longer on the page`);
+
+  if (decision.kind === 'click') {
+    await page.evaluate((node) => {
+      window.__dshBrowserAgent.nodes.get(node).click();
+    }, descriptor.node);
+    await page.waitForTimeout(SETTLE_MS);
+    return `clicked ${descriptor.label}`;
+  }
+
+  if (decision.kind === 'fill') {
+    const value = await valueProvider(descriptor);
+    await page.evaluate(
+      ({ node, text }) => {
+        const element = window.__dshBrowserAgent.nodes.get(node);
+        element.focus();
+        if (element.isContentEditable) element.textContent = text;
+        else element.value = text;
+        element.dispatchEvent(new Event('input', { bubbles: true }));
+        element.dispatchEvent(new Event('change', { bubbles: true }));
+      },
+      { node: descriptor.node, text: value },
+    );
+    await page.waitForTimeout(SETTLE_MS);
+    return `typed "${value}" into ${descriptor.label}`;
+  }
+
+  if (decision.kind === 'select') {
+    await page.evaluate(
+      ({ node, optionIndex }) => {
+        const element = window.__dshBrowserAgent.nodes.get(node);
+        const option = element.options[optionIndex - 1];
+        if (option === undefined) throw new Error('the option is no longer offered');
+        element.value = option.value;
+        element.dispatchEvent(new Event('input', { bubbles: true }));
+        element.dispatchEvent(new Event('change', { bubbles: true }));
+      },
+      { node: descriptor.node, optionIndex: descriptor.optionIndex },
+    );
+    await page.waitForTimeout(OPTION_SETTLE_MS);
+    return `selected ${descriptor.label}`;
+  }
+
+  throw new Error(`unsupported operation ${decision.operation}`);
+}

package/lib/index.js

@@ -0,0 +1,104 @@
+/**
+ * dsh-browser-agent — host half.
+ *
+ * Registers two things: the `browser-agent` settings namespace, and the
+ * `browser_agent` model tool that runs the decision loop over CDP against the
+ * user's own Chrome.
+ *
+ * The tool is registered through `ctx.get('tools')` rather than the `ctx.tools`
+ * property. Cordis requires a declared `inject` before a service property may
+ * be read, and `tools` is genuinely optional here — a headless deployment that
+ * composes no tool registry still has a working settings namespace — so it must
+ * not be injected. Reading it optionally is the same choice
+ * `dsh-config-manager`'s `registerModelTools` makes, for the same reason.
+ *
+ * @module @logictan/dsh-browser-agent
+ */
+import { defineTool } from '@deepseek-ai/dsh-tools';
+
+import { attach, activePage } from './cdp.js';
+import { Config, SETTINGS_NAMESPACE, resolveTextRoute } from './config.js';
+import { run } from './loop.js';
+
+/** Plugin row id; must equal the row id in `cordis.patch.yml`. */
+export const name = 'browser-agent';
+
+/**
+ * Services this plugin needs before it can mount.
+ *
+ * `llm` is required: the TYPE_TEXT step cannot work without it, and a silent
+ * degradation there would be worse than a load failure. `settings` is required
+ * for the same reason — every run reads its configuration from it.
+ */
+export const inject = ['settings', 'llm'];
+
+/**
+ * Register the `browser_agent` tool.
+ *
+ * @param ctx - the plugin context.
+ */
+export function apply(ctx) {
+  const settings = ctx.settings;
+  const llm = ctx.llm;
+  const tools = ctx.get('tools');
+
+  // The namespace must be registered before anything reads or writes it: an
+  // unregistered namespace makes `settings.get` return undefined and every
+  // Plugins-page write fail with "settings namespace ... is not registered".
+  settings.register(SETTINGS_NAMESPACE, Config);
+
+  const definition = defineTool({
+    name: 'browser_agent',
+    description:
+      "Drive the user's own Chrome to accomplish a goal. Navigates to `url` and runs an observation/decision loop " +
+      'until the goal is met, then returns a structured trace. Requires Chrome to be running with a debugging port ' +
+      '(the dedicated-profile command is in the plugin README) and a TypeSafe API key in the plugin settings.',
+    parameters: {
+      url: { type: 'string', description: 'Page to open before starting.', required: true },
+      goal: { type: 'string', description: 'What must be true when the task is finished.', required: true },
+    },
+    output: {
+      schema: { type: 'json', description: 'Structured trace: status, steps, decision count, final url.' },
+      render: (_args, value) => [{ type: 'text', text: JSON.stringify(value, null, 2) }],
+    },
+    async execute(args, exec) {
+      const current = Config(settings.get(SETTINGS_NAMESPACE) ?? {});
+      // An unset optional field resolves to undefined; a cleared one to ''.
+      if (current.typesafeApiKey === undefined || current.typesafeApiKey === '') {
+        throw new Error(
+          'browser_agent: no TypeSafe API key is configured. ' +
+            'Open Settings → Plugins → browser-agent and paste a key from https://console.typesafe.ai/keys.',
+        );
+      }
+
+      const browser = await attach(current.cdpEndpoint);
+      try {
+        const page = await activePage(browser);
+        return await run({
+          url: args.url,
+          goal: args.goal,
+          page,
+          llm,
+          config: {
+            apiKey: current.typesafeApiKey,
+            endpoint: current.typesafeEndpoint,
+            model: current.typesafeModel,
+            maxSteps: current.maxSteps,
+            textRoute: resolveTextRoute(current, exec),
+          },
+          signal: exec.signal,
+        });
+      } finally {
+        // Disconnect, never close: the browser is the user's, and closing it
+        // would take their windows and tabs with it.
+        await browser.close();
+      }
+    },
+  });
+
+  if (tools === null || tools === undefined || typeof tools !== 'object') {
+    ctx.logger?.warn?.('browser_agent: the tools service is unavailable; the tool was not registered.');
+    return;
+  }
+  ctx.effect(() => tools.register(definition), 'browser-agent: browser_agent tool');
+}

package/lib/loop.js

@@ -0,0 +1,190 @@
+/**
+ * The decision loop.
+ *
+ * Observe, ask, validate, execute — repeated until the model answers `DONE` or
+ * `BLOCKED`, or a cap is reached. Every iteration re-observes rather than
+ * reusing the previous table, because the page is the only source of truth
+ * about what is currently clickable and an identity from a stale observation
+ * may no longer resolve.
+ *
+ * Both caps are hard stops that still return a trace: a run that hits one has
+ * not failed, it has run out of budget, and the caller needs the trace to see
+ * where it got to. `DONE` is the model's own claim and is reported as such —
+ * this plugin has no independent way to verify the goal was met.
+ *
+ * @module @logictan/dsh-browser-agent/loop
+ */
+import { actionSpace } from './actions.js';
+import { buildRequest, resolveDecision } from './request.js';
+import { ask } from './typesafe.js';
+import { execute } from './execute.js';
+import { SNAPSHOT_LIMITS, snapshot } from './observe.js';
+import { fieldValue } from './value.js';
+
+/** Browser actions allowed per run. */
+export const DEFAULT_MAX_STEPS = 60;
+
+/** Decision requests allowed per run; two per step is the observed ratio. */
+export const MAX_DECISIONS = 120;
+
+/** How many times an observation is retried while a navigation is in flight. */
+const OBSERVE_ATTEMPTS = 5;
+
+/** How long to wait between those retries, in milliseconds. */
+const OBSERVE_RETRY_MS = 120;
+
+/**
+ * Whether a failed evaluation was the old document being torn down.
+ *
+ * A click that navigates destroys the document the evaluation was bound to, so
+ * the call is rejected rather than returning a value. Playwright reports that
+ * as a context-destroyed error; any other rejection is a real fault.
+ *
+ * @param cause - the rejection from `page.evaluate`.
+ * @returns whether the failure was a destroyed execution context.
+ */
+function isContextDestroyed(cause) {
+  const message = cause instanceof Error ? cause.message : String(cause);
+  return message.includes('Execution context was destroyed') || message.includes('Cannot find context with specified id');
+}
+
+/**
+ * Observe the page, tolerating a navigation that is still in flight.
+ *
+ * A click may navigate, and the loop re-observes immediately afterwards. That
+ * observation can land while the old document is gone and the new one is not
+ * ready — the expected consequence of the action just taken, not a failure. It
+ * is retried against the document that replaces it; a run that treated it as
+ * fatal would end the moment the agent clicked its first link.
+ *
+ * Exported because the retry policy is a seam worth testing on its own: a
+ * non-navigating rejection must propagate unchanged.
+ *
+ * @param page - the attached page.
+ * @returns the page-side snapshot, or `null` when the document has no body.
+ * @throws {Error} any rejection that is not a destroyed execution context, and
+ *   a context-destroyed rejection that outlives {@link OBSERVE_ATTEMPTS}.
+ */
+export async function observe(page) {
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      return await page.evaluate(snapshot, SNAPSHOT_LIMITS);
+    } catch (cause) {
+      if (attempt >= OBSERVE_ATTEMPTS || !isContextDestroyed(cause)) throw cause;
+      await page.waitForTimeout(OBSERVE_RETRY_MS);
+    }
+  }
+}
+
+/**
+ * Run one task to completion.
+ *
+ * @param input - the page, the goal, the settings, the `ctx.llm` service, and
+ *   an abort signal.
+ * @returns a structured trace: the steps taken, how the run ended, and the
+ *   decision count.
+ */
+export async function run(input) {
+  const { page, goal, config, llm, signal } = input;
+
+  await page.goto(input.url, { waitUntil: 'domcontentloaded' });
+  await page.waitForTimeout(200);
+
+  const history = [];
+  const steps = [];
+  let decisions = 0;
+  let status = 'max-steps';
+
+  for (let step = 1; step <= config.maxSteps; step += 1) {
+    if (signal?.aborted) {
+      status = 'cancelled';
+      break;
+    }
+
+    const observation = await observe(page);
+    if (observation === null) {
+      status = 'blocked';
+      history.push({ action: 'the page has no body', kind: 'BLOCKED', text: '', page_changed: false });
+      break;
+    }
+
+    const table = actionSpace(observation.actions, { controls: observation.controls });
+    const body = buildRequest({
+      goal,
+      page: observation,
+      elements: table.elements,
+      targets: table.targets,
+      controls: table.controls,
+      history,
+      model: config.model,
+    });
+
+    if (decisions >= MAX_DECISIONS) {
+      status = 'max-decisions';
+      break;
+    }
+    decisions += 1;
+
+    const response = await ask({ body, endpoint: config.endpoint, apiKey: config.apiKey, signal });
+    const decision = resolveDecision({ answers: response.answers, targets: table.targets, controls: table.controls });
+
+    if (decision.kind === 'done' || decision.kind === 'blocked') {
+      status = decision.kind;
+      steps.push({
+        step,
+        operation: decision.operation,
+        target: decision.target,
+        label: decision.descriptor?.label ?? null,
+        confidence: decision.confidence,
+        result: decision.kind,
+      });
+      history.push({ action: decision.operation, kind: decision.operation, text: '', page_changed: false });
+      break;
+    }
+
+    let result;
+    try {
+      result = await execute({
+        page,
+        decision,
+        valueProvider: (descriptor) =>
+          fieldValue({
+            llm,
+            route: config.textRoute,
+            goal,
+            descriptor,
+            page: observation,
+            history,
+            signal,
+          }),
+      });
+    } catch (cause) {
+      // A target that vanished between observation and execution is a normal
+      // race, not a fault: record it and let the next observation re-decide.
+      result = `failed: ${cause instanceof Error ? cause.message : String(cause)}`;
+    }
+
+    steps.push({
+      step,
+      operation: decision.operation,
+      target: decision.target,
+      label: decision.descriptor?.label ?? null,
+      confidence: decision.confidence,
+      result,
+    });
+    history.push({
+      action: `${decision.operation} ${decision.descriptor?.label ?? ''}`.trim(),
+      kind: decision.operation,
+      text: decision.kind === 'fill' ? result : '',
+      page_changed: true,
+    });
+  }
+
+  return {
+    status,
+    goal,
+    url: page.url(),
+    steps,
+    decisions,
+  };
+}