dsh-browser-playwright-codex 0.5.5 → 0.6.0

package/lib/tool.js

@@ -3,7 +3,8 @@ import { defineTool } from '@deepseek-ai/dsh-tools';
 import { BlockAssembler, createUserMessage } from '@deepseek-ai/dsh-llm';
 import { BrowserError } from "./errors.js";
 import { registeredSuffixes } from "./contract.js";
-import { getEnabled, publishGates } from "./runtime-state.js";
+import { getEnabled, publishGates, publishLoginSites } from "./runtime-state.js";
+import { summarizeLoginSites } from "./login-sites.js";
 import { isCdpMethodAllowed, cdpDenialMessage } from "./cdp-policy.js";
 import { SCREENSHOT_SCHEMA, SNAPSHOT_SCHEMA, TABS_SCHEMA } from "./tool-schemas.js";
 import { parseJsonText, extractionPrompt } from "./extract.js";
@@ -68,11 +69,37 @@ export function apply(ctx, config) {
         return id === undefined ? 'anonymous' : String(id);
     };
     /** Acquire the calling session's live browser session. */
+    /**
+     * Publish the "which sites look logged in" summary for the settings card. Throttled, and
+     * every failure is swallowed: this is a display nicety and must never slow down or break an
+     * action. Nothing here is model-visible — it travels only to the settings route.
+     */
+    let lastLoginPublish = 0;
+    const publishLogins = (session) => {
+        const now = Date.now();
+        if (now - lastLoginPublish < 5000)
+            return;
+        lastLoginPublish = now;
+        void session
+            .cookieJar()
+            .then(async (rows) => {
+            publishLoginSites(summarizeLoginSites(rows, 'live', now, {
+                visited: session.visitedHosts(),
+                titles: await session.siteTitles(),
+            }));
+        })
+            .catch(() => {
+            /* display-only: a failed cookie read must not surface as a tool failure */
+        });
+    };
     const acquire = (exec) => {
         if (!getEnabled()) {
             throw new BrowserError('DISABLED', 'dsh-browser-playwright-codex is disabled; open Settings → Plugins → 浏览器 and turn it on to use the browser');
         }
-        return ctx.browser.acquire(ownerFor(exec), exec.signal);
+        return ctx.browser.acquire(ownerFor(exec), exec.signal).then((session) => {
+            publishLogins(session);
+            return session;
+        });
     };
     const services = () => {
         const out = {};
@@ -137,46 +164,69 @@ export function apply(ctx, config) {
     }));
     ctx.tools.register(defineTool({
         name: p('click'),
-        description: 'Click the element with the given ref from the latest snapshot and return the new snapshot. ' +
-            'Refs go stale after navigation or DOM changes — take a fresh ' +
-            p('snapshot') +
-            ' if a ref fails. ' +
-            'Prefer ref when the snapshot has one; for a target shown WITHOUT a ref (plain text, e.g. generic "设置", or a menu item ' +
-            p('hover') +
-            ' just revealed), pass text instead, copied exactly as rendered — a revealed item may need ' +
-            p('hover') +
-            ' first to stay on screen. ' +
-            'IRON RULE: after the click, verify in the returned snapshot that the intended effect actually happened (URL, selection, toast, dialog, new content). ' +
-            'A click can also succeed without changing this page: when the snapshot shows no change, check ' +
+        description: 'Click the target from the latest snapshot — or reveal it without clicking (mode:"hover") — and return the new snapshot. ' +
+            'Best target first: `ref` when the snapshot has one; `text`, a visible label copied exactly as rendered, for a target shown WITHOUT a ref; `x`+`y` viewport CSS pixels only as a last resort (canvas-drawn hit areas). Refs go stale after navigation or DOM changes: take a fresh snapshot if a ref fails. ' +
+            'mode:"hover" only REVEALS content and never activates: use it when a control hides content until the pointer rests on it, especially when the snapshot lacks an element the page obviously should have. A hover returning only means the pointer moved: read the note comparing the actionable ref count before and after; new refs mean pre-hover refs are stale, so act from the NEW refs before an unrelated click closes the menu. If that note reports no new refs, do NOT repeat the hover and do NOT fall back to raw coordinates — hover the parent/container or the sibling that owns the menu, otherwise use a visible alternative or tell the user. ' +
+            'precise:true takes a ref click down the geometric path instead: the element is scrolled into view and clicked at its exact centre through native input, with no screenshot measuring, no devicePixelRatio guesswork. It is the FALLBACK, not a routine alternative — use it only after a ref click reports the element is not actionable, or that the click could not be delivered. ' +
+            'IRON RULE: verify the effect in the returned snapshot (URL, selection, toast, dialog, new content). No change may still be success: check ' +
             p('tabs') +
-            ' for a new tab (a target="_blank" link) before concluding failure, and treat a same-page overlay or popup, URL unchanged and no new tab, as success — the returned snapshot is the evidence: compare its refs and content with the pre-click snapshot. ' +
-            'The snapshot may open with a landing note naming what received the click: if that is not your target, the click was intercepted — treat it as failed and retarget. ' +
-            'If there was no visible effect, do NOT blindly repeat the click and do NOT fall back to ' +
-            p('click_at') +
-            ' raw coordinates — inspect the visible state for the blocker or interception, resolve it, then retry the most direct semantic action. ' +
-            'If the click raises a native dialog, it stays PENDING and this call returns immediately with the dialog text instead of the page tree — answer it with ' +
+            ' for a new tab (a target="_blank" link) before concluding, and treat a same-page overlay as success. ' +
+            'The landing note names what received the click: if that is not your target the click was intercepted — treat it as failed and retarget; do NOT blindly repeat it and do NOT throw raw coordinates at it, but inspect the visible state for the blocker or interception first, resolve that, then retry the most direct semantic action. ' +
+            'A native dialog stays PENDING and this returns immediately with the dialog text; this tool cannot accept it — ' +
             p('dialog') +
-            '. This tool has no way to accept a dialog: accepting is a separate, deliberate call.',
+            ' is a separate, deliberate call.',
         parameters: {
             ref: {
                 type: 'string',
-                description: 'Element ref (e.g. e12) from the latest snapshot. Use this whenever the snapshot has a ref for the target.',
+                description: 'Element ref (e.g. e12) from the latest snapshot. Preferred whenever the snapshot has one.',
             },
             text: {
                 type: 'string',
-                description: 'Visible label of a target shown WITHOUT a ref (plain text, e.g. generic "搜索设置"). Copied exactly as the snapshot renders it; the first visible element whose own text matches is clicked and the landing note names it.',
+                description: 'Visible label of a target shown WITHOUT a ref, copied exactly as the snapshot renders it; the first visible element whose own text matches is used and the note names it.',
+            },
+            x: {
+                type: 'integer',
+                description: 'Viewport x in CSS pixels, with y: a raw click, mode "click" only, last resort.',
+            },
+            y: {
+                type: 'integer',
+                description: 'Viewport y in CSS pixels, with x: a raw click, mode "click" only, last resort.',
+            },
+            mode: {
+                type: 'string',
+                enum: ['click', 'hover'],
+                description: 'click (default) activates the target; hover only reveals it.',
+            },
+            precise: {
+                type: 'boolean',
+                description: 'Ref clicks only: click the geometric centre through native input instead of the standard click.',
             },
         },
         output: { schema: SNAPSHOT_SCHEMA, render: renderSnapshotValue },
         async execute(args, exec) {
             const session = await acquire(exec);
-            if (args.ref !== undefined && args.ref !== '') {
-                return snapshotValue(await session.click(args.ref, exec.signal));
+            const mode = args.mode ?? 'click';
+            if (mode !== 'click' && mode !== 'hover') {
+                throw new BrowserError('PARAM_MISSING', 'browser_click takes mode "click" or "hover", got: ' + String(args.mode));
+            }
+            const ref = args.ref !== undefined && args.ref !== '' ? args.ref : undefined;
+            const text = args.text !== undefined && args.text !== '' ? args.text : undefined;
+            const point = typeof args.x === 'number' && typeof args.y === 'number' ? { x: args.x, y: args.y } : undefined;
+            if (mode === 'hover') {
+                if (ref !== undefined)
+                    return snapshotValue(await session.hover(ref, exec.signal));
+                if (text !== undefined)
+                    return snapshotValue(await session.hoverText(text, exec.signal));
+                throw new BrowserError('PARAM_MISSING', 'browser_click with mode "hover" needs a ref from the latest snapshot, or the visible text label of a trigger that has no ref');
             }
-            if (args.text !== undefined && args.text !== '') {
-                return snapshotValue(await session.clickText(args.text, exec.signal));
+            if (ref !== undefined) {
+                return snapshotValue(args.precise === true ? await session.clickAtRef(ref, exec.signal) : await session.click(ref, exec.signal));
             }
-            throw new BrowserError('PARAM_MISSING', 'browser_click needs either a ref from the latest snapshot or the visible text label of a target that has no ref');
+            if (text !== undefined)
+                return snapshotValue(await session.clickText(text, exec.signal));
+            if (point !== undefined)
+                return snapshotValue(await session.clickAt(point.x, point.y, exec.signal));
+            throw new BrowserError('PARAM_MISSING', 'browser_click needs a target: a ref from the latest snapshot, the visible text label of a target that has no ref, or both x and y viewport coordinates');
         },
     }));
     ctx.tools.register(defineTool({
@@ -207,71 +257,10 @@ export function apply(ctx, config) {
             return snapshotValue(await session.resolveDialog(args.accept, args.promptText, exec.signal));
         },
     }));
-    ctx.tools.register(defineTool({
-        name: p('click_at'),
-        description: 'Precise native click, two modes. Ref mode (prefer it): pass ref — the element is scrolled into view, awaited for geometry stability, and clicked at its exact geometric centre through native input: no screenshot measuring, no devicePixelRatio guesswork. ' +
-            'Raw mode: pass x/y viewport CSS pixels to click a raw point with no element involvement. ' +
-            'This is the FALLBACK, not a routine alternative: try ' +
-            p('click') +
-            ' (including its text mode) first, and use ref mode only after a click reports the element is not actionable or the click could not be delivered. Raw x/y is a last resort for targets no ref resolves (canvas-drawn hit areas): raw clicks still return a landing note saying what they hit, so re-snapshot and confirm. ' +
-            'IRON RULE: verify the effect in the returned snapshot; if nothing changed, do NOT repeat blindly and do NOT throw more raw coordinates at it — find the blocker or interception first, then retry the most direct semantic action. ' +
-            'The snapshot may open with a landing note naming what received the click, or — when an intercepted ref-mode target had to fall back to a bare click at its centre — a CAUTION note: if the click was intercepted, treat it as failed and retarget.',
-        parameters: {
-            ref: {
-                type: 'string',
-                description: 'Element ref (e.g. e12) from the latest snapshot. If given, its geometric centre is clicked; x/y are ignored.',
-            },
-            x: { type: 'integer', description: 'Raw-mode viewport x in CSS pixels (required only when ref is omitted).' },
-            y: { type: 'integer', description: 'Raw-mode viewport y in CSS pixels (required only when ref is omitted).' },
-        },
-        output: { schema: SNAPSHOT_SCHEMA, render: renderSnapshotValue },
-        async execute(args, exec) {
-            const session = await acquire(exec);
-            if (args.ref !== undefined && args.ref !== '') {
-                return snapshotValue(await session.clickAtRef(args.ref, exec.signal));
-            }
-            if (typeof args.x !== 'number' || typeof args.y !== 'number') {
-                throw new BrowserError('PARAM_MISSING', 'browser_click_at needs a ref, or both x and y viewport coordinates');
-            }
-            return snapshotValue(await session.clickAt(args.x, args.y, exec.signal));
-        },
-    }));
-    ctx.tools.register(defineTool({
-        name: p('hover'),
-        description: 'Hover the element with the given ref — or, for a trigger the snapshot gave no ref, the visible element whose own label equals text — and return the new snapshot. ' +
-            'Hover only REVEALS content and never activates anything: use ' +
-            p('click') +
-            ' to open, select or submit. Use it when a control hides content until the pointer rests on it (submenus, dropdowns, account menus, tooltips, previews, chart values, player controls) — and especially when the snapshot lacks an element the page obviously should have. ' +
-            'Prefer ref when the snapshot has one; use text only for a label shown WITHOUT a ref, copied exactly as rendered. Do not hover plain links/buttons you mean to activate, and never hover "just in case". ' +
-            'A hover returning only means the pointer moved: the snapshot opens with a note comparing the actionable ref count before and after — read it. New refs mean the revealed content is in this snapshot and pre-hover refs are stale, so pick from the NEW refs and act before any unrelated click closes the menu. ' +
-            'IRON RULE: if the note reports no new refs and what you expected is still missing, do NOT repeat the hover and do NOT fall back to ' +
-            p('click_at') +
-            ' raw coordinates — hover the parent/container or the sibling that owns the menu; if that also reveals nothing, hover is not implemented here, so use a visible alternative or tell the user.',
-        parameters: {
-            ref: {
-                type: 'string',
-                description: 'Element ref (e.g. e12) from the latest snapshot. Use this whenever the snapshot has a ref for the trigger.',
-            },
-            text: {
-                type: 'string',
-                description: 'Visible label of a trigger shown WITHOUT a ref (plain text, e.g. generic "设置"). Copied exactly as the snapshot renders it; the first visible element whose own text matches is hovered and the note names it.',
-            },
-        },
-        output: { schema: SNAPSHOT_SCHEMA, render: renderSnapshotValue },
-        async execute(args, exec) {
-            const session = await acquire(exec);
-            if (args.ref !== undefined && args.ref !== '') {
-                return snapshotValue(await session.hover(args.ref, exec.signal));
-            }
-            if (args.text !== undefined && args.text !== '') {
-                return snapshotValue(await session.hoverText(args.text, exec.signal));
-            }
-            throw new BrowserError('PARAM_MISSING', 'browser_hover needs either a ref from the latest snapshot or the visible text label of a trigger that has no ref');
-        },
-    }));
     ctx.tools.register(defineTool({
         name: p('fill'),
         description: 'Replace the value of the input or select referenced by the latest snapshot. ' +
+            '`value` is accepted as an alias of `text`. ' +
             'For <select> elements the text selects the matching option label. ' +
             'IRON RULE: after filling, verify in the returned snapshot that the field actually holds the value. ' +
             'Frameworks can silently reject or transform input (controlled components, contenteditable editors, ' +
@@ -284,12 +273,27 @@ export function apply(ctx, config) {
             'say so) — understand why it was ignored, then act.',
         parameters: {
             ref: { type: 'string', required: true, description: 'Input or select ref (e.g. e7) from the latest snapshot.' },
-            text: { type: 'string', required: true, description: 'Exact value to fill, or option label for selects.' },
+            text: {
+                type: 'string',
+                description: 'Exact value to fill, or option label for selects. (In browser_click, `text` is a label to click instead.)',
+            },
+            value: { type: 'string', description: 'Alias of `text`; if both are given, `text` wins.' },
         },
         output: { schema: SNAPSHOT_SCHEMA, render: renderSnapshotValue },
         async execute(args, exec) {
+            // `text` deliberately is NOT `required: true` here, even though in practice you must give it
+            // (or `value`). The parameter DSL is a flat property map whose only requiredness is
+            // per-property, so it cannot express "exactly one of text/value" — the `oneOf` the compiled
+            // JSON Schema supports is value-level, not object-level. The requirement is therefore
+            // enforced below, before a session is acquired (a malformed call must not launch a browser),
+            // and the message names both spellings. 2026-10-16: a reviewing model passed `value` first
+            // and only got `invalid arguments: missing required property "text"` — no way forward.
+            const value = args.text ?? args.value;
+            if (value === undefined) {
+                throw new BrowserError('PARAM_MISSING', 'browser_fill needs the value to type: pass `text` (alias `value`). The `text` here is the value you type — for a visible label to click, use browser_click instead.');
+            }
             const session = await acquire(exec);
-            return snapshotValue(await session.fill(args.ref, args.text, exec.signal));
+            return snapshotValue(await session.fill(args.ref, value, exec.signal));
         },
     }));
     ctx.tools.register(defineTool({

package/lib/types.d.ts

@@ -2,6 +2,7 @@
  * Domain types for the browser capability: snapshots, sessions, providers.
  * @module dsh-browser-playwright-codex/types
  */
+import type { CookieRow } from './login-sites.ts';
 /** Page load condition a navigation waits for. */
 export type LoadState = 'load' | 'domcontentloaded' | 'networkidle';
 /** One snapshot element. Refs identify actionable elements for later calls. */
@@ -191,6 +192,18 @@ export interface BrowserSession {
     } | undefined, signal?: AbortSignal): Promise<ScreenshotCapture>;
     /** Extract the page's bounded data: text, links, forms, frames. */
     pageData(): Promise<unknown>;
+    /**
+     * Read this session's cookies for display-only login inference. Implementations must drop
+     * `value` before returning: the inference only needs names, domains and lifetime.
+     */
+    cookieJar(): Promise<readonly CookieRow[]>;
+    /**
+     * Registrable domains of the pages this browser has open, used to attribute cookies to the site
+     * that was visited rather than to whoever set them.
+     */
+    visitedHosts(): string[];
+    /** Page titles by registrable domain, for display only; empty when nothing is open. */
+    siteTitles(): Promise<Map<string, string>>;
     /** Evaluate one JavaScript expression and return its JSON result. */
     evaluate(expression: string, signal?: AbortSignal): Promise<unknown>;
     /** Click the element whose visible text matches, by semantic locator. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-browser-playwright-codex",
-  "version": "0.5.5",
+  "version": "0.6.0",
   "description": "Browser automation for DeepSeek Harness: fill in forms, scrape lists and work across tabs in real pages, using the logins you already have. Runs in a visible window that stays quiet when minimised, and stops for sign-ins, CAPTCHAs and native dialogs instead of confirming for you.",
   "license": "MIT",
   "type": "module",
@@ -42,6 +42,7 @@
   ],
   "scripts": {
     "build": "node node_modules/typescript/bin/tsc -p tsconfig.json",
+    "typecheck": "node node_modules/typescript/bin/tsc -p tsconfig.tests.json",
     "test": "node --test \"tests/*.test.ts\"",
     "test:scenarios": "node --test tests/scenarios.test.ts tests/tool-scenarios.test.ts",
     "test:live": "node --test tests/live-smoke.test.ts",
@@ -51,7 +52,7 @@
     "cost": "node node_modules/tsx/dist/cli.mjs scripts/measure-prompt-cost.mjs",
     "format": "node node_modules/prettier/bin/prettier.cjs --write \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.mjs\"",
     "format:check": "node node_modules/prettier/bin/prettier.cjs --check \"src/**/*.ts\" \"tests/**/*.ts\" \"scripts/**/*.mjs\"",
-    "prepublishOnly": "npm run build && npm test && npm run verify && npm run preflight",
+    "prepublishOnly": "npm run build && npm run typecheck && npm test && npm run verify && npm run preflight",
     "preflight": "node scripts/preflight.mjs ."
   },
   "dependencies": {