dsh-browser-view 0.4.0 → 0.7.0

package/lib/types/browser/types.d.ts

@@ -1,4 +1,4 @@
-import type { Browser, BrowserContext, Page } from "playwright-core";
+import type { Browser, BrowserContext, Frame, Locator, Page } from "playwright-core";
 export interface BrowserConfig {
     enabled?: boolean;
     channel?: string;
@@ -55,11 +55,28 @@ export interface BrowserConfig {
      * 需要拦输入类动作时显式加入)。
      */
     sensitiveTools?: string[];
+    /** 多维环境与路径自适应策略规则链 (v0.6 Policy Engine) */
+    confirmPolicies?: import("./policy/types.js").PolicyRule[];
+    /** 拟人化操作引擎: auto(遇风控/登录页自动激活) | true(全局开启) | false(默认关闭) */
+    humanize?: boolean | "auto";
+    /** 拟人化操作速率: slow | normal (默认) | fast */
+    humanizeSpeed?: "slow" | "normal" | "fast";
     /**
      * replay/batch 是否同样走 auto 审批闸(默认 true)。设为 false 表示回放
      * 不再弹审批 —— 数据驱动/定时回归的逃生口,风险自担(见 README)。
      */
     confirmInReplay?: boolean;
+    /**
+     * what 语义定位零命中时是否自动附 SoM 标徽截图(默认 true):多模态模型
+     * 可看图后 browser_click { mark: N } 一次往返完成定位。仅失败路径触发。
+     */
+    somFallback?: boolean;
+    /**
+     * SPA ref 软失效重锚定(默认 true):主 frame 导航不清空 ref 表,推进
+     * navSeq 代际;定位失败时全量重扫按 role+name 指纹重映射(路由切换场景
+     * DOM 大部分未变,免整轮快照往返)。false 回退「导航即清空」旧行为。
+     */
+    refReanchor?: boolean;
     /**
      * 工具集:all(默认,全部工具)| core(只注册高频核心集,低频的
      * 录制/回放/报告/抽取/profile 等不注册,降 systemPrompt 常驻 token)。
@@ -104,6 +121,58 @@ export interface RefEntry {
     name: string;
     /** 生成快照时计算的稳定 CSS 路径(优先 #id / [data-testid] / nth-of-type 链)。 */
     selector: string;
+    /** 元素所在 Frame 的全局索引(0 为主 Frame, >=1 为子 Frame)。 */
+    frameIndex?: number;
+    /** 元素所在 Frame 的当前 URL(用于导航防漂移二次校验)。 */
+    frameUrl?: string;
+    /** 快照时的视口内包围盒(仅视口内元素记录);ref 路径与 role 降级全部失灵时的坐标兜底。 */
+    rect?: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    /** 快照时的滚动偏移;坐标兜底按当前 scroll 修正视口位置。 */
+    scroll?: {
+        x: number;
+        y: number;
+    };
+    /** 注册时的导航代际;落后于 pageNavSeq = 疑似过期(SPA 路由),定位失败时按指纹重锚定。 */
+    navSeq?: number;
+}
+/** SoM 标徽(browser_screenshot { som: true })的注册项(id → 定位信息)。 */
+export interface MarkEntry {
+    role: string;
+    name: string;
+    selector: string;
+    /** 标徽快照时的视口内包围盒与滚动偏移(坐标兜底语义同 RefEntry)。 */
+    rect?: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+    scroll?: {
+        x: number;
+        y: number;
+    };
+}
+/** 统一目标定位解析结果(无论在主 Frame 还是子 Frame 均可统一操作)。 */
+export interface TargetResolution {
+    /** 绑定的 Playwright Locator 实例。 */
+    locator: Locator;
+    /** 执行上下文容器(主页面 Page 或子 Frame)。 */
+    container: Page | Frame;
+    /** 原始 RefEntry 元数据(仅 ref 定位时有效)。 */
+    entry?: RefEntry;
+    /** 是否位于子 Frame 中。 */
+    isIFrame: boolean;
+    /** 匹配到的总元素数量 (用于排障诊断) */
+    matchedCount?: number;
+    /** 匹配到的可见元素数量 */
+    visibleCount?: number;
+    /** 语义定位的备选目标(按打分次序,最多 2 个);点击效果验证无变化时自动换选。 */
+    alternates?: Locator[];
 }
 /** 录制时采集的选择器指纹(回放自愈的降级候选来源)。 */
 export interface SelectorHint {

package/lib/types/browser/walker.d.ts

@@ -7,6 +7,7 @@ declare function __walkerCore(): {
     nameOf: (el: Element, role: string) => string;
     cssPath: (el: Element) => string;
     isVisible: (el: Element) => boolean;
+    isElementInViewport: (el: Element, padding?: number) => boolean;
     describeElement: (el: Element) => {
         role: string;
         name: string;
@@ -17,6 +18,8 @@ declare function __walkerCore(): {
         text: string;
         visible: boolean;
     };
+    composedParent: (el: Element) => Element | null;
+    deepChildren: (el: Element) => Element[];
 };
 /** walker 核心注入后暴露给 entry 的 API(类型层)。 */
 export type WalkerApi = ReturnType<typeof __walkerCore>;

package/lib/types/client/api.d.ts

@@ -0,0 +1,24 @@
+import type { BrowserOp, BrowserStatus, LatestRun, NavHistoryItem, PendingConfirm, PendingPick, RecordingDetail, RecordingRow } from "./types.js";
+/** 获取插件后端 API 基路径 (默认为 /browser) */
+export declare function getBasePath(): Promise<string>;
+export declare function fetchStatus(base: string): Promise<BrowserStatus | null>;
+export declare function fetchHistory(base: string): Promise<NavHistoryItem[]>;
+export declare function fetchOperations(base: string): Promise<BrowserOp[]>;
+export declare function fetchLatestRun(base: string): Promise<{
+    run: LatestRun | null;
+    busy: boolean;
+}>;
+export declare function fetchPendingConfirm(base: string): Promise<PendingConfirm | null>;
+export declare function submitConfirmDecision(base: string, id: string, decision: "allowed" | "denied"): Promise<boolean>;
+export declare function fetchPendingPick(base: string): Promise<PendingPick | null>;
+/** 提交点选坐标(x/y 为截图位图像素坐标,imgW/imgH 为截图尺寸;服务端换算成视口 CSS 坐标)。 */
+export declare function submitPick(base: string, id: string, x: number, y: number, imgW: number, imgH: number): Promise<boolean>;
+export declare function triggerRerun(base: string, source: string, retryOf?: string): Promise<{
+    ok: boolean;
+    status?: number;
+    error?: string;
+}>;
+export declare function fetchRecordings(base: string): Promise<RecordingRow[]>;
+export declare function fetchRecordingDetail(base: string, name: string): Promise<RecordingDetail | null>;
+export declare function deleteRecordingApi(base: string, name: string): Promise<boolean>;
+//# sourceMappingURL=api.d.ts.map

package/lib/types/client/components/ConfirmCard.d.ts

@@ -0,0 +1,20 @@
+import type { PendingConfirm } from "../types.js";
+interface ConfirmCardProps {
+    confirm: PendingConfirm;
+    dark: boolean;
+    onDecision: (decision: "allowed" | "denied") => void;
+}
+export declare function ConfirmCard({ confirm, dark, onDecision }: ConfirmCardProps): import("react").DetailedReactHTMLElement<{
+    style: {
+        padding: string;
+        borderRadius: string;
+        border: string;
+        background: string;
+        color: "#fca5a5" | "#b91c1c";
+        fontSize: string;
+        lineHeight: string;
+    };
+    title: string;
+}, HTMLElement>;
+export {};
+//# sourceMappingURL=ConfirmCard.d.ts.map

package/lib/types/client/components/FloatingOverlay.d.ts

@@ -0,0 +1,10 @@
+export declare function BrowserFloatingOverlay(): import("react").DetailedReactHTMLElement<{
+    id: string;
+    style: {
+        position: "fixed";
+        inset: number;
+        pointerEvents: "none";
+        zIndex: number;
+    };
+}, HTMLElement>;
+//# sourceMappingURL=FloatingOverlay.d.ts.map

package/lib/types/client/components/FloatingPanel.d.ts

@@ -0,0 +1,40 @@
+import type { BrowserOp, BrowserStatus, LatestRun, Palette, PendingConfirm, PendingPick } from "../types.js";
+interface FloatingPanelProps {
+    status: BrowserStatus | null;
+    ops: BrowserOp[];
+    lastRun: LatestRun | null;
+    agentBusy: boolean;
+    rerunMsg: string;
+    pendingConfirm: PendingConfirm | null;
+    pendingPick?: PendingPick | null;
+    shotSrc: string;
+    streamSrc?: string;
+    dark: boolean;
+    palette: Palette;
+    onOpenModal: () => void;
+    onOpenReport: () => void;
+    onConfirmDecision: (decision: "allowed" | "denied") => void;
+    onRerunLast: () => void;
+}
+export declare function FloatingPanel({ status, ops, lastRun, agentBusy, rerunMsg, pendingConfirm, pendingPick, shotSrc, streamSrc, dark, palette: p, onOpenModal, onOpenReport, onConfirmDecision, onRerunLast, }: FloatingPanelProps): import("react").DetailedReactHTMLElement<{
+    style: {
+        position: "fixed";
+        right: string;
+        bottom: string;
+        width: string;
+        maxHeight: string;
+        overflowY: "auto";
+        overflowX: "hidden";
+        display: "flex";
+        flexDirection: "column";
+        gap: string;
+        border: string;
+        borderRadius: string;
+        background: string;
+        boxShadow: "0 8px 32px rgba(0, 0, 0, 0.3)";
+        padding: string;
+        pointerEvents: "auto";
+    };
+}, HTMLElement>;
+export {};
+//# sourceMappingURL=FloatingPanel.d.ts.map

package/lib/types/client/components/InspectionModal.d.ts

@@ -0,0 +1,51 @@
+import { type MouseEvent } from "react";
+import type { NavHistoryItem, Palette, BrowserStatus, PendingPick } from "../types.js";
+interface InspectionModalProps {
+    status: BrowserStatus | null;
+    history: NavHistoryItem[];
+    shotSrc: string;
+    streamSrc?: string;
+    dark: boolean;
+    palette: Palette;
+    onClose: () => void;
+    /** 面板点选教学:等待中非空,大图切换为十字光标点选模式。 */
+    pendingPick?: PendingPick | null;
+    /** 用户点选回调(x/y 为截图位图像素坐标,imgW/imgH 为截图尺寸)。 */
+    onPick?: (x: number, y: number, imgW: number, imgH: number) => void;
+}
+export declare function InspectionModal({ status, history, shotSrc, streamSrc, dark, palette: p, onClose, pendingPick, onPick, }: InspectionModalProps): import("react").DetailedReactHTMLElement<{
+    onClick: (e: MouseEvent) => void;
+    style: {
+        position: "fixed";
+        inset: number;
+        zIndex: number;
+        display: "flex";
+        alignItems: "center";
+        justifyContent: "center";
+        padding: string;
+        background: string;
+        pointerEvents: "auto";
+    };
+}, HTMLElement>;
+interface ReportModalProps {
+    base: string;
+    dark: boolean;
+    palette: Palette;
+    onClose: () => void;
+}
+export declare function ReportModal({ base, dark, palette: p, onClose }: ReportModalProps): import("react").DetailedReactHTMLElement<{
+    onClick: (e: MouseEvent) => void;
+    style: {
+        position: "fixed";
+        inset: number;
+        zIndex: number;
+        display: "flex";
+        alignItems: "center";
+        justifyContent: "center";
+        padding: string;
+        background: string;
+        pointerEvents: "auto";
+    };
+}, HTMLElement>;
+export {};
+//# sourceMappingURL=InspectionModal.d.ts.map

package/lib/types/client/components/SettingsSection.d.ts

@@ -0,0 +1,5 @@
+import type { SettingsSectionOwnerProps } from "@deepseek-ai/dsh-client-ui-settings/client";
+export declare function BrowserSettingsSection(_props: SettingsSectionOwnerProps): import("react").DetailedReactHTMLElement<{
+    style: import("react").CSSProperties;
+}, HTMLElement>;
+//# sourceMappingURL=SettingsSection.d.ts.map

package/lib/types/client/components/styles.d.ts

@@ -0,0 +1,18 @@
+import type React from "react";
+import type { Palette } from "../types.js";
+export declare function styleFor(p: Palette): {
+    row: React.CSSProperties;
+    label: React.CSSProperties;
+    meta: React.CSSProperties;
+    btn: React.CSSProperties;
+    delBtn: React.CSSProperties;
+    itemBox: React.CSSProperties;
+    stepsBox: React.CSSProperties;
+    stepLine: React.CSSProperties;
+    toolName: React.CSSProperties;
+    stepArgs: React.CSSProperties;
+    empty: React.CSSProperties;
+    root: React.CSSProperties;
+    heading: React.CSSProperties;
+};
+//# sourceMappingURL=styles.d.ts.map

package/lib/types/client/hooks/useBrowserStatus.d.ts

@@ -0,0 +1,11 @@
+import type { BrowserOp, BrowserStatus, LatestRun } from "../types.js";
+export declare function useBrowserStatus(base: string): {
+    status: BrowserStatus | null;
+    ops: BrowserOp[];
+    shotVersion: number;
+    lastRun: LatestRun | null;
+    agentBusy: boolean;
+    rerunMsg: string;
+    rerunLast: () => Promise<void>;
+};
+//# sourceMappingURL=useBrowserStatus.d.ts.map

package/lib/types/client/hooks/useConfirmPolling.d.ts

@@ -0,0 +1,6 @@
+import type { PendingConfirm } from "../types.js";
+export declare function useConfirmPolling(base: string): {
+    pendingConfirm: PendingConfirm | null;
+    resolveConfirm: (id: string, decision: "allowed" | "denied") => Promise<void>;
+};
+//# sourceMappingURL=useConfirmPolling.d.ts.map

package/lib/types/client/hooks/usePickPolling.d.ts

@@ -0,0 +1,5 @@
+import type { PendingPick } from "../types.js";
+export declare function usePickPolling(base: string): {
+    pendingPick: PendingPick | null;
+};
+//# sourceMappingURL=usePickPolling.d.ts.map

package/lib/types/client/hooks/useRecordings.d.ts

@@ -0,0 +1,9 @@
+import type { RecordingDetail, RecordingRow } from "../types.js";
+export declare function useRecordings(base: string): {
+    list: RecordingRow[] | null;
+    error: string;
+    expanded: Record<string, "failed" | RecordingDetail | "loading" | undefined>;
+    toggleDetail: (name: string) => void;
+    remove: (name: string) => Promise<void>;
+};
+//# sourceMappingURL=useRecordings.d.ts.map

package/lib/types/client/index.d.ts

@@ -1,17 +1,14 @@
 /**
  * dsh-browser-view client half:
- * 1. "settings.section" -> A "🌐 浏览器" page in the official dsh settings
- *    dialog managing saved browser recordings with polished header & icon.
- * 2. "shell.overlay" -> Universal floating widget (FAB + live screenshot
- *    thumbnail panel + full-size inspection modal) rendered natively in both
- *    Electron desktop and web client.
+ * 1. "settings.section" -> A "🌐 浏览器" page in official dsh settings dialog.
+ * 2. "shell.overlay" -> Universal floating widget (FAB + live screenshot + modal).
  */
-import type { ClientContext } from '@deepseek-ai/dsh-client-runtime/client';
-declare module '@deepseek-ai/dsh-client-ui-slots' {
+import type { ClientContext } from "@deepseek-ai/dsh-client-runtime/client";
+declare module "@deepseek-ai/dsh-client-ui-slots" {
     interface SlotMap {
-        'shell.overlay': {
-            kind: 'list';
-            scope: 'root';
+        "shell.overlay": {
+            kind: "list";
+            scope: "root";
         };
     }
 }

package/lib/types/client/theme.d.ts

@@ -0,0 +1,7 @@
+import type { Palette } from "./types.js";
+export declare const LIGHT: Palette;
+export declare const DARK: Palette;
+/** 跟随宿主主题 (body[data-ds-dark-theme]) */
+export declare function useDarkTheme(): boolean;
+export declare function fmtTime(ts: number): string;
+//# sourceMappingURL=theme.d.ts.map

package/lib/types/client/types.d.ts

@@ -0,0 +1,74 @@
+export interface Palette {
+    primary: string;
+    secondary: string;
+    tertiary: string;
+    border: string;
+    hoverBg: string;
+    danger: string;
+}
+export interface RecordingRow {
+    name: string;
+    steps: number;
+    savedAt: number;
+    preview: string;
+}
+export interface RecordingDetail {
+    name: string;
+    steps: Array<{
+        tool: string;
+        args: unknown;
+    }>;
+}
+export interface BrowserStatus {
+    open: boolean;
+    tabs?: number;
+    url?: string;
+    title?: string;
+    channel?: string;
+    pause?: {
+        instruction: string;
+        startedAt: number;
+    } | null;
+}
+export interface BrowserOp {
+    tool: string;
+    ok: boolean;
+    ms: number;
+    ts: number;
+    error?: string;
+}
+export interface NavHistoryItem {
+    url: string;
+    title?: string;
+    ts: number;
+}
+export interface PendingConfirm {
+    id: string;
+    reason: string;
+    tool?: string;
+    args?: Record<string, unknown>;
+    startedAt: number;
+    decision?: "allowed" | "denied";
+}
+/** 面板点选教学请求(browser_pick_element 等待期间轮询到)。 */
+export interface PendingPick {
+    id: string;
+    instruction: string;
+    startedAt: number;
+}
+export interface LatestRun {
+    runId: string;
+    ts: number;
+    source: string;
+    passed: number;
+    total: number;
+    failed: number;
+    healedCount: number;
+    row?: number;
+    retries?: {
+        attempted: number;
+        recovered: number;
+    };
+    retryOf?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/lib/types/lib/config.d.ts

@@ -8,10 +8,8 @@ export interface FieldSchema {
     min?: number;
 }
 export type ConfigSchema = Record<string, FieldSchema>;
-declare const TYPES: string[];
 /** 校验配置;返回错误列表(空 = 通过)。 */
 export declare function validateConfig(schema: ConfigSchema, config: Record<string, any>): string[];
 /** 合并默认值 + 校验;抛错时带模块名前缀。 */
 export declare function resolveConfig(moduleName: string, schema: ConfigSchema, defaults: Record<string, any>, raw: Record<string, any> | undefined): Record<string, any>;
-export { TYPES };
 //# sourceMappingURL=config.d.ts.map

package/lib/types/lib/http.d.ts

@@ -1,8 +1,6 @@
 import type { IncomingMessage, ServerResponse } from "node:http";
 /** Parse the request URL pathname (query strings are ignored). */
 export declare function urlPath(req: IncomingMessage): string;
-/** Read the request body as a UTF-8 string (bounded). */
-export declare function readRawBody(req: IncomingMessage, limitBytes?: number): Promise<string>;
 /** Read and parse a JSON request body; `undefined` when the body is empty. */
 export declare function readJsonBody(req: IncomingMessage, limitBytes?: number): Promise<unknown>;
 /** Send a JSON response. */

package/lib/types/lib/types.d.ts

@@ -3,6 +3,7 @@ export interface ReefContext {
     get<T = any>(name: string): T | undefined;
     effect(fn: () => (() => void) | void, label?: string): void;
     on?(event: string, listener: (...args: unknown[]) => unknown): (() => void) | void;
+    inject?(names: string[], callback: (ctx: ReefContext) => void): unknown;
     logger?: {
         info?: (message: string) => void;
         warn?: (message: string) => void;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dsh-browser-view",
-  "version": "0.4.0",
-  "description": "DSH 插件:共享 Playwright 浏览器自动化(36 个 browser_* 工具,含语义元素查找、批处理、视觉回归、选择器自愈、参数化/数据驱动回放、失败自动重试、结构化抽取与字段建议、增量快照历史版本、回归报告与隔离 context)+ 人机协作(人工接管等待、敏感操作审批(confirmMode:auto 宿主层硬拦截)、人机混合录制、面板一键重跑、面板双向化:操作预告高亮/内嵌报告)+ 实时画面面板与录制管理。仅依赖 playwright-core,不依赖任何 @deepseek-ai 运行时包。",
+  "version": "0.7.0",
+  "description": "DSH 插件:共享 Playwright 浏览器自动化(38 个 browser_* 工具,含 what 意图语义定位(一次调用替代快照→查找→点击)、SoM 标徽看图点击与零命中截图兜底、SPA ref 软失效重锚定、open Shadow DOM 组合树穿透、跨 Frame/IFrame 定位、视口感知动态快照、批量表单智能填充、点击效果验证与定位失败遥测、Playwright 规范代码导出、批处理、视觉回归、选择器自愈、增量快照)+ 人机协作(面板点选教学 browser_pick_element、人工接管等待、敏感操作审批硬拦截、人机混合录制、面板一键重跑)+ CDP 流式推流、拟人化轨迹与高斯击键、全景时光机、企业富控件矩阵。仅依赖 playwright-core。",
   "type": "module",
   "main": "./lib/index.js",
   "types": "./lib/types/index.d.ts",
@@ -27,8 +27,16 @@
   "scripts": {
     "build": "tsc -p tsconfig.json && tsdown",
     "typecheck": "tsc -p tsconfig.json --noEmit --pretty false && tsc -p tsconfig.test.json --noEmit --pretty false",
+    "lint": "biome check src test",
+    "format": "biome format --write src test",
     "test": "vitest run",
-    "verify": "npm run typecheck && npm run test && npm run build",
+    "test:algo": "tsx scripts/test-dev-server-algo-config.ts",
+    "test:enterprise": "tsx scripts/test-dev-server-enterprise.ts",
+    "test:energy": "tsx scripts/test-dev-server-energy.ts",
+    "test:shadow": "tsx scripts/test-dev-server-shadow.ts",
+    "test:pick": "tsx scripts/test-panel-pick.ts",
+    "dev:server": "tsx scripts/dev-server.ts",
+    "verify": "npm run typecheck && npm run lint && npm run test && npm run build",
     "prepublishOnly": "npm run verify"
   },
   "repository": {
@@ -56,9 +64,11 @@
     "playwright-core": "^1.49.0"
   },
   "devDependencies": {
+    "@biomejs/biome": "^2.5.15",
     "@types/node": "^22.0.0",
     "@types/react": "^18.2.0",
     "tsdown": "^0.22.0",
+    "tsx": "^4.23.15",
     "typescript": "^5.9.2",
     "vitest": "^3.0.0"
   },

package/lib/types/embed.d.ts

@@ -1,7 +0,0 @@
-import type { ReefContext, WebRoute } from "./lib/types.js";
-export declare function applyEmbed(ctx: ReefContext, webServer: {
-    register(route: WebRoute): () => void;
-    tapIndex(transform: (html: string) => string): () => void;
-    port?: number;
-}, base: string): () => void;
-//# sourceMappingURL=embed.d.ts.map