@dyyz1993/agent-browser 0.13.2 → 0.24.0

package/dist/browser.js

@@ -4,7 +4,8 @@ import os from 'node:os';
 import { existsSync, mkdirSync, rmSync, readFileSync, writeFileSync, statSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-import { getEnhancedSnapshot, parseRef } from './snapshot.js';
+import { getEnhancedSnapshot, generateStableSelectors, parseRef, } from './snapshot.js';
+import { SnapshotStore } from './snapshot-store.js';
 import { getEventCallbacks } from './actions.js';
 /**
  * Manages the Playwright browser lifecycle with multiple tabs/windows
@@ -46,6 +47,7 @@ export class BrowserManager {
     isRecordingHar = false;
     refMap = {};
     lastSnapshot = '';
+    snapshotStore = new SnapshotStore();
     scopedHeaderRoutes = new Map();
     commandHistory = [];
     // CDP session for screencast and input injection
@@ -93,7 +95,63 @@ export class BrowserManager {
         const snapshot = await getEnhancedSnapshot(frame, options);
         this.refMap = snapshot.refs;
         this.lastSnapshot = snapshot.tree;
-        return snapshot;
+        let snapshotId;
+        const url = this.pages.length > 0 ? this.getPage().url() : '';
+        const elements = [];
+        let index = 1;
+        for (const [ref, data] of Object.entries(snapshot.refs)) {
+            elements.push({
+                ref,
+                index: index,
+                role: data.role,
+                name: data.name,
+                cssSelector: '',
+                xpath: '',
+            });
+            index++;
+        }
+        snapshotId = this.snapshotStore.create(url, elements, options?.framePath);
+        const elementCount = elements.length;
+        const header = `Snapshot #${snapshotId} (${elementCount} interactive elements)\n---`;
+        const tips = `---\nTips:\n  Get selector:  snapshot --selector-for ${snapshotId}:@e1\n  Or by index:   snapshot --selector-for ${snapshotId}:1\n  List all:      snapshot --selectors-of ${snapshotId}\n  Validate:      snapshot --validate ${snapshotId}`;
+        snapshot.tree = `${header}\n${snapshot.tree}\n${tips}`;
+        this.lastSnapshot = snapshot.tree;
+        return { ...snapshot, snapshotId };
+    }
+    /**
+     * Ensure selectors have been lazily generated for a snapshot.
+     * Generates them on first call, then caches in the store.
+     */
+    async ensureSelectorsGenerated(snapId) {
+        const store = this.snapshotStore;
+        if (store.isSelectorsGenerated(snapId))
+            return true;
+        const entry = store.get(snapId);
+        if (!entry)
+            return false;
+        const refs = {};
+        for (const [ref, el] of entry.elements) {
+            refs[ref] = {
+                selector: `getByRole('${el.role}'${el.name ? `, { name: "${el.name}", exact: true }` : ''})`,
+                role: el.role,
+                name: el.name,
+            };
+        }
+        const frame = entry.framePath ? this.getFrame(entry.framePath) : this.getFrame();
+        let stableSelectors = {};
+        try {
+            stableSelectors = await generateStableSelectors(frame, refs);
+        }
+        catch { }
+        for (const [ref, sel] of Object.entries(stableSelectors)) {
+            const el = entry.elements.get(ref);
+            if (el) {
+                el.cssSelector = sel.cssSelector;
+                el.xpath = sel.xpath;
+            }
+        }
+        store.markSelectorsGenerated(snapId);
+        return true;
     }
     /**
      * Get the cached ref map from last snapshot
@@ -101,6 +159,9 @@ export class BrowserManager {
     getRefMap() {
         return this.refMap;
     }
+    getSnapshotStore() {
+        return this.snapshotStore;
+    }
     recordCommand(action, selector, value, success) {
         this.commandHistory.push({ action, selector, value, success, timestamp: Date.now() });
     }
@@ -133,7 +194,10 @@ export class BrowserManager {
         }
         let locator;
         if (refData.name) {
-            locator = frame.getByRole(refData.role, { name: refData.name, exact: true });
+            locator = frame.getByRole(refData.role, {
+                name: refData.name,
+                exact: true,
+            });
         }
         else {
             locator = frame.getByRole(refData.role);
@@ -2306,7 +2370,7 @@ export class BrowserManager {
         // 这个会第一个执行（后添加的先执行）
         // 注意：必须设置 xyzSessionId，否则 inject.js 会跳过初始化
         // 使用时间戳来确保只有最新的会话 ID 被设置
-        const sessionIdTimestamp = parseInt(this.recorderSessionId.replace('recorder-', '')) || Date.now();
+        const sessionIdTimestamp = parseInt(this.recorderSessionId.replace('recorder-', ''), 10) || Date.now();
         await context.addInitScript({
             content: `
         // 只有当新的会话 ID 更新时才设置
@@ -2455,7 +2519,7 @@ export class BrowserManager {
                 const pageIndex = this.getPageIndex(newPage);
                 const newTabIndex = pageIndex >= 0 ? pageIndex : this.pages.length;
                 this.recorderSteps.push({
-                    id: this.recorderSteps.length + 1,
+                    id: `step-${this.recorderSteps.length + 1}`,
                     timestamp: Date.now(),
                     action: 'tab_new',
                     url: newPage.url(),
@@ -2464,7 +2528,7 @@ export class BrowserManager {
                 setTimeout(() => {
                     if (this.recorderSessionId && this.activePageIndex !== previousActiveIndex) {
                         this.recorderSteps.push({
-                            id: this.recorderSteps.length + 1,
+                            id: `step-${this.recorderSteps.length + 1}`,
                             timestamp: Date.now(),
                             action: 'tab_switch',
                             index: this.activePageIndex,
@@ -2475,7 +2539,7 @@ export class BrowserManager {
                     if (this.recorderSessionId) {
                         const closeIndex = this.getPageIndex(newPage);
                         this.recorderSteps.push({
-                            id: this.recorderSteps.length + 1,
+                            id: `step-${this.recorderSteps.length + 1}`,
                             timestamp: Date.now(),
                             action: 'tab_close',
                             index: closeIndex >= 0 ? closeIndex : -1,
@@ -2531,8 +2595,6 @@ export class BrowserManager {
                     const hasCloseFunc = typeof win.xyzClose === 'function';
                     const hasFlushFunc = typeof win.xyzFlushPending === 'function';
                     console.log('[stopRecorder] hasFlushFunc:', hasFlushFunc, 'hasCloseFunc:', hasCloseFunc, 'hasPanel:', hasPanel);
-                    // 重要：先调用 xyzFlushPending，再设置 xyzStopped
-                    // 因为 recordStep 会检查 xyzStopped，如果为 true 就不记录
                     if (hasFlushFunc) {
                         console.log('[stopRecorder] Calling xyzFlushPending');
                         win.xyzFlushPending();
@@ -2540,15 +2602,10 @@ export class BrowserManager {
                     else {
                         console.log('[stopRecorder] xyzFlushPending not found');
                     }
-                    // 然后再设置停止标志
                     win.xyzActive = false;
                     win.xyzStopped = true;
-                    // 重置初始化标志，允许新的录制会话重新初始化
                     win.xyzInited = false;
                     win.xyzInitializedSessionId = undefined;
-                    // 注意：不要清除 xyzSessionId，因为旧的监听器需要用它来检查是否应该跳过
-                    // 新的录制会话会设置新的 xyzSessionId，旧的监听器会检测到时间戳更新并跳过
-                    // win.xyzSessionId = undefined;
                     if (hasCloseFunc) {
                         win.xyzClose();
                     }
@@ -2665,13 +2722,13 @@ export class BrowserManager {
             if (step.from && typeof step.from === 'string') {
                 lines.push(`    from: "${step.from}"`);
             }
-            else if (step.from) {
+            else if (step.from && typeof step.from === 'object') {
                 lines.push(`    from: { width: ${step.from.width}, height: ${step.from.height} }`);
             }
             if (step.to && typeof step.to === 'string') {
                 lines.push(`    to: "${step.to}"`);
             }
-            else if (step.to) {
+            else if (step.to && typeof step.to === 'object') {
                 lines.push(`    to: { width: ${step.to.width}, height: ${step.to.height} }`);
             }
             // 备注信息 - 添加重点提示
@@ -2702,7 +2759,7 @@ export class BrowserManager {
                 lines.push(`      # User marked this as: "${step.annotation.label}"`);
             }
             // 只在特定操作类型时携带 URL
-            if (step.url && urlRequiredActions.includes(step.action)) {
+            if (step.url && step.action && urlRequiredActions.includes(step.action)) {
                 lines.push(`    url: "${step.url}"`);
             }
             if (step.index !== undefined)
@@ -2758,6 +2815,7 @@ export class BrowserManager {
             if (s.altKey)
                 parts.push('Alt');
             if (s.shiftKey &&
+                s.key &&
                 !['Shift', 'ArrowUp', 'ArrowDown', 'ArrowLeft', 'ArrowRight'].includes(s.key)) {
                 parts.push('Shift');
             }
@@ -2831,7 +2889,6 @@ export class BrowserManager {
                         sampled = step.points;
                     }
                     else {
-                        // 均匀采样
                         sampled = [];
                         const step_size = (step.points.length - 1) / (maxPoints - 1);
                         for (let i = 0; i < maxPoints; i++) {
@@ -2839,7 +2896,6 @@ export class BrowserManager {
                             sampled.push(step.points[idx]);
                         }
                     }
-                    // 格式化为 x:y:delay 字符串
                     const segments = sampled.map((p, i) => {
                         const x = Math.round(p.x);
                         const y = Math.round(p.y);