@jackwener/opencli 1.1.0 → 1.2.0

package/src/browser/index.ts

@@ -9,6 +9,8 @@ export { Page } from './page.js';
 export { BrowserBridge, BrowserBridge as PlaywrightMCP } from './mcp.js';
 export { CDPBridge } from './cdp.js';
 export { isDaemonRunning } from './daemon-client.js';
+export { generateSnapshotJs, scrollToRefJs, getFormStateJs } from './dom-snapshot.js';
+export type { SnapshotOptions } from './dom-snapshot.js';
 
 import { extractTabEntries, diffTabIndexes, appendLimited } from './tabs.js';
 import { __test__ as cdpTest } from './cdp.js';

package/src/browser/page.ts

@@ -11,9 +11,10 @@
  */
 
 import { formatSnapshot } from '../snapshotFormatter.js';
-import type { IPage } from '../types.js';
+import type { BrowserCookie, IPage, ScreenshotOptions, SnapshotOptions, WaitOptions } from '../types.js';
 import { sendCommand } from './daemon-client.js';
 import { wrapForEval } from './utils.js';
+import { generateSnapshotJs, scrollToRefJs, getFormStateJs } from './dom-snapshot.js';
 import {
   clickJs,
   typeTextJs,
@@ -42,7 +43,7 @@ export class Page implements IPage {
     return { workspace: this.workspace };
   }
 
-  async goto(url: string): Promise<void> {
+  async goto(url: string, options?: { waitUntil?: 'load' | 'none'; settleMs?: number }): Promise<void> {
     const result = await sendCommand('navigate', {
       url,
       ...this._workspaceOpt(),
@@ -52,6 +53,12 @@ export class Page implements IPage {
     if (result?.tabId) {
       this._tabId = result.tabId;
     }
+    // Post-load settle: the extension already waits for tab.status === 'complete',
+    // but SPA frameworks (React/Vue) need extra time to render after DOM load.
+    if (options?.waitUntil !== 'none') {
+      const settleMs = options?.settleMs ?? 1000;
+      await new Promise(resolve => setTimeout(resolve, settleMs));
+    }
   }
 
   /** Close the automation window in the extension */
@@ -63,17 +70,40 @@ export class Page implements IPage {
     }
   }
 
-  async evaluate(js: string): Promise<any> {
+  async evaluate(js: string): Promise<unknown> {
     const code = wrapForEval(js);
     return sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
   }
 
-  async getCookies(opts: { domain?: string; url?: string } = {}): Promise<any[]> {
+  async getCookies(opts: { domain?: string; url?: string } = {}): Promise<BrowserCookie[]> {
     const result = await sendCommand('cookies', { ...this._workspaceOpt(), ...opts });
     return Array.isArray(result) ? result : [];
   }
 
-  async snapshot(opts: { interactive?: boolean; compact?: boolean; maxDepth?: number; raw?: boolean } = {}): Promise<any> {
+  async snapshot(opts: SnapshotOptions = {}): Promise<unknown> {
+    // Primary: use the advanced DOM snapshot engine with multi-layer pruning
+    const snapshotJs = generateSnapshotJs({
+      viewportExpand: opts.viewportExpand ?? 800,
+      maxDepth: Math.max(1, Math.min(Number(opts.maxDepth) || 50, 200)),
+      interactiveOnly: opts.interactive ?? false,
+      maxTextLength: opts.maxTextLength ?? 120,
+      includeScrollInfo: true,
+      bboxDedup: true,
+    });
+
+    try {
+      const result = await sendCommand('exec', { code: snapshotJs, ...this._workspaceOpt(), ...this._tabOpt() });
+      // The advanced engine already produces a clean, pruned, LLM-friendly output.
+      // Do NOT pass through formatSnapshot — its format is incompatible.
+      return result;
+    } catch {
+      // Fallback: basic DOM snapshot (original implementation)
+      return this._basicSnapshot(opts);
+    }
+  }
+
+  /** Fallback basic snapshot — original buildTree approach */
+  private async _basicSnapshot(opts: Pick<SnapshotOptions, 'interactive' | 'compact' | 'maxDepth' | 'raw'> = {}): Promise<unknown> {
     const maxDepth = Math.max(1, Math.min(Number(opts.maxDepth) || 50, 200));
     const code = `
       (async () => {
@@ -87,7 +117,7 @@ export class Page implements IPage {
 
           let indent = '  '.repeat(depth);
           let line = indent + role;
-          if (name) line += ' "' + name.replace(/"/g, '\\\\"') + '"';
+          if (name) line += ' "' + name.replace(/"/g, '\\\\\\"') + '"';
           if (node.tagName?.toLowerCase() === 'a' && node.href) line += ' [' + node.href + ']';
           if (node.tagName?.toLowerCase() === 'input') line += ' [' + (node.type || 'text') + ']';
 
@@ -123,7 +153,17 @@ export class Page implements IPage {
     await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
   }
 
-  async wait(options: number | { text?: string; time?: number; timeout?: number }): Promise<void> {
+  async scrollTo(ref: string): Promise<unknown> {
+    const code = scrollToRefJs(ref);
+    return sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
+  }
+
+  async getFormState(): Promise<Record<string, unknown>> {
+    const code = getFormStateJs();
+    return (await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() })) as Record<string, unknown>;
+  }
+
+  async wait(options: number | WaitOptions): Promise<void> {
     if (typeof options === 'number') {
       await new Promise(resolve => setTimeout(resolve, options * 1000));
       return;
@@ -139,8 +179,9 @@ export class Page implements IPage {
     }
   }
 
-  async tabs(): Promise<any> {
-    return sendCommand('tabs', { op: 'list', ...this._workspaceOpt() });
+  async tabs(): Promise<unknown[]> {
+    const result = await sendCommand('tabs', { op: 'list', ...this._workspaceOpt() });
+    return Array.isArray(result) ? result : [];
   }
 
   async closeTab(index?: number): Promise<void> {
@@ -155,9 +196,10 @@ export class Page implements IPage {
     await sendCommand('tabs', { op: 'select', index, ...this._workspaceOpt() });
   }
 
-  async networkRequests(includeStatic: boolean = false): Promise<any> {
+  async networkRequests(includeStatic: boolean = false): Promise<unknown[]> {
     const code = networkRequestsJs(includeStatic);
-    return sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
+    const result = await sendCommand('exec', { code, ...this._workspaceOpt(), ...this._tabOpt() });
+    return Array.isArray(result) ? result : [];
   }
 
   /**
@@ -165,7 +207,7 @@ export class Page implements IPage {
    * Would require CDP Runtime.consoleAPICalled event listener.
    * @returns Always returns empty array.
    */
-  async consoleMessages(_level: string = 'info'): Promise<any> {
+  async consoleMessages(_level: string = 'info'): Promise<unknown[]> {
     return [];
   }
 
@@ -176,12 +218,7 @@ export class Page implements IPage {
    * @param options.fullPage - capture full scrollable page
    * @param options.path - save to file path (returns base64 if omitted)
    */
-  async screenshot(options: {
-    format?: 'png' | 'jpeg';
-    quality?: number;
-    fullPage?: boolean;
-    path?: string;
-  } = {}): Promise<string> {
+  async screenshot(options: ScreenshotOptions = {}): Promise<string> {
     const base64 = await sendCommand('screenshot', {
       ...this._workspaceOpt(),
       format: options.format,
@@ -223,11 +260,11 @@ export class Page implements IPage {
     }));
   }
 
-  async getInterceptedRequests(): Promise<any[]> {
+  async getInterceptedRequests(): Promise<unknown[]> {
     const { generateReadInterceptedJs } = await import('../interceptor.js');
     // Same as installInterceptor: must go through evaluate() for IIFE wrapping
     const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr'));
-    return (result as any[]) || [];
+    return Array.isArray(result) ? result : [];
   }
 }
 

package/src/build-manifest.test.ts

@@ -1,5 +1,8 @@
-import { describe, expect, it } from 'vitest';
-import { parseTsArgsBlock } from './build-manifest.js';
+import { afterEach, describe, expect, it } from 'vitest';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { parseTsArgsBlock, scanTs, shouldReplaceManifestEntry } from './build-manifest.js';
 
 describe('parseTsArgsBlock', () => {
   it('keeps args with nested choices arrays', () => {
@@ -62,3 +65,68 @@ describe('parseTsArgsBlock', () => {
     ]);
   });
 });
+
+describe('manifest helper rules', () => {
+  const tempDirs: string[] = [];
+
+  afterEach(() => {
+    for (const dir of tempDirs.splice(0)) {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it('prefers TS adapters over duplicate YAML adapters', () => {
+    expect(shouldReplaceManifestEntry(
+      {
+        site: 'demo',
+        name: 'search',
+        description: 'yaml',
+        strategy: 'public',
+        browser: false,
+        args: [],
+        type: 'yaml',
+      },
+      {
+        site: 'demo',
+        name: 'search',
+        description: 'ts',
+        strategy: 'public',
+        browser: false,
+        args: [],
+        type: 'ts',
+        modulePath: 'demo/search.js',
+      },
+    )).toBe(true);
+
+    expect(shouldReplaceManifestEntry(
+      {
+        site: 'demo',
+        name: 'search',
+        description: 'ts',
+        strategy: 'public',
+        browser: false,
+        args: [],
+        type: 'ts',
+        modulePath: 'demo/search.js',
+      },
+      {
+        site: 'demo',
+        name: 'search',
+        description: 'yaml',
+        strategy: 'public',
+        browser: false,
+        args: [],
+        type: 'yaml',
+      },
+    )).toBe(false);
+  });
+
+  it('skips TS files that do not register a cli', () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'opencli-manifest-'));
+    tempDirs.push(dir);
+    const file = path.join(dir, 'utils.ts');
+    fs.writeFileSync(file, `export function helper() { return 'noop'; }`);
+
+    expect(scanTs(file, 'demo')).toBeNull();
+  });
+});

package/src/build-manifest.ts

@@ -18,7 +18,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const CLIS_DIR = path.resolve(__dirname, 'clis');
 const OUTPUT = path.resolve(__dirname, '..', 'dist', 'cli-manifest.json');
 
-interface ManifestEntry {
+export interface ManifestEntry {
   site: string;
   name: string;
   description: string;
@@ -28,19 +28,53 @@ interface ManifestEntry {
   args: Array<{
     name: string;
     type?: string;
-    default?: any;
+    default?: unknown;
     required?: boolean;
     positional?: boolean;
     help?: string;
     choices?: string[];
   }>;
   columns?: string[];
-  pipeline?: any[];
+  pipeline?: Record<string, unknown>[];
   timeout?: number;
   /** 'yaml' or 'ts' — determines how executeCommand loads the handler */
   type: 'yaml' | 'ts';
   /** Relative path from clis/ dir, e.g. 'bilibili/hot.yaml' or 'bilibili/search.js' */
   modulePath?: string;
+  /** Pre-navigation control — see CliCommand.navigateBefore */
+  navigateBefore?: boolean | string;
+}
+
+interface YamlArgDefinition {
+  type?: string;
+  default?: unknown;
+  required?: boolean;
+  positional?: boolean;
+  description?: string;
+  help?: string;
+  choices?: string[];
+}
+
+interface YamlCliDefinition {
+  site?: string;
+  name?: string;
+  description?: string;
+  domain?: string;
+  strategy?: string;
+  browser?: boolean;
+  args?: Record<string, YamlArgDefinition>;
+  columns?: string[];
+  pipeline?: Record<string, unknown>[];
+  timeout?: number;
+  navigateBefore?: boolean | string;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function getErrorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
 }
 
 function extractBalancedBlock(
@@ -127,7 +161,7 @@ export function parseTsArgsBlock(argsBlock: string): ManifestEntry['args'] {
     const helpMatch = body.match(/help\s*:\s*['"`]([^'"`]*)['"`]/);
     const positionalMatch = body.match(/positional\s*:\s*(true|false)/);
 
-    let defaultVal: any = undefined;
+    let defaultVal: unknown = undefined;
     if (defaultMatch) {
       const raw = defaultMatch[1].trim();
       if (raw === 'true') defaultVal = true;
@@ -156,21 +190,23 @@ export function parseTsArgsBlock(argsBlock: string): ManifestEntry['args'] {
 function scanYaml(filePath: string, site: string): ManifestEntry | null {
   try {
     const raw = fs.readFileSync(filePath, 'utf-8');
-    const def = yaml.load(raw) as any;
-    if (!def || typeof def !== 'object') return null;
+    const def = yaml.load(raw) as YamlCliDefinition | null;
+    if (!isRecord(def)) return null;
+    const cliDef = def as YamlCliDefinition;
 
-    const strategyStr = def.strategy ?? (def.browser === false ? 'public' : 'cookie');
+    const strategyStr = cliDef.strategy ?? (cliDef.browser === false ? 'public' : 'cookie');
     const strategy = strategyStr.toUpperCase();
-    const browser = def.browser ?? (strategy !== 'PUBLIC');
+    const browser = cliDef.browser ?? (strategy !== 'PUBLIC');
 
     const args: ManifestEntry['args'] = [];
-    if (def.args && typeof def.args === 'object') {
-      for (const [argName, argDef] of Object.entries(def.args as Record<string, any>)) {
+    if (cliDef.args && typeof cliDef.args === 'object') {
+      for (const [argName, argDef] of Object.entries(cliDef.args)) {
         args.push({
           name: argName,
           type: argDef?.type ?? 'str',
           default: argDef?.default,
           required: argDef?.required ?? false,
+          positional: argDef?.positional === true || undefined,
           help: argDef?.description ?? argDef?.help ?? '',
           choices: argDef?.choices,
         });
@@ -178,44 +214,48 @@ function scanYaml(filePath: string, site: string): ManifestEntry | null {
     }
 
     return {
-      site: def.site ?? site,
-      name: def.name ?? path.basename(filePath, path.extname(filePath)),
-      description: def.description ?? '',
-      domain: def.domain,
+      site: cliDef.site ?? site,
+      name: cliDef.name ?? path.basename(filePath, path.extname(filePath)),
+      description: cliDef.description ?? '',
+      domain: cliDef.domain,
       strategy: strategy.toLowerCase(),
       browser,
       args,
-      columns: def.columns,
-      pipeline: def.pipeline,
-      timeout: def.timeout,
+      columns: cliDef.columns,
+      pipeline: cliDef.pipeline,
+      timeout: cliDef.timeout,
       type: 'yaml',
+      navigateBefore: cliDef.navigateBefore,
     };
-  } catch (err: any) {
-    process.stderr.write(`Warning: failed to parse ${filePath}: ${err.message}\n`);
+  } catch (err) {
+    process.stderr.write(`Warning: failed to parse ${filePath}: ${getErrorMessage(err)}\n`);
     return null;
   }
 }
 
-function scanTs(filePath: string, site: string): ManifestEntry {
+export function scanTs(filePath: string, site: string): ManifestEntry | null {
   // TS adapters self-register via cli() at import time.
   // We statically parse the source to extract metadata for the manifest stub.
   const baseName = path.basename(filePath, path.extname(filePath));
   const relativePath = `${site}/${baseName}.js`;
 
-  const entry: ManifestEntry = {
-    site,
-    name: baseName,
-    description: '',
-    strategy: 'cookie',
-    browser: true,
-    args: [],
-    type: 'ts',
-    modulePath: relativePath,
-  };
-
   try {
     const src = fs.readFileSync(filePath, 'utf-8');
 
+    // Helper/test modules should not appear as CLI commands in the manifest.
+    if (!/\bcli\s*\(/.test(src)) return null;
+
+    const entry: ManifestEntry = {
+      site,
+      name: baseName,
+      description: '',
+      strategy: 'cookie',
+      browser: true,
+      args: [],
+      type: 'ts',
+      modulePath: relativePath,
+    };
+
     // Extract description
     const descMatch = src.match(/description\s*:\s*['"`]([^'"`]*)['"`]/);
     if (descMatch) entry.description = descMatch[1];
@@ -244,15 +284,30 @@ function scanTs(filePath: string, site: string): ManifestEntry {
     if (argsBlock) {
       entry.args = parseTsArgsBlock(argsBlock);
     }
-  } catch {
-    // If parsing fails, fall back to empty metadata — module will self-register at runtime
+
+    // Extract navigateBefore: false
+    const navMatch = src.match(/navigateBefore\s*:\s*(true|false)/);
+    if (navMatch) entry.navigateBefore = navMatch[1] === 'true' ? true : false;
+
+    return entry;
+  } catch (err) {
+    // If parsing fails, log a warning (matching scanYaml behaviour) and skip the entry.
+    process.stderr.write(`Warning: failed to scan ${filePath}: ${getErrorMessage(err)}\n`);
+    return null;
   }
+}
 
-  return entry;
+/**
+ * When both YAML and TS adapters exist for the same site/name,
+ * prefer the TS version (it self-registers and typically has richer logic).
+ */
+export function shouldReplaceManifestEntry(current: ManifestEntry, next: ManifestEntry): boolean {
+  if (current.type === next.type) return true;
+  return current.type === 'yaml' && next.type === 'ts';
 }
 
 export function buildManifest(): ManifestEntry[] {
-  const manifest: ManifestEntry[] = [];
+  const manifest = new Map<string, ManifestEntry>();
 
   if (fs.existsSync(CLIS_DIR)) {
     for (const site of fs.readdirSync(CLIS_DIR)) {
@@ -262,18 +317,37 @@ export function buildManifest(): ManifestEntry[] {
         const filePath = path.join(siteDir, file);
         if (file.endsWith('.yaml') || file.endsWith('.yml')) {
           const entry = scanYaml(filePath, site);
-          if (entry) manifest.push(entry);
+          if (entry) {
+            const key = `${entry.site}/${entry.name}`;
+            const existing = manifest.get(key);
+            if (!existing || shouldReplaceManifestEntry(existing, entry)) {
+              if (existing && existing.type !== entry.type) {
+                process.stderr.write(`⚠️  Duplicate adapter ${key}: ${existing.type} superseded by ${entry.type}\n`);
+              }
+              manifest.set(key, entry);
+            }
+          }
         } else if (
-          (file.endsWith('.ts') && !file.endsWith('.d.ts') && file !== 'index.ts') ||
-          (file.endsWith('.js') && !file.endsWith('.d.js') && file !== 'index.js')
+          (file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts') && file !== 'index.ts') ||
+          (file.endsWith('.js') && !file.endsWith('.d.js') && !file.endsWith('.test.js') && file !== 'index.js')
         ) {
-          manifest.push(scanTs(filePath, site));
+          const entry = scanTs(filePath, site);
+          if (entry) {
+            const key = `${entry.site}/${entry.name}`;
+            const existing = manifest.get(key);
+            if (!existing || shouldReplaceManifestEntry(existing, entry)) {
+              if (existing && existing.type !== entry.type) {
+                process.stderr.write(`⚠️  Duplicate adapter ${key}: ${existing.type} superseded by ${entry.type}\n`);
+              }
+              manifest.set(key, entry);
+            }
+          }
         }
       }
     }
   }
 
-  return manifest;
+  return [...manifest.values()];
 }
 
 function main(): void {