@jackwener/opencli 0.7.0 → 0.7.3

package/dist/clis/youtube/video.js

@@ -0,0 +1,114 @@
+/**
+ * YouTube video metadata — read ytInitialPlayerResponse + ytInitialData from video page.
+ */
+import { cli, Strategy } from '../../registry.js';
+import { parseVideoId } from './utils.js';
+cli({
+    site: 'youtube',
+    name: 'video',
+    description: 'Get YouTube video metadata (title, views, description, etc.)',
+    domain: 'www.youtube.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'url', required: true, help: 'YouTube video URL or video ID' },
+    ],
+    columns: ['field', 'value'],
+    func: async (page, kwargs) => {
+        const videoId = parseVideoId(kwargs.url);
+        const videoUrl = `https://www.youtube.com/watch?v=${videoId}`;
+        await page.goto(videoUrl);
+        await page.wait(3);
+        const data = await page.evaluate(`
+      (async () => {
+        const player = window.ytInitialPlayerResponse;
+        const yt = window.ytInitialData;
+        if (!player) return { error: 'ytInitialPlayerResponse not found' };
+
+        const details = player.videoDetails || {};
+        const microformat = player.microformat?.playerMicroformatRenderer || {};
+
+        // Try to get full description from ytInitialData
+        let fullDescription = details.shortDescription || '';
+        try {
+          const contents = yt?.contents?.twoColumnWatchNextResults
+            ?.results?.results?.contents;
+          if (contents) {
+            for (const c of contents) {
+              const desc = c.videoSecondaryInfoRenderer?.attributedDescription?.content;
+              if (desc) { fullDescription = desc; break; }
+            }
+          }
+        } catch {}
+
+        // Get like count if available
+        let likes = '';
+        try {
+          const contents = yt?.contents?.twoColumnWatchNextResults
+            ?.results?.results?.contents;
+          if (contents) {
+            for (const c of contents) {
+              const buttons = c.videoPrimaryInfoRenderer?.videoActions
+                ?.menuRenderer?.topLevelButtons;
+              if (buttons) {
+                for (const b of buttons) {
+                  const toggle = b.segmentedLikeDislikeButtonViewModel
+                    ?.likeButtonViewModel?.likeButtonViewModel?.toggleButtonViewModel
+                    ?.toggleButtonViewModel?.defaultButtonViewModel?.buttonViewModel;
+                  if (toggle?.title) { likes = toggle.title; break; }
+                }
+              }
+            }
+          }
+        } catch {}
+
+        // Get publish date
+        const publishDate = microformat.publishDate
+          || microformat.uploadDate
+          || details.publishDate || '';
+
+        // Get category
+        const category = microformat.category || '';
+
+        // Get channel subscriber count if available
+        let subscribers = '';
+        try {
+          const contents = yt?.contents?.twoColumnWatchNextResults
+            ?.results?.results?.contents;
+          if (contents) {
+            for (const c of contents) {
+              const owner = c.videoSecondaryInfoRenderer?.owner
+                ?.videoOwnerRenderer?.subscriberCountText?.simpleText;
+              if (owner) { subscribers = owner; break; }
+            }
+          }
+        } catch {}
+
+        return {
+          title: details.title || '',
+          channel: details.author || '',
+          channelId: details.channelId || '',
+          videoId: details.videoId || '',
+          views: details.viewCount || '',
+          likes,
+          subscribers,
+          duration: details.lengthSeconds ? details.lengthSeconds + 's' : '',
+          publishDate,
+          category,
+          description: fullDescription,
+          keywords: (details.keywords || []).join(', '),
+          isLive: details.isLiveContent || false,
+          thumbnail: details.thumbnail?.thumbnails?.slice(-1)?.[0]?.url || '',
+        };
+      })()
+    `);
+        if (!data || typeof data !== 'object')
+            throw new Error('Failed to extract video metadata from page');
+        if (data.error)
+            throw new Error(data.error);
+        // Return as field/value pairs for table display
+        return Object.entries(data).map(([field, value]) => ({
+            field,
+            value: String(value),
+        }));
+    },
+});

package/dist/engine.js

@@ -100,7 +100,8 @@ async function discoverClisFromFs(dir) {
             if (file.endsWith('.yaml') || file.endsWith('.yml')) {
                 registerYamlCli(filePath, site);
             }
-            else if (file.endsWith('.js') && !file.endsWith('.d.js')) {
+            else if ((file.endsWith('.js') && !file.endsWith('.d.js')) ||
+                (file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts'))) {
                 promises.push(import(`file://${filePath}`).catch((err) => {
                     process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
                 }));

package/dist/main.js

@@ -65,9 +65,17 @@ program.command('list').description('List all available CLI commands').option('-
     console.log();
 });
 program.command('validate').description('Validate CLI definitions').argument('[target]', 'site or site/name')
-    .action(async (target) => { const { validateClisWithTarget, renderValidationReport } = await import('./validate.js'); console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target))); });
+    .action(async (target) => {
+    const { validateClisWithTarget, renderValidationReport } = await import('./validate.js');
+    console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target)));
+});
 program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
-    .action(async (target, opts) => { const { verifyClis, renderVerifyReport } = await import('./verify.js'); const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); console.log(renderVerifyReport(r)); process.exitCode = r.ok ? 0 : 1; });
+    .action(async (target, opts) => {
+    const { verifyClis, renderVerifyReport } = await import('./verify.js');
+    const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke });
+    console.log(renderVerifyReport(r));
+    process.exitCode = r.ok ? 0 : 1;
+});
 program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click <labels>', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")')
     .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s) => s.trim()) : undefined; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); });
 program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')

package/dist/output.js

@@ -85,7 +85,8 @@ function renderCsv(data, opts) {
     for (const row of rows) {
         console.log(columns.map(c => {
             const v = String(row[c] ?? '');
-            return v.includes(',') || v.includes('"') ? `"${v.replace(/"/g, '""')}"` : v;
+            return v.includes(',') || v.includes('"') || v.includes('\n')
+                ? `"${v.replace(/"/g, '""')}"` : v;
         }).join(','));
     }
 }

package/dist/registry.d.ts

@@ -37,18 +37,11 @@ export interface InternalCliCommand extends CliCommand {
     _lazy?: boolean;
     _modulePath?: string;
 }
-export interface CliOptions {
+export interface CliOptions extends Partial<Omit<CliCommand, 'args' | 'description'>> {
     site: string;
     name: string;
     description?: string;
-    domain?: string;
-    strategy?: Strategy;
-    browser?: boolean;
     args?: Arg[];
-    columns?: string[];
-    func?: (page: IPage, kwargs: Record<string, any>, debug?: boolean) => Promise<any>;
-    pipeline?: any[];
-    timeoutSeconds?: number;
 }
 export declare function cli(opts: CliOptions): CliCommand;
 export declare function getRegistry(): Map<string, CliCommand>;

package/dist/snapshotFormatter.d.ts

@@ -1,9 +1,18 @@
 /**
  * Aria snapshot formatter: parses Playwright MCP snapshot text into clean format.
+ *
+ * Multi-pass pipeline:
+ * 1. Parse & filter: strip annotations, metadata, noise roles, ads, decorators
+ * 2. Deduplicate: generic/text child matching parent label
+ * 3. Deduplicate: heading + link with identical labels
+ * 4. Deduplicate: nested identical links
+ * 5. Prune: empty containers (iterative bottom-up)
+ * 6. Collapse: single-child containers
  */
 export interface FormatOptions {
     interactive?: boolean;
     compact?: boolean;
     maxDepth?: number;
+    maxTextLength?: number;
 }
 export declare function formatSnapshot(raw: string, opts?: FormatOptions): string;

package/dist/snapshotFormatter.js

@@ -1,41 +1,378 @@
 /**
  * Aria snapshot formatter: parses Playwright MCP snapshot text into clean format.
+ *
+ * Multi-pass pipeline:
+ * 1. Parse & filter: strip annotations, metadata, noise roles, ads, decorators
+ * 2. Deduplicate: generic/text child matching parent label
+ * 3. Deduplicate: heading + link with identical labels
+ * 4. Deduplicate: nested identical links
+ * 5. Prune: empty containers (iterative bottom-up)
+ * 6. Collapse: single-child containers
  */
+const DEFAULT_MAX_TEXT_LENGTH = 200;
+// Roles that are pure noise and should always be filtered
+const NOISE_ROLES = new Set([
+    'none', 'presentation', 'separator', 'paragraph', 'tooltip', 'status',
+]);
+// Roles whose entire subtree should be removed (footer boilerplate, etc.)
+const SUBTREE_NOISE_ROLES = new Set([
+    'contentinfo',
+]);
+// Roles considered interactive (clickable/typeable)
+const INTERACTIVE_ROLES = new Set([
+    'button', 'link', 'textbox', 'checkbox', 'radio',
+    'combobox', 'tab', 'menuitem', 'option', 'switch',
+    'slider', 'spinbutton', 'searchbox',
+]);
+// Structural landmark roles kept even in interactive mode
+const LANDMARK_ROLES = new Set([
+    'main', 'navigation', 'banner', 'heading', 'search',
+    'region', 'list', 'listitem', 'article', 'complementary',
+    'group', 'toolbar', 'tablist',
+]);
+// Container roles eligible for pruning and collapse
+const CONTAINER_ROLES = new Set([
+    'list', 'listitem', 'group', 'toolbar', 'tablist',
+    'navigation', 'region', 'complementary',
+    'search', 'article', 'paragraph', 'figure',
+]);
+// Decorator / separator text that adds no semantic value
+const DECORATOR_TEXT = new Set(['•', '·', '|', '—', '-', '/', '\\']);
+// Ad-related URL patterns
+const AD_URL_PATTERNS = [
+    'googleadservices.com/pagead/',
+    'alb.reddit.com/cr?',
+    'doubleclick.net/',
+    'cm.bilibili.com/cm/api/fees/',
+];
+// Boilerplate button labels to filter (back-to-top, etc.)
+const BOILERPLATE_LABELS = [
+    '回到顶部', 'back to top', 'scroll to top', 'go to top',
+];
+/**
+ * Parse role and text from a trimmed snapshot line.
+ * Handles quoted labels and trailing text after colon correctly,
+ * including lines wrapped in single quotes by Playwright.
+ */
+function parseLine(trimmed) {
+    // Unwrap outer single quotes if present (Playwright wraps lines with special chars)
+    let line = trimmed;
+    if (line.startsWith("'") && line.endsWith("':")) {
+        line = line.slice(1, -2) + ':';
+    }
+    else if (line.startsWith("'") && line.endsWith("'")) {
+        line = line.slice(1, -1);
+    }
+    // Role is the first word
+    const roleMatch = line.match(/^([a-zA-Z]+)\b/);
+    const role = roleMatch ? roleMatch[1].toLowerCase() : '';
+    // Extract quoted text content (the semantic label)
+    const textMatch = line.match(/"([^"]*)"/);
+    const text = textMatch ? textMatch[1] : '';
+    // For trailing text: strip annotations and quoted strings first, then check after last colon
+    // This avoids matching colons inside quoted labels like "Account: user@email.com"
+    let stripped = line;
+    // Remove all quoted strings
+    stripped = stripped.replace(/"[^"]*"/g, '""');
+    // Remove all bracket annotations
+    stripped = stripped.replace(/\[[^\]]*\]/g, '');
+    const colonIdx = stripped.lastIndexOf(':');
+    let trailingText = '';
+    if (colonIdx !== -1) {
+        const afterColon = stripped.slice(colonIdx + 1).trim();
+        if (afterColon.length > 0) {
+            // Get the actual trailing text from original line at same position
+            const origColonIdx = line.lastIndexOf(':');
+            if (origColonIdx !== -1) {
+                trailingText = line.slice(origColonIdx + 1).trim();
+            }
+        }
+    }
+    return { role, text, hasText: text.length > 0 || trailingText.length > 0, trailingText };
+}
+/**
+ * Strip ALL bracket annotations from a content line, preserving quoted strings.
+ * Handles both double-quoted and outer single-quoted lines from Playwright.
+ */
+function stripAnnotations(content) {
+    // Unwrap outer single quotes first
+    let line = content;
+    if (line.startsWith("'") && (line.endsWith("':") || line.endsWith("'"))) {
+        if (line.endsWith("':")) {
+            line = line.slice(1, -2) + ':';
+        }
+        else {
+            line = line.slice(1, -1);
+        }
+    }
+    // Split by double quotes to protect quoted content
+    const parts = line.split('"');
+    for (let i = 0; i < parts.length; i += 2) {
+        // Only strip annotations from non-quoted parts (even indices)
+        parts[i] = parts[i].replace(/\s*\[[^\]]*\]/g, '');
+    }
+    let result = parts.join('"').replace(/\s{2,}/g, ' ').trim();
+    return result;
+}
+/**
+ * Check if a line is a metadata-only line (like /url: ...).
+ */
+function isMetadataLine(trimmed) {
+    return /^\/[a-zA-Z]+:/.test(trimmed);
+}
+/**
+ * Check if text content is purely decorative (separators, dots, etc.)
+ */
+function isDecoratorText(text) {
+    return DECORATOR_TEXT.has(text.trim());
+}
+/**
+ * Check if a node is ad-related based on its text content.
+ */
+function isAdNode(text, trailingText) {
+    const t = (text + ' ' + trailingText).toLowerCase();
+    if (t.includes('sponsored') || t.includes('advertisement'))
+        return true;
+    if (t.includes('广告'))
+        return true;
+    // Check for ad tracking URLs in the label
+    for (const pattern of AD_URL_PATTERNS) {
+        if (text.includes(pattern) || trailingText.includes(pattern))
+            return true;
+    }
+    return false;
+}
+/**
+ * Check if a node is boilerplate UI (back-to-top, etc.)
+ */
+function isBoilerplateNode(text) {
+    const t = text.toLowerCase();
+    return BOILERPLATE_LABELS.some(label => t.includes(label));
+}
+/**
+ * Check if a role is noise that should be filtered.
+ */
+function isNoiseNode(role, hasText, text, trailingText) {
+    if (NOISE_ROLES.has(role))
+        return true;
+    // generic without text is a wrapper
+    if (role === 'generic' && !hasText)
+        return true;
+    // img without alt text is noise
+    if (role === 'img' && !hasText)
+        return true;
+    // Decorator-only text nodes
+    if ((role === 'generic' || role === 'text') && hasText) {
+        const content = trailingText || text;
+        if (isDecoratorText(content))
+            return true;
+    }
+    return false;
+}
 export function formatSnapshot(raw, opts = {}) {
     if (!raw || typeof raw !== 'string')
         return '';
+    const maxTextLen = opts.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH;
     const lines = raw.split('\n');
-    const result = [];
+    // === Pass 1: Parse, filter, and collect entries ===
+    const entries = [];
     let refCounter = 0;
-    for (const line of lines) {
+    let skipUntilDepth = -1; // When >= 0, skip all nodes at depth > this value
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
         if (!line.trim())
             continue;
         const indent = line.length - line.trimStart().length;
         const depth = Math.floor(indent / 2);
-        if (opts.maxDepth && depth > opts.maxDepth)
-            continue;
+        // If we're in a subtree skip zone, check depth
+        if (skipUntilDepth >= 0) {
+            if (depth > skipUntilDepth)
+                continue; // still inside subtree
+            skipUntilDepth = -1; // exited subtree
+        }
         let content = line.trimStart();
-        // Skip non-interactive elements in interactive mode
-        if (opts.interactive) {
-            const interactiveRoles = ['button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'tab', 'menuitem', 'option'];
-            const role = content.split(/[\s[]/)[0]?.toLowerCase() ?? '';
-            if (!interactiveRoles.some(r => role.includes(r)) && depth > 1)
-                continue;
+        // Strip leading "- "
+        if (content.startsWith('- ')) {
+            content = content.slice(2);
+        }
+        // Skip metadata lines
+        if (isMetadataLine(content))
+            continue;
+        // Apply maxDepth filter
+        if (opts.maxDepth !== undefined && depth > opts.maxDepth)
+            continue;
+        const { role, text, hasText, trailingText } = parseLine(content);
+        // Skip noise nodes
+        if (isNoiseNode(role, hasText, text, trailingText))
+            continue;
+        // Skip subtree noise roles (contentinfo footer, etc.) — skip entire subtree
+        if (SUBTREE_NOISE_ROLES.has(role)) {
+            skipUntilDepth = depth;
+            continue;
         }
-        // Compact: strip verbose role descriptions
+        // Strip annotations
+        content = stripAnnotations(content);
+        // Check if node should trigger subtree skip (ads, boilerplate)
+        const isSubtreeSkip = isAdNode(text, trailingText) || isBoilerplateNode(text);
+        // Interactive mode filter
+        const isInteractive = INTERACTIVE_ROLES.has(role);
+        const isLandmark = LANDMARK_ROLES.has(role);
+        if (opts.interactive && !isInteractive && !isLandmark && !hasText)
+            continue;
+        // Compact mode
         if (opts.compact) {
             content = content
                 .replace(/\s*\[.*?\]\s*/g, ' ')
                 .replace(/\s+/g, ' ')
                 .trim();
         }
+        // Text truncation
+        if (maxTextLen > 0 && content.length > maxTextLen) {
+            content = content.slice(0, maxTextLen) + '…';
+        }
         // Assign refs to interactive elements
-        const interactivePattern = /^(button|link|textbox|checkbox|radio|combobox|tab|menuitem|option)\b/i;
-        if (interactivePattern.test(content)) {
+        if (isInteractive) {
             refCounter++;
             content = `[@${refCounter}] ${content}`;
         }
-        result.push('  '.repeat(depth) + content);
+        entries.push({ depth, content, role, text, trailingText, isInteractive, isLandmark, isSubtreeSkip });
+    }
+    // === Pass 2: Remove subtree-skip nodes (ads, boilerplate, contentinfo) ===
+    let noAds = [];
+    for (let i = 0; i < entries.length; i++) {
+        const entry = entries[i];
+        if (entry.isSubtreeSkip) {
+            const skipDepth = entry.depth;
+            i++;
+            while (i < entries.length && entries[i].depth > skipDepth) {
+                i++;
+            }
+            i--;
+            continue;
+        }
+        noAds.push(entry);
+    }
+    // === Pass 3: Deduplicate child generic/text matching parent label ===
+    let deduped = [];
+    for (let i = 0; i < noAds.length; i++) {
+        const entry = noAds[i];
+        if (entry.role === 'generic' || entry.role === 'text') {
+            let parent;
+            for (let j = deduped.length - 1; j >= 0; j--) {
+                if (deduped[j].depth < entry.depth) {
+                    parent = deduped[j];
+                    break;
+                }
+                if (deduped[j].depth === entry.depth)
+                    break;
+            }
+            if (parent) {
+                const childText = entry.trailingText || entry.text;
+                if (childText && parent.text && childText === parent.text) {
+                    continue;
+                }
+            }
+        }
+        deduped.push(entry);
+    }
+    // === Pass 4: Deduplicate heading + child link with identical label ===
+    // Pattern: heading "Title": → link "Title": (same text) → skip the link
+    const deduped2 = [];
+    for (let i = 0; i < deduped.length; i++) {
+        const entry = deduped[i];
+        if (entry.role === 'heading' && entry.text) {
+            const next = deduped[i + 1];
+            if (next && next.role === 'link' && next.text === entry.text && next.depth === entry.depth + 1) {
+                // Keep the heading, skip the link. But preserve link's children re-parented.
+                deduped2.push(entry);
+                i++; // skip the link
+                continue;
+            }
+        }
+        deduped2.push(entry);
+    }
+    // === Pass 5: Deduplicate nested identical links ===
+    const deduped3 = [];
+    for (let i = 0; i < deduped2.length; i++) {
+        const entry = deduped2[i];
+        if (entry.role === 'link' && entry.text) {
+            const next = deduped2[i + 1];
+            if (next && next.role === 'link' && next.text === entry.text && next.depth === entry.depth + 1) {
+                continue; // Skip parent, keep child
+            }
+        }
+        deduped3.push(entry);
+    }
+    // === Pass 6: Iteratively prune empty containers (bottom-up) ===
+    let current = deduped3;
+    let changed = true;
+    while (changed) {
+        changed = false;
+        const next = [];
+        for (let i = 0; i < current.length; i++) {
+            const entry = current[i];
+            if (CONTAINER_ROLES.has(entry.role) && !entry.text && !entry.trailingText) {
+                let hasChildren = false;
+                for (let j = i + 1; j < current.length; j++) {
+                    if (current[j].depth <= entry.depth)
+                        break;
+                    if (current[j].depth > entry.depth) {
+                        hasChildren = true;
+                        break;
+                    }
+                }
+                if (!hasChildren) {
+                    changed = true;
+                    continue;
+                }
+            }
+            next.push(entry);
+        }
+        current = next;
+    }
+    // === Pass 7: Collapse single-child containers ===
+    const collapsed = [];
+    for (let i = 0; i < current.length; i++) {
+        const entry = current[i];
+        if (CONTAINER_ROLES.has(entry.role) && !entry.text && !entry.trailingText) {
+            let childCount = 0;
+            let childIdx = -1;
+            for (let j = i + 1; j < current.length; j++) {
+                if (current[j].depth <= entry.depth)
+                    break;
+                if (current[j].depth === entry.depth + 1) {
+                    childCount++;
+                    if (childCount === 1)
+                        childIdx = j;
+                }
+            }
+            if (childCount === 1 && childIdx !== -1) {
+                const child = current[childIdx];
+                let hasGrandchildren = false;
+                for (let j = childIdx + 1; j < current.length; j++) {
+                    if (current[j].depth <= child.depth)
+                        break;
+                    if (current[j].depth > child.depth) {
+                        hasGrandchildren = true;
+                        break;
+                    }
+                }
+                if (!hasGrandchildren) {
+                    const mergedContent = entry.content.replace(/:$/, '') + ' > ' + child.content;
+                    collapsed.push({
+                        ...entry,
+                        content: mergedContent,
+                        role: child.role,
+                        text: child.text,
+                        trailingText: child.trailingText,
+                        isInteractive: child.isInteractive,
+                    });
+                    i++;
+                    continue;
+                }
+            }
+        }
+        collapsed.push(entry);
     }
-    return result.join('\n');
+    return collapsed.map(e => '  '.repeat(e.depth) + e.content).join('\n');
 }

package/dist/snapshotFormatter.test.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Tests for snapshotFormatter.ts: Playwright MCP snapshot tree filtering.
+ *
+ * Uses sanitized excerpts from real websites (GitHub, Bilibili, Twitter)
+ * to validate noise filtering, annotation stripping, and output quality.
+ */
+export {};