@jackwener/opencli 0.1.1 → 0.2.0

package/dist/clis/youtube/search.js

@@ -0,0 +1,60 @@
+/**
+ * YouTube search — innertube API via browser session.
+ * Source: bb-sites/youtube/search.js
+ */
+import { cli, Strategy } from '../../registry.js';
+cli({
+    site: 'youtube',
+    name: 'search',
+    description: 'Search YouTube videos',
+    domain: 'www.youtube.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'query', required: true, help: 'Search query' },
+        { name: 'limit', type: 'int', default: 20, help: 'Max results (max 50)' },
+    ],
+    columns: ['rank', 'title', 'channel', 'views', 'duration', 'url'],
+    func: async (page, kwargs) => {
+        const limit = Math.min(kwargs.limit || 20, 50);
+        await page.goto('https://www.youtube.com');
+        await page.wait(2);
+        const data = await page.evaluate(`
+      (async () => {
+        const cfg = window.ytcfg?.data_ || {};
+        const apiKey = cfg.INNERTUBE_API_KEY;
+        const context = cfg.INNERTUBE_CONTEXT;
+        if (!apiKey || !context) return {error: 'YouTube config not found'};
+
+        const resp = await fetch('/youtubei/v1/search?key=' + apiKey + '&prettyPrint=false', {
+          method: 'POST', credentials: 'include',
+          headers: {'Content-Type': 'application/json'},
+          body: JSON.stringify({context, query: '${kwargs.query.replace(/'/g, "\\'")}'})
+        });
+        if (!resp.ok) return {error: 'HTTP ' + resp.status};
+
+        const data = await resp.json();
+        const contents = data.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents || [];
+        const videos = [];
+        for (const section of contents) {
+          for (const item of (section.itemSectionRenderer?.contents || [])) {
+            if (item.videoRenderer && videos.length < ${limit}) {
+              const v = item.videoRenderer;
+              videos.push({
+                rank: videos.length + 1,
+                title: v.title?.runs?.[0]?.text || '',
+                channel: v.ownerText?.runs?.[0]?.text || '',
+                views: v.viewCountText?.simpleText || v.shortViewCountText?.simpleText || '',
+                duration: v.lengthText?.simpleText || 'LIVE',
+                url: 'https://www.youtube.com/watch?v=' + v.videoId
+              });
+            }
+          }
+        }
+        return videos;
+      })()
+    `);
+        if (!Array.isArray(data))
+            return [];
+        return data;
+    },
+});

package/dist/engine.d.ts

@@ -2,5 +2,6 @@
  * CLI discovery: finds YAML/TS CLI definitions and registers them.
  */
 import { type CliCommand } from './registry.js';
+import type { IPage } from './types.js';
 export declare function discoverClis(...dirs: string[]): void;
-export declare function executeCommand(cmd: CliCommand, page: any, kwargs: Record<string, any>, debug?: boolean): Promise<any>;
+export declare function executeCommand(cmd: CliCommand, page: IPage | null, kwargs: Record<string, any>, debug?: boolean): Promise<any>;

package/dist/explore.js

@@ -463,7 +463,7 @@ export function renderExploreSummary(result) {
     lines.push(`Output: ${result.out_dir}`);
     return lines.join('\n');
 }
-async function readPageMetadata(page) {
+async function readPageMetadata(page /* IPage */) {
     try {
         const result = await page.evaluate(`() => ({ url: window.location.href, title: document.title || '' })`);
         if (result && typeof result === 'object')

package/dist/generate.js

@@ -9,7 +9,8 @@
  */
 import { exploreUrl } from './explore.js';
 import { synthesizeFromExplore } from './synthesize.js';
-import { registerCandidates } from './register.js';
+// TODO: implement real CLI registration (copy candidate YAML to user clis dir)
+function registerCandidates(_opts) { return { ok: true, count: 0 }; }
 const CAPABILITY_ALIASES = {
     search: ['search', '搜索', '查找', 'query', 'keyword'],
     hot: ['hot', '热门', '热榜', '热搜', 'popular', 'top', 'ranking'],

package/dist/main.js

@@ -2,6 +2,7 @@
 /**
  * opencli — Make any website your CLI. AI-powered.
  */
+import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -17,9 +18,12 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 const BUILTIN_CLIS = path.resolve(__dirname, 'clis');
 const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
+// Read version from package.json (single source of truth)
+const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
+const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
 discoverClis(BUILTIN_CLIS, USER_CLIS);
 const program = new Command();
-program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version('0.1.0');
+program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
 // ── Built-in commands ──────────────────────────────────────────────────────
 program.command('list').description('List all available CLI commands').option('--json', 'JSON output')
     .action((opts) => {
@@ -53,9 +57,7 @@ program.command('validate').description('Validate CLI definitions').argument('[t
     .action(async (target) => { const { validateClisWithTarget, renderValidationReport } = await import('./validate.js'); console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target))); });
 program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
     .action(async (target, opts) => { const { verifyClis, renderVerifyReport } = await import('./verify.js'); const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); console.log(renderVerifyReport(r)); process.exitCode = r.ok ? 0 : 1; });
-program.command('explore').description('Explore a website').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
-    .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); });
-program.command('probe').description('Probe a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
+program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
     .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); });
 program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
     .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });

package/dist/output.d.ts

@@ -1,5 +1,5 @@
 /**
- * Output formatting: table, JSON, Markdown, CSV.
+ * Output formatting: table, JSON, Markdown, CSV, YAML.
  */
 export interface RenderOptions {
     fmt?: string;

package/dist/output.js

@@ -1,8 +1,9 @@
 /**
- * Output formatting: table, JSON, Markdown, CSV.
+ * Output formatting: table, JSON, Markdown, CSV, YAML.
  */
 import chalk from 'chalk';
 import Table from 'cli-table3';
+import yaml from 'js-yaml';
 export function render(data, opts = {}) {
     const fmt = opts.fmt ?? 'table';
     if (data === null || data === undefined) {
@@ -20,6 +21,10 @@ export function render(data, opts = {}) {
         case 'csv':
             renderCsv(data, opts);
             break;
+        case 'yaml':
+        case 'yml':
+            renderYaml(data);
+            break;
         default:
             renderTable(data, opts);
             break;
@@ -32,19 +37,15 @@ function renderTable(data, opts) {
         return;
     }
     const columns = opts.columns ?? Object.keys(rows[0]);
-    const header = columns.map((c, i) => i === 0 ? '#' : capitalize(c));
+    const header = columns.map(c => capitalize(c));
     const table = new Table({
         head: header.map(h => chalk.bold(h)),
         style: { head: [], border: [] },
         wordWrap: true,
         wrapOnWordBoundary: true,
-        colWidths: columns.map((c, i) => {
+        colWidths: columns.map((_c, i) => {
             if (i === 0)
-                return 4;
-            if (c === 'url' || c === 'description')
-                return null;
-            if (c === 'title' || c === 'name' || c === 'repo')
-                return null;
+                return 6;
             return null;
         }).filter(() => true),
     });
@@ -93,6 +94,9 @@ function renderCsv(data, opts) {
         }).join(','));
     }
 }
+function renderYaml(data) {
+    console.log(yaml.dump(data, { sortKeys: false, lineWidth: 120, noRefs: true }));
+}
 function capitalize(s) {
     return s.charAt(0).toUpperCase() + s.slice(1);
 }

package/dist/pipeline/executor.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Pipeline executor: runs YAML pipeline steps sequentially.
+ */
+import type { IPage } from '../types.js';
+export interface PipelineContext {
+    args?: Record<string, any>;
+    debug?: boolean;
+}
+export declare function executePipeline(page: IPage | null, pipeline: any[], ctx?: PipelineContext): Promise<any>;

package/dist/pipeline/executor.js

@@ -0,0 +1,88 @@
+/**
+ * Pipeline executor: runs YAML pipeline steps sequentially.
+ */
+import chalk from 'chalk';
+import { stepNavigate, stepClick, stepType, stepWait, stepPress, stepSnapshot, stepEvaluate } from './steps/browser.js';
+import { stepFetch } from './steps/fetch.js';
+import { stepSelect, stepMap, stepFilter, stepSort, stepLimit } from './steps/transform.js';
+import { stepIntercept } from './steps/intercept.js';
+import { stepTap } from './steps/tap.js';
+/** Registry of all available step handlers */
+const STEP_HANDLERS = {
+    navigate: stepNavigate,
+    fetch: stepFetch,
+    select: stepSelect,
+    evaluate: stepEvaluate,
+    snapshot: stepSnapshot,
+    click: stepClick,
+    type: stepType,
+    wait: stepWait,
+    press: stepPress,
+    map: stepMap,
+    filter: stepFilter,
+    sort: stepSort,
+    limit: stepLimit,
+    intercept: stepIntercept,
+    tap: stepTap,
+};
+export async function executePipeline(page, pipeline, ctx = {}) {
+    const args = ctx.args ?? {};
+    const debug = ctx.debug ?? false;
+    let data = null;
+    const total = pipeline.length;
+    for (let i = 0; i < pipeline.length; i++) {
+        const step = pipeline[i];
+        if (!step || typeof step !== 'object')
+            continue;
+        for (const [op, params] of Object.entries(step)) {
+            if (debug)
+                debugStepStart(i + 1, total, op, params);
+            const handler = STEP_HANDLERS[op];
+            if (handler) {
+                data = await handler(page, params, data, args);
+            }
+            else {
+                if (debug)
+                    process.stderr.write(`  ${chalk.yellow('⚠')}  Unknown step: ${op}\n`);
+            }
+            // Detect error objects returned by steps (e.g. tap store not found)
+            if (data && typeof data === 'object' && !Array.isArray(data) && data.error) {
+                process.stderr.write(`  ${chalk.yellow('⚠')}  ${chalk.yellow(op)}: ${data.error}\n`);
+                if (data.hint)
+                    process.stderr.write(`  ${chalk.dim('💡')} ${chalk.dim(data.hint)}\n`);
+            }
+            if (debug)
+                debugStepResult(op, data);
+        }
+    }
+    return data;
+}
+function debugStepStart(stepNum, total, op, params) {
+    let preview = '';
+    if (typeof params === 'string') {
+        preview = params.length <= 80 ? ` → ${params}` : ` → ${params.slice(0, 77)}...`;
+    }
+    else if (params && typeof params === 'object' && !Array.isArray(params)) {
+        preview = ` (${Object.keys(params).join(', ')})`;
+    }
+    process.stderr.write(`  ${chalk.dim(`[${stepNum}/${total}]`)} ${chalk.bold.cyan(op)}${preview}\n`);
+}
+function debugStepResult(op, data) {
+    if (data === null || data === undefined) {
+        process.stderr.write(`       ${chalk.dim('→ (no data)')}\n`);
+    }
+    else if (Array.isArray(data)) {
+        process.stderr.write(`       ${chalk.dim(`→ ${data.length} items`)}\n`);
+    }
+    else if (typeof data === 'object') {
+        const keys = Object.keys(data).slice(0, 5);
+        process.stderr.write(`       ${chalk.dim(`→ dict (${keys.join(', ')}${Object.keys(data).length > 5 ? '...' : ''})`)}\n`);
+    }
+    else if (typeof data === 'string') {
+        const p = data.slice(0, 60).replace(/\n/g, '\\n');
+        process.stderr.write(`       ${chalk.dim(`→ "${p}${data.length > 60 ? '...' : ''}"`)}\n`);
+    }
+    else {
+        process.stderr.write(`       ${chalk.dim(`→ ${typeof data}`)}\n`);
+    }
+}

package/dist/pipeline/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Pipeline module: re-exports for backward compatibility.
+ */
+export { executePipeline, type PipelineContext } from './executor.js';
+export { render, evalExpr, resolvePath, normalizeEvaluateSource, type RenderContext } from './template.js';

package/dist/pipeline/index.js

@@ -0,0 +1,5 @@
+/**
+ * Pipeline module: re-exports for backward compatibility.
+ */
+export { executePipeline } from './executor.js';
+export { render, evalExpr, resolvePath, normalizeEvaluateSource } from './template.js';

package/dist/pipeline/steps/browser.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Pipeline step: navigate, click, type, wait, press, snapshot.
+ * Browser interaction primitives.
+ */
+import type { IPage } from '../../types.js';
+export declare function stepNavigate(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
+export declare function stepClick(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
+export declare function stepType(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
+export declare function stepWait(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
+export declare function stepPress(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;
+export declare function stepSnapshot(page: IPage, params: any, _data: any, _args: Record<string, any>): Promise<any>;
+export declare function stepEvaluate(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;

package/dist/pipeline/steps/browser.js

@@ -0,0 +1,68 @@
+/**
+ * Pipeline step: navigate, click, type, wait, press, snapshot.
+ * Browser interaction primitives.
+ */
+import { render, normalizeEvaluateSource } from '../template.js';
+export async function stepNavigate(page, params, data, args) {
+    const url = render(params, { args, data });
+    await page.goto(String(url));
+    return data;
+}
+export async function stepClick(page, params, data, args) {
+    await page.click(String(render(params, { args, data })).replace(/^@/, ''));
+    return data;
+}
+export async function stepType(page, params, data, args) {
+    if (typeof params === 'object' && params) {
+        const ref = String(render(params.ref ?? '', { args, data })).replace(/^@/, '');
+        const text = String(render(params.text ?? '', { args, data }));
+        await page.typeText(ref, text);
+        if (params.submit)
+            await page.pressKey('Enter');
+    }
+    return data;
+}
+export async function stepWait(page, params, data, args) {
+    if (typeof params === 'number')
+        await page.wait(params);
+    else if (typeof params === 'object' && params) {
+        if ('text' in params) {
+            const timeout = params.timeout ?? 10;
+            const start = Date.now();
+            while ((Date.now() - start) / 1000 < timeout) {
+                const snap = await page.snapshot({ raw: true });
+                if (typeof snap === 'string' && snap.includes(params.text))
+                    break;
+                await page.wait(0.5);
+            }
+        }
+        else if ('time' in params)
+            await page.wait(Number(params.time));
+    }
+    else if (typeof params === 'string')
+        await page.wait(Number(render(params, { args, data })));
+    return data;
+}
+export async function stepPress(page, params, data, args) {
+    await page.pressKey(String(render(params, { args, data })));
+    return data;
+}
+export async function stepSnapshot(page, params, _data, _args) {
+    const opts = (typeof params === 'object' && params) ? params : {};
+    return page.snapshot({ interactive: opts.interactive ?? false, compact: opts.compact ?? false, maxDepth: opts.max_depth, raw: opts.raw ?? false });
+}
+export async function stepEvaluate(page, params, data, args) {
+    const js = String(render(params, { args, data }));
+    let result = await page.evaluate(normalizeEvaluateSource(js));
+    // MCP may return JSON as a string — auto-parse it
+    if (typeof result === 'string') {
+        const trimmed = result.trim();
+        if ((trimmed.startsWith('[') && trimmed.endsWith(']')) || (trimmed.startsWith('{') && trimmed.endsWith('}'))) {
+            try {
+                result = JSON.parse(trimmed);
+            }
+            catch { }
+        }
+    }
+    return result;
+}

package/dist/pipeline/steps/fetch.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Pipeline step: fetch — HTTP API requests.
+ */
+import type { IPage } from '../../types.js';
+export declare function stepFetch(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any>;

package/dist/pipeline/steps/fetch.js

@@ -0,0 +1,50 @@
+/**
+ * Pipeline step: fetch — HTTP API requests.
+ */
+import { render } from '../template.js';
+/** Single URL fetch helper */
+async function fetchSingle(page, url, method, queryParams, headers, args, data) {
+    const renderedParams = {};
+    for (const [k, v] of Object.entries(queryParams))
+        renderedParams[k] = String(render(v, { args, data }));
+    const renderedHeaders = {};
+    for (const [k, v] of Object.entries(headers))
+        renderedHeaders[k] = String(render(v, { args, data }));
+    let finalUrl = url;
+    if (Object.keys(renderedParams).length > 0) {
+        const qs = new URLSearchParams(renderedParams).toString();
+        finalUrl = `${finalUrl}${finalUrl.includes('?') ? '&' : '?'}${qs}`;
+    }
+    if (page === null) {
+        const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
+        return resp.json();
+    }
+    const headersJs = JSON.stringify(renderedHeaders);
+    const escapedUrl = finalUrl.replace(/"/g, '\\"');
+    return page.evaluate(`
+    async () => {
+      const resp = await fetch("${escapedUrl}", {
+        method: "${method}", headers: ${headersJs}, credentials: "include"
+      });
+      return await resp.json();
+    }
+  `);
+}
+export async function stepFetch(page, params, data, args) {
+    const urlOrObj = typeof params === 'string' ? params : (params?.url ?? '');
+    const method = params?.method ?? 'GET';
+    const queryParams = params?.params ?? {};
+    const headers = params?.headers ?? {};
+    const urlTemplate = String(urlOrObj);
+    // Per-item fetch when data is array and URL references item
+    if (Array.isArray(data) && urlTemplate.includes('item')) {
+        const results = [];
+        for (let i = 0; i < data.length; i++) {
+            const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i }));
+            results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data));
+        }
+        return results;
+    }
+    const url = render(urlOrObj, { args, data });
+    return fetchSingle(page, String(url), method, queryParams, headers, args, data);
+}

package/dist/pipeline/steps/intercept.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Pipeline step: intercept — declarative XHR interception.
+ */
+import type { IPage } from '../../types.js';
+export declare function stepIntercept(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;

package/dist/pipeline/steps/intercept.js

@@ -0,0 +1,75 @@
+/**
+ * Pipeline step: intercept — declarative XHR interception.
+ */
+import { render } from '../template.js';
+export async function stepIntercept(page, params, data, args) {
+    const cfg = typeof params === 'object' ? params : {};
+    const trigger = cfg.trigger ?? '';
+    const capturePattern = cfg.capture ?? '';
+    const timeout = cfg.timeout ?? 8;
+    const selectPath = cfg.select ?? null;
+    if (!capturePattern)
+        return data;
+    // Step 1: Execute the trigger action
+    if (trigger.startsWith('navigate:')) {
+        const url = render(trigger.slice('navigate:'.length), { args, data });
+        await page.goto(String(url));
+    }
+    else if (trigger.startsWith('evaluate:')) {
+        const js = trigger.slice('evaluate:'.length);
+        const { normalizeEvaluateSource } = await import('../template.js');
+        await page.evaluate(normalizeEvaluateSource(render(js, { args, data })));
+    }
+    else if (trigger.startsWith('click:')) {
+        const ref = render(trigger.slice('click:'.length), { args, data });
+        await page.click(String(ref).replace(/^@/, ''));
+    }
+    else if (trigger === 'scroll') {
+        await page.scroll('down');
+    }
+    // Step 2: Wait a bit for network requests to fire
+    await page.wait(Math.min(timeout, 3));
+    // Step 3: Get network requests and find matching ones
+    const rawNetwork = await page.networkRequests(false);
+    const matchingResponses = [];
+    if (typeof rawNetwork === 'string') {
+        const lines = rawNetwork.split('\n');
+        for (const line of lines) {
+            const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
+            if (match) {
+                const [, , url, status] = match;
+                if (url.includes(capturePattern) && status === '200') {
+                    try {
+                        const body = await page.evaluate(`
+              async () => {
+                try {
+                  const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
+                  if (!resp.ok) return null;
+                  return await resp.json();
+                } catch { return null; }
+              }
+            `);
+                        if (body)
+                            matchingResponses.push(body);
+                    }
+                    catch { }
+                }
+            }
+        }
+    }
+    // Step 4: Select from response if specified
+    let result = matchingResponses.length === 1 ? matchingResponses[0] :
+        matchingResponses.length > 1 ? matchingResponses : data;
+    if (selectPath && result) {
+        let current = result;
+        for (const part of String(selectPath).split('.')) {
+            if (current && typeof current === 'object' && !Array.isArray(current)) {
+                current = current[part];
+            }
+            else
+                break;
+        }
+        result = current ?? result;
+    }
+    return result;
+}

package/dist/pipeline/steps/tap.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Pipeline step: tap — declarative Store Action Bridge.
+ *
+ * Generates a self-contained IIFE that:
+ * 1. Injects fetch + XHR dual interception proxy
+ * 2. Finds the Pinia/Vuex store and calls the action
+ * 3. Captures the response matching the URL pattern
+ * 4. Auto-cleans up interception in finally block
+ * 5. Returns the captured data (optionally sub-selected)
+ */
+import type { IPage } from '../../types.js';
+export declare function stepTap(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any>;