@jackwener/opencli 0.1.0

package/dist/synthesize.js

@@ -0,0 +1,191 @@
+/**
+ * Synthesize: turn explore capabilities into ready-to-use CLI definitions.
+ *
+ * Takes the structured capabilities from Deep Explore and generates
+ * YAML pipeline files that can be directly registered as CLI commands.
+ *
+ * This is the bridge between discovery (explore) and usability (CLI).
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import yaml from 'js-yaml';
+export function synthesizeFromExplore(target, opts = {}) {
+    const exploreDir = fs.existsSync(target) ? target : path.join('.opencli', 'explore', target);
+    if (!fs.existsSync(exploreDir))
+        throw new Error(`Explore dir not found: ${target}`);
+    const manifest = JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8'));
+    const capabilities = JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8'));
+    const endpoints = JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8'));
+    const auth = JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8'));
+    const targetDir = opts.outDir ?? path.join(exploreDir, 'candidates');
+    fs.mkdirSync(targetDir, { recursive: true });
+    const site = manifest.site;
+    const topN = opts.top ?? 5;
+    const candidates = [];
+    // Sort capabilities by confidence
+    const sortedCaps = [...capabilities]
+        .sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0))
+        .slice(0, topN);
+    for (const cap of sortedCaps) {
+        // Find the matching endpoint for more detail
+        const endpoint = endpoints.find((ep) => ep.pattern === cap.endpoint) ??
+            endpoints[0];
+        const candidate = buildCandidateYaml(site, manifest, cap, endpoint);
+        const fileName = `${cap.name}.yaml`;
+        const filePath = path.join(targetDir, fileName);
+        fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 }));
+        candidates.push({
+            name: cap.name,
+            path: filePath,
+            strategy: cap.strategy,
+            endpoint: cap.endpoint,
+            confidence: cap.confidence,
+            columns: candidate.yaml.columns,
+        });
+    }
+    const index = {
+        site,
+        target_url: manifest.target_url,
+        generated_from: exploreDir,
+        candidate_count: candidates.length,
+        candidates,
+    };
+    fs.writeFileSync(path.join(targetDir, 'candidates.json'), JSON.stringify(index, null, 2));
+    return {
+        site,
+        explore_dir: exploreDir,
+        out_dir: targetDir,
+        candidate_count: candidates.length,
+        candidates,
+    };
+}
+/** Volatile params to strip from generated URLs */
+const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']);
+const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']);
+const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']);
+const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']);
+/**
+ * Build a clean templated URL from a raw API URL.
+ * - Strips volatile params (w_rid, wts, etc.)
+ * - Templates search, limit, and pagination params
+ * - Builds URL string manually to avoid URL encoding of ${{ }} expressions
+ */
+function buildTemplatedUrl(rawUrl, cap, endpoint) {
+    try {
+        const u = new URL(rawUrl);
+        const base = `${u.protocol}//${u.host}${u.pathname}`;
+        const params = [];
+        const hasKeyword = cap.recommendedArgs?.some((a) => a.name === 'keyword');
+        u.searchParams.forEach((v, k) => {
+            // Skip volatile params
+            if (VOLATILE_PARAMS.has(k))
+                return;
+            // Template known param types
+            if (hasKeyword && SEARCH_PARAM_NAMES.has(k)) {
+                params.push([k, '${{ args.keyword }}']);
+            }
+            else if (LIMIT_PARAM_NAMES.has(k)) {
+                params.push([k, '${{ args.limit | default(20) }}']);
+            }
+            else if (PAGE_PARAM_NAMES.has(k)) {
+                params.push([k, '${{ args.page | default(1) }}']);
+            }
+            else {
+                params.push([k, v]);
+            }
+        });
+        if (params.length === 0)
+            return base;
+        return base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&');
+    }
+    catch {
+        return rawUrl;
+    }
+}
+/**
+ * Build a YAML pipeline definition from a capability + endpoint.
+ */
+function buildCandidateYaml(site, manifest, cap, endpoint) {
+    const needsBrowser = cap.strategy !== 'public';
+    const pipeline = [];
+    // Step 1: Navigate (if browser-based)
+    if (needsBrowser) {
+        pipeline.push({ navigate: manifest.target_url });
+    }
+    // Step 2: Fetch the API — build a clean URL with templates
+    const rawUrl = endpoint?.url ?? manifest.target_url;
+    const fetchStep = { url: buildTemplatedUrl(rawUrl, cap, endpoint) };
+    pipeline.push({ fetch: fetchStep });
+    // Step 3: Select the item path
+    if (cap.itemPath) {
+        pipeline.push({ select: cap.itemPath });
+    }
+    // Step 4: Map fields to columns
+    const mapStep = {};
+    const columns = cap.recommendedColumns ?? ['title', 'url'];
+    // Add a rank column if not doing search
+    if (!cap.recommendedArgs?.some((a) => a.name === 'keyword')) {
+        mapStep['rank'] = '${{ index + 1 }}';
+    }
+    // Build field mappings from the endpoint's detected fields
+    const detectedFields = endpoint?.detectedFields ?? {};
+    for (const col of columns) {
+        const fieldPath = detectedFields[col];
+        if (fieldPath) {
+            mapStep[col] = `\${{ item.${fieldPath} }}`;
+        }
+        else {
+            mapStep[col] = `\${{ item.${col} }}`;
+        }
+    }
+    pipeline.push({ map: mapStep });
+    // Step 5: Limit
+    pipeline.push({ limit: '${{ args.limit | default(20) }}' });
+    // Build args definition
+    const argsDef = {};
+    for (const arg of cap.recommendedArgs ?? []) {
+        const def = { type: arg.type ?? 'str' };
+        if (arg.required)
+            def.required = true;
+        if (arg.default != null)
+            def.default = arg.default;
+        if (arg.name === 'keyword')
+            def.description = 'Search keyword';
+        else if (arg.name === 'limit')
+            def.description = 'Number of items to return';
+        else if (arg.name === 'page')
+            def.description = 'Page number';
+        argsDef[arg.name] = def;
+    }
+    // Ensure limit arg always exists
+    if (!argsDef['limit']) {
+        argsDef['limit'] = { type: 'int', default: 20, description: 'Number of items to return' };
+    }
+    const allColumns = Object.keys(mapStep);
+    return {
+        name: cap.name,
+        yaml: {
+            site,
+            name: cap.name,
+            description: `${site} ${cap.name} (auto-generated)`,
+            domain: manifest.final_url ? new URL(manifest.final_url).hostname : undefined,
+            strategy: cap.strategy,
+            browser: needsBrowser,
+            args: argsDef,
+            pipeline,
+            columns: allColumns,
+        },
+    };
+}
+export function renderSynthesizeSummary(r) {
+    const lines = [
+        'opencli synthesize: OK',
+        `Site: ${r.site}`,
+        `Source: ${r.explore_dir}`,
+        `Candidates: ${r.candidate_count}`,
+    ];
+    for (const c of r.candidates ?? []) {
+        lines.push(`  • ${c.name} (${c.strategy}, ${(c.confidence * 100).toFixed(0)}% confidence) → ${c.path}`);
+    }
+    return lines.join('\n');
+}

package/dist/validate.d.ts

@@ -0,0 +1,2 @@
+export declare function validateClisWithTarget(dirs: string[], target?: string): any;
+export declare function renderValidationReport(report: any): string;

package/dist/validate.js

@@ -0,0 +1,73 @@
+/** Validate CLI definitions. */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import yaml from 'js-yaml';
+export function validateClisWithTarget(dirs, target) {
+    const results = [];
+    let errors = 0;
+    let warnings = 0;
+    let files = 0;
+    for (const dir of dirs) {
+        if (!fs.existsSync(dir))
+            continue;
+        for (const site of fs.readdirSync(dir)) {
+            if (target && site !== target && !target.startsWith(site + '/'))
+                continue;
+            const siteDir = path.join(dir, site);
+            if (!fs.statSync(siteDir).isDirectory())
+                continue;
+            for (const file of fs.readdirSync(siteDir)) {
+                if (!file.endsWith('.yaml') && !file.endsWith('.yml'))
+                    continue;
+                if (target && target.includes('/') && !target.endsWith(file.replace(/\.(yaml|yml)$/, '')))
+                    continue;
+                files++;
+                const filePath = path.join(siteDir, file);
+                const r = validateYamlFile(filePath);
+                results.push(r);
+                errors += r.errors.length;
+                warnings += r.warnings.length;
+            }
+        }
+    }
+    return { ok: errors === 0, results, errors, warnings, files };
+}
+function validateYamlFile(filePath) {
+    const errors = [];
+    const warnings = [];
+    try {
+        const raw = fs.readFileSync(filePath, 'utf-8');
+        const def = yaml.load(raw);
+        if (!def || typeof def !== 'object') {
+            errors.push('Not a valid YAML object');
+            return { path: filePath, errors, warnings };
+        }
+        if (!def.site)
+            errors.push('Missing "site"');
+        if (!def.name)
+            errors.push('Missing "name"');
+        if (def.pipeline && !Array.isArray(def.pipeline))
+            errors.push('"pipeline" must be an array');
+        if (def.columns && !Array.isArray(def.columns))
+            errors.push('"columns" must be an array');
+        if (def.args && typeof def.args !== 'object')
+            errors.push('"args" must be an object');
+    }
+    catch (e) {
+        errors.push(`YAML parse error: ${e.message}`);
+    }
+    return { path: filePath, errors, warnings };
+}
+export function renderValidationReport(report) {
+    const lines = [`opencli validate: ${report.ok ? 'PASS' : 'FAIL'}`, `Checked ${report.results.length} CLI(s) in ${report.files} file(s)`, `Errors: ${report.errors}  Warnings: ${report.warnings}`];
+    for (const r of report.results) {
+        if (r.errors.length > 0 || r.warnings.length > 0) {
+            lines.push(`\n${r.path}:`);
+            for (const e of r.errors)
+                lines.push(`  ❌ ${e}`);
+            for (const w of r.warnings)
+                lines.push(`  ⚠️  ${w}`);
+        }
+    }
+    return lines.join('\n');
+}

package/dist/verify.d.ts

@@ -0,0 +1,2 @@
+export declare function verifyClis(opts: any): Promise<any>;
+export declare function renderVerifyReport(report: any): string;

package/dist/verify.js

@@ -0,0 +1,9 @@
+/** Verification: validate + smoke. */
+import { validateClisWithTarget, renderValidationReport } from './validate.js';
+export async function verifyClis(opts) {
+    const report = validateClisWithTarget([opts.builtinClis, opts.userClis], opts.target);
+    return { ok: report.ok, validation: report, smoke: null };
+}
+export function renderVerifyReport(report) {
+    return renderValidationReport(report.validation);
+}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@jackwener/opencli",
+  "version": "0.1.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "Make any website your CLI. AI-powered.",
+  "type": "module",
+  "main": "dist/main.js",
+  "bin": {
+    "opencli": "dist/main.js"
+  },
+  "scripts": {
+    "dev": "tsx src/main.ts",
+    "build": "tsc",
+    "start": "node dist/main.js",
+    "typecheck": "tsc --noEmit",
+    "lint": "tsc --noEmit",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "cli",
+    "browser",
+    "web",
+    "ai",
+    "playwright"
+  ],
+  "author": "jackwener",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jackwener/opencli.git"
+  },
+  "dependencies": {
+    "chalk": "^5.3.0",
+    "cli-table3": "^0.6.5",
+    "commander": "^13.1.0",
+    "js-yaml": "^4.1.0"
+  },
+  "devDependencies": {
+    "@playwright/mcp": "^0.0.68",
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.13.10",
+    "tsx": "^4.19.3",
+    "typescript": "^5.8.2"
+  }
+}

package/src/bilibili.ts

@@ -0,0 +1,111 @@
+/**
+ * Bilibili shared helpers: WBI signing, authenticated fetch, nav data, UID resolution.
+ */
+
+const MIXIN_KEY_ENC_TAB = [
+  46,47,18,2,53,8,23,32,15,50,10,31,58,3,45,35,27,43,5,49,
+  33,9,42,19,29,28,14,39,12,38,41,13,37,48,7,16,24,55,40,
+  61,26,17,0,1,60,51,30,4,22,25,54,21,56,59,6,63,57,62,11,
+  36,20,34,44,52,
+];
+
+export function stripHtml(s: string): string {
+  return s.replace(/<[^>]+>/g, '').replace(/&[a-z]+;/gi, ' ').trim();
+}
+
+export function payloadData(payload: any): any {
+  return payload?.data ?? payload;
+}
+
+async function getNavData(page: any): Promise<any> {
+  return page.evaluate(`
+    async () => {
+      const res = await fetch('https://api.bilibili.com/x/web-interface/nav', { credentials: 'include' });
+      return await res.json();
+    }
+  `);
+}
+
+async function getWbiKeys(page: any): Promise<{ imgKey: string; subKey: string }> {
+  const nav = await getNavData(page);
+  const wbiImg = nav?.data?.wbi_img ?? {};
+  const imgUrl = wbiImg.img_url ?? '';
+  const subUrl = wbiImg.sub_url ?? '';
+  const imgKey = imgUrl.split('/').pop()?.split('.')[0] ?? '';
+  const subKey = subUrl.split('/').pop()?.split('.')[0] ?? '';
+  return { imgKey, subKey };
+}
+
+function getMixinKey(imgKey: string, subKey: string): string {
+  const raw = imgKey + subKey;
+  return MIXIN_KEY_ENC_TAB.map(i => raw[i] || '').join('').slice(0, 32);
+}
+
+async function md5(text: string): Promise<string> {
+  const { createHash } = await import('node:crypto');
+  return createHash('md5').update(text).digest('hex');
+}
+
+export async function wbiSign(
+  page: any,
+  params: Record<string, any>,
+): Promise<Record<string, string>> {
+  const { imgKey, subKey } = await getWbiKeys(page);
+  const mixinKey = getMixinKey(imgKey, subKey);
+  const wts = Math.floor(Date.now() / 1000);
+  const sorted: Record<string, string> = {};
+  const allParams = { ...params, wts: String(wts) };
+  for (const key of Object.keys(allParams).sort()) {
+    sorted[key] = String(allParams[key]).replace(/[!'()*]/g, '');
+  }
+  const query = new URLSearchParams(sorted).toString();
+  const wRid = await md5(query + mixinKey);
+  sorted.w_rid = wRid;
+  return sorted;
+}
+
+export async function apiGet(
+  page: any,
+  path: string,
+  opts: { params?: Record<string, any>; signed?: boolean } = {},
+): Promise<any> {
+  const baseUrl = 'https://api.bilibili.com';
+  let params = opts.params ?? {};
+  if (opts.signed) {
+    params = await wbiSign(page, params);
+  }
+  const qs = new URLSearchParams(
+    Object.fromEntries(Object.entries(params).map(([k, v]) => [k, String(v)])),
+  );
+  const url = `${baseUrl}${path}?${qs}`;
+  return fetchJson(page, url);
+}
+
+export async function fetchJson(page: any, url: string): Promise<any> {
+  const escapedUrl = url.replace(/"/g, '\\"');
+  return page.evaluate(`
+    async () => {
+      const res = await fetch("${escapedUrl}", { credentials: "include" });
+      return await res.json();
+    }
+  `);
+}
+
+export async function getSelfUid(page: any): Promise<string> {
+  const nav = await getNavData(page);
+  const mid = nav?.data?.mid;
+  if (!mid) throw new Error('Not logged in to Bilibili');
+  return String(mid);
+}
+
+export async function resolveUid(page: any, input: string): Promise<string> {
+  if (/^\d+$/.test(input)) return input;
+  // Search for user by name
+  const payload = await apiGet(page, '/x/web-interface/wbi/search/type', {
+    params: { search_type: 'bili_user', keyword: input },
+    signed: true,
+  });
+  const results = payload?.data?.result ?? [];
+  if (results.length > 0) return String(results[0].mid);
+  throw new Error(`Cannot resolve UID for: ${input}`);
+}