@jackwener/opencli 0.1.0

package/src/registry.ts

@@ -0,0 +1,87 @@
+/**
+ * Core registry: Strategy enum, Arg/CliCommand interfaces, cli() registration.
+ */
+
+export enum Strategy {
+  PUBLIC = 'public',
+  COOKIE = 'cookie',
+  HEADER = 'header',
+  INTERCEPT = 'intercept',
+  UI = 'ui',
+}
+
+export interface Arg {
+  name: string;
+  type?: string;
+  default?: any;
+  required?: boolean;
+  help?: string;
+  choices?: string[];
+}
+
+export interface CliCommand {
+  site: string;
+  name: string;
+  description: string;
+  domain?: string;
+  strategy?: Strategy;
+  browser?: boolean;
+  args: Arg[];
+  columns?: string[];
+  func?: (page: any, kwargs: Record<string, any>, debug?: boolean) => Promise<any>;
+  pipeline?: any[];
+  timeoutSeconds?: number;
+  source?: string;
+}
+
+export interface CliOptions {
+  site: string;
+  name: string;
+  description?: string;
+  domain?: string;
+  strategy?: Strategy;
+  browser?: boolean;
+  args?: Arg[];
+  columns?: string[];
+  func?: (page: any, kwargs: Record<string, any>, debug?: boolean) => Promise<any>;
+  pipeline?: any[];
+  timeoutSeconds?: number;
+}
+
+const _registry = new Map<string, CliCommand>();
+
+export function cli(opts: CliOptions): CliCommand {
+  const cmd: CliCommand = {
+    site: opts.site,
+    name: opts.name,
+    description: opts.description ?? '',
+    domain: opts.domain,
+    strategy: opts.strategy ?? (opts.browser === false ? Strategy.PUBLIC : Strategy.COOKIE),
+    browser: opts.browser ?? (opts.strategy === Strategy.PUBLIC ? false : true),
+    args: opts.args ?? [],
+    columns: opts.columns,
+    func: opts.func,
+    pipeline: opts.pipeline,
+    timeoutSeconds: opts.timeoutSeconds,
+  };
+
+  const key = fullName(cmd);
+  _registry.set(key, cmd);
+  return cmd;
+}
+
+export function getRegistry(): Map<string, CliCommand> {
+  return _registry;
+}
+
+export function fullName(cmd: CliCommand): string {
+  return `${cmd.site}/${cmd.name}`;
+}
+
+export function strategyLabel(cmd: CliCommand): string {
+  return cmd.strategy ?? 'public';
+}
+
+export function registerCommand(cmd: CliCommand): void {
+  _registry.set(fullName(cmd), cmd);
+}

package/src/runtime.ts

@@ -0,0 +1,36 @@
+/**
+ * Runtime utilities: timeouts and browser session management.
+ */
+
+export const DEFAULT_BROWSER_CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
+export const DEFAULT_BROWSER_COMMAND_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_COMMAND_TIMEOUT ?? '45', 10);
+export const DEFAULT_BROWSER_EXPLORE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_EXPLORE_TIMEOUT ?? '120', 10);
+export const DEFAULT_BROWSER_SMOKE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_SMOKE_TIMEOUT ?? '60', 10);
+
+export async function runWithTimeout<T>(
+  promise: Promise<T>,
+  opts: { timeout: number; label?: string },
+): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`${opts.label ?? 'Operation'} timed out after ${opts.timeout}s`));
+    }, opts.timeout * 1000);
+
+    promise
+      .then((result) => { clearTimeout(timer); resolve(result); })
+      .catch((err) => { clearTimeout(timer); reject(err); });
+  });
+}
+
+export async function browserSession<T>(
+  BrowserFactory: new () => any,
+  fn: (page: any) => Promise<T>,
+): Promise<T> {
+  const mcp = new BrowserFactory();
+  try {
+    const page = await mcp.connect({ timeout: DEFAULT_BROWSER_CONNECT_TIMEOUT });
+    return await fn(page);
+  } finally {
+    await mcp.close().catch(() => {});
+  }
+}

package/src/scaffold.ts

@@ -0,0 +1,2 @@
+/** Scaffold a draft CLI from endpoint evidence. */
+export function scaffoldCli(opts: any): any { return { ok: true }; }

package/src/smoke.ts

@@ -0,0 +1,2 @@
+/** Smoke testing, verification, register, promote, explore, synthesize, scaffold, generate — stubs. */
+export async function runSmoke(cmd: any, page: any, args?: any): Promise<any> { return { ok: true, result: null }; }

package/src/snapshotFormatter.ts

@@ -0,0 +1,51 @@
+/**
+ * Aria snapshot formatter: parses Playwright MCP snapshot text into clean format.
+ */
+
+export interface FormatOptions {
+  interactive?: boolean;
+  compact?: boolean;
+  maxDepth?: number;
+}
+
+export function formatSnapshot(raw: string, opts: FormatOptions = {}): string {
+  if (!raw || typeof raw !== 'string') return '';
+  const lines = raw.split('\n');
+  const result: string[] = [];
+  let refCounter = 0;
+
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    const indent = line.length - line.trimStart().length;
+    const depth = Math.floor(indent / 2);
+    if (opts.maxDepth && depth > opts.maxDepth) continue;
+
+    let content = line.trimStart();
+
+    // Skip non-interactive elements in interactive mode
+    if (opts.interactive) {
+      const interactiveRoles = ['button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'tab', 'menuitem', 'option'];
+      const role = content.split(/[\s[]/)[0]?.toLowerCase() ?? '';
+      if (!interactiveRoles.some(r => role.includes(r)) && depth > 1) continue;
+    }
+
+    // Compact: strip verbose role descriptions
+    if (opts.compact) {
+      content = content
+        .replace(/\s*\[.*?\]\s*/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+    }
+
+    // Assign refs to interactive elements
+    const interactivePattern = /^(button|link|textbox|checkbox|radio|combobox|tab|menuitem|option)\b/i;
+    if (interactivePattern.test(content)) {
+      refCounter++;
+      content = `[@${refCounter}] ${content}`;
+    }
+
+    result.push('  '.repeat(depth) + content);
+  }
+
+  return result.join('\n');
+}

package/src/synthesize.ts

@@ -0,0 +1,210 @@
+/**
+ * Synthesize: turn explore capabilities into ready-to-use CLI definitions.
+ *
+ * Takes the structured capabilities from Deep Explore and generates
+ * YAML pipeline files that can be directly registered as CLI commands.
+ *
+ * This is the bridge between discovery (explore) and usability (CLI).
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import yaml from 'js-yaml';
+
+export function synthesizeFromExplore(target: string, opts: any = {}): any {
+  const exploreDir = fs.existsSync(target) ? target : path.join('.opencli', 'explore', target);
+  if (!fs.existsSync(exploreDir)) throw new Error(`Explore dir not found: ${target}`);
+
+  const manifest = JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8'));
+  const capabilities = JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8'));
+  const endpoints = JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8'));
+  const auth = JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8'));
+
+  const targetDir = opts.outDir ?? path.join(exploreDir, 'candidates');
+  fs.mkdirSync(targetDir, { recursive: true });
+
+  const site = manifest.site;
+  const topN = opts.top ?? 5;
+  const candidates: any[] = [];
+
+  // Sort capabilities by confidence
+  const sortedCaps = [...capabilities]
+    .sort((a: any, b: any) => (b.confidence ?? 0) - (a.confidence ?? 0))
+    .slice(0, topN);
+
+  for (const cap of sortedCaps) {
+    // Find the matching endpoint for more detail
+    const endpoint = endpoints.find((ep: any) => ep.pattern === cap.endpoint) ??
+                     endpoints[0];
+
+    const candidate = buildCandidateYaml(site, manifest, cap, endpoint);
+    const fileName = `${cap.name}.yaml`;
+    const filePath = path.join(targetDir, fileName);
+    fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 }));
+
+    candidates.push({
+      name: cap.name,
+      path: filePath,
+      strategy: cap.strategy,
+      endpoint: cap.endpoint,
+      confidence: cap.confidence,
+      columns: candidate.yaml.columns,
+    });
+  }
+
+  const index = {
+    site,
+    target_url: manifest.target_url,
+    generated_from: exploreDir,
+    candidate_count: candidates.length,
+    candidates,
+  };
+  fs.writeFileSync(path.join(targetDir, 'candidates.json'), JSON.stringify(index, null, 2));
+
+  return {
+    site,
+    explore_dir: exploreDir,
+    out_dir: targetDir,
+    candidate_count: candidates.length,
+    candidates,
+  };
+}
+
+/** Volatile params to strip from generated URLs */
+const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']);
+const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']);
+const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']);
+const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']);
+
+/**
+ * Build a clean templated URL from a raw API URL.
+ * - Strips volatile params (w_rid, wts, etc.)
+ * - Templates search, limit, and pagination params
+ * - Builds URL string manually to avoid URL encoding of ${{ }} expressions
+ */
+function buildTemplatedUrl(rawUrl: string, cap: any, endpoint: any): string {
+  try {
+    const u = new URL(rawUrl);
+    const base = `${u.protocol}//${u.host}${u.pathname}`;
+    const params: Array<[string, string]> = [];
+
+    const hasKeyword = cap.recommendedArgs?.some((a: any) => a.name === 'keyword');
+
+    u.searchParams.forEach((v, k) => {
+      // Skip volatile params
+      if (VOLATILE_PARAMS.has(k)) return;
+
+      // Template known param types
+      if (hasKeyword && SEARCH_PARAM_NAMES.has(k)) {
+        params.push([k, '${{ args.keyword }}']);
+      } else if (LIMIT_PARAM_NAMES.has(k)) {
+        params.push([k, '${{ args.limit | default(20) }}']);
+      } else if (PAGE_PARAM_NAMES.has(k)) {
+        params.push([k, '${{ args.page | default(1) }}']);
+      } else {
+        params.push([k, v]);
+      }
+    });
+
+    if (params.length === 0) return base;
+    return base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&');
+  } catch {
+    return rawUrl;
+  }
+}
+
+/**
+ * Build a YAML pipeline definition from a capability + endpoint.
+ */
+function buildCandidateYaml(site: string, manifest: any, cap: any, endpoint: any): { name: string; yaml: any } {
+  const needsBrowser = cap.strategy !== 'public';
+  const pipeline: any[] = [];
+
+  // Step 1: Navigate (if browser-based)
+  if (needsBrowser) {
+    pipeline.push({ navigate: manifest.target_url });
+  }
+
+  // Step 2: Fetch the API — build a clean URL with templates
+  const rawUrl = endpoint?.url ?? manifest.target_url;
+  const fetchStep: any = { url: buildTemplatedUrl(rawUrl, cap, endpoint) };
+
+  pipeline.push({ fetch: fetchStep });
+
+  // Step 3: Select the item path
+  if (cap.itemPath) {
+    pipeline.push({ select: cap.itemPath });
+  }
+
+  // Step 4: Map fields to columns
+  const mapStep: Record<string, string> = {};
+  const columns = cap.recommendedColumns ?? ['title', 'url'];
+
+  // Add a rank column if not doing search
+  if (!cap.recommendedArgs?.some((a: any) => a.name === 'keyword')) {
+    mapStep['rank'] = '${{ index + 1 }}';
+  }
+
+  // Build field mappings from the endpoint's detected fields
+  const detectedFields = endpoint?.detectedFields ?? {};
+  for (const col of columns) {
+    const fieldPath = detectedFields[col];
+    if (fieldPath) {
+      mapStep[col] = `\${{ item.${fieldPath} }}`;
+    } else {
+      mapStep[col] = `\${{ item.${col} }}`;
+    }
+  }
+
+  pipeline.push({ map: mapStep });
+
+  // Step 5: Limit
+  pipeline.push({ limit: '${{ args.limit | default(20) }}' });
+
+  // Build args definition
+  const argsDef: Record<string, any> = {};
+  for (const arg of cap.recommendedArgs ?? []) {
+    const def: any = { type: arg.type ?? 'str' };
+    if (arg.required) def.required = true;
+    if (arg.default != null) def.default = arg.default;
+    if (arg.name === 'keyword') def.description = 'Search keyword';
+    else if (arg.name === 'limit') def.description = 'Number of items to return';
+    else if (arg.name === 'page') def.description = 'Page number';
+    argsDef[arg.name] = def;
+  }
+
+  // Ensure limit arg always exists
+  if (!argsDef['limit']) {
+    argsDef['limit'] = { type: 'int', default: 20, description: 'Number of items to return' };
+  }
+
+  const allColumns = Object.keys(mapStep);
+
+  return {
+    name: cap.name,
+    yaml: {
+      site,
+      name: cap.name,
+      description: `${site} ${cap.name} (auto-generated)`,
+      domain: manifest.final_url ? new URL(manifest.final_url).hostname : undefined,
+      strategy: cap.strategy,
+      browser: needsBrowser,
+      args: argsDef,
+      pipeline,
+      columns: allColumns,
+    },
+  };
+}
+
+export function renderSynthesizeSummary(r: any): string {
+  const lines = [
+    'opencli synthesize: OK',
+    `Site: ${r.site}`,
+    `Source: ${r.explore_dir}`,
+    `Candidates: ${r.candidate_count}`,
+  ];
+  for (const c of r.candidates ?? []) {
+    lines.push(`  • ${c.name} (${c.strategy}, ${(c.confidence * 100).toFixed(0)}% confidence) → ${c.path}`);
+  }
+  return lines.join('\n');
+}

package/src/validate.ts

@@ -0,0 +1,55 @@
+/** Validate CLI definitions. */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import yaml from 'js-yaml';
+
+export function validateClisWithTarget(dirs: string[], target?: string): any {
+  const results: any[] = [];
+  let errors = 0; let warnings = 0; let files = 0;
+  for (const dir of dirs) {
+    if (!fs.existsSync(dir)) continue;
+    for (const site of fs.readdirSync(dir)) {
+      if (target && site !== target && !target.startsWith(site + '/')) continue;
+      const siteDir = path.join(dir, site);
+      if (!fs.statSync(siteDir).isDirectory()) continue;
+      for (const file of fs.readdirSync(siteDir)) {
+        if (!file.endsWith('.yaml') && !file.endsWith('.yml')) continue;
+        if (target && target.includes('/') && !target.endsWith(file.replace(/\.(yaml|yml)$/, ''))) continue;
+        files++;
+        const filePath = path.join(siteDir, file);
+        const r = validateYamlFile(filePath);
+        results.push(r);
+        errors += r.errors.length;
+        warnings += r.warnings.length;
+      }
+    }
+  }
+  return { ok: errors === 0, results, errors, warnings, files };
+}
+
+function validateYamlFile(filePath: string): any {
+  const errors: string[] = []; const warnings: string[] = [];
+  try {
+    const raw = fs.readFileSync(filePath, 'utf-8');
+    const def = yaml.load(raw) as any;
+    if (!def || typeof def !== 'object') { errors.push('Not a valid YAML object'); return { path: filePath, errors, warnings }; }
+    if (!def.site) errors.push('Missing "site"');
+    if (!def.name) errors.push('Missing "name"');
+    if (def.pipeline && !Array.isArray(def.pipeline)) errors.push('"pipeline" must be an array');
+    if (def.columns && !Array.isArray(def.columns)) errors.push('"columns" must be an array');
+    if (def.args && typeof def.args !== 'object') errors.push('"args" must be an object');
+  } catch (e: any) { errors.push(`YAML parse error: ${e.message}`); }
+  return { path: filePath, errors, warnings };
+}
+
+export function renderValidationReport(report: any): string {
+  const lines = [`opencli validate: ${report.ok ? 'PASS' : 'FAIL'}`, `Checked ${report.results.length} CLI(s) in ${report.files} file(s)`, `Errors: ${report.errors}  Warnings: ${report.warnings}`];
+  for (const r of report.results) {
+    if (r.errors.length > 0 || r.warnings.length > 0) {
+      lines.push(`\n${r.path}:`);
+      for (const e of r.errors) lines.push(`  ❌ ${e}`);
+      for (const w of r.warnings) lines.push(`  ⚠️  ${w}`);
+    }
+  }
+  return lines.join('\n');
+}

package/src/verify.ts

@@ -0,0 +1,9 @@
+/** Verification: validate + smoke. */
+import { validateClisWithTarget, renderValidationReport } from './validate.js';
+export async function verifyClis(opts: any): Promise<any> {
+  const report = validateClisWithTarget([opts.builtinClis, opts.userClis], opts.target);
+  return { ok: report.ok, validation: report, smoke: null };
+}
+export function renderVerifyReport(report: any): string {
+  return renderValidationReport(report.validation);
+}

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "Node16",
+    "moduleResolution": "Node16",
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": false,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "incremental": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "dist"]
+}