@jackwener/opencli 1.6.0 → 1.6.2

package/autoresearch/eval-publish.ts

@@ -0,0 +1,238 @@
+#!/usr/bin/env npx tsx
+/**
+ * Layer 5: Publish Testing — end-to-end content creation via operate commands
+ *
+ * Tests the full chain: read content → navigate to platform → fill title+body → (optionally) publish → verify → cleanup
+ *
+ * Task types:
+ *   fill-only: navigate + fill fields + verify content was entered (safe, no side effects)
+ *   publish:   full publish + verify + cleanup (deletes the post after verification)
+ *
+ * Usage:
+ *   npx tsx autoresearch/eval-publish.ts                        # Run all tasks
+ *   npx tsx autoresearch/eval-publish.ts --task twitter-fill     # Run single task
+ *   npx tsx autoresearch/eval-publish.ts --type fill-only        # Run only fill tasks (safe)
+ *   npx tsx autoresearch/eval-publish.ts --type publish          # Run only publish tasks (destructive)
+ *   npx tsx autoresearch/eval-publish.ts --platform twitter      # Run only twitter tasks
+ */
+
+import { execSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PROJECT_ROOT = join(__dirname, '..');
+const TASKS_FILE = join(__dirname, 'publish-tasks.json');
+const RESULTS_DIR = join(__dirname, 'results');
+
+interface PublishTask {
+  name: string;
+  platform: string;
+  type: 'fill-only' | 'publish';
+  description: string;
+  steps: string[];
+  judge: JudgeCriteria;
+  cleanup?: string[];
+  note?: string;
+}
+
+type JudgeCriteria =
+  | { type: 'contains'; value: string }
+  | { type: 'arrayMinLength'; minLength: number }
+  | { type: 'nonEmpty' }
+  | { type: 'matchesPattern'; pattern: string };
+
+interface TaskResult {
+  name: string;
+  platform: string;
+  taskType: 'fill-only' | 'publish';
+  passed: boolean;
+  duration: number;
+  cleanupResult?: string;
+  error?: string;
+}
+
+function judge(criteria: JudgeCriteria, output: string): boolean {
+  try {
+    switch (criteria.type) {
+      case 'contains':
+        return output.toLowerCase().includes(criteria.value.toLowerCase());
+      case 'arrayMinLength': {
+        try {
+          const arr = JSON.parse(output);
+          if (Array.isArray(arr)) return arr.length >= criteria.minLength;
+        } catch { /* not JSON */ }
+        return false;
+      }
+      case 'nonEmpty':
+        return output.trim().length > 0 && output.trim() !== 'null' && output.trim() !== 'undefined';
+      case 'matchesPattern':
+        return new RegExp(criteria.pattern, 'i').test(output);
+      default:
+        return false;
+    }
+  } catch {
+    return false;
+  }
+}
+
+function runCommand(cmd: string, timeout = 30000): string {
+  const localCmd = cmd.replace(/^opencli /, `node dist/main.js `);
+  try {
+    return execSync(localCmd, {
+      cwd: PROJECT_ROOT,
+      timeout,
+      encoding: 'utf-8',
+      env: process.env,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    }).trim();
+  } catch (err: any) {
+    return err.stdout?.trim() || err.stderr?.trim() || '';
+  }
+}
+
+function runTask(task: PublishTask): TaskResult {
+  const start = Date.now();
+
+  try {
+    // Run main steps
+    let lastOutput = '';
+    for (let i = 0; i < task.steps.length; i++) {
+      const step = task.steps[i];
+      process.stderr.write(`    step ${i + 1}/${task.steps.length}: ${step.slice(0, 60)}...\n`);
+      lastOutput = runCommand(step, 45000);
+    }
+
+    const passed = judge(task.judge, lastOutput);
+
+    // Run cleanup steps (if publish type and cleanup defined)
+    let cleanupResult: string | undefined;
+    if (task.cleanup && task.cleanup.length > 0) {
+      process.stderr.write(`    cleanup: ${task.cleanup.length} steps...\n`);
+      let cleanupOutput = '';
+      for (const step of task.cleanup) {
+        cleanupOutput = runCommand(step, 30000);
+      }
+      cleanupResult = cleanupOutput.slice(0, 100);
+    }
+
+    return {
+      name: task.name,
+      platform: task.platform,
+      taskType: task.type,
+      passed,
+      duration: Date.now() - start,
+      cleanupResult,
+      error: passed ? undefined : `Output: ${lastOutput.slice(0, 150)}`,
+    };
+  } catch (err: any) {
+    return {
+      name: task.name,
+      platform: task.platform,
+      taskType: task.type,
+      passed: false,
+      duration: Date.now() - start,
+      error: err.message?.slice(0, 150),
+    };
+  }
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const singleTask = args.includes('--task') ? args[args.indexOf('--task') + 1] : null;
+  const filterType = args.includes('--type') ? args[args.indexOf('--type') + 1] : null;
+  const filterPlatform = args.includes('--platform') ? args[args.indexOf('--platform') + 1] : null;
+
+  const allTasks: PublishTask[] = JSON.parse(readFileSync(TASKS_FILE, 'utf-8'));
+  let tasks = allTasks;
+
+  if (singleTask) tasks = tasks.filter(t => t.name === singleTask);
+  if (filterType) tasks = tasks.filter(t => t.type === filterType);
+  if (filterPlatform) tasks = tasks.filter(t => t.platform === filterPlatform);
+
+  if (tasks.length === 0) {
+    console.error(`No tasks matched filters: task=${singleTask}, type=${filterType}, platform=${filterPlatform}`);
+    process.exit(1);
+  }
+
+  const fillTasks = tasks.filter(t => t.type === 'fill-only');
+  const publishTasks = tasks.filter(t => t.type === 'publish');
+
+  console.log(`\n📝 Layer 5: Publish Testing — ${tasks.length} tasks`);
+  console.log(`   fill-only: ${fillTasks.length} | publish: ${publishTasks.length}`);
+  console.log(`   platforms: ${[...new Set(tasks.map(t => t.platform))].join(', ')}\n`);
+
+  const results: TaskResult[] = [];
+
+  for (let i = 0; i < tasks.length; i++) {
+    const task = tasks[i];
+    const icon = task.type === 'publish' ? '🚀' : '📋';
+    process.stdout.write(`  [${i + 1}/${tasks.length}] ${icon} ${task.name} (${task.platform})...`);
+
+    const result = runTask(task);
+    results.push(result);
+
+    const status = result.passed ? '✓' : '✗';
+    const cleanup = result.cleanupResult ? ` [cleanup: ${result.cleanupResult.slice(0, 30)}]` : '';
+    console.log(` ${status} (${(result.duration / 1000).toFixed(1)}s)${cleanup}`);
+
+    // Close browser between tasks for clean state
+    if (i < tasks.length - 1) {
+      try { runCommand('opencli operate close'); } catch { /* ignore */ }
+    }
+  }
+
+  // Final close
+  try { runCommand('opencli operate close'); } catch { /* ignore */ }
+
+  // Summary
+  const totalPassed = results.filter(r => r.passed).length;
+  const fillPassed = results.filter(r => r.taskType === 'fill-only' && r.passed).length;
+  const publishPassed = results.filter(r => r.taskType === 'publish' && r.passed).length;
+  const totalDuration = results.reduce((s, r) => s + r.duration, 0);
+
+  const fillTotal = results.filter(r => r.taskType === 'fill-only').length;
+  const publishTotal = results.filter(r => r.taskType === 'publish').length;
+
+  console.log(`\n${'─'.repeat(50)}`);
+  console.log(`  Score:  ${totalPassed}/${results.length}`);
+  console.log(`    fill-only: ${fillPassed}/${fillTotal}`);
+  console.log(`    publish:   ${publishPassed}/${publishTotal}`);
+  console.log(`  Time:   ${Math.round(totalDuration / 1000)}s`);
+
+  // Platform breakdown
+  const platforms = [...new Set(results.map(r => r.platform))];
+  for (const p of platforms) {
+    const pr = results.filter(r => r.platform === p);
+    const pp = pr.filter(r => r.passed).length;
+    console.log(`    ${p}: ${pp}/${pr.length}`);
+  }
+
+  const failures = results.filter(r => !r.passed);
+  if (failures.length > 0) {
+    console.log(`\n  Failures:`);
+    for (const f of failures) {
+      console.log(`    ✗ ${f.name} [${f.platform}/${f.taskType}]: ${f.error ?? 'unknown'}`);
+    }
+  }
+  console.log('');
+
+  // Save result
+  mkdirSync(RESULTS_DIR, { recursive: true });
+  const existing = readdirSync(RESULTS_DIR).filter(f => f.startsWith('publish-')).length;
+  const roundNum = String(existing + 1).padStart(3, '0');
+  const resultPath = join(RESULTS_DIR, `publish-${roundNum}.json`);
+  writeFileSync(resultPath, JSON.stringify({
+    timestamp: new Date().toISOString(),
+    score: `${totalPassed}/${results.length}`,
+    fillScore: `${fillPassed}/${fillTotal}`,
+    publishScore: `${publishPassed}/${publishTotal}`,
+    duration: `${Math.round(totalDuration / 1000)}s`,
+    tasks: results,
+  }, null, 2), 'utf-8');
+  console.log(`  Results saved to: ${resultPath}`);
+  console.log(`\nSCORE=${totalPassed}/${results.length}`);
+}
+
+main();

package/autoresearch/eval-save.ts

@@ -0,0 +1,249 @@
+#!/usr/bin/env npx tsx
+/**
+ * Layer 4: Save as CLI Testing — "Save as CLI" Pipeline
+ *
+ * Tests the full operate init → write adapter → operate verify flow.
+ * Validates that browser exploration can be crystallized into reusable CLI adapters.
+ *
+ * Usage:
+ *   npx tsx autoresearch/eval-save.ts              # Run all tasks
+ *   npx tsx autoresearch/eval-save.ts --task hn-top # Run single task
+ */
+
+import { execSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdirSync, readdirSync, existsSync, rmSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { homedir } from 'node:os';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const TASKS_FILE = join(__dirname, 'save-tasks.json');
+const RESULTS_DIR = join(__dirname, 'results');
+const USER_CLIS_DIR = join(homedir(), '.opencli', 'clis');
+
+interface SaveTask {
+  name: string;
+  site: string;
+  command: string;
+  /** Inline adapter code (simple tasks) */
+  adapter?: string;
+  /** Path to adapter file relative to autoresearch/ dir (complex tasks — avoids JSON escape issues) */
+  adapterFile?: string;
+  judge: JudgeCriteria;
+  set?: 'test';
+  note?: string;
+}
+
+type JudgeCriteria =
+  | { type: 'contains'; value: string }
+  | { type: 'arrayMinLength'; minLength: number }
+  | { type: 'nonEmpty' }
+  | { type: 'matchesPattern'; pattern: string };
+
+interface TaskResult {
+  name: string;
+  phase: 'init' | 'write' | 'verify' | 'judge';
+  passed: boolean;
+  duration: number;
+  error?: string;
+  set: 'train' | 'test';
+}
+
+function judge(criteria: JudgeCriteria, output: string): boolean {
+  try {
+    switch (criteria.type) {
+      case 'contains':
+        return output.toLowerCase().includes(criteria.value.toLowerCase());
+      case 'arrayMinLength': {
+        // operate verify outputs table text; try JSON parse first, then count non-empty lines
+        try {
+          const arr = JSON.parse(output);
+          if (Array.isArray(arr)) return arr.length >= criteria.minLength;
+        } catch { /* not JSON — try line counting */ }
+        // Table output: count data rows (skip header, separator, empty lines)
+        const lines = output.split('\n').filter(l => l.trim() && !l.startsWith('─') && !l.startsWith('┌') && !l.startsWith('└') && !l.startsWith('├'));
+        // Subtract header row
+        const dataLines = lines.length > 1 ? lines.length - 1 : 0;
+        return dataLines >= criteria.minLength;
+      }
+      case 'nonEmpty':
+        return output.trim().length > 0 && output.trim() !== 'null' && output.trim() !== 'undefined';
+      case 'matchesPattern':
+        return new RegExp(criteria.pattern).test(output);
+      default:
+        return false;
+    }
+  } catch {
+    return false;
+  }
+}
+
+const PROJECT_ROOT = join(__dirname, '..');
+
+/** Run a command, using local dist/main.js instead of global opencli for consistency */
+function runCommand(cmd: string, timeout = 30000): string {
+  // Use local build so tests always run against the current source
+  const localCmd = cmd.replace(/^opencli /, `node dist/main.js `);
+  try {
+    return execSync(localCmd, {
+      cwd: PROJECT_ROOT,
+      timeout,
+      encoding: 'utf-8',
+      env: process.env,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    }).trim();
+  } catch (err: any) {
+    return err.stdout?.trim() || err.stderr?.trim() || '';
+  }
+}
+
+function cleanupAdapter(site: string, command: string): void {
+  const siteDir = join(USER_CLIS_DIR, site);
+  const filePath = join(siteDir, `${command}.ts`);
+  try {
+    if (existsSync(filePath)) rmSync(filePath);
+    // Remove site dir if empty
+    if (existsSync(siteDir)) {
+      const remaining = readdirSync(siteDir);
+      if (remaining.length === 0) rmSync(siteDir, { recursive: true });
+    }
+  } catch { /* best effort */ }
+}
+
+function runTask(task: SaveTask): TaskResult {
+  const start = Date.now();
+  const { site, command } = task;
+  const adapterDir = join(USER_CLIS_DIR, site);
+  const adapterPath = join(adapterDir, `${command}.ts`);
+
+  // Cleanup any leftover from previous runs
+  cleanupAdapter(site, command);
+
+  try {
+    // Phase 1: init — create scaffold
+    const initOutput = runCommand(`opencli operate init ${site}/${command}`);
+    if (!existsSync(adapterPath)) {
+      return {
+        name: task.name, phase: 'init', passed: false,
+        duration: Date.now() - start,
+        error: `init failed: file not created. Output: ${initOutput.slice(0, 100)}`,
+        set: task.set === 'test' ? 'test' : 'train',
+      };
+    }
+
+    // Phase 2: write — overwrite scaffold with real adapter code
+    if (task.adapterFile) {
+      // Read from file (complex adapters — avoids JSON string escape issues)
+      const srcPath = join(__dirname, task.adapterFile);
+      const code = readFileSync(srcPath, 'utf-8');
+      writeFileSync(adapterPath, code, 'utf-8');
+    } else if (task.adapter) {
+      writeFileSync(adapterPath, task.adapter, 'utf-8');
+    }
+
+    // Phase 3: verify — run the adapter via operate verify
+    const verifyOutput = runCommand(
+      `opencli operate verify ${site}/${command}`,
+      45000, // longer timeout for network calls
+    );
+
+    if (verifyOutput.includes('✗ Adapter failed')) {
+      return {
+        name: task.name, phase: 'verify', passed: false,
+        duration: Date.now() - start,
+        error: `verify failed: ${verifyOutput.slice(0, 200)}`,
+        set: task.set === 'test' ? 'test' : 'train',
+      };
+    }
+
+    // Phase 4: judge — check output quality
+    const passed = judge(task.judge, verifyOutput);
+
+    return {
+      name: task.name,
+      phase: 'judge',
+      passed,
+      duration: Date.now() - start,
+      error: passed ? undefined : `Judge failed on output: ${verifyOutput.slice(0, 150)}`,
+      set: task.set === 'test' ? 'test' : 'train',
+    };
+  } catch (err: any) {
+    return {
+      name: task.name, phase: 'verify', passed: false,
+      duration: Date.now() - start,
+      error: err.message?.slice(0, 150),
+      set: task.set === 'test' ? 'test' : 'train',
+    };
+  } finally {
+    // Always cleanup test adapters
+    cleanupAdapter(site, command);
+  }
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const singleTask = args.includes('--task') ? args[args.indexOf('--task') + 1] : null;
+
+  const allTasks: SaveTask[] = JSON.parse(readFileSync(TASKS_FILE, 'utf-8'));
+  const tasks = singleTask ? allTasks.filter(t => t.name === singleTask) : allTasks;
+
+  if (tasks.length === 0) {
+    console.error(`Task "${singleTask}" not found.`);
+    process.exit(1);
+  }
+
+  console.log(`\n🧪 Layer 4: Save as CLI — ${tasks.length} tasks\n`);
+
+  const results: TaskResult[] = [];
+
+  for (let i = 0; i < tasks.length; i++) {
+    const task = tasks[i];
+    process.stdout.write(`  [${i + 1}/${tasks.length}] ${task.name}...`);
+
+    const result = runTask(task);
+    results.push(result);
+
+    const icon = result.passed ? '✓' : '✗';
+    const phase = result.passed ? '' : ` (${result.phase})`;
+    console.log(` ${icon}${phase} (${(result.duration / 1000).toFixed(1)}s)`);
+  }
+
+  // Summary
+  const trainResults = results.filter(r => r.set === 'train');
+  const testResults = results.filter(r => r.set === 'test');
+  const totalPassed = results.filter(r => r.passed).length;
+  const trainPassed = trainResults.filter(r => r.passed).length;
+  const testPassed = testResults.filter(r => r.passed).length;
+  const totalDuration = results.reduce((s, r) => s + r.duration, 0);
+
+  console.log(`\n${'─'.repeat(50)}`);
+  console.log(`  Score:  ${totalPassed}/${results.length} (train: ${trainPassed}/${trainResults.length}, test: ${testPassed}/${testResults.length})`);
+  console.log(`  Time:   ${Math.round(totalDuration / 1000)}s`);
+
+  const failures = results.filter(r => !r.passed);
+  if (failures.length > 0) {
+    console.log(`\n  Failures:`);
+    for (const f of failures) {
+      console.log(`    ✗ ${f.name} [${f.phase}]: ${f.error ?? 'unknown'}`);
+    }
+  }
+  console.log('');
+
+  // Save result
+  mkdirSync(RESULTS_DIR, { recursive: true });
+  const existing = readdirSync(RESULTS_DIR).filter(f => f.startsWith('save-')).length;
+  const roundNum = String(existing + 1).padStart(3, '0');
+  const resultPath = join(RESULTS_DIR, `save-${roundNum}.json`);
+  writeFileSync(resultPath, JSON.stringify({
+    timestamp: new Date().toISOString(),
+    score: `${totalPassed}/${results.length}`,
+    trainScore: `${trainPassed}/${trainResults.length}`,
+    testScore: `${testPassed}/${testResults.length}`,
+    duration: `${Math.round(totalDuration / 1000)}s`,
+    tasks: results,
+  }, null, 2), 'utf-8');
+  console.log(`  Results saved to: ${resultPath}`);
+  console.log(`\nSCORE=${totalPassed}/${results.length}`);
+}
+
+main();

package/autoresearch/eval-skill.ts

@@ -18,7 +18,6 @@ import { join, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
-const TASKS_FILE = join(__dirname, 'skill-tasks.yaml');
 const RESULTS_DIR = join(__dirname, 'results');
 const SKILL_PATH = join(__dirname, '..', 'skills', 'opencli-operate', 'SKILL.md');
 
@@ -160,13 +159,20 @@ Always close the browser with 'opencli operate close' when done.`;
 }
 
 function extractVerdict(text: string): { success: boolean; explanation: string } {
-  // Try to find {"success": ...} JSON in the text
-  const jsonMatches = text.match(/\{"success"\s*:\s*(true|false)\s*,\s*"explanation"\s*:\s*"([^"]*)"\s*\}/g);
-  if (jsonMatches) {
-    const last = jsonMatches[jsonMatches.length - 1];
-    try {
-      return JSON.parse(last);
-    } catch { /* fall through */ }
+  // Try to find and parse {"success": ...} JSON from the last occurrence
+  const idx = text.lastIndexOf('{"success"');
+  if (idx !== -1) {
+    // Find the matching closing brace (handle escaped quotes in explanation)
+    const sub = text.slice(idx);
+    let braceCount = 0;
+    let end = -1;
+    for (let i = 0; i < sub.length; i++) {
+      if (sub[i] === '{') braceCount++;
+      else if (sub[i] === '}') { braceCount--; if (braceCount === 0) { end = i + 1; break; } }
+    }
+    if (end > 0) {
+      try { return JSON.parse(sub.slice(0, end)); } catch { /* fall through */ }
+    }
   }
 
   // Fallback: check for success indicators in text