@jackwener/opencli 1.6.1 → 1.6.2

package/autoresearch/eval-v2ex.ts

@@ -0,0 +1,220 @@
+#!/usr/bin/env npx tsx
+/**
+ * V2EX Test Suite: Deterministic command testing against v2ex.com.
+ *
+ * 40 tasks across 5 difficulty layers:
+ *   L1 Atomic (10) → L2 Single Page (10) → L3 Multi-Step (10)
+ *   → L4 Write Ops (5) → L5 Complex Chain (5)
+ *
+ * Usage:
+ *   npx tsx autoresearch/eval-v2ex.ts                    # Run all tasks
+ *   npx tsx autoresearch/eval-v2ex.ts --task v2ex-hot-topics  # Run single task
+ *   npx tsx autoresearch/eval-v2ex.ts --layer 1          # Run only Layer 1 (atomic)
+ */
+
+import { execSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const TASKS_FILE = join(__dirname, 'v2ex-tasks.json');
+const RESULTS_DIR = join(__dirname, 'results');
+
+interface BrowseTask {
+  name: string;
+  steps: string[];
+  judge: JudgeCriteria;
+  set?: 'test';
+  note?: string;
+  _comment?: string;
+}
+
+type JudgeCriteria =
+  | { type: 'contains'; value: string }
+  | { type: 'arrayMinLength'; minLength: number }
+  | { type: 'nonEmpty' }
+  | { type: 'matchesPattern'; pattern: string };
+
+interface TaskResult {
+  name: string;
+  passed: boolean;
+  duration: number;
+  error?: string;
+  layer: string;
+}
+
+// Layer classification by task name prefix pattern
+function getLayer(name: string): string {
+  if (['v2ex-open-', 'v2ex-state-', 'v2ex-get-title', 'v2ex-click-tab', 'v2ex-scroll-down',
+       'v2ex-get-first-', 'v2ex-eval-extract', 'v2ex-get-url', 'v2ex-back-nav', 'v2ex-wait-'].some(p => name.startsWith(p)))
+    return 'L1-atomic';
+  if (['v2ex-hot-topics', 'v2ex-node-list', 'v2ex-topic-meta', 'v2ex-node-topics',
+       'v2ex-node-pagination', 'v2ex-tab-content', 'v2ex-topic-replies-extract',
+       'v2ex-topic-reply-count', 'v2ex-member-info', 'v2ex-search-results'].includes(name))
+    return 'L2-single-page';
+  if (['v2ex-click-topic-read', 'v2ex-click-author', 'v2ex-navigate-node', 'v2ex-pagination-page2',
+       'v2ex-topic-and-back', 'v2ex-tab-then-topic', 'v2ex-scroll-find-more',
+       'v2ex-node-to-topic', 'v2ex-multi-tab-compare', 'v2ex-topic-reply-to-author'].some(p => name.startsWith(p)))
+    return 'L3-multi-step';
+  if (['v2ex-reply-', 'v2ex-favorite-', 'v2ex-thank-', 'v2ex-create-'].some(p => name.startsWith(p)))
+    return 'L4-write';
+  if (['v2ex-collect-', 'v2ex-multi-node-', 'v2ex-topic-deep-', 'v2ex-cross-page-', 'v2ex-full-'].some(p => name.startsWith(p)))
+    return 'L5-complex';
+  return 'unknown';
+}
+
+function judge(criteria: JudgeCriteria, output: string): boolean {
+  try {
+    switch (criteria.type) {
+      case 'contains':
+        return output.toLowerCase().includes(criteria.value.toLowerCase());
+      case 'arrayMinLength': {
+        try {
+          const arr = JSON.parse(output);
+          if (Array.isArray(arr)) return arr.length >= criteria.minLength;
+        } catch { /* not JSON array */ }
+        return false;
+      }
+      case 'nonEmpty':
+        return output.trim().length > 0 && output.trim() !== 'null' && output.trim() !== 'undefined';
+      case 'matchesPattern':
+        return new RegExp(criteria.pattern).test(output);
+      default:
+        return false;
+    }
+  } catch {
+    return false;
+  }
+}
+
+function runCommand(cmd: string, timeout = 30000): string {
+  try {
+    return execSync(cmd, {
+      cwd: join(__dirname, '..'),
+      timeout,
+      encoding: 'utf-8',
+      env: process.env,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    }).trim();
+  } catch (err: any) {
+    return err.stdout?.trim() ?? '';
+  }
+}
+
+function runTask(task: BrowseTask): TaskResult {
+  const start = Date.now();
+  let lastOutput = '';
+
+  try {
+    for (const step of task.steps) {
+      lastOutput = runCommand(step);
+    }
+
+    const passed = judge(task.judge, lastOutput);
+
+    return {
+      name: task.name,
+      passed,
+      duration: Date.now() - start,
+      error: passed ? undefined : `Output: ${lastOutput.slice(0, 150)}`,
+      layer: getLayer(task.name),
+    };
+  } catch (err: any) {
+    return {
+      name: task.name,
+      passed: false,
+      duration: Date.now() - start,
+      error: err.message?.slice(0, 100),
+      layer: getLayer(task.name),
+    };
+  }
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const singleTask = args.includes('--task') ? args[args.indexOf('--task') + 1] : null;
+  const layerFilter = args.includes('--layer') ? args[args.indexOf('--layer') + 1] : null;
+
+  const raw = JSON.parse(readFileSync(TASKS_FILE, 'utf-8')) as (BrowseTask | { _comment: string })[];
+  const allTasks = raw.filter((t): t is BrowseTask => 'name' in t && 'steps' in t);
+
+  let tasks = allTasks;
+  if (singleTask) {
+    tasks = allTasks.filter(t => t.name === singleTask);
+  } else if (layerFilter) {
+    const prefix = `L${layerFilter}`;
+    tasks = allTasks.filter(t => getLayer(t.name).startsWith(prefix));
+  }
+
+  if (tasks.length === 0) {
+    console.error(singleTask ? `Task "${singleTask}" not found.` : `No tasks for layer ${layerFilter}.`);
+    process.exit(1);
+  }
+
+  console.log(`\n🔬 V2EX Test Suite — ${tasks.length} tasks\n`);
+
+  const results: TaskResult[] = [];
+
+  for (let i = 0; i < tasks.length; i++) {
+    const task = tasks[i];
+    process.stdout.write(`  [${i + 1}/${tasks.length}] ${task.name}...`);
+
+    const result = runTask(task);
+    results.push(result);
+
+    const icon = result.passed ? '✓' : '✗';
+    console.log(` ${icon} (${(result.duration / 1000).toFixed(1)}s)`);
+
+    // Close browser between tasks for clean state
+    if (i < tasks.length - 1) {
+      try { runCommand('opencli operate close'); } catch { /* ignore */ }
+    }
+  }
+
+  // Final close
+  try { runCommand('opencli operate close'); } catch { /* ignore */ }
+
+  // Summary by layer
+  const layers = [...new Set(results.map(r => r.layer))].sort();
+  const totalPassed = results.filter(r => r.passed).length;
+  const totalDuration = results.reduce((s, r) => s + r.duration, 0);
+
+  console.log(`\n${'─'.repeat(50)}`);
+  console.log(`  Total:  ${totalPassed}/${results.length}`);
+  for (const layer of layers) {
+    const layerResults = results.filter(r => r.layer === layer);
+    const layerPassed = layerResults.filter(r => r.passed).length;
+    console.log(`  ${layer}: ${layerPassed}/${layerResults.length}`);
+  }
+  console.log(`  Time:   ${Math.round(totalDuration / 60000)}min`);
+
+  const failures = results.filter(r => !r.passed);
+  if (failures.length > 0) {
+    console.log(`\n  Failures:`);
+    for (const f of failures) {
+      console.log(`    ✗ [${f.layer}] ${f.name}: ${f.error ?? 'unknown'}`);
+    }
+  }
+  console.log('');
+
+  // Save result
+  mkdirSync(RESULTS_DIR, { recursive: true });
+  const existing = readdirSync(RESULTS_DIR).filter(f => f.startsWith('v2ex-')).length;
+  const roundNum = String(existing + 1).padStart(3, '0');
+  const resultPath = join(RESULTS_DIR, `v2ex-${roundNum}.json`);
+  writeFileSync(resultPath, JSON.stringify({
+    timestamp: new Date().toISOString(),
+    score: `${totalPassed}/${results.length}`,
+    layers: Object.fromEntries(layers.map(l => {
+      const lr = results.filter(r => r.layer === l);
+      return [l, `${lr.filter(r => r.passed).length}/${lr.length}`];
+    })),
+    duration: `${Math.round(totalDuration / 60000)}min`,
+    tasks: results,
+  }, null, 2), 'utf-8');
+  console.log(`  Results saved to: ${resultPath}`);
+  console.log(`\nSCORE=${totalPassed}/${results.length}`);
+}
+
+main();

package/autoresearch/eval-zhihu.ts

@@ -0,0 +1,230 @@
+#!/usr/bin/env npx tsx
+/**
+ * Zhihu Test Suite: Deterministic command testing against v2ex.com.
+ *
+ * 40 tasks across 5 difficulty layers:
+ *   L1 Atomic (10) → L2 Single Page (10) → L3 Multi-Step (10)
+ *   → L4 Write Ops (5) → L5 Complex Chain (5)
+ *
+ * Usage:
+ *   npx tsx autoresearch/eval-v2ex.ts                    # Run all tasks
+ *   npx tsx autoresearch/eval-v2ex.ts --task zhihu-hot-topics  # Run single task
+ *   npx tsx autoresearch/eval-v2ex.ts --layer 1          # Run only Layer 1 (atomic)
+ */
+
+import { execSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdirSync, readdirSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const TASKS_FILE = join(__dirname, 'zhihu-tasks.json');
+const RESULTS_DIR = join(__dirname, 'results');
+
+interface BrowseTask {
+  name: string;
+  steps: string[];
+  judge: JudgeCriteria;
+  set?: 'test';
+  note?: string;
+  _comment?: string;
+}
+
+type JudgeCriteria =
+  | { type: 'contains'; value: string }
+  | { type: 'arrayMinLength'; minLength: number }
+  | { type: 'nonEmpty' }
+  | { type: 'matchesPattern'; pattern: string };
+
+interface TaskResult {
+  name: string;
+  passed: boolean;
+  duration: number;
+  error?: string;
+  layer: string;
+}
+
+// Layer classification by task name
+function getLayer(name: string): string {
+  const l1 = ['zhihu-open-home', 'zhihu-get-title', 'zhihu-state', 'zhihu-get-url', 'zhihu-scroll-down',
+    'zhihu-click-tab-hot', 'zhihu-back-navigation', 'zhihu-wait-page-load', 'zhihu-keys-escape', 'zhihu-screenshot'];
+  const l2 = ['zhihu-feed-titles', 'zhihu-hot-list', 'zhihu-hot-metrics', 'zhihu-nav-tabs',
+    'zhihu-feed-with-authors', 'zhihu-feed-types', 'zhihu-user-avatar', 'zhihu-search-input-exists'];
+  const l3 = ['zhihu-question-title', 'zhihu-question-meta', 'zhihu-first-answer', 'zhihu-answer-votes',
+    'zhihu-question-buttons', 'zhihu-multiple-answers', 'zhihu-question-description', 'zhihu-answer-count-number'];
+  const l4 = ['zhihu-hot-to-question', 'zhihu-feed-to-question', 'zhihu-question-to-author',
+    'zhihu-search-navigate', 'zhihu-topic-page', 'zhihu-user-profile', 'zhihu-question-and-back', 'zhihu-scroll-load-more'];
+  const l5 = ['zhihu-upvote-button-find', 'zhihu-follow-question-find', 'zhihu-comment-button-find',
+    'zhihu-bookmark-find', 'zhihu-write-answer-btn', 'zhihu-share-find'];
+  const l6 = ['zhihu-hot-read-answer-author', 'zhihu-hot-to-author-profile', 'zhihu-multi-hot-topics',
+    'zhihu-search-then-read', 'zhihu-question-scroll-answers', 'zhihu-compare-tabs', 'zhihu-user-answers', 'zhihu-topic-questions'];
+  const l7 = ['zhihu-search-basic', 'zhihu-search-people', 'zhihu-search-topic',
+    'zhihu-search-click-result', 'zhihu-search-filter-answers', 'zhihu-search-and-back'];
+  const l8 = ['zhihu-full-browse-workflow', 'zhihu-deep-author-chain', 'zhihu-cross-question-compare',
+    'zhihu-search-read-chain', 'zhihu-3-page-chain', 'zhihu-hot-scroll-deep-read'];
+
+  if (l1.includes(name)) return 'L1-atomic';
+  if (l2.includes(name)) return 'L2-feed';
+  if (l3.includes(name)) return 'L3-question';
+  if (l4.includes(name)) return 'L4-navigation';
+  if (l5.includes(name)) return 'L5-write';
+  if (l6.includes(name)) return 'L6-chain';
+  if (l7.includes(name)) return 'L7-search';
+  if (l8.includes(name)) return 'L8-complex';
+  return 'unknown';
+}
+
+function judge(criteria: JudgeCriteria, output: string): boolean {
+  try {
+    switch (criteria.type) {
+      case 'contains':
+        return output.toLowerCase().includes(criteria.value.toLowerCase());
+      case 'arrayMinLength': {
+        try {
+          const arr = JSON.parse(output);
+          if (Array.isArray(arr)) return arr.length >= criteria.minLength;
+        } catch { /* not JSON array */ }
+        return false;
+      }
+      case 'nonEmpty':
+        return output.trim().length > 0 && output.trim() !== 'null' && output.trim() !== 'undefined';
+      case 'matchesPattern':
+        return new RegExp(criteria.pattern).test(output);
+      default:
+        return false;
+    }
+  } catch {
+    return false;
+  }
+}
+
+function runCommand(cmd: string, timeout = 30000): string {
+  try {
+    return execSync(cmd, {
+      cwd: join(__dirname, '..'),
+      timeout,
+      encoding: 'utf-8',
+      env: process.env,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    }).trim();
+  } catch (err: any) {
+    return err.stdout?.trim() ?? '';
+  }
+}
+
+function runTask(task: BrowseTask): TaskResult {
+  const start = Date.now();
+  let lastOutput = '';
+
+  try {
+    for (const step of task.steps) {
+      lastOutput = runCommand(step);
+    }
+
+    const passed = judge(task.judge, lastOutput);
+
+    return {
+      name: task.name,
+      passed,
+      duration: Date.now() - start,
+      error: passed ? undefined : `Output: ${lastOutput.slice(0, 150)}`,
+      layer: getLayer(task.name),
+    };
+  } catch (err: any) {
+    return {
+      name: task.name,
+      passed: false,
+      duration: Date.now() - start,
+      error: err.message?.slice(0, 100),
+      layer: getLayer(task.name),
+    };
+  }
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const singleTask = args.includes('--task') ? args[args.indexOf('--task') + 1] : null;
+  const layerFilter = args.includes('--layer') ? args[args.indexOf('--layer') + 1] : null;
+
+  const raw = JSON.parse(readFileSync(TASKS_FILE, 'utf-8')) as (BrowseTask | { _comment: string })[];
+  const allTasks = raw.filter((t): t is BrowseTask => 'name' in t && 'steps' in t);
+
+  let tasks = allTasks;
+  if (singleTask) {
+    tasks = allTasks.filter(t => t.name === singleTask);
+  } else if (layerFilter) {
+    const prefix = `L${layerFilter}`;
+    tasks = allTasks.filter(t => getLayer(t.name).startsWith(prefix));
+  }
+
+  if (tasks.length === 0) {
+    console.error(singleTask ? `Task "${singleTask}" not found.` : `No tasks for layer ${layerFilter}.`);
+    process.exit(1);
+  }
+
+  console.log(`\n🔬 Zhihu Test Suite — ${tasks.length} tasks\n`);
+
+  const results: TaskResult[] = [];
+
+  for (let i = 0; i < tasks.length; i++) {
+    const task = tasks[i];
+    process.stdout.write(`  [${i + 1}/${tasks.length}] ${task.name}...`);
+
+    const result = runTask(task);
+    results.push(result);
+
+    const icon = result.passed ? '✓' : '✗';
+    console.log(` ${icon} (${(result.duration / 1000).toFixed(1)}s)`);
+
+    // Close browser between tasks for clean state
+    if (i < tasks.length - 1) {
+      try { runCommand('opencli operate close'); } catch { /* ignore */ }
+    }
+  }
+
+  // Final close
+  try { runCommand('opencli operate close'); } catch { /* ignore */ }
+
+  // Summary by layer
+  const layers = [...new Set(results.map(r => r.layer))].sort();
+  const totalPassed = results.filter(r => r.passed).length;
+  const totalDuration = results.reduce((s, r) => s + r.duration, 0);
+
+  console.log(`\n${'─'.repeat(50)}`);
+  console.log(`  Total:  ${totalPassed}/${results.length}`);
+  for (const layer of layers) {
+    const layerResults = results.filter(r => r.layer === layer);
+    const layerPassed = layerResults.filter(r => r.passed).length;
+    console.log(`  ${layer}: ${layerPassed}/${layerResults.length}`);
+  }
+  console.log(`  Time:   ${Math.round(totalDuration / 60000)}min`);
+
+  const failures = results.filter(r => !r.passed);
+  if (failures.length > 0) {
+    console.log(`\n  Failures:`);
+    for (const f of failures) {
+      console.log(`    ✗ [${f.layer}] ${f.name}: ${f.error ?? 'unknown'}`);
+    }
+  }
+  console.log('');
+
+  // Save result
+  mkdirSync(RESULTS_DIR, { recursive: true });
+  const existing = readdirSync(RESULTS_DIR).filter(f => f.startsWith('zhihu-')).length;
+  const roundNum = String(existing + 1).padStart(3, '0');
+  const resultPath = join(RESULTS_DIR, `zhihu-${roundNum}.json`);
+  writeFileSync(resultPath, JSON.stringify({
+    timestamp: new Date().toISOString(),
+    score: `${totalPassed}/${results.length}`,
+    layers: Object.fromEntries(layers.map(l => {
+      const lr = results.filter(r => r.layer === l);
+      return [l, `${lr.filter(r => r.passed).length}/${lr.length}`];
+    })),
+    duration: `${Math.round(totalDuration / 60000)}min`,
+    tasks: results,
+  }, null, 2), 'utf-8');
+  console.log(`  Results saved to: ${resultPath}`);
+  console.log(`\nSCORE=${totalPassed}/${results.length}`);
+}
+
+main();

package/autoresearch/logger.ts

@@ -0,0 +1,69 @@
+/**
+ * AutoResearch TSV Logger — append-only results log with metadata header.
+ */
+
+import { writeFileSync, readFileSync, existsSync, appendFileSync } from 'node:fs';
+import type { AutoResearchConfig, IterationResult } from './config.js';
+
+const COLUMNS = ['iteration', 'commit', 'metric', 'delta', 'guard', 'status', 'description'];
+
+export class Logger {
+  constructor(private path: string) {}
+
+  /** Create the TSV file with metadata header */
+  init(config: AutoResearchConfig): void {
+    const header = [
+      `# metric_direction: ${config.direction === 'higher' ? 'higher_is_better' : 'lower_is_better'}`,
+      `# goal: ${config.goal}`,
+      `# scope: ${config.scope.join(', ')}`,
+      `# verify: ${config.verify}`,
+      config.guard ? `# guard: ${config.guard}` : null,
+      COLUMNS.join('\t'),
+    ].filter(Boolean).join('\n');
+    writeFileSync(this.path, header + '\n', 'utf-8');
+  }
+
+  /** Append one iteration result */
+  append(result: IterationResult): void {
+    const row = [
+      result.iteration,
+      result.commit,
+      result.metric,
+      result.delta >= 0 ? `+${result.delta}` : result.delta,
+      result.guard,
+      result.status,
+      result.description,
+    ].join('\t');
+    appendFileSync(this.path, row + '\n', 'utf-8');
+  }
+
+  /** Read last N entries for pattern recognition */
+  readLast(n: number): IterationResult[] {
+    if (!existsSync(this.path)) return [];
+    const lines = readFileSync(this.path, 'utf-8').split('\n')
+      .filter(l => l && !l.startsWith('#') && !l.startsWith('iteration'));
+    return lines.slice(-n).map(line => {
+      const [iteration, commit, metric, delta, guard, status, ...desc] = line.split('\t');
+      return {
+        iteration: parseInt(iteration, 10),
+        commit,
+        metric: parseFloat(metric),
+        delta: parseFloat(delta),
+        guard: guard as 'pass' | 'fail' | '-',
+        status: status as IterationResult['status'],
+        description: desc.join('\t'),
+      };
+    });
+  }
+
+  /** Count consecutive discards from the end */
+  consecutiveDiscards(): number {
+    const entries = this.readLast(20);
+    let count = 0;
+    for (let i = entries.length - 1; i >= 0; i--) {
+      if (entries[i].status === 'discard') count++;
+      else break;
+    }
+    return count;
+  }
+}

package/autoresearch/presets/combined-reliability.ts

@@ -0,0 +1,27 @@
+/**
+ * Preset: Combined Reliability (browse + V2EX + Zhihu)
+ *
+ * Optimizes across ALL test suites simultaneously.
+ * Current baseline: 57/59 + 60/60 + 60/60 = 177/179
+ * Target: 179/179 (100%)
+ */
+
+import type { AutoResearchConfig } from '../config.js';
+
+export const combinedReliability: AutoResearchConfig = {
+  goal: 'Fix all remaining test failures across browse + V2EX + Zhihu (177/179 → 179/179)',
+  scope: [
+    'src/browser/dom-snapshot.ts',
+    'src/browser/dom-helpers.ts',
+    'src/browser/base-page.ts',
+    'src/browser/page.ts',
+    'src/cli.ts',
+    'autoresearch/browse-tasks.json',
+  ],
+  metric: 'pass_count',
+  direction: 'higher',
+  verify: 'npx tsx autoresearch/eval-all.ts 2>&1 | tail -1',
+  guard: 'npm run build',
+  iterations: 10,
+  minDelta: 1,
+};

package/autoresearch/presets/index.ts

@@ -0,0 +1,23 @@
+export { operateReliability } from './operate-reliability.js';
+export { skillQuality } from './skill-quality.js';
+export { v2exReliability } from './v2ex-reliability.js';
+export { zhihuReliability } from './zhihu-reliability.js';
+export { combinedReliability } from './combined-reliability.js';
+export { saveReliability } from './save-reliability.js';
+
+import type { AutoResearchConfig } from '../config.js';
+import { operateReliability } from './operate-reliability.js';
+import { skillQuality } from './skill-quality.js';
+import { v2exReliability } from './v2ex-reliability.js';
+import { zhihuReliability } from './zhihu-reliability.js';
+import { combinedReliability } from './combined-reliability.js';
+import { saveReliability } from './save-reliability.js';
+
+export const PRESETS: Record<string, AutoResearchConfig> = {
+  'operate-reliability': operateReliability,
+  'skill-quality': skillQuality,
+  'v2ex-reliability': v2exReliability,
+  'zhihu-reliability': zhihuReliability,
+  'combined': combinedReliability,
+  'save-reliability': saveReliability,
+};

package/autoresearch/presets/operate-reliability.ts

@@ -0,0 +1,24 @@
+/**
+ * Preset: Operate Command Reliability
+ *
+ * Optimizes opencli operate commands against the Layer 1 deterministic test suite.
+ * Metric: number of passing browse-tasks (out of 59).
+ */
+
+import type { AutoResearchConfig } from '../config.js';
+
+export const operateReliability: AutoResearchConfig = {
+  goal: 'Increase operate command pass rate to 59/59 (100%)',
+  scope: [
+    'src/browser/dom-snapshot.ts',
+    'src/browser/dom-helpers.ts',
+    'src/browser/base-page.ts',
+    'src/browser/page.ts',
+    'src/cli.ts',
+  ],
+  metric: 'pass_count',
+  direction: 'higher',
+  verify: 'npx tsx autoresearch/eval-browse.ts 2>&1 | tail -1',
+  guard: 'npm run build',
+  minDelta: 1,
+};

package/autoresearch/presets/save-reliability.ts

@@ -0,0 +1,26 @@
+/**
+ * Preset: Save as CLI Reliability
+ *
+ * Optimizes the "Save as CLI" pipeline: operate init → write adapter → run.
+ * Covers PUBLIC (no auth) and COOKIE (browser session) strategies.
+ * Metric: number of passing save-tasks.
+ */
+
+import type { AutoResearchConfig } from '../config.js';
+
+export const saveReliability: AutoResearchConfig = {
+  goal: 'Increase "Save as CLI" pipeline pass rate to 100%. The flow is: operate init creates a scaffold, user writes adapter code, opencli discovers and runs it. Covers both PUBLIC (fetch API) and COOKIE (browser session) strategies. Focus on: init template correctness, user CLI discovery, adapter loading, verify command robustness, and browser session handling.',
+  scope: [
+    'src/cli.ts',
+    'src/discovery.ts',
+    'src/registry.ts',
+    'skills/opencli-operate/SKILL.md',
+    'autoresearch/save-tasks.json',
+    'autoresearch/save-adapters/*.ts',
+  ],
+  metric: 'pass_count',
+  direction: 'higher',
+  verify: 'npx tsx autoresearch/eval-save.ts 2>&1 | tail -1',
+  guard: 'npm run build',
+  minDelta: 1,
+};

package/autoresearch/presets/skill-quality.ts

@@ -0,0 +1,20 @@
+/**
+ * Preset: Skill E2E Quality
+ *
+ * Optimizes the opencli-operate SKILL.md against the Layer 2 LLM E2E test suite.
+ * Metric: number of passing skill-tasks (out of 35).
+ */
+
+import type { AutoResearchConfig } from '../config.js';
+
+export const skillQuality: AutoResearchConfig = {
+  goal: 'Increase skill E2E pass rate to 35/35 (100%)',
+  scope: [
+    'skills/opencli-operate/SKILL.md',
+  ],
+  metric: 'pass_count',
+  direction: 'higher',
+  verify: 'npx tsx autoresearch/eval-skill.ts 2>&1 | tail -1',
+  guard: 'npm run build',
+  iterations: 20,
+};

package/autoresearch/presets/v2ex-reliability.ts

@@ -0,0 +1,24 @@
+/**
+ * Preset: V2EX Command Reliability
+ *
+ * Optimizes opencli operate commands against the V2EX-specific test suite.
+ * 40 tasks across 5 difficulty layers (atomic → complex chain).
+ */
+
+import type { AutoResearchConfig } from '../config.js';
+
+export const v2exReliability: AutoResearchConfig = {
+  goal: 'Increase V2EX operate command pass rate to 40/40 (100%)',
+  scope: [
+    'src/browser/dom-snapshot.ts',
+    'src/browser/dom-helpers.ts',
+    'src/browser/base-page.ts',
+    'src/browser/page.ts',
+    'src/cli.ts',
+  ],
+  metric: 'pass_count',
+  direction: 'higher',
+  verify: 'npx tsx autoresearch/eval-v2ex.ts 2>&1 | tail -1',
+  guard: 'npm run build',
+  minDelta: 1,
+};