deepspider 0.2.11 → 0.3.0

package/src/agent/subagents/factory.js

@@ -0,0 +1,60 @@
+/**
+ * DeepSpider - 子代理工厂函数
+ * 统一子代理创建，自动注入公共配置
+ */
+
+import { createSkillsMiddleware } from 'deepagents';
+import { SKILLS, skillsBackend } from '../skills/config.js';
+import { createFilterToolsMiddleware } from '../middleware/filterTools.js';
+import { evolveTools } from '../tools/evolve.js';
+
+/**
+ * 创建子代理配置
+ * @param {Object} config - 子代理配置
+ * @param {string} config.name - 子代理名称
+ * @param {string} config.description - 子代理描述
+ * @param {string} config.systemPrompt - 系统提示
+ * @param {Array} config.tools - 工具列表
+ * @param {Array} [config.skills] - 技能源列表（SKILLS 枚举值）
+ * @param {Array} [config.middleware] - 额外的中间件
+ * @param {boolean} [config.includeEvolve=true] - 是否包含 evolve 工具
+ */
+export function createSubagent(config) {
+  const {
+    name,
+    description,
+    systemPrompt,
+    tools,
+    skills = [],
+    middleware = [],
+    includeEvolve = true,
+  } = config;
+
+  // 合并工具列表
+  const finalTools = includeEvolve
+    ? [...tools, ...evolveTools]
+    : tools;
+
+  // 构建中间件列表
+  const finalMiddleware = [
+    createFilterToolsMiddleware(),
+    createSkillsMiddleware({
+      backend: skillsBackend,
+      sources: skills,
+    }),
+    ...middleware,
+  ];
+
+  return {
+    name,
+    description,
+    systemPrompt,
+    tools: finalTools,
+    middleware: finalMiddleware,
+  };
+}
+
+/**
+ * 预定义的技能映射
+ */
+export { SKILLS };

package/src/agent/subagents/index.js

@@ -2,6 +2,9 @@
  * DeepSpider - 子代理索引
  */
 
+// 工厂函数
+export { createSubagent, SKILLS } from './factory.js';
+
 // 编排层
 export { crawlerSubagent } from './crawler.js';
 

package/src/agent/tools/report.js

@@ -102,8 +102,24 @@ function generateHtmlPage(title, markdown, pythonCode, jsCode) {
  * 2. 传入代码内容（兼容）- pythonCode/jsCode
  */
 export const saveAnalysisReport = tool(
-  async ({ domain, title, markdown, pythonCode, pythonCodeFile, jsCode, jsCodeFile }) => {
+  async ({ domain, title, markdown, pythonCode, pythonCodeFile, jsCode, jsCodeFile, validationResult }) => {
     try {
+      // 验证状态检查
+      let validationWarning = '';
+      let validationStatus = 'unknown';
+
+      if (!validationResult) {
+        validationWarning = '⚠️ 警告：未提供端到端验证结果，建议先使用 verify_encryption 或 run_python 验证代码正确性';
+        validationStatus = 'not_verified';
+        console.warn('[report]', validationWarning);
+      } else if (validationResult.success === false) {
+        validationWarning = `⚠️ 警告：验证失败 - ${validationResult.error || '未知错误'}`;
+        validationStatus = 'failed';
+        console.warn('[report]', validationWarning);
+      } else {
+        validationStatus = 'passed';
+      }
+
       const domainDir = join(OUTPUT_DIR, extractDomain(domain));
       ensureDir(domainDir);
 
@@ -153,18 +169,30 @@ export const saveAnalysisReport = tool(
       const fileList = Object.entries(paths)
         .map(([type, p]) => `- ${type}: \`${p}\``)
         .join('\n');
-      const finalMarkdown = markdown + '\n\n## 生成文件\n\n' + fileList;
+
+      // 添加验证状态标记
+      const validationSection = validationWarning
+        ? `\n\n## 验证状态\n\n${validationWarning}\n`
+        : '\n\n## 验证状态\n\n✅ 端到端验证通过\n';
+
+      const finalMarkdown = markdown + validationSection + '\n## 生成文件\n\n' + fileList;
       writeFileSync(paths.markdown, finalMarkdown, 'utf-8');
 
       console.log('[report] 已保存:', domainDir);
-      return JSON.stringify({ success: true, paths, dir: domainDir });
+      return JSON.stringify({
+        success: true,
+        paths,
+        dir: domainDir,
+        validationStatus,
+        validationWarning: validationWarning || null,
+      });
     } catch (e) {
       return JSON.stringify({ success: false, error: e.message });
     }
   },
   {
     name: 'save_analysis_report',
-    description: `保存分析报告。推荐先用 artifact_save 保存代码文件，再传入文件路径。`,
+    description: `保存分析报告。推荐先用 artifact_save 保存代码文件，再传入文件路径。建议先验证代码正确性再保存报告。`,
     schema: z.object({
       domain: z.string().describe('网站域名'),
       title: z.string().optional().describe('报告标题'),
@@ -173,6 +201,10 @@ export const saveAnalysisReport = tool(
       pythonCode: z.string().optional().describe('Python 代码内容（不推荐）'),
       jsCodeFile: z.string().optional().describe('JS 代码文件路径'),
       jsCode: z.string().optional().describe('JS 代码内容'),
+      validationResult: z.object({
+        success: z.boolean().describe('验证是否成功'),
+        error: z.string().optional().describe('错误信息'),
+      }).optional().describe('端到端验证结果（推荐提供）'),
     }),
   }
 );

package/src/browser/client.js

@@ -4,13 +4,15 @@
  */
 
 import { chromium } from 'patchright';
+import { EventEmitter } from 'events';
 import { getDefaultHookScript } from './defaultHooks.js';
 import { NetworkInterceptor } from './interceptors/NetworkInterceptor.js';
 import { ScriptInterceptor } from './interceptors/ScriptInterceptor.js';
 import { getDataStore } from '../store/DataStore.js';
 
-export class BrowserClient {
+export class BrowserClient extends EventEmitter {
   constructor() {
+    super();
     this.browser = null;
     this.context = null;
     this.page = null;
@@ -20,6 +22,7 @@ export class BrowserClient {
     this.scriptInterceptor = null;
     this.hookScript = null;
     this.onMessage = null;
+    this._isCleaningUp = false;
   }
 
   /**
@@ -51,6 +54,8 @@ export class BrowserClient {
     }
 
     this.browser = await chromium.launch(launchOptions);
+    this.emit('launched', { headless });
+
     this.context = await this.browser.newContext({
       ignoreHTTPSErrors: true,
     });
@@ -183,15 +188,47 @@ export class BrowserClient {
    * 关闭浏览器
    */
   async close() {
-    if (this.cdpSession) {
-      await this.cdpSession.detach().catch(() => {});
-      this.cdpSession = null;
-    }
-    if (this.browser) {
-      await this.browser.close();
-      this.browser = null;
-      this.context = null;
-      this.page = null;
+    await this.cleanup();
+  }
+
+  /**
+   * 清理所有资源
+   */
+  async cleanup() {
+    if (this._isCleaningUp) return;
+    this._isCleaningUp = true;
+
+    try {
+      // 停止拦截器
+      if (this.networkInterceptor) {
+        await this.networkInterceptor.stop?.().catch(() => {});
+        this.networkInterceptor = null;
+      }
+      if (this.scriptInterceptor) {
+        await this.scriptInterceptor.stop?.().catch(() => {});
+        this.scriptInterceptor = null;
+      }
+
+      // 分离 CDP session
+      if (this.cdpSession) {
+        await this.cdpSession.detach().catch(() => {});
+        this.cdpSession = null;
+      }
+
+      // 关闭浏览器
+      if (this.browser) {
+        await this.browser.close();
+        this.browser = null;
+        this.context = null;
+        this.page = null;
+        this.pages = [];
+      }
+
+      this.emit('closed');
+    } catch (e) {
+      this.emit('error', e);
+    } finally {
+      this._isCleaningUp = false;
     }
   }
 }

package/src/cli/commands/config.js

@@ -0,0 +1,94 @@
+/**
+ * deepspider config 子命令
+ */
+
+import {
+  loadConfig,
+  saveConfig,
+  getEffectiveConfig,
+  resetConfig,
+  CONFIG_FILE,
+  ENV_MAP,
+  DEFAULTS,
+} from '../config.js';
+
+export function run(args) {
+  const sub = args[0] || 'list';
+
+  switch (sub) {
+    case 'list':
+      return list();
+    case 'get':
+      return get(args[1]);
+    case 'set':
+      return set(args[1], args[2]);
+    case 'reset':
+      return reset();
+    case 'path':
+      return path();
+    default:
+      console.error(`未知子命令: ${sub}\n可用: list, get, set, reset, path`);
+      process.exit(1);
+  }
+}
+
+function list() {
+  const effective = getEffectiveConfig();
+  console.log('配置项:');
+  for (const [key, { value, source }] of Object.entries(effective)) {
+    const display = key === 'apiKey' && value ? maskKey(value) : value || '(未设置)';
+    console.log(`  ${key} = ${display}  [${source}]`);
+  }
+}
+
+function get(key) {
+  if (!key) {
+    console.error('用法: deepspider config get <key>');
+    process.exit(1);
+  }
+  if (!Object.hasOwn(DEFAULTS, key)) {
+    console.error(`未知配置项: ${key}\n可用: ${Object.keys(DEFAULTS).join(', ')}`);
+    process.exit(1);
+  }
+  const effective = getEffectiveConfig();
+  const { value, source } = effective[key];
+  const display = key === 'apiKey' && value ? maskKey(value) : value || '(未设置)';
+  console.log(`${key} = ${display}  [${source}]`);
+}
+
+function set(key, value) {
+  if (!key || value === undefined) {
+    console.error('用法: deepspider config set <key> <value>');
+    process.exit(1);
+  }
+  if (!Object.hasOwn(DEFAULTS, key)) {
+    console.error(`未知配置项: ${key}\n可用: ${Object.keys(DEFAULTS).join(', ')}`);
+    process.exit(1);
+  }
+  const config = loadConfig();
+  config[key] = value;
+  saveConfig(config);
+
+  const envVar = ENV_MAP[key];
+  console.log(`已设置 ${key} = ${key === 'apiKey' ? maskKey(value) : value}`);
+  if (process.env[envVar]) {
+    console.log(`注意: 环境变量 ${envVar} 已设置，将优先使用环境变量的值`);
+  }
+}
+
+function reset() {
+  if (resetConfig()) {
+    console.log('配置已重置');
+  } else {
+    console.log('配置文件不存在，无需重置');
+  }
+}
+
+function path() {
+  console.log(CONFIG_FILE);
+}
+
+function maskKey(key) {
+  if (key.length <= 8) return '****';
+  return key.slice(0, 4) + '****' + key.slice(-4);
+}

package/src/cli/commands/help.js

@@ -0,0 +1,34 @@
+/**
+ * deepspider --help
+ */
+
+import { getVersion } from '../../config/settings.js';
+
+export function run() {
+  console.log(`
+deepspider v${getVersion()} - 智能爬虫工程平台
+
+用法:
+  deepspider                        启动交互式 Agent
+  deepspider <url>                  打开目标网站并启动 Agent
+  deepspider config                 管理配置
+  deepspider update                 检查更新
+
+选项:
+  -v, --version                     显示版本号
+  -h, --help                        显示帮助信息
+  --debug                           启用调试模式
+
+配置命令:
+  deepspider config list            列出所有配置
+  deepspider config get <key>       获取配置项
+  deepspider config set <key> <val> 设置配置项
+  deepspider config reset           重置配置
+  deepspider config path            显示配置文件路径
+
+示例:
+  deepspider https://example.com    分析目标网站
+  deepspider config set apiKey sk-xxx
+  deepspider config set model gpt-4o
+`.trim());
+}

package/src/cli/commands/update.js

@@ -0,0 +1,78 @@
+/**
+ * deepspider update
+ */
+
+import readline from 'readline';
+import { existsSync } from 'fs';
+import { execSync } from 'child_process';
+import { join } from 'path';
+import { getVersion } from '../../config/settings.js';
+
+export async function run() {
+  const current = getVersion();
+
+  console.log(`当前版本: v${current}`);
+  console.log('检查更新...');
+
+  let latest;
+  try {
+    const resp = await fetch('https://registry.npmjs.org/deepspider/latest');
+    if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+    const data = await resp.json();
+    latest = data.version;
+  } catch (e) {
+    console.error(`检查更新失败: ${e.message}`);
+    process.exit(1);
+  }
+
+  if (current === latest) {
+    console.log(`已是最新版本 v${current}`);
+    return;
+  }
+
+  console.log(`发现新版本: v${latest}`);
+
+  const isGlobal = detectGlobalInstall();
+
+  if (isGlobal) {
+    const confirmed = await confirm(`是否更新到 v${latest}？(y/N) `);
+    if (!confirmed) {
+      console.log('已取消');
+      return;
+    }
+    console.log('正在更新...');
+    try {
+      execSync('npm install -g deepspider@latest', { stdio: 'inherit' });
+      console.log(`已更新到 v${latest}`);
+    } catch {
+      console.error('更新失败，请手动执行: npm install -g deepspider@latest');
+      process.exit(1);
+    }
+  } else {
+    console.log('当前为本地安装，请手动更新:');
+    console.log('  git pull && pnpm install');
+  }
+}
+
+function confirm(question) {
+  return new Promise((resolve) => {
+    const rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout,
+    });
+    rl.question(question, (answer) => {
+      rl.close();
+      resolve(answer.toLowerCase() === 'y');
+    });
+  });
+}
+
+function detectGlobalInstall() {
+  try {
+    const globalDir = execSync('npm root -g', { encoding: 'utf-8' }).trim();
+    const globalPkg = join(globalDir, 'deepspider');
+    return existsSync(globalPkg);
+  } catch {
+    return false;
+  }
+}

package/src/cli/commands/version.js

@@ -0,0 +1,9 @@
+/**
+ * deepspider --version
+ */
+
+import { getVersion } from '../../config/settings.js';
+
+export function run() {
+  console.log(`deepspider v${getVersion()}`);
+}

package/src/cli/config.js

@@ -0,0 +1,15 @@
+/**
+ * CLI 配置模块 - 从 config/ 核心层 re-export
+ */
+
+export {
+  CONFIG_FILE,
+  DEFAULTS,
+  ENV_MAP,
+  loadConfig,
+  saveConfig,
+  getEffectiveConfig,
+  getConfigValues,
+  resetConfig,
+  getVersion,
+} from '../config/settings.js';

package/src/config/settings.js

@@ -0,0 +1,102 @@
+/**
+ * DeepSpider 配置管理（核心层）
+ * 配置文件：~/.deepspider/config/settings.json
+ * 优先级：环境变量 > 配置文件 > 默认值
+ *
+ * 此模块位于 config/ 层，供 cli/ 和 agent/ 共同依赖
+ */
+
+import { join } from 'path';
+import { readFileSync, writeFileSync, existsSync, unlinkSync } from 'fs';
+import { PATHS, ensureDir } from './paths.js';
+
+export const CONFIG_FILE = join(PATHS.CONFIG_DIR, 'settings.json');
+
+export const DEFAULTS = {
+  apiKey: '',
+  baseUrl: 'https://api.openai.com/v1',
+  model: 'gpt-4o',
+};
+
+export const ENV_MAP = {
+  apiKey: 'DEEPSPIDER_API_KEY',
+  baseUrl: 'DEEPSPIDER_BASE_URL',
+  model: 'DEEPSPIDER_MODEL',
+};
+
+/**
+ * 从配置文件加载
+ */
+export function loadConfig() {
+  if (!existsSync(CONFIG_FILE)) return {};
+  try {
+    return JSON.parse(readFileSync(CONFIG_FILE, 'utf-8'));
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * 保存到配置文件
+ */
+export function saveConfig(config) {
+  ensureDir(PATHS.CONFIG_DIR);
+  writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2) + '\n');
+}
+
+/**
+ * 获取有效配置（合并环境变量、配置文件、默认值）
+ * 返回 { key: { value, source } }
+ */
+export function getEffectiveConfig() {
+  const fileConfig = loadConfig();
+  const result = {};
+
+  for (const key of Object.keys(DEFAULTS)) {
+    const envVar = ENV_MAP[key];
+    const envVal = process.env[envVar];
+
+    if (envVal !== undefined && envVal !== '') {
+      result[key] = { value: envVal, source: 'env' };
+    } else if (fileConfig[key] !== undefined && fileConfig[key] !== '') {
+      result[key] = { value: fileConfig[key], source: 'file' };
+    } else {
+      result[key] = { value: DEFAULTS[key], source: 'default' };
+    }
+  }
+
+  return result;
+}
+
+/**
+ * 获取配置值（纯值，用于运行时）
+ */
+export function getConfigValues() {
+  const effective = getEffectiveConfig();
+  const values = {};
+  for (const [key, { value }] of Object.entries(effective)) {
+    values[key] = value;
+  }
+  return values;
+}
+
+/**
+ * 重置配置文件
+ */
+export function resetConfig() {
+  if (existsSync(CONFIG_FILE)) {
+    unlinkSync(CONFIG_FILE);
+    return true;
+  }
+  return false;
+}
+
+/**
+ * 读取 package.json 版本号
+ * 基于 import.meta.url 定位，不依赖文件调用位置
+ */
+export function getVersion() {
+  const __dirname = new URL('.', import.meta.url).pathname;
+  const pkg = JSON.parse(readFileSync(join(__dirname, '../../package.json'), 'utf-8'));
+  return pkg.version;
+}

package/.claude/agents/check.md

@@ -1,122 +0,0 @@
----
-name: check
-description: |
-  Code quality check expert. Reviews code changes against specs and self-fixes issues.
-tools: Read, Write, Edit, Bash, Glob, Grep, mcp__exa__web_search_exa, mcp__exa__get_code_context_exa
-model: opus
----
-# Check Agent
-
-You are the Check Agent in the Trellis workflow.
-
-## Context
-
-Before checking, read:
-- `.trellis/spec/` - Development guidelines
-- Pre-commit checklist for quality standards
-
-## Core Responsibilities
-
-1. **Get code changes** - Use git diff to get uncommitted code
-2. **Check against specs** - Verify code follows guidelines
-3. **Self-fix** - Fix issues yourself, not just report them
-4. **Run verification** - typecheck and lint
-
-## Important
-
-**Fix issues yourself**, don't just report them.
-
-You have write and edit tools, you can modify code directly.
-
----
-
-## Workflow
-
-### Step 1: Get Changes
-
-```bash
-git diff --name-only  # List changed files
-git diff              # View specific changes
-```
-
-### Step 2: Check Against Specs
-
-Read relevant specs in `.trellis/spec/` to check code:
-
-- Does it follow directory structure conventions
-- Does it follow naming conventions
-- Does it follow code patterns
-- Are there missing types
-- Are there potential bugs
-
-### Step 3: Self-Fix
-
-After finding issues:
-
-1. Fix the issue directly (use edit tool)
-2. Record what was fixed
-3. Continue checking other issues
-
-### Step 4: Run Verification
-
-Run project's lint and typecheck commands to verify changes.
-
-If failed, fix issues and re-run.
-
----
-
-## Completion Markers (Ralph Loop)
-
-**CRITICAL**: You are in a loop controlled by the Ralph Loop system.
-The loop will NOT stop until you output ALL required completion markers.
-
-Completion markers are generated from `check.jsonl` in the task directory.
-Each entry's `reason` field becomes a marker: `{REASON}_FINISH`
-
-For example, if check.jsonl contains:
-```json
-{"file": "...", "reason": "TypeCheck"}
-{"file": "...", "reason": "Lint"}
-{"file": "...", "reason": "CodeReview"}
-```
-
-You MUST output these markers when each check passes:
-- `TYPECHECK_FINISH` - After typecheck passes
-- `LINT_FINISH` - After lint passes
-- `CODEREVIEW_FINISH` - After code review passes
-
-If check.jsonl doesn't exist or has no reasons, output: `ALL_CHECKS_FINISH`
-
-**The loop will block you from stopping until all markers are present in your output.**
-
----
-
-## Report Format
-
-```markdown
-## Self-Check Complete
-
-### Files Checked
-
-- src/components/Feature.tsx
-- src/hooks/useFeature.ts
-
-### Issues Found and Fixed
-
-1. `<file>:<line>` - <what was fixed>
-2. `<file>:<line>` - <what was fixed>
-
-### Issues Not Fixed
-
-(If there are issues that cannot be self-fixed, list them here with reasons)
-
-### Verification Results
-
-- TypeCheck: Passed TYPECHECK_FINISH
-- Lint: Passed LINT_FINISH
-
-### Summary
-
-Checked X files, found Y issues, all fixed.
-ALL_CHECKS_FINISH
-```