deepspider 0.1.0

package/src/browser/ui/confirmDialog.js

@@ -0,0 +1,158 @@
+/**
+ * DeepSpider - 确认弹窗
+ * 显示选中内容，支持编辑后确认
+ */
+
+export function generateConfirmDialogScript() {
+  return `
+(function() {
+  const deepspider = window.__deepspider__;
+  if (!deepspider || deepspider._confirmDialog) return;
+  deepspider._confirmDialog = true;
+
+  let dialog = null;
+
+  function createDialog() {
+    dialog = document.createElement('div');
+    dialog.id = 'deepspider-confirm-dialog';
+    dialog.innerHTML = \`
+      <div class="deepspider-dialog-mask"></div>
+      <div class="deepspider-dialog-content">
+        <div class="deepspider-dialog-header">
+          <span>确认分析内容</span>
+          <button class="deepspider-dialog-close">&times;</button>
+        </div>
+        <div class="deepspider-dialog-body">
+          <label>选中内容：</label>
+          <textarea class="deepspider-dialog-text" rows="5"></textarea>
+          <label>XPath：</label>
+          <input class="deepspider-dialog-xpath" type="text" readonly />
+        </div>
+        <div class="deepspider-dialog-footer">
+          <button class="deepspider-btn-cancel">取消</button>
+          <button class="deepspider-btn-confirm">发送分析</button>
+        </div>
+      </div>
+    \`;
+
+    // 样式
+    const style = document.createElement('style');
+    style.textContent = \`
+      #deepspider-confirm-dialog { display: none; }
+      #deepspider-confirm-dialog.active { display: block; }
+      .deepspider-dialog-mask {
+        position: fixed; inset: 0;
+        background: rgba(0,0,0,0.5);
+        z-index: 2147483640;
+      }
+      .deepspider-dialog-content {
+        position: fixed;
+        top: 50%; left: 50%;
+        transform: translate(-50%, -50%);
+        background: #fff; border-radius: 8px;
+        width: 450px; max-width: 90vw;
+        z-index: 2147483641;
+        box-shadow: 0 4px 20px rgba(0,0,0,0.3);
+      }
+      .deepspider-dialog-header {
+        padding: 12px 16px;
+        border-bottom: 1px solid #eee;
+        display: flex; justify-content: space-between;
+        align-items: center; font-weight: 600;
+      }
+      .deepspider-dialog-close {
+        background: none; border: none;
+        font-size: 20px; cursor: pointer;
+        color: #666;
+      }
+      .deepspider-dialog-body {
+        padding: 16px;
+      }
+      .deepspider-dialog-body label {
+        display: block; margin-bottom: 4px;
+        font-size: 12px; color: #666;
+      }
+      .deepspider-dialog-text {
+        width: 100%; padding: 8px;
+        border: 1px solid #ddd; border-radius: 4px;
+        font-size: 13px; resize: vertical;
+        margin-bottom: 12px; box-sizing: border-box;
+      }
+      .deepspider-dialog-xpath {
+        width: 100%; padding: 8px;
+        border: 1px solid #ddd; border-radius: 4px;
+        font-size: 12px; color: #666;
+        background: #f5f5f5; box-sizing: border-box;
+      }
+      .deepspider-dialog-footer {
+        padding: 12px 16px;
+        border-top: 1px solid #eee;
+        text-align: right;
+      }
+      .deepspider-dialog-footer button {
+        padding: 8px 16px; margin-left: 8px;
+        border-radius: 4px; cursor: pointer;
+        font-size: 14px;
+      }
+      .deepspider-btn-cancel {
+        background: #f5f5f5; border: 1px solid #ddd;
+        color: #666;
+      }
+      .deepspider-btn-confirm {
+        background: #4A90D9; border: none;
+        color: #fff;
+      }
+    \`;
+
+    document.head.appendChild(style);
+    document.body.appendChild(dialog);
+    bindEvents();
+  }
+
+  function bindEvents() {
+    dialog.querySelector('.deepspider-dialog-close').onclick = hideDialog;
+    dialog.querySelector('.deepspider-btn-cancel').onclick = hideDialog;
+    dialog.querySelector('.deepspider-dialog-mask').onclick = hideDialog;
+
+    dialog.querySelector('.deepspider-btn-confirm').onclick = () => {
+      const text = dialog.querySelector('.deepspider-dialog-text').value;
+      const xpath = dialog.querySelector('.deepspider-dialog-xpath').value;
+
+      // 设置待分析数据
+      deepspider.pendingAnalysis = {
+        text,
+        xpath,
+        timestamp: Date.now()
+      };
+
+      hideDialog();
+
+      // 打开分析面板
+      if (deepspider.showAnalysisPanel) {
+        deepspider.showAnalysisPanel();
+        deepspider.addPanelMessage('user', '分析这段数据：' + text.slice(0, 100));
+      }
+    };
+  }
+
+  function showDialog(data) {
+    if (!dialog) createDialog();
+
+    dialog.querySelector('.deepspider-dialog-text').value = data.text || '';
+    dialog.querySelector('.deepspider-dialog-xpath').value = data.xpath || '';
+    dialog.classList.add('active');
+  }
+
+  function hideDialog() {
+    if (dialog) dialog.classList.remove('active');
+  }
+
+  deepspider.showConfirmDialog = showDialog;
+  deepspider.hideConfirmDialog = hideDialog;
+
+  console.log('[DeepSpider] ConfirmDialog 已加载');
+})();
+`;
+}
+
+export default { generateConfirmDialogScript };

package/src/browser/ui/panel.html

@@ -0,0 +1,152 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="UTF-8">
+  <title>DeepSpider Panel</title>
+  <style>
+    * { box-sizing: border-box; margin: 0; padding: 0; }
+    body {
+      font-family: -apple-system, BlinkMacSystemFont, sans-serif;
+      background: #1e1e1e;
+      color: #d4d4d4;
+      height: 100vh;
+      display: flex;
+      flex-direction: column;
+    }
+    .header {
+      padding: 12px 16px;
+      background: #2d2d2d;
+      border-bottom: 1px solid #3c3c3c;
+      display: flex;
+      justify-content: space-between;
+      align-items: center;
+    }
+    .header-title { font-weight: 600; color: #4fc3f7; font-size: 14px; }
+    .header-btns button {
+      background: none; border: none;
+      color: #808080; font-size: 14px;
+      cursor: pointer; margin-left: 8px;
+      padding: 4px 8px;
+    }
+    .header-btns button:hover { color: #fff; }
+    .header-btns button.active { color: #4fc3f7; background: #3c3c3c; border-radius: 4px; }
+    .messages {
+      flex: 1;
+      overflow-y: auto;
+      padding: 12px;
+    }
+    .msg {
+      margin-bottom: 10px;
+      padding: 10px 12px;
+      border-radius: 6px;
+      line-height: 1.5;
+      word-break: break-word;
+      white-space: pre-wrap;
+      font-size: 13px;
+    }
+    .msg-user { background: #094771; margin-left: 40px; }
+    .msg-assistant { background: #2d2d2d; margin-right: 40px; }
+    .msg-system { background: #3c3c3c; text-align: center; font-size: 12px; color: #808080; }
+    .input-area {
+      padding: 12px;
+      border-top: 1px solid #3c3c3c;
+      display: flex;
+      gap: 8px;
+    }
+    .input-area textarea {
+      flex: 1;
+      padding: 10px;
+      background: #2d2d2d;
+      border: 1px solid #3c3c3c;
+      border-radius: 4px;
+      color: #d4d4d4;
+      resize: none;
+      font-size: 13px;
+      font-family: inherit;
+    }
+    .input-area textarea:focus { outline: none; border-color: #4fc3f7; }
+    .input-area button {
+      padding: 10px 16px;
+      background: #4fc3f7;
+      border: none;
+      border-radius: 4px;
+      color: #1e1e1e;
+      cursor: pointer;
+      font-weight: 500;
+    }
+    .input-area button:hover { background: #81d4fa; }
+  </style>
+</head>
+<body>
+  <div class="header">
+    <span class="header-title">DeepSpider</span>
+    <div class="header-btns">
+      <button id="btn-select" title="选择元素">⦿ 选择</button>
+      <button id="btn-clear" title="清空消息">清空</button>
+    </div>
+  </div>
+  <div class="messages" id="messages"></div>
+  <div class="input-area">
+    <textarea id="chat-input" placeholder="输入问题..." rows="2"></textarea>
+    <button id="btn-send">发送</button>
+  </div>
+  <script>
+    const messages = [];
+    const messagesEl = document.getElementById('messages');
+    const chatInput = document.getElementById('chat-input');
+
+    function escapeHtml(str) {
+      return String(str).replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+    }
+
+    function renderMessages() {
+      messagesEl.innerHTML = messages.map(m =>
+        '<div class="msg msg-' + m.role + '">' + escapeHtml(m.content) + '</div>'
+      ).join('');
+      messagesEl.scrollTop = messagesEl.scrollHeight;
+    }
+
+    function addMessage(role, content) {
+      messages.push({ role, content, time: Date.now() });
+      renderMessages();
+    }
+
+    // 暴露给后端调用
+    window.__panel__ = {
+      addMessage,
+      clearMessages: () => { messages.length = 0; renderMessages(); }
+    };
+
+    // 发送消息
+    function sendChat() {
+      const text = chatInput.value.trim();
+      if (!text) return;
+      chatInput.value = '';
+      addMessage('user', text);
+      if (typeof __deepspider_send__ === 'function') {
+        __deepspider_send__(JSON.stringify({ type: 'chat', text }));
+      }
+    }
+
+    document.getElementById('btn-send').onclick = sendChat;
+    chatInput.onkeydown = (e) => {
+      if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); sendChat(); }
+    };
+
+    // 选择模式
+    document.getElementById('btn-select').onclick = () => {
+      if (typeof __deepspider_send__ === 'function') {
+        __deepspider_send__(JSON.stringify({ type: 'start-select' }));
+      }
+    };
+
+    // 清空
+    document.getElementById('btn-clear').onclick = () => {
+      messages.length = 0;
+      renderMessages();
+    };
+
+    addMessage('system', '面板已就绪，点击「选择」按钮选择页面元素进行分析');
+  </script>
+</body>
+</html>

package/src/browser/ui/selector.js

@@ -0,0 +1,170 @@
+/**
+ * DeepSpider - 元素选择器模式
+ * 类似开发者工具的元素选择功能
+ */
+
+export function generateSelectorScript() {
+  return `
+(function() {
+  const deepspider = window.__deepspider__;
+  if (!deepspider || deepspider._selector) return;
+  deepspider._selector = true;
+
+  let isActive = false;
+  let overlay = null;
+  let infoBox = null;
+  let currentElement = null;
+
+  // 创建高亮覆盖层
+  function createOverlay() {
+    overlay = document.createElement('div');
+    overlay.id = 'deepspider-selector-overlay';
+    overlay.style.cssText = \`
+      position: fixed;
+      pointer-events: none;
+      border: 2px solid #4A90D9;
+      background: rgba(74, 144, 217, 0.1);
+      z-index: 2147483646;
+      display: none;
+      transition: all 0.1s ease;
+    \`;
+    document.body.appendChild(overlay);
+
+    // 元素信息框
+    infoBox = document.createElement('div');
+    infoBox.id = 'deepspider-selector-info';
+    infoBox.style.cssText = \`
+      position: fixed;
+      background: #333;
+      color: #fff;
+      padding: 4px 8px;
+      font-size: 12px;
+      font-family: monospace;
+      border-radius: 3px;
+      z-index: 2147483647;
+      display: none;
+      pointer-events: none;
+    \`;
+    document.body.appendChild(infoBox);
+  }
+
+  // 生成 XPath
+  function getXPath(element) {
+    if (!element) return '';
+    if (element.id) return '//*[@id="' + element.id + '"]';
+
+    const parts = [];
+    while (element && element.nodeType === 1) {
+      let index = 1;
+      let sibling = element.previousSibling;
+      while (sibling) {
+        if (sibling.nodeType === 1 && sibling.tagName === element.tagName) {
+          index++;
+        }
+        sibling = sibling.previousSibling;
+      }
+      const tagName = element.tagName.toLowerCase();
+      parts.unshift(tagName + '[' + index + ']');
+      element = element.parentNode;
+    }
+    return '/' + parts.join('/');
+  }
+
+  // 鼠标移动处理
+  function onMouseMove(e) {
+    if (!isActive) return;
+
+    const target = document.elementFromPoint(e.clientX, e.clientY);
+    if (!target || target === overlay || target === infoBox) return;
+    if (target.id?.startsWith('deepspider-')) return;
+
+    currentElement = target;
+    const rect = target.getBoundingClientRect();
+
+    // 更新高亮位置
+    overlay.style.left = rect.left + 'px';
+    overlay.style.top = rect.top + 'px';
+    overlay.style.width = rect.width + 'px';
+    overlay.style.height = rect.height + 'px';
+    overlay.style.display = 'block';
+
+    // 更新信息框
+    const tagName = target.tagName.toLowerCase();
+    const className = target.className ? '.' + target.className.split(' ')[0] : '';
+    const id = target.id ? '#' + target.id : '';
+    infoBox.textContent = tagName + id + className + ' | ' + Math.round(rect.width) + 'x' + Math.round(rect.height);
+    infoBox.style.left = rect.left + 'px';
+    infoBox.style.top = Math.max(0, rect.top - 24) + 'px';
+    infoBox.style.display = 'block';
+  }
+
+  // 点击选中
+  function onClick(e) {
+    if (!isActive || !currentElement) return;
+
+    e.preventDefault();
+    e.stopPropagation();
+
+    const element = currentElement;
+    const text = element.innerText?.trim().slice(0, 500) || '';
+    const xpath = getXPath(element);
+
+    // 停止选择模式
+    stopSelector();
+
+    // 显示确认弹窗
+    deepspider.showConfirmDialog({
+      text,
+      xpath,
+      tagName: element.tagName.toLowerCase(),
+      element
+    });
+  }
+
+  // ESC 退出
+  function onKeyDown(e) {
+    if (e.key === 'Escape' && isActive) {
+      stopSelector();
+    }
+  }
+
+  // 启动选择器
+  function startSelector() {
+    if (isActive) return;
+    isActive = true;
+
+    if (!overlay) createOverlay();
+
+    document.addEventListener('mousemove', onMouseMove, true);
+    document.addEventListener('click', onClick, true);
+    document.addEventListener('keydown', onKeyDown, true);
+    document.body.style.cursor = 'crosshair';
+
+    console.log('[DeepSpider] 选择器模式已开启，点击选择元素，ESC 退出');
+  }
+
+  // 停止选择器
+  function stopSelector() {
+    isActive = false;
+
+    document.removeEventListener('mousemove', onMouseMove, true);
+    document.removeEventListener('click', onClick, true);
+    document.removeEventListener('keydown', onKeyDown, true);
+    document.body.style.cursor = '';
+
+    if (overlay) overlay.style.display = 'none';
+    if (infoBox) infoBox.style.display = 'none';
+    currentElement = null;
+  }
+
+  // 暴露 API
+  deepspider.startSelector = startSelector;
+  deepspider.stopSelector = stopSelector;
+  deepspider.isSelectingMode = () => isActive;
+
+  console.log('[DeepSpider] Selector 已加载');
+})();
+`;
+}
+
+export default { generateSelectorScript };

package/src/config/index.js

@@ -0,0 +1,5 @@
+/**
+ * DeepSpider - 配置模块入口
+ */
+
+export * from './paths.js';

package/src/config/paths.js

@@ -0,0 +1,71 @@
+/**
+ * DeepSpider - 统一路径配置
+ * 所有存储路径统一到 ~/.deepspider/ 目录
+ */
+
+import { join } from 'path';
+import { homedir } from 'os';
+import { mkdirSync, existsSync } from 'fs';
+
+// 基础目录：~/.deepspider/
+export const DEEPSPIDER_HOME = join(homedir(), '.deepspider');
+
+// 子目录
+export const PATHS = {
+  // 采集数据（响应、脚本）
+  DATA_DIR: join(DEEPSPIDER_HOME, 'data'),
+  SITES_DIR: join(DEEPSPIDER_HOME, 'data', 'sites'),
+
+  // 知识库（加密模式、环境补丁等）
+  STORE_DIR: join(DEEPSPIDER_HOME, 'store'),
+
+  // Agent 输出
+  OUTPUT_DIR: join(DEEPSPIDER_HOME, 'output'),
+  REPORTS_DIR: join(DEEPSPIDER_HOME, 'output', 'reports'),
+  SCREENSHOTS_DIR: join(DEEPSPIDER_HOME, 'output', 'screenshots'),
+  UNPACKED_DIR: join(DEEPSPIDER_HOME, 'output', 'unpacked'),
+
+  // 配置（预留）
+  CONFIG_DIR: join(DEEPSPIDER_HOME, 'config'),
+};
+
+/**
+ * 确保目录存在
+ */
+export function ensureDir(dir) {
+  if (dir && !existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+}
+
+/**
+ * 初始化所有目录
+ */
+export function initDirectories() {
+  Object.values(PATHS).forEach(dir => ensureDir(dir));
+}
+
+/**
+ * 获取站点数据目录
+ */
+export function getSiteDataDir(hostname) {
+  return join(PATHS.SITES_DIR, hostname);
+}
+
+/**
+ * 获取报告目录（按域名）
+ */
+export function getReportDir(domain) {
+  const safeDomain = domain.replace(/[^a-zA-Z0-9.-]/g, '_');
+  return join(PATHS.REPORTS_DIR, safeDomain);
+}
+
+/**
+ * 生成带时间戳的文件名
+ */
+export function generateFilename(prefix, ext) {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  return `${prefix}_${timestamp}.${ext}`;
+}
+
+export default PATHS;

package/src/config/patterns/crypto.js

@@ -0,0 +1,36 @@
+/**
+ * DeepSpider - 加密模式配置
+ */
+
+export const cryptoPatterns = {
+  md5: {
+    name: 'MD5',
+    type: 'hash',
+    signatures: [/md5\s*\(/i, /[0-9a-f]{32}/i],
+    outputLength: 32
+  },
+  sha256: {
+    name: 'SHA256',
+    type: 'hash',
+    signatures: [/sha256|SHA256/, /[0-9a-f]{64}/i],
+    outputLength: 64
+  },
+  aes: {
+    name: 'AES',
+    type: 'symmetric',
+    signatures: [/CryptoJS\.AES/, /aes.*encrypt/i],
+    keyLengths: [16, 24, 32]
+  },
+  rsa: {
+    name: 'RSA',
+    type: 'asymmetric',
+    signatures: [/JSEncrypt/, /RSAKey/]
+  },
+  base64: {
+    name: 'Base64',
+    type: 'encoding',
+    signatures: [/btoa|atob/]
+  }
+};
+
+export default cryptoPatterns;

package/src/config/profiles/chrome.json

@@ -0,0 +1,71 @@
+{
+  "name": "Chrome 120 Windows",
+  "browser": "chrome",
+  "version": "120.0.0.0",
+  "platform": "Win32",
+
+  "navigator": {
+    "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "appVersion": "5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "platform": "Win32",
+    "language": "zh-CN",
+    "languages": ["zh-CN", "zh", "en"],
+    "cookieEnabled": true,
+    "doNotTrack": null,
+    "maxTouchPoints": 0,
+    "hardwareConcurrency": 8,
+    "deviceMemory": 8,
+    "vendor": "Google Inc.",
+    "vendorSub": "",
+    "productSub": "20030107",
+    "webdriver": false,
+    "plugins": [
+      { "name": "PDF Viewer", "filename": "internal-pdf-viewer" },
+      { "name": "Chrome PDF Viewer", "filename": "internal-pdf-viewer" },
+      { "name": "Chromium PDF Viewer", "filename": "internal-pdf-viewer" }
+    ],
+    "mimeTypes": [
+      { "type": "application/pdf", "suffixes": "pdf" },
+      { "type": "text/pdf", "suffixes": "pdf" }
+    ]
+  },
+
+  "screen": {
+    "width": 1920,
+    "height": 1080,
+    "availWidth": 1920,
+    "availHeight": 1040,
+    "colorDepth": 24,
+    "pixelDepth": 24
+  },
+
+  "window": {
+    "innerWidth": 1920,
+    "innerHeight": 937,
+    "outerWidth": 1920,
+    "outerHeight": 1040,
+    "devicePixelRatio": 1
+  },
+
+  "document": {
+    "characterSet": "UTF-8",
+    "compatMode": "CSS1Compat",
+    "contentType": "text/html"
+  },
+
+  "chrome": {
+    "app": {},
+    "runtime": {
+      "id": null
+    }
+  },
+
+  "canvas": {
+    "fingerprint": "canvas-fp-chrome-120"
+  },
+
+  "webgl": {
+    "vendor": "Google Inc. (NVIDIA)",
+    "renderer": "ANGLE (NVIDIA, NVIDIA GeForce GTX 1060 Direct3D11 vs_5_0 ps_5_0)"
+  }
+}