argo-search 1.0.1 → 2.9.7

package/scripts/crawl.py

@@ -1,63 +1,94 @@
 #!/usr/bin/env python3
-"""crawl.py — 站点级爬取"""
-import json, re, sys, time
-from concurrent.futures import ThreadPoolExecutor, as_completed
+"""crawl.py — 站点级爬取（fetch_v3 降级链：增强 HTTP → TLS 指纹 → Wayback）"""
+import re, time
 from urllib.parse import urljoin, urlparse
-from fetch import fetch_page
+from bounded_run import run_bounded, TaskError
+from cli_io import dumps
+# fetch_v3 而非裸 urllib fetch：UA 轮换 + Cookie 积累 + curl_cffi 指纹伪造，
+# 反爬站（CF 保护等）从必然失败到大概率成功。批量场景禁浏览器降级
+# （BFS 可能爬几十页，每页启动 Chrome CDP 会拖慢整体）。
+from fetch_v3 import fetch_page_v3 as _fetch_page_v3
+
+
+def _crawl_fetch(url: str, max_chars: int, timeout: int, raw: bool = True) -> dict:
+    """批量爬取用轻量降级链（不升浏览器）：HTTP → TLS 指纹 → Wayback。
+
+    raw 默认 True：爬取需要原始 HTML 提取链接，且 fetch_v3 缓存不存 html，
+    必须绕过缓存才能拿到完整页面。通过模块级 fetch_page 转发，
+    测试可 monkeypatch crawl.fetch_page 注入假抓取。
+    """
+    return fetch_page(url, max_chars=max_chars, timeout=timeout, raw=raw)
+
+
+# 默认实现指向 fetch_v3 的兼容入口；测试可整体替换此名字
+fetch_page = _fetch_page_v3
 
 def crawl_sitemap(url, max_pages=20, timeout=10):
     """从 sitemap.xml 爬取"""
+    t0 = time.monotonic()
     sitemap_url = urljoin(url, '/sitemap.xml')
-    result = fetch_page(sitemap_url, max_chars=50000, timeout=timeout, raw=True)
+    result = _crawl_fetch(sitemap_url, 50000, timeout)
     if not result['success']:
         # 尝试 robots.txt 指定的 sitemap
         robots_url = urljoin(url, '/robots.txt')
-        robots_result = fetch_page(robots_url, max_chars=10000, timeout=timeout, raw=True)
+        robots_result = _crawl_fetch(robots_url, 10000, timeout)
         if robots_result['success'] and 'Sitemap:' in robots_result.get('html', ''):
             import re as _re
             sitemap_urls = _re.findall(r'Sitemap:\s*(.+)', robots_result['html'])
             if sitemap_urls:
                 sitemap_url = sitemap_urls[0].strip()
-                result = fetch_page(sitemap_url, max_chars=50000, timeout=timeout, raw=True)
+                result = _crawl_fetch(sitemap_url, 50000, timeout)
     if not result.get('success') or not result.get('html'):
         return {'url': url, 'pages': [], 'total': 0, 'error': 'sitemap not found or empty'}
-    html = result.get('html', result.get('content', ''))
-    # 同时尝试从 raw content 提取（兼容 XML 被 ContentExtractor 过滤的情况）
+    html = result.get('html') or result.get('content') or ''
     urls = re.findall(r'<loc>\s*(.*?)\s*</loc>', html)
-    if not urls:
-        urls = re.findall(r'<loc>(.*?)</loc>', result.get('content', ''))
     urls = urls[:max_pages]
     pages = []
-    with ThreadPoolExecutor(max_workers=5) as ex:
-        futures = {ex.submit(fetch_page, u, 2000, timeout): u for u in urls}
-        for fut in as_completed(futures, timeout=timeout*2):
-            try:
-                r = fut.result()
-                if r['success']:
-                    pages.append({'url': r['url'], 'content': r['content'][:500], 'depth': 0})
-            except: pass
-    return {'url': url, 'pages': pages, 'total': len(pages), 'elapsed_ms': int((time.time())*1000)}
+    # 总时限 timeout*2：到点未抓到的站点跳过，不再被卡住的单个 URL 拖住，也不会
+    # 因超时异常中断整次 sitemap 爬取。
+    finished, _unfinished = run_bounded(
+        urls, lambda u: _crawl_fetch(u, 2000, timeout), timeout * 2, max_workers=5)
+    for _u, r in finished:
+        if isinstance(r, TaskError) or not r.get('success'):
+            continue
+        pages.append({'url': r['url'], 'content': r['content'][:500], 'depth': 0})
+    return {'url': url, 'pages': pages, 'total': len(pages), 'elapsed_ms': int((time.monotonic() - t0) * 1000)}
 
 def crawl_bfs(url, max_pages=10, max_depth=2, timeout=8):
-    """BFS 爬取"""
+    """BFS 爬取（按层并行 frontier，去重后入队）"""
+    from collections import deque
     visited = set()
     pages = []
-    queue = [(url, 0)]
+    queue = deque([(url, 0)])
     while queue and len(pages) < max_pages:
-        current_url, depth = queue.pop(0)
-        if current_url in visited or depth > max_depth:
-            continue
-        visited.add(current_url)
-        result = fetch_page(current_url, 2000, timeout)
-        if result['success']:
-            pages.append({'url': current_url, 'content': result['content'][:500], 'depth': depth})
-            links = re.findall(r'href=["\']([^"\'#]+)', result['content'])
+        # 取一层（最多 5 个）并行抓取
+        frontier = []
+        while queue and len(frontier) < 5:
+            u, d = queue.popleft()
+            if u in visited or d > max_depth:
+                continue
+            visited.add(u)
+            frontier.append((u, d))
+        if not frontier:
+            break
+        # 一层最多 5 个，总时限 timeout*2；到点未完成的 URL 跳过，不被单个慢站拖住。
+        finished, _unfinished = run_bounded(
+            frontier, lambda ud: _crawl_fetch(ud[0], 2000, timeout),
+            timeout * 2, max_workers=min(len(frontier), 5))
+        for (u, d), result in finished:
+            if isinstance(result, TaskError) or not result.get('success'):
+                continue
+            pages.append({'url': u, 'content': result['content'][:500], 'depth': d})
+            if len(pages) >= max_pages:
+                break
+            html = result.get('html') or result.get('content') or ''
+            links = re.findall(r'href=["\']([^"\'#]+)', html)
             for link in links[:5]:
-                full = urljoin(current_url, link)
-                if urlparse(full).netloc == urlparse(url).netloc and full not in visited:
-                    queue.append((full, depth+1))
+                full = urljoin(u, link)
+                if (urlparse(full).netloc == urlparse(url).netloc
+                        and full not in visited):
+                    queue.append((full, d + 1))
     return {'url': url, 'pages': pages, 'total': len(pages)}
-
 if __name__ == '__main__':
     import argparse
     p = argparse.ArgumentParser()
@@ -65,9 +96,17 @@ if __name__ == '__main__':
     p.add_argument('--strategy', default='bfs', choices=['sitemap','bfs'])
     p.add_argument('--max-pages', type=int, default=10)
     p.add_argument('--max-depth', type=int, default=2)
+    # bin/argo 的 usage 承诺了 Common flags「--json」，此前 crawl/extract 的
+    # parser 里没有它，Agent 照文档传参直接撞墙（error: unrecognized
+    # arguments，exit 2）。这里接住该开关并给出真实语义：默认仍是截断预览
+    # （防上下文膨胀），--json 输出完整可解析 JSON——截断的 JSON 不是合法
+    # JSON，调用方 json.loads 必失败，等于「拿不到数据还得自己猜」。
+    p.add_argument('--json', action='store_true',
+                   help='输出完整 JSON（默认为 2000 字符截断预览）')
     args = p.parse_args()
     if args.strategy == 'sitemap':
         r = crawl_sitemap(args.url, args.max_pages)
     else:
         r = crawl_bfs(args.url, args.max_pages, args.max_depth)
-    print(json.dumps(r, ensure_ascii=False, indent=2)[:2000])
+    out = dumps(r)
+    print(out if args.json else out[:2000])