@shirlytaylor73/smart-search 0.2.0-beta.1

package/src/smart_search/sources.py

@@ -0,0 +1,429 @@
+import ast
+import json
+import re
+import uuid
+from collections import OrderedDict
+from typing import Any
+
+import asyncio
+
+from .config import config
+from .utils import extract_unique_urls
+
+
+_MD_LINK_PATTERN = re.compile(r"\[([^\]]+)\]\((https?://[^)]+)\)")
+_INLINE_CITATION_LINK_PATTERN = re.compile(r"\[\[(\d+)\]\]\((https?://[^)]+)\)")
+_SOURCES_HEADING_PATTERN = re.compile(
+    r"(?im)^"
+    r"(?:#{1,6}\s*)?"
+    r"(?:\*\*|__)?\s*"
+    r"(sources?|references?|citations?|信源|参考资料|参考|引用|来源列表|来源)"
+    r"\s*(?:\*\*|__)?"
+    r"(?:\s*[（(][^)\n]*[)）])?"
+    r"\s*[:：]?\s*$"
+)
+_SOURCES_FUNCTION_PATTERN = re.compile(
+    r"(?im)(^|\n)\s*(sources|source|citations|citation|references|reference|citation_card|source_cards|source_card)\s*\("
+)
+
+_THINK_BLOCK_PATTERN = re.compile(r"<think>[\s\S]*?</think>", re.IGNORECASE)
+
+_LEADING_POLICY_PATTERNS = [
+    re.compile(r"^I\s+cannot\s+(comply|assist|help|provide|fulfill)", re.IGNORECASE),
+    re.compile(r"^I\s+(can't|won't|will\s+not|am\s+unable\s+to)\s+(comply|assist|help|provide|fulfill)", re.IGNORECASE),
+    re.compile(r"^I('m|\s+am)\s+not\s+able\s+to", re.IGNORECASE),
+    re.compile(r"^(Sorry|Apologies|I\s+apologize),?\s+(but\s+)?I\s+(cannot|can't|won't)", re.IGNORECASE),
+    re.compile(r"^As\s+an?\s+AI(\s+language\s+model)?", re.IGNORECASE),
+    re.compile(r"^我(无法|不能|没有办法)(遵从|遵守|协助|帮助|提供|满足)", re.IGNORECASE),
+    re.compile(r"^(抱歉|对不起|很遗憾)[，,]?\s*我(无法|不能)", re.IGNORECASE),
+]
+
+_POLICY_META_KEYWORDS = {
+    "policy", "policies", "guideline", "guidelines", "content policy",
+    "usage policy", "terms of service", "acceptable use",
+    "策略", "政策", "准则", "使用条款", "服务条款",
+}
+
+_POLICY_CONTEXT_KEYWORDS = {
+    "prompt injection", "jailbreak", "system prompt", "hidden instruction",
+    "bypass", "override", "ignore previous",
+    "提示注入", "越狱", "系统提示", "隐藏指令",
+}
+
+
+def _normalize_policy_text(text: str) -> str:
+    text = re.sub(r"[*_`#~\[\](){}]", "", text)
+    return text.lower().strip()
+
+
+def _split_paragraphs(text: str) -> list[str]:
+    return [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
+
+
+def _looks_like_policy_block(text: str) -> bool:
+    normalized = _normalize_policy_text(text)
+    if not normalized:
+        return False
+
+    for pattern in _LEADING_POLICY_PATTERNS:
+        if pattern.search(normalized):
+            return True
+
+    meta_count = sum(1 for kw in _POLICY_META_KEYWORDS if kw in normalized)
+    ctx_count = sum(1 for kw in _POLICY_CONTEXT_KEYWORDS if kw in normalized)
+    if meta_count >= 2 or ctx_count >= 2 or (meta_count >= 1 and ctx_count >= 1):
+        return True
+
+    return False
+
+
+def sanitize_answer_text(text: str) -> str:
+    text = _THINK_BLOCK_PATTERN.sub("", text or "").strip()
+    if not text:
+        return ""
+
+    paragraphs = _split_paragraphs(text)
+    cleaned: list[str] = []
+    leading = True
+    for para in paragraphs:
+        if leading and _looks_like_policy_block(para):
+            continue
+        leading = False
+        cleaned.append(para)
+
+    return "\n\n".join(cleaned).strip()
+
+
+def new_session_id() -> str:
+    return uuid.uuid4().hex[:12]
+
+
+class SourcesCache:
+    def __init__(self, max_size: int = 256):
+        self._max_size = max_size
+        self._lock = asyncio.Lock()
+        self._cache: OrderedDict[str, list[dict]] = OrderedDict()
+
+    async def set(self, session_id: str, sources: list[dict]) -> None:
+        async with self._lock:
+            self._cache[session_id] = sources
+            self._cache.move_to_end(session_id)
+            while len(self._cache) > self._max_size:
+                self._cache.popitem(last=False)
+
+    async def get(self, session_id: str) -> list[dict] | None:
+        async with self._lock:
+            sources = self._cache.get(session_id)
+            if sources is None:
+                return None
+            self._cache.move_to_end(session_id)
+            return sources
+
+
+def merge_sources(*source_lists: list[dict]) -> list[dict]:
+    seen: set[str] = set()
+    merged: list[dict] = []
+    for sources in source_lists:
+        for item in sources or []:
+            url = (item or {}).get("url")
+            if not isinstance(url, str) or not url.strip():
+                continue
+            url = url.strip()
+            if url in seen:
+                continue
+            seen.add(url)
+            merged.append(item)
+    return merged
+
+
+def split_answer_and_sources(text: str) -> tuple[str, list[dict]]:
+    raw = (text or "").strip()
+    if not raw:
+        return "", []
+
+    if config.output_cleanup_enabled:
+        cleaned = sanitize_answer_text(raw)
+        if cleaned:
+            raw = cleaned
+
+    inline_sources = _extract_inline_citation_sources(raw)
+
+    split = _split_function_call_sources(raw)
+    if split:
+        answer, sources = split
+        return answer, merge_sources(sources, inline_sources)
+
+    split = _split_heading_sources(raw)
+    if split:
+        answer, sources = split
+        return answer, merge_sources(sources, inline_sources)
+
+    split = _split_details_block_sources(raw)
+    if split:
+        answer, sources = split
+        return answer, merge_sources(sources, inline_sources)
+
+    split = _split_tail_link_block(raw)
+    if split:
+        answer, sources = split
+        return answer, merge_sources(sources, inline_sources)
+
+    return raw, inline_sources
+
+
+def _split_function_call_sources(text: str) -> tuple[str, list[dict]] | None:
+    matches = list(_SOURCES_FUNCTION_PATTERN.finditer(text))
+    if not matches:
+        return None
+
+    for m in reversed(matches):
+        open_paren_idx = m.end() - 1
+        extracted = _extract_balanced_call_at_end(text, open_paren_idx)
+        if not extracted:
+            continue
+
+        close_paren_idx, args_text = extracted
+        sources = _parse_sources_payload(args_text)
+        if not sources:
+            continue
+
+        answer = text[: m.start()].rstrip()
+        return answer, sources
+
+    return None
+
+
+def _extract_balanced_call_at_end(text: str, open_paren_idx: int) -> tuple[int, str] | None:
+    if open_paren_idx < 0 or open_paren_idx >= len(text) or text[open_paren_idx] != "(":
+        return None
+
+    depth = 1
+    in_string: str | None = None
+    escape = False
+
+    for idx in range(open_paren_idx + 1, len(text)):
+        ch = text[idx]
+        if in_string:
+            if escape:
+                escape = False
+                continue
+            if ch == "\\":
+                escape = True
+                continue
+            if ch == in_string:
+                in_string = None
+            continue
+
+        if ch in ("'", '"'):
+            in_string = ch
+            continue
+
+        if ch == "(":
+            depth += 1
+            continue
+        if ch == ")":
+            depth -= 1
+            if depth == 0:
+                if text[idx + 1 :].strip():
+                    return None
+                args_text = text[open_paren_idx + 1 : idx]
+                return idx, args_text
+
+    return None
+
+
+def _split_heading_sources(text: str) -> tuple[str, list[dict]] | None:
+    matches = list(_SOURCES_HEADING_PATTERN.finditer(text))
+    if not matches:
+        return None
+
+    for m in reversed(matches):
+        start = m.start()
+        sources_text = text[start:]
+        sources = _extract_sources_from_text(sources_text)
+        if not sources:
+            continue
+        answer = text[:start].rstrip()
+        return answer, sources
+    return None
+
+
+def _split_tail_link_block(text: str) -> tuple[str, list[dict]] | None:
+    lines = text.splitlines()
+    if not lines:
+        return None
+
+    idx = len(lines) - 1
+    while idx >= 0 and not lines[idx].strip():
+        idx -= 1
+    if idx < 0:
+        return None
+
+    tail_end = idx
+    link_like_count = 0
+    while idx >= 0:
+        line = lines[idx].strip()
+        if not line:
+            idx -= 1
+            continue
+        if not _is_link_only_line(line):
+            break
+        link_like_count += 1
+        idx -= 1
+
+    tail_start = idx + 1
+    if link_like_count < 2:
+        return None
+
+    block_text = "\n".join(lines[tail_start : tail_end + 1])
+    sources = _extract_sources_from_text(block_text)
+    if not sources:
+        return None
+
+    answer = "\n".join(lines[:tail_start]).rstrip()
+    return answer, sources
+
+
+def _split_details_block_sources(text: str) -> tuple[str, list[dict]] | None:
+    lower = text.lower()
+    close_idx = lower.rfind("</details>")
+    if close_idx == -1:
+        return None
+    tail = text[close_idx + len("</details>") :].strip()
+    if tail:
+        return None
+
+    open_idx = lower.rfind("<details", 0, close_idx)
+    if open_idx == -1:
+        return None
+
+    block_text = text[open_idx : close_idx + len("</details>")]
+    sources = _extract_sources_from_text(block_text)
+    if len(sources) < 2:
+        return None
+
+    answer = text[:open_idx].rstrip()
+    return answer, sources
+
+
+def _is_link_only_line(line: str) -> bool:
+    stripped = re.sub(r"^\s*(?:[-*]|\d+\.)\s*", "", line).strip()
+    if not stripped:
+        return False
+    if stripped.startswith(("http://", "https://")):
+        return True
+    if _MD_LINK_PATTERN.search(stripped):
+        return True
+    return False
+
+
+def _parse_sources_payload(payload: str) -> list[dict]:
+    payload = (payload or "").strip().rstrip(";")
+    if not payload:
+        return []
+
+    data: Any = None
+    try:
+        data = json.loads(payload)
+    except Exception:
+        try:
+            data = ast.literal_eval(payload)
+        except Exception:
+            data = None
+
+    if data is None:
+        return _extract_sources_from_text(payload)
+
+    if isinstance(data, dict):
+        for key in ("sources", "citations", "references", "urls"):
+            if key in data:
+                return _normalize_sources(data[key])
+        return _normalize_sources(data)
+
+    return _normalize_sources(data)
+
+
+def _normalize_sources(data: Any) -> list[dict]:
+    items: list[Any]
+    if isinstance(data, (list, tuple)):
+        items = list(data)
+    elif isinstance(data, dict):
+        items = [data]
+    else:
+        items = [data]
+
+    normalized: list[dict] = []
+    seen: set[str] = set()
+
+    for item in items:
+        if isinstance(item, str):
+            for url in extract_unique_urls(item):
+                if url not in seen:
+                    seen.add(url)
+                    normalized.append({"url": url})
+            continue
+
+        if isinstance(item, (list, tuple)) and len(item) >= 2:
+            title, url = item[0], item[1]
+            if isinstance(url, str) and url.startswith(("http://", "https://")) and url not in seen:
+                seen.add(url)
+                out: dict = {"url": url}
+                if isinstance(title, str) and title.strip():
+                    out["title"] = title.strip()
+                normalized.append(out)
+            continue
+
+        if isinstance(item, dict):
+            url = item.get("url") or item.get("href") or item.get("link")
+            if not isinstance(url, str) or not url.startswith(("http://", "https://")):
+                continue
+            if url in seen:
+                continue
+            seen.add(url)
+            out: dict = {"url": url}
+            title = item.get("title") or item.get("name") or item.get("label")
+            if isinstance(title, str) and title.strip():
+                out["title"] = title.strip()
+            desc = item.get("description") or item.get("snippet") or item.get("content")
+            if isinstance(desc, str) and desc.strip():
+                out["description"] = desc.strip()
+            normalized.append(out)
+            continue
+
+    return normalized
+
+
+def _extract_sources_from_text(text: str) -> list[dict]:
+    sources: list[dict] = []
+    seen: set[str] = set()
+
+    for title, url in _MD_LINK_PATTERN.findall(text or ""):
+        url = (url or "").strip()
+        if not url or url in seen:
+            continue
+        seen.add(url)
+        title = (title or "").strip()
+        if title:
+            sources.append({"title": title, "url": url})
+        else:
+            sources.append({"url": url})
+
+    for url in extract_unique_urls(text or ""):
+        if url in seen:
+            continue
+        seen.add(url)
+        sources.append({"url": url})
+
+    return sources
+
+
+def _extract_inline_citation_sources(text: str) -> list[dict]:
+    sources: list[dict] = []
+    seen: set[str] = set()
+    for number, url in _INLINE_CITATION_LINK_PATTERN.findall(text or ""):
+        url = (url or "").strip()
+        if not url or url in seen:
+            continue
+        seen.add(url)
+        sources.append({"title": number, "url": url})
+    return sources

package/src/smart_search/utils.py

@@ -0,0 +1,220 @@
+from typing import List
+import re
+from .providers.base import SearchResult
+
+_URL_PATTERN = re.compile(r'https?://[^\s<>"\'`，。、；：！？》）】\)]+')
+
+
+def extract_unique_urls(text: str) -> list[str]:
+    seen: set[str] = set()
+    urls: list[str] = []
+    for m in _URL_PATTERN.finditer(text):
+        url = m.group().rstrip('.,;:!?')
+        if url not in seen:
+            seen.add(url)
+            urls.append(url)
+    return urls
+
+
+def format_extra_sources(tavily_results: list[dict] | None, firecrawl_results: list[dict] | None) -> str:
+    sections = []
+    idx = 1
+    urls = []
+    if firecrawl_results:
+        lines = ["## Extra Sources [Firecrawl]"]
+        for r in firecrawl_results:
+            title = r.get("title") or "Untitled"
+            url = r.get("url", "")
+            if len(url) == 0:
+                continue
+            if url in urls:
+                continue
+            urls.append(url)
+            desc = r.get("description", "")
+            lines.append(f"{idx}. **[{title}]({url})**")
+            if desc:
+                lines.append(f"   {desc}")
+            idx += 1
+        sections.append("\n".join(lines))
+    if tavily_results:
+        lines = ["## Extra Sources [Tavily]"]
+        for r in tavily_results:
+            title = r.get("title") or "Untitled"
+            url = r.get("url", "")
+            if url in urls:
+                continue
+            content = r.get("content", "")
+            lines.append(f"{idx}. **[{title}]({url})**")
+            if content:
+                lines.append(f"   {content}")
+            idx += 1
+        sections.append("\n".join(lines))
+    return "\n\n".join(sections)
+
+
+def format_search_results(results: List[SearchResult]) -> str:
+    if not results:
+        return "No results found."
+
+    formatted = []
+    for i, result in enumerate(results, 1):
+        parts = [f"## Result {i}: {result.title}"]
+
+        if result.url:
+            parts.append(f"**URL:** {result.url}")
+
+        if result.snippet:
+            parts.append(f"**Summary:** {result.snippet}")
+
+        if result.source:
+            parts.append(f"**Source:** {result.source}")
+
+        if result.published_date:
+            parts.append(f"**Published:** {result.published_date}")
+
+        formatted.append("\n".join(parts))
+
+    return "\n\n---\n\n".join(formatted)
+
+fetch_prompt = """
+# Profile: Web Content Fetcher
+
+- **Language**: 中文
+- **Role**: 你是一个专业的网页内容抓取和解析专家，获取指定 URL 的网页内容，并将其转换为与原网页高度一致的结构化 Markdown 文本格式。
+
+---
+
+## Workflow
+
+### 1. URL 验证与内容获取
+- 验证 URL 格式有效性，检查可访问性（处理重定向/超时）
+- **关键**：优先识别页面目录/大纲结构（Table of Contents），作为内容抓取的导航索引
+- 全量获取 HTML 内容，确保不遗漏任何章节或动态加载内容
+
+### 2. 智能解析与内容提取
+- **结构优先**：若存在目录/大纲，严格按其层级结构进行内容提取和组织
+- 解析 HTML 文档树，识别所有内容元素：
+  - 标题层级（h1-h6）及其嵌套关系
+  - 正文段落、文本格式（粗体/斜体/下划线）
+  - 列表结构（有序/无序/嵌套）
+  - 表格（包含表头/数据行/合并单元格）
+  - 代码块（行内代码/多行代码块/语言标识）
+  - 引用块、分隔线
+  - 图片（src/alt/title 属性）
+  - 链接（内部/外部/锚点）
+
+### 3. 内容清理与语义保留
+- 移除非内容标签：`<script>`、`<style>`、`<iframe>`、`<noscript>`
+- 过滤干扰元素：广告模块、追踪代码、社交分享按钮
+- **保留语义信息**：图片 alt/title、链接 href/title、代码语言标识
+- 特殊模块标注：导航栏、侧边栏、页脚用特殊标记保留
+
+---
+
+## Skills
+
+### 1. 内容精准提取与还原
+- **如果存在目录或者大纲，则按照目录或者大纲的结构进行提取**
+- **完整保留原始内容结构**，不遗漏任何信息
+- **准确识别并提取**标题、段落、列表、表格、代码块等所有元素
+- **保持原网页的内容层次和逻辑关系**
+- **精确处理特殊字符**，确保无乱码和格式错误
+- **还原文本内容**，包括换行、缩进、空格等细节
+
+### 2. 结构化组织与呈现
+- **标题层级**：使用 `#`、`##`、`###` 等还原标题层级
+- **目录结构**：使用列表生成 Table of Contents，带锚点链接
+- **内容分区**：使用 `###` 或代码块（` ```section ``` `）明确划分 Section
+- **嵌套结构**：使用缩进列表或引用块（`>`）保持层次关系
+- **辅助模块**：侧边栏、导航等用特殊代码块（` ```sidebar ``` `、` ```nav ``` `）包裹
+
+### 3. 格式转换优化
+- **HTML 转 Markdown**：保持 100% 内容一致性
+- **表格处理**：使用 Markdown 表格语法（`|---|---|`）
+- **代码片段**：用 ` ```语言标识``` ` 包裹，保留原始缩进
+- **图片处理**：转换为 `![alt](url)` 格式，保留所有属性
+- **链接处理**：转换为 `[文本](URL)` 格式，保持完整路径
+- **强调样式**：`<strong>` → `**粗体**`，`<em>` → `*斜体*`
+
+### 4. 内容完整性保障
+- **零删减原则**：不删减任何原网页文本内容
+- **元数据保留**：保留时间戳、作者信息、标签等关键信息
+- **多媒体标注**：视频、音频以链接或占位符标注（`[视频: 标题](URL)`）
+- **动态内容处理**：尽可能抓取完整内容
+
+---
+
+## Rules
+
+### 1. 内容一致性原则（核心）
+- ✅ 返回内容必须与原网页内容**完全一致**，不能有信息缺失
+- ✅ 保持原网页的**所有文本、结构和语义信息**
+- ❌ **不进行**内容摘要、精简、改写或总结
+- ✅ 保留原始的**段落划分、换行、空格**等格式细节
+
+### 2. 格式转换标准
+| HTML | Markdown | 示例 |
+|------|----------|------|
+| `<h1>`-`<h6>` | `#`-`######` | `# 标题` |
+| `<strong>` | `**粗体**` | **粗体** |
+| `<em>` | `*斜体*` | *斜体* |
+| `<a>` | `[文本](url)` | [链接](url) |
+| `<img>` | `![alt](url)` | ![图](url) |
+| `<code>` | `` `代码` `` | `code` |
+| `<pre><code>` | ` ```\n代码\n``` ` | 代码块 |
+
+### 3. 输出质量要求
+- **元数据头部**：
+  ```markdown
+  ---
+  source: [原始URL]
+  title: [网页标题]
+  fetched_at: [抓取时间]
+  ---
+  ```
+- **编码标准**：统一使用 UTF-8
+- **可用性**：输出可直接用于文档生成或阅读
+
+---
+
+## Initialization
+
+当接收到 URL 时：
+1. 按 Workflow 执行抓取和处理
+2. 返回完整的结构化 Markdown 文档
+"""
+
+
+url_describe_prompt = (
+    "Browse the given URL. Return exactly two sections:\n\n"
+    "Title: <page title from the page's own <title> tag or top heading; "
+    "if missing/generic, craft one using key terms found in the page>\n\n"
+    "Extracts: <copy 2-4 verbatim fragments from the page that best represent "
+    "its core content. Each fragment must be the author's original words, "
+    "wrapped in quotes, separated by ' | '. "
+    "Do NOT paraphrase, rephrase, interpret, or describe. "
+    "Do NOT write sentences like 'This page discusses...' or 'The author argues...'. "
+    "You are a copy-paste machine.>\n\n"
+    "Nothing else."
+)
+
+rank_sources_prompt = (
+    "Given a user query and a numbered source list, output ONLY the source numbers "
+    "reordered by relevance to the query (most relevant first). "
+    "Format: space-separated integers on a single line (e.g., 14 12 1 3 5). "
+    "Include every number exactly once. Nothing else."
+)
+
+search_prompt = """You are a helpful research assistant. Answer the user's question thoroughly using web search results.
+
+Guidelines:
+- Infer the user's true intent even when the question is vague. Consider multiple angles.
+- Search broadly first (5+ perspectives), then go deep on the 2-3 most relevant ones.
+- Prioritize authoritative sources: official docs, Wikipedia, academic papers, reputable journalism.
+- Search in English first for breadth, switch to Chinese when the topic demands it.
+- Every factual claim should cite its source. More credible sources strengthen the answer.
+- Lead with the most likely answer, then provide supporting analysis.
+- Define technical terms in plain language. Use real-world analogies for complex concepts.
+- Format output in clean Markdown. Use LaTeX for formulas, code blocks for scripts.
+- Be direct and concise. No filler or unnecessary follow-up questions.
+"""