bookmark-research 0.6.0 → 0.7.0

package/src/netscape.py

@@ -0,0 +1,124 @@
+"""Read the canvas card's HTML export (Netscape bookmark format).
+
+The canvas right-click export writes ``<!DOCTYPE NETSCAPE-Bookmark-file-1>`` with
+``<DT><H3>`` folders and ``<DT><A HREF>`` bookmarks, so the same reader also
+accepts a Chrome bookmark export.  Anything else falls back to plain ``<a href>``
+links, mirroring the canvas app's own import.
+"""
+import json
+from html.parser import HTMLParser
+
+MAX_ITEMS = 20000
+
+
+class _NetscapeParser(HTMLParser):
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.stack = [[]]
+        self.pending_folder = None
+        self.mode = None
+        self.buffer = []
+        self.href = ""
+        self.pending_title = ""
+        self.count = 0
+
+    def handle_starttag(self, tag, attrs):
+        tag = tag.lower()
+        if tag == "dl":
+            if self.pending_folder is not None:
+                self.stack.append(self.pending_folder["children"])
+                self.pending_folder = None
+            else:
+                self.stack.append(self.stack[-1])
+        elif tag == "h3":
+            self.mode, self.buffer = "h3", []
+        elif tag == "a":
+            self.mode, self.buffer = "a", []
+            values = dict(attrs)
+            self.href = (values.get("href") or "").strip()
+        elif tag == "title" and not self.pending_title:
+            self.mode, self.buffer = "title", []
+
+    def handle_endtag(self, tag):
+        tag = tag.lower()
+        if tag == "dl":
+            if len(self.stack) > 1:
+                self.stack.pop()
+        elif tag == "h3" and self.mode == "h3":
+            title = "".join(self.buffer).strip()
+            folder = {"title": title or "Folder", "url": "", "type": "folder", "children": []}
+            self.stack[-1].append(folder)
+            self.pending_folder = folder
+            self.mode, self.buffer = None, []
+        elif tag == "a" and self.mode == "a":
+            title = "".join(self.buffer).strip()
+            if self.href:
+                self.stack[-1].append({"title": title or self.href, "url": self.href,
+                                       "type": "bookmark", "children": []})
+                self.count += 1
+                if self.count > MAX_ITEMS:
+                    raise ValueError("HTML export has too many bookmarks")
+            self.mode, self.buffer, self.href = None, [], ""
+        elif tag == "title" and self.mode == "title":
+            self.pending_title = "".join(self.buffer).strip()
+            self.mode, self.buffer = None, []
+
+    def handle_data(self, data):
+        if self.mode is not None:
+            self.buffer.append(data)
+
+
+def section_from_html(text, title=""):
+    """Parse a Netscape bookmark export into a temporary section payload."""
+    parser = _NetscapeParser()
+    parser.feed(text)
+    parser.close()
+    items = parser.stack[0]
+    if not items:
+        # Some exports are plain HTML; keep every anchor as a flat bookmark.
+        fallback = _links_only(text)
+        items = fallback
+    if not items:
+        raise ValueError("No bookmark links found in this HTML export")
+    return {"format": "bookmark-canvas-section", "schemaVersion": 2,
+            "sectionType": "temporary", "title": title or "Imported bookmarks",
+            "label": "导入", "tempKind": "special", "source": "import-html",
+            "items": items}
+
+
+class _LinkParser(HTMLParser):
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.items = []
+        self.href = None
+        self.buffer = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag.lower() == "a":
+            values = dict(attrs)
+            href = (values.get("href") or "").strip()
+            if href.startswith(("http://", "https://")):
+                self.href, self.buffer = href, []
+
+    def handle_endtag(self, tag):
+        if tag.lower() == "a" and self.href:
+            title = "".join(self.buffer).strip() or self.href
+            self.items.append({"title": title, "url": self.href, "type": "bookmark",
+                               "children": []})
+            self.href, self.buffer = None, []
+
+    def handle_data(self, data):
+        if self.href is not None:
+            self.buffer.append(data)
+
+
+def _links_only(text):
+    parser = _LinkParser()
+    parser.feed(text)
+    parser.close()
+    return parser.items
+
+
+def encode(section):
+    return json.dumps(section, ensure_ascii=False, allow_nan=False,
+                      separators=(",", ":")).encode("utf-8")

package/src/onboarding.py

@@ -3,6 +3,7 @@
 from contextlib import contextmanager
 import json
 import os
+import re
 import shlex
 import shutil
 import sys
@@ -351,13 +352,13 @@ class Onboarding:
         hints = {"exa": ("Exa search and reading", "Exa 搜索与阅读"), "parallel": ("Parallel search and extract", "Parallel 搜索与提取"),
                  "tavily": ("Tavily search and extract", "Tavily 搜索与提取"),
                  "jina": ("Jina Reader; search needs a key", "Jina Reader；搜索需要密钥"),
-                 "deepseek": ("DeepSeek Search / Fetch; needs a key", "DeepSeek 搜索／读取；需要密钥")}
-        # One ordered list answers the former search, fallback-search and reader
-        # questions: selected providers run concurrently and also read pages,
-        # unselected providers stay as the fallback search tier.
+                 "deepseek": ("DeepSeek Search / Fetch; needs a key", "DeepSeek 搜索／读取；需要密钥"),
+                 "perplexity": ("Perplexity structured Search; needs a key; page reading uses another provider", "Perplexity 结构化搜索；需要密钥；网页读取需使用其他服务")}
+        # The selected sources are the concurrent primary tier; the remaining
+        # providers form the fallback tier. Search-only providers are labeled.
         options = [Option(name, name, name, *hints.get(name, ("", None))) for name in Settings.PROVIDERS]
-        primary = console.multiselect("Search and page-reading services (concurrent, in this order)",
-                                      "搜索与正文读取服务（并发执行，按此顺序）", options, current["search"]["providers"])
+        primary = console.multiselect("Search sources (concurrent, in this order)",
+                                      "搜索来源（并发执行，按此顺序）", options, current["search"]["providers"])
         providers = list(primary)
         fallbacks = [name for name in Settings.PROVIDERS if name not in providers]
         storage = self._storage(current)
@@ -382,12 +383,59 @@ class Onboarding:
         elif service == "parallel":
             console.say("Parallel processor remains " + config["parallel"]["processor"] + "; change it with config set if needed.",
                         "Parallel processor 沿用 " + config["parallel"]["processor"] + "；可通过 config set 修改。")
+        custom = current["custom_agentic"]
+        custom_choice = console.select("Custom OpenAI-compatible model endpoint (optional)",
+            "自定义 OpenAI 兼容模型端点（可选）", [
+                Option("disabled", "Disabled", "关闭", "keep custom model search unavailable", "不启用自定义模型搜索"),
+                Option("enabled", "Configure endpoint", "配置端点",
+                       "separate answer channel; not merged into URL search results",
+                       "独立回答通道；不并入 URL 搜索结果")],
+            "enabled" if custom["enabled"] else "disabled")
+        if custom_choice == "enabled":
+            from grok_search import normalize_compatible_base_url
+
+            def check_base_url(value):
+                try:
+                    normalize_compatible_base_url(value)
+                except ValueError as error:
+                    return (str(error), "请输入有效的 HTTP(S) Base URL，不要包含密钥、查询参数或片段。")
+                return None
+
+            base_url = normalize_compatible_base_url(console.input(
+                "OpenAI-compatible Base URL (for example https://gateway.example/v1)",
+                "OpenAI 兼容 Base URL（例如 https://gateway.example/v1）",
+                custom["base_url"], check_base_url))
+
+            def check_model(value):
+                return None if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:/+-]{0,191}", value) else (
+                    "Use a model ID with letters, numbers, '.', '_', ':', '/', '+' or '-'.",
+                    "模型 ID 仅可包含字母、数字及 . _ : / + -。")
+
+            model = console.input("Model ID", "模型 ID", custom["model"], check_model)
+            api_mode = console.select("API protocol", "API 协议", [
+                Option("chat_completions", "Chat Completions", "Chat Completions"),
+                Option("responses", "Responses", "Responses")], custom["api_mode"])
+            search_options = [Option("prompt", "Ask the upstream model to use its own web search",
+                                     "提示上游模型使用其自身联网搜索",
+                                     "the endpoint must provide this capability",
+                                     "端点必须自己具备联网能力")]
+            if api_mode == "responses":
+                search_options.append(Option("native", "Request the Responses web_search tool",
+                                             "请求 Responses web_search 工具",
+                                             "the upstream must implement this tool",
+                                             "上游必须支持此工具"))
+            search_mode = console.select("Search capability", "搜索能力", search_options,
+                                         custom["search_mode"] if any(row.value == custom["search_mode"] for row in search_options) else "prompt")
+            custom_agentic = {"enabled": True, "base_url": base_url, "model": model,
+                              "api_mode": api_mode, "search_mode": search_mode}
+        else:
+            custom_agentic = {"enabled": False}
         # Only changed values are returned, so an unchanged answer keeps the saved
         # Wiki follow-up, readiness policy and archive toggle exactly as they were.
         result = {"research": {"depth": depth, "response_language": language},
                   "search": {"providers": providers, "fallback_providers": fallbacks},
                   "fetch": {"provider": providers[0], "providers": providers},
-                  "professional_research": services}
+                  "professional_research": services, "custom_agentic": custom_agentic}
         result.update(storage)
         return result
 
@@ -432,14 +480,16 @@ class Onboarding:
         console.note("API keys", "API 密钥", [
             ("Retrieval adapters are included; anonymous search/reading works where the provider permits it.",
              "检索适配器已包含在插件中；服务方允许时可使用匿名搜索和阅读。"),
+            ("Select keys for services enabled above; use Space to select, Enter to continue. You can skip this step.",
+             "选择上面启用的服务所需的密钥；空格勾选、回车继续，也可以跳过。"),
             ("Host MCP OAuth login stays in the host. An OpenAI API key is separate from a ChatGPT/Codex login.",
              "宿主 MCP 的 OAuth 登录由宿主管理；OpenAI API Key 与 ChatGPT/Codex 登录分开。"),
             ("Keys are hidden and saved in a local, unencrypted file readable only by your user: " + str(self.credentials.path),
              "密钥隐藏输入，并保存为仅当前用户可读的本地明文文件：" + str(self.credentials.path)),
             ("Environment variables override saved keys. Select nothing to configure credentials later.",
              "环境变量优先于保存的密钥；不选择即可稍后配置。"),
-            ("Guidance: exa.ai/docs/reference/exa-mcp · docs.parallel.ai · docs.tavily.com · jina.ai/reader · platform.openai.com/api-keys",
-             "说明：exa.ai/docs/reference/exa-mcp · docs.parallel.ai · docs.tavily.com · jina.ai/reader · platform.openai.com/api-keys")])
+            ("Key names and storage: https://github.com/Browser-bookmark-hub/Bookmark-Research/blob/main/docs/configuration.en.md",
+             "Key 名称与保存位置说明：https://github.com/Browser-bookmark-hub/Bookmark-Research/blob/main/docs/configuration.md")])
         status = {"environment": ("set in environment (overrides saved key)", "环境变量已设置（优先于保存的密钥）"),
                   "private_file": ("configured, not yet verified", "已配置，尚未验证"), None: ("missing", "未配置")}
         options = [Option(row["name"], row["name"].split("_")[0].lower(), None, *status[row["source"] if row["configured"] else None], hint_always=True)

package/src/perplexity.py

@@ -0,0 +1,78 @@
+"""Perplexity Search API client; returns structured sources, not a model answer."""
+
+import http.client
+import json
+import threading
+import urllib.error
+import urllib.request
+
+from release import user_agent
+from remote_mcp import McpError, McpHttpClient
+from service_http import ResearchHttp, _NoRedirect
+
+
+class PerplexityClient:
+    SEARCH_URL = "https://api.perplexity.ai/search"
+
+    def __init__(self, headers=None, timeout=30, search_url=None):
+        self.headers = dict(headers or {})
+        self.timeout = timeout
+        self.search_url = search_url or self.SEARCH_URL
+        self._lock = threading.Lock()
+        self._requests = 0
+
+    def snapshot_usage(self):
+        with self._lock:
+            return {"http_requests": self._requests, "tool_calls": self._requests}
+
+    def _request(self, query, limit):
+        payload = json.dumps({"query": query, "max_results": max(1, min(int(limit), 20))},
+                             ensure_ascii=False, allow_nan=False).encode("utf-8")
+        request = urllib.request.Request(self.search_url, data=payload, method="POST", headers={
+            "Accept": "application/json", "Content-Type": "application/json",
+            "User-Agent": user_agent(), **self.headers})
+        with self._lock:
+            self._requests += 1
+        try:
+            with urllib.request.build_opener(_NoRedirect()).open(request, timeout=self.timeout) as response:
+                body = response.read(McpHttpClient.MAX_BYTES + 1)
+            if len(body) > McpHttpClient.MAX_BYTES:
+                raise McpError("Perplexity response exceeds size limit", "response_too_large")
+            value = json.loads(body, object_pairs_hook=ResearchHttp._unique_object,
+                               parse_constant=ResearchHttp._invalid_constant,
+                               parse_float=ResearchHttp._finite_float)
+            if not isinstance(value, dict) or not isinstance(value.get("results"), list):
+                raise McpError("Perplexity returned invalid search results", "result_format")
+            rows = []
+            for item in value["results"]:
+                if not isinstance(item, dict) or not isinstance(item.get("url"), str):
+                    continue
+                snippet = item.get("snippet")
+                rows.append({"url": item["url"], "title": item.get("title") or item["url"],
+                             "snippet": snippet if isinstance(snippet, str) else "",
+                             "date": item.get("date"), "last_updated": item.get("last_updated")})
+            return {"structuredContent": {"results": rows}}
+        except urllib.error.HTTPError as error:
+            status = error.code
+            kind = {400: "invalid_request", 401: "authentication_required", 403: "access_denied",
+                    408: "timeout", 429: "rate_limited"}.get(status, "http_error")
+            retry = McpHttpClient._retry_after((error.headers or {}).get("Retry-After"))
+            try:
+                error.close()
+            except OSError:
+                pass
+            raise McpError("Perplexity Search HTTP " + str(status), kind,
+                           status == 429 or status >= 500, http_status=status, retry_after=retry) from None
+        except TimeoutError:
+            raise McpError("Perplexity Search timed out", "timeout", True) from None
+        except (urllib.error.URLError, OSError, http.client.HTTPException):
+            raise McpError("Perplexity Search connection failed", "connection_error", True) from None
+        except (ValueError, UnicodeError):
+            raise McpError("Perplexity returned invalid JSON", "result_format") from None
+
+    def call(self, purpose, query=None, urls=None, limit=5):
+        if purpose != "search":
+            raise McpError("Perplexity Search does not provide URL extraction", "capability_unavailable")
+        if not self.headers.get("Authorization"):
+            raise McpError("Perplexity Search requires PERPLEXITY_API_KEY", "authentication_required")
+        return self._request(query, limit)

package/src/raw_library.py

@@ -22,9 +22,12 @@ import re
 import sqlite3
 from pathlib import Path
 
+import storage
 from settings import Settings
 
 SCHEMA_VERSION = 1
+_EMPTY_REPORT = {"missing_tasks": 0, "scanned_units": 0, "removed_units": 0,
+                 "new_documents": 0, "errors": []}
 MAX_BODY_BYTES = 50 * 1024 * 1024
 MAX_TERMS = 16
 TRIGRAM_MINIMUM = 3
@@ -80,7 +83,19 @@ class RawLibrary:
 
     # Storage -----------------------------------------------------------------
 
-    def _connect(self):
+    def _connect(self, readonly=False):
+        if readonly:
+            if not self.db_path.is_file():
+                raise sqlite3.OperationalError("Raw library index is not built yet: %s" % self.db_path)
+            connection = storage.connect(self.db_path, readonly=True, timeout=30)
+            connection.row_factory = sqlite3.Row
+            try:
+                stored = dict(connection.execute("SELECT key, value FROM meta").fetchall())
+            except sqlite3.DatabaseError:
+                stored = {}
+            if stored.get("mode") in ("trigram", "substring"):
+                self.mode = stored["mode"]
+            return storage.mark_query_only(connection)
         self.db_path.parent.mkdir(parents=True, exist_ok=True)
         for attempt in (0, 1):
             connection = sqlite3.connect(str(self.db_path), timeout=30, isolation_level=None)
@@ -88,9 +103,9 @@ class RawLibrary:
                 connection.execute("PRAGMA journal_mode=WAL")
                 self._schema(connection)
                 return connection
-            except sqlite3.DatabaseError:
+            except sqlite3.DatabaseError as error:
                 connection.close()
-                if attempt:
+                if attempt or storage.is_readonly_failure(error):
                     raise
                 # A derived cache: a damaged file is discarded and rebuilt.
                 for suffix in ("", "-wal", "-shm"):
@@ -98,6 +113,16 @@ class RawLibrary:
                     if candidate.exists():
                         candidate.unlink()
 
+    def _open_for_search(self):
+        """Prefer the writable index (it self-heals); a read-only one stays searchable."""
+        try:
+            connection = self._connect()
+        except (sqlite3.DatabaseError, OSError) as error:
+            if not storage.is_readonly_failure(error) or not self.db_path.is_file():
+                raise
+            return self._connect(readonly=True), False
+        return connection, True
+
     def _schema(self, connection):
         connection.execute("CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL)")
         stored = dict(connection.execute("SELECT key, value FROM meta").fetchall())
@@ -343,15 +368,17 @@ class RawLibrary:
         text = re.sub(r"\s+", " ", text).strip()
         return ("…" if start > 0 else "") + text + ("…" if start + width < len(body) else "")
 
-    def search(self, query, limit=10, offset=0, url=None, occurrence_limit=50, occurrence_offset=0):
+    def search(self, query, limit=10, offset=0, url=None, occurrence_limit=50, occurrence_offset=0, expand_versions=False):
         """Literal search over saved page text; every term must occur (quoted query = one phrase).
 
-        Identical bodies are merged by SHA-256, so one result can have many
-        occurrences. Occurrences are paged with ``occurrence_limit`` /
-        ``occurrence_offset``: every result reports the true ``occurrence_count``,
-        whether the page is complete, and the offset that continues it.
+        By default, results are grouped by URL, reporting version counts,
+        time spans, and all captured versions. When expand_versions=True,
+        bodies are merged by SHA-256. Occurrences are paged with
+        occurrence_limit / occurrence_offset.
         """
         terms = self._terms(query)
+        if type(expand_versions) is not bool:
+            raise ValueError("expand_versions must be a boolean")
         for value, label, minimum, maximum in ((limit, "limit", 1, 50), (offset, "offset", 0, 10000000),
                                                (occurrence_limit, "occurrence_limit", 1, 1000),
                                                (occurrence_offset, "occurrence_offset", 0, 10000000)):
@@ -359,9 +386,20 @@ class RawLibrary:
                 raise ValueError("%s must be between %s and %s" % (label, minimum, maximum))
         if url is not None and (not isinstance(url, str) or not url or len(url) > 2048 or "\x00" in url):
             raise ValueError("url filter must be 1 to 2048 characters")
-        connection = self._connect()
+        connection, writable = self._open_for_search()
         try:
-            report = self._refresh(connection)
+            if writable:
+                try:
+                    report = self._refresh(connection)
+                except sqlite3.DatabaseError as error:
+                    if not storage.is_readonly_failure(error):
+                        raise
+                    # Querying a shared index on a read-only mount: nothing was written.
+                    connection.close()
+                    connection, writable = self._connect(readonly=True), False
+                    report = _EMPTY_REPORT
+            else:
+                report = _EMPTY_REPORT
             indexed = self._statistics(connection, report)
             fts = self.mode == "trigram" and all(len(term) >= TRIGRAM_MINIMUM for term in terms)
             clauses, parameters = [], []
@@ -376,49 +414,149 @@ class RawLibrary:
                 clauses.append("d.sha256 IN (SELECT sha256 FROM occurrences WHERE instr(lower(url), lower(?)) > 0)")
                 parameters.append(url)
             base = ("FROM page_text JOIN documents d ON d.id = page_text.rowid WHERE " + " AND ".join(clauses))
-            total = connection.execute("SELECT COUNT(*) " + base, parameters).fetchone()[0]
             order = "bm25(page_text), d.id DESC" if fts else "d.id DESC"
-            rows = connection.execute("SELECT d.sha256, d.characters, page_text.body " + base +
-                                      " ORDER BY " + order + " LIMIT ? OFFSET ?",
-                                      parameters + [limit, offset]).fetchall()
-            results = []
-            for sha256, characters, body in rows:
-                count = connection.execute("SELECT COUNT(*) FROM occurrences WHERE sha256 = ?",
-                                           (sha256,)).fetchone()[0]
-                occurrences = []
-                for row in connection.execute(
-                        "SELECT kind, capture_id, research_id, source_id, path, url, title, retrieved_at "
-                        "FROM occurrences WHERE sha256 = ? "
-                        "ORDER BY (retrieved_at IS NULL), retrieved_at DESC, unit, position "
-                        "LIMIT ? OFFSET ?", (sha256, occurrence_limit, occurrence_offset)):
-                    kind, capture_id, research_id, source_id, path, found_url, title, retrieved_at = row
-                    entry = {"kind": kind}
-                    if kind == "archive":
-                        entry["capture_id"] = capture_id
-                    else:
-                        entry.update(research_id=research_id, source_id=source_id)
-                    entry.update(path=path, url=found_url, title=title, retrieved_at=retrieved_at)
-                    occurrences.append(entry)
-                shown = occurrence_offset + len(occurrences)
-                results.append({"sha256": sha256, "characters": characters,
-                                "url": next((entry["url"] for entry in occurrences if entry["url"]), None),
-                                "title": next((entry["title"] for entry in occurrences if entry["title"]), None),
-                                "snippet": self._snippet(body, terms),
-                                "occurrence_count": count,
-                                "occurrence_offset": occurrence_offset,
-                                "occurrences_truncated": shown < count,
-                                "occurrences_next_offset": shown if shown < count else None,
-                                "occurrences": occurrences})
+
+            if expand_versions:
+                total = connection.execute("SELECT COUNT(*) " + base, parameters).fetchone()[0]
+                rows = connection.execute("SELECT d.sha256, d.characters, page_text.body " + base +
+                                          " ORDER BY " + order + " LIMIT ? OFFSET ?",
+                                          parameters + [limit, offset]).fetchall()
+                results = []
+                for sha256, characters, body in rows:
+                    count = connection.execute("SELECT COUNT(*) FROM occurrences WHERE sha256 = ?",
+                                               (sha256,)).fetchone()[0]
+                    occurrences = []
+                    for row in connection.execute(
+                            "SELECT kind, capture_id, research_id, source_id, path, url, title, retrieved_at "
+                            "FROM occurrences WHERE sha256 = ? "
+                            "ORDER BY (retrieved_at IS NULL), retrieved_at DESC, unit, position "
+                            "LIMIT ? OFFSET ?", (sha256, occurrence_limit, occurrence_offset)):
+                        kind, capture_id, research_id, source_id, path, found_url, title, retrieved_at = row
+                        entry = {"kind": kind}
+                        if kind == "archive":
+                            entry["capture_id"] = capture_id
+                        else:
+                            entry.update(research_id=research_id, source_id=source_id)
+                        entry.update(path=path, url=found_url, title=title, retrieved_at=retrieved_at)
+                        occurrences.append(entry)
+                    shown = occurrence_offset + len(occurrences)
+                    results.append({"sha256": sha256, "characters": characters,
+                                    "url": next((entry["url"] for entry in occurrences if entry["url"]), None),
+                                    "title": next((entry["title"] for entry in occurrences if entry["title"]), None),
+                                    "snippet": self._snippet(body, terms),
+                                    "occurrence_count": count,
+                                    "occurrence_offset": occurrence_offset,
+                                    "occurrences_truncated": shown < count,
+                                    "occurrences_next_offset": shown if shown < count else None,
+                                    "occurrences": occurrences})
+            else:
+                doc_rows = connection.execute("SELECT d.sha256, d.characters " + base +
+                                              " ORDER BY " + order, parameters).fetchall()
+                shas = [row[0] for row in doc_rows]
+                sha_chars = {row[0]: row[1] for row in doc_rows}
+
+                occ_map = {}
+                if shas:
+                    for i in range(0, len(shas), 900):
+                        chunk = shas[i : i + 900]
+                        placeholders = ",".join("?" for _ in chunk)
+                        q = (
+                            "SELECT sha256, kind, capture_id, research_id, source_id, path, url, title, retrieved_at "
+                            "FROM occurrences WHERE sha256 IN (" + placeholders + ") "
+                            "ORDER BY (retrieved_at IS NULL), retrieved_at DESC, unit, position"
+                        )
+                        for r in connection.execute(q, chunk):
+                            occ_map.setdefault(r[0], []).append(r[1:])
+
+                groups = {}
+                for sha256 in shas:
+                    characters = sha_chars[sha256]
+                    occ_rows = occ_map.get(sha256, [])
+                    found_url = next((r[5] for r in occ_rows if r[5]), None)
+                    grp_key = found_url if found_url else sha256
+                    if grp_key not in groups:
+                        groups[grp_key] = {
+                            "url": found_url,
+                            "versions": [],
+                            "all_occurrences": [],
+                        }
+                    v_title = next((r[6] for r in occ_rows if r[6]), None)
+                    v_retrieved = next((r[7] for r in occ_rows if r[7]), None)
+                    groups[grp_key]["versions"].append({
+                        "sha256": sha256,
+                        "characters": characters,
+                        "title": v_title,
+                        "retrieved_at": v_retrieved,
+                        "occurrence_count": len(occ_rows),
+                    })
+                    for r in occ_rows:
+                        k, cap_id, res_id, src_id, p, u, t, ret_at = r
+                        entry = {"kind": k}
+                        if k == "archive":
+                            entry["capture_id"] = cap_id
+                        else:
+                            entry.update(research_id=res_id, source_id=src_id)
+                        entry.update(path=p, url=u, title=t, retrieved_at=ret_at)
+                        groups[grp_key]["all_occurrences"].append(entry)
+
+                total = len(groups)
+                paged_keys = list(groups.keys())[offset : offset + limit]
+
+                needed_shas = {v["sha256"] for k in paged_keys for v in groups[k]["versions"]}
+                body_map = {}
+                if needed_shas:
+                    placeholders = ",".join("?" for _ in needed_shas)
+                    bq = (
+                        "SELECT d.sha256, page_text.body FROM page_text "
+                        "JOIN documents d ON d.id = page_text.rowid "
+                        "WHERE d.sha256 IN (" + placeholders + ")"
+                    )
+                    for s_hash, body in connection.execute(bq, list(needed_shas)).fetchall():
+                        body_map[s_hash] = self._snippet(body, terms)
+
+                results = []
+                for key in paged_keys:
+                    grp = groups[key]
+                    versions = grp["versions"]
+                    for v in versions:
+                        v["snippet"] = body_map.get(v["sha256"], "")
+                    versions.sort(key=lambda v: (v["retrieved_at"] is not None, v["retrieved_at"] or ""), reverse=True)
+                    all_occs = grp["all_occurrences"]
+                    all_occs.sort(key=lambda o: (o.get("retrieved_at") is not None, o.get("retrieved_at") or ""), reverse=True)
+                    total_occ_count = len(all_occs)
+                    paged_occs = all_occs[occurrence_offset : occurrence_offset + occurrence_limit]
+                    shown = occurrence_offset + len(paged_occs)
+                    dates = [v["retrieved_at"] for v in versions if v["retrieved_at"]]
+                    earliest = min(dates) if dates else None
+                    latest = max(dates) if dates else None
+                    top_version = versions[0]
+                    results.append({
+                        "url": grp["url"],
+                        "title": top_version["title"],
+                        "sha256": top_version["sha256"],
+                        "characters": top_version["characters"],
+                        "snippet": top_version["snippet"],
+                        "version_count": len(versions),
+                        "time_span": {"earliest": earliest, "latest": latest},
+                        "versions": versions,
+                        "occurrence_count": total_occ_count,
+                        "occurrence_offset": occurrence_offset,
+                        "occurrences_truncated": shown < total_occ_count,
+                        "occurrences_next_offset": shown if shown < total_occ_count else None,
+                        "occurrences": paged_occs,
+                    })
         finally:
             connection.close()
         next_offset = offset + len(results)
         return {"query": query, "total": total, "offset": offset,
                 "next_offset": next_offset if next_offset < total else None,
                 "occurrence_limit": occurrence_limit, "occurrence_offset": occurrence_offset,
+                "expand_versions": expand_versions, "grouped_by": "hash" if expand_versions else "url",
                 "match": "fts_trigram" if fts else "substring", "results": results, "indexed": indexed,
-                "note": "Literal matching over saved page text (knowledge archive and research evidence); "
-                        "not web search and not semantic retrieval. Identical bodies are merged by SHA-256 and "
-                        "each result pages its occurrences with occurrence_offset/occurrence_limit."}
+                "index_refresh": "performed" if writable else "skipped_read_only",
+                "note": ("Literal matching over saved page text (knowledge archive and research evidence); "
+                         "not web search and not semantic retrieval. " +
+                         ("Results are grouped by URL with version counts and time spans. Use expand_versions=true to view distinct content hashes." if not expand_versions else "Identical bodies are merged by SHA-256 and each result pages its occurrences with occurrence_offset/occurrence_limit."))}
 
     def search_terms(self, terms, limit=5):
         """Search several literal terms and merge them into one preflight list.