bookmark-research 0.6.0 → 0.7.0

package/src/settings.py

@@ -16,16 +16,13 @@ from package_paths import is_package_dir
 class Settings:
     """Load on each operation so MCP clients can change preferences without restart."""
 
-    PROVIDERS = ("exa", "parallel", "tavily", "jina", "deepseek")
+    PROVIDERS = ("exa", "parallel", "tavily", "jina", "deepseek", "perplexity")
+    FETCH_PROVIDERS = ("exa", "parallel", "tavily", "jina", "deepseek", "browser")
     TOOL_SCOPE_PRESETS = ("all", "quick", "deep")
     # Daily context, retrieval, knowledge storage and provider-managed research.
     # Persistent inventories and diagnostics remain in the explicit full catalog.
     QUICK_TOOLS = (
-        "sync_package", "index_status", "search_bookmarks", "get_context", "plan_research",
-        "search_web", "fetch_web", "search_archive", "wiki_write", "wiki_get", "wiki_list", "wiki_search",
-        "research_services", "research_service_prepare", "research_service_start", "research_service_status",
-        "research_service_result", "research_service_cancel", "research_service_attach",
-        "get_settings", "update_settings")
+        "locate", "read", "walk", "search", "fetch", "ingest", "save", "spawn", "job", "health")
     OPENAI_RESEARCH_MODELS = ("gpt-5.6-sol", "o3-deep-research", "o4-mini-deep-research")
 
     def __init__(self, path=None):
@@ -83,8 +80,12 @@ class Settings:
                     "openai": {"model": "gpt-5.6-sol", "max_tool_calls": 24,
                                "allow_untested_model": False},
                     "parallel": {"processor": "pro"}},
-                "wiki": {"directory": str(cls.data_directory() / "wiki"), "after_research": "suggest",
-                         "max_source_age_days": 30},
+                "custom_agentic": {"enabled": False, "base_url": "", "model": "",
+                                   "api_mode": "chat_completions", "search_mode": "prompt"},
+                "browser": {"enabled": False, "driver": "auto", "cdp_url": "http://127.0.0.1:9222",
+                            "headless": True, "timeout_seconds": 20, "max_concurrency": 2},
+                "wiki": {"directory": str(cls.data_directory() / "wiki"), "after_research": "auto",
+                         "max_source_age_days": 30, "aliases": []},
                 "wiki_preflight": True,
                 "freshness_days": 30,
                 "plan": {"max_characters": 8000, "min_relevance": 0.05, "homepage_limit": 2},
@@ -112,7 +113,7 @@ class Settings:
         defaults = cls.defaults()
         if set(value) - set(defaults):
             raise ValueError("Unknown settings fields; credentials belong in the environment or private credential file")
-        for section in ("search", "fetch", "server", "archive", "research", "readiness", "professional_research", "wiki", "plan", "output"):
+        for section in ("search", "fetch", "server", "archive", "research", "readiness", "professional_research", "custom_agentic", "browser", "wiki", "plan", "output"):
             if not isinstance(value.get(section), dict) or set(value[section]) != set(defaults[section]):
                 raise ValueError("Invalid settings fields in " + section)
         for label, number, minimum, maximum in (
@@ -163,14 +164,20 @@ class Settings:
             raise ValueError("search.strategy must be hedged, staggered, sequential or wait_all")
         if value["fetch"]["strategy"] not in ("sequential", "staggered", "hedged"):
             raise ValueError("fetch.strategy must be sequential, staggered or hedged")
-        for section, field, minimum in (("search", "providers", 1), ("fetch", "providers", 1),
-                                        ("search", "fallback_providers", 0)):
+        for section, field, allowed, minimum in (("search", "providers", cls.PROVIDERS, 1),
+                                                ("fetch", "providers", cls.FETCH_PROVIDERS, 1),
+                                                ("search", "fallback_providers", cls.PROVIDERS, 0)):
             providers = value[section][field]
-            if (not isinstance(providers, list) or not minimum <= len(providers) <= len(cls.PROVIDERS)
-                    or any(p not in cls.PROVIDERS for p in providers) or len(set(providers)) != len(providers)):
+            if (not isinstance(providers, list) or not minimum <= len(providers) <= len(allowed)
+                    or any(p not in allowed for p in providers) or len(set(providers)) != len(providers)):
                 raise ValueError(section + "." + field + " must select supported providers without duplicates")
-        if value["fetch"]["provider"] not in cls.PROVIDERS:
+        if value["fetch"]["provider"] not in cls.FETCH_PROVIDERS:
             raise ValueError("fetch.provider must select a supported provider")
+        for field in ("provider", "providers"):
+            configured = value["fetch"][field]
+            names = [configured] if field == "provider" else configured
+            if any(name not in cls.FETCH_PROVIDERS for name in names):
+                raise ValueError("fetch settings may select only providers that support page extraction")
         if type(value["archive"]["enabled"]) is not bool:
             raise ValueError("archive.enabled must be a boolean")
         directory = value["archive"]["directory"]
@@ -213,12 +220,50 @@ class Settings:
         processor = services["parallel"]["processor"]
         if not isinstance(processor, str) or not re.fullmatch(r"[a-z][a-z0-9-]{0,63}", processor):
             raise ValueError("professional_research.parallel.processor must be a processor name")
+        custom = value["custom_agentic"]
+        if type(custom["enabled"]) is not bool:
+            raise ValueError("custom_agentic.enabled must be a boolean")
+        if custom["api_mode"] not in ("chat_completions", "responses"):
+            raise ValueError("custom_agentic.api_mode must be chat_completions or responses")
+        if custom["search_mode"] not in ("prompt", "native"):
+            raise ValueError("custom_agentic.search_mode must be prompt or native")
+        if custom["search_mode"] == "native" and custom["api_mode"] != "responses":
+            raise ValueError("custom_agentic native web search requires the Responses API")
+        if not isinstance(custom["base_url"], str) or len(custom["base_url"]) > 1024:
+            raise ValueError("custom_agentic.base_url must be a string up to 1024 characters")
+        if not isinstance(custom["model"], str) or len(custom["model"]) > 192:
+            raise ValueError("custom_agentic.model must be a string up to 192 characters")
+        if custom["enabled"]:
+            from grok_search import normalize_compatible_base_url
+            normalize_compatible_base_url(custom["base_url"])
+            if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:/+-]{0,191}", custom["model"]):
+                raise ValueError("custom_agentic.model must be a valid model identifier")
+        browser = value["browser"]
+        if type(browser["enabled"]) is not bool:
+            raise ValueError("browser.enabled must be a boolean")
+        if browser["driver"] not in ("auto", "playwright", "cdp", "mock"):
+            raise ValueError("browser.driver must be auto, playwright, cdp or mock")
+        if not isinstance(browser["cdp_url"], str) or len(browser["cdp_url"]) > 1024:
+            raise ValueError("browser.cdp_url must be a string up to 1024 characters")
+        if type(browser["headless"]) is not bool:
+            raise ValueError("browser.headless must be a boolean")
+        for label, number, minimum, maximum in (
+                ("browser.timeout_seconds", browser["timeout_seconds"], 1, 60),
+                ("browser.max_concurrency", browser["max_concurrency"], 1, 8)):
+            if type(number) is not int or not minimum <= number <= maximum:
+                raise ValueError("%s must be an integer between %s and %s" % (label, minimum, maximum))
         wiki_directory = value["wiki"]["directory"]
         if not isinstance(wiki_directory, str) or not wiki_directory.strip() or "\x00" in wiki_directory:
             raise ValueError("wiki.directory must be a nonempty path")
         value["wiki"]["directory"] = str(cls.external_path(wiki_directory, "Wiki directory"))
         if value["wiki"]["after_research"] not in ("suggest", "auto", "off"):
             raise ValueError("wiki.after_research must be suggest, auto or off")
+        aliases = value["wiki"].get("aliases", [])
+        if not isinstance(aliases, list):
+            raise ValueError("wiki.aliases must be a list of alias groups")
+        for group in aliases:
+            if not isinstance(group, (list, tuple)) or not all(isinstance(t, str) for t in group):
+                raise ValueError("each alias group in wiki.aliases must be a list of strings")
         output = value["output"]
         if output["mode"] not in ("central", "beside_input"):
             raise ValueError("output.mode must be central or beside_input")
@@ -233,9 +278,9 @@ class Settings:
 
     @staticmethod
     def fetch_call_count(provider, urls):
-        # Reader has one HTTP request per URL; the other providers batch URLs
+        # Reader and Browser have one request/render per URL; the other providers batch URLs
         # in one MCP tool call. Reserve this before any network work begins.
-        return len(urls) if provider == "jina" else 1
+        return len(urls) if provider in ("jina", "browser") else 1
 
     @staticmethod
     def _unique_object(pairs):

package/src/source_inputs.py

@@ -36,6 +36,53 @@ def _safe_name(name):
     return parts
 
 
+def _looks_like_section(obj):
+    return isinstance(obj, dict) and (
+        obj.get("format") == "bookmark-canvas-section"
+        or (obj.get("sectionType") in ("permanent", "temporary")
+            and ("items" in obj or "tree" in obj)))
+
+
+def _unwrap_export(obj, depth=0):
+    """Mirror the canvas app: accept {data|tree|bookmarkTree|bookmarks|items} wrappers."""
+    if depth > 5 or not isinstance(obj, dict) or _looks_like_section(obj):
+        return obj
+    for key in ("data", "tree", "bookmarkTree", "bookmarks", "items"):
+        value = obj.get(key)
+        if isinstance(value, (dict, list)):
+            return _unwrap_export(value, depth + 1)
+    return obj
+
+
+def _wrap_items(items, name):
+    return {"format": "bookmark-canvas-section", "schemaVersion": 2,
+            "sectionType": "temporary", "title": Path(name).stem,
+            "label": "导入", "tempKind": "special", "source": "import-json",
+            "items": items}
+
+
+def _normalize_section_bytes(raw, name):
+    try:
+        obj = json.loads(raw.decode("utf-8"))
+    except (UnicodeDecodeError, ValueError):
+        return raw
+    if _looks_like_section(obj) and obj.get("format") == "bookmark-canvas-section":
+        return raw  # already a section: keep bytes identical for idempotent re-import
+    if _looks_like_section(obj):
+        obj = dict(obj, format="bookmark-canvas-section")
+        return json.dumps(obj, ensure_ascii=False).encode("utf-8")
+    candidate = _unwrap_export(obj)
+    if _looks_like_section(candidate):
+        if "format" not in candidate:
+            candidate = dict(candidate, format="bookmark-canvas-section")
+        return json.dumps(candidate, ensure_ascii=False).encode("utf-8")
+    if isinstance(candidate, list):
+        return json.dumps(_wrap_items(candidate, name), ensure_ascii=False).encode("utf-8")
+    if isinstance(candidate, dict) and isinstance(candidate.get("children"), list):
+        return json.dumps(_wrap_items([candidate], name), ensure_ascii=False).encode("utf-8")
+    return raw
+
+
 def _section_name(name, raw, section_paths):
     obj = json_object(raw)
     prefix = "永久栏目" if obj.get("sectionType") == "permanent" else "临时栏目"
@@ -175,6 +222,36 @@ def _zip(path, section_paths):
         raise ValueError("Cannot read ZIP export: " + str(exc)) from exc
 
 
+def scan_stats(path):
+    path = Path(path).expanduser().resolve()
+    if path.is_file():
+        if path.suffix.lower() in (".zip", ".json"):
+            st = path.stat()
+            return {path.name: [st.st_mtime_ns, st.st_size]}
+        raise ValueError("Input file must be ZIP or section JSON: " + str(path))
+    if not path.is_dir():
+        raise ValueError("Input directory does not exist: " + str(path))
+
+    stats = {}
+    for canvas in path.glob("*.canvas"):
+        if not canvas.is_symlink() and canvas.is_file():
+            st = canvas.stat()
+            stats[canvas.relative_to(path).as_posix()] = [st.st_mtime_ns, st.st_size]
+    for loose in path.glob("*.json"):
+        if not loose.is_symlink() and loose.is_file():
+            st = loose.stat()
+            stats[loose.relative_to(path).as_posix()] = [st.st_mtime_ns, st.st_size]
+    for directory in PROTOCOL_DIRS:
+        branch = path / directory
+        if branch.is_symlink() or not branch.is_dir():
+            continue
+        for entry in branch.rglob("*"):
+            if not entry.is_symlink() and entry.is_file() and entry.suffix == ".json":
+                st = entry.stat()
+                stats[entry.relative_to(path).as_posix()] = [st.st_mtime_ns, st.st_size]
+    return stats
+
+
 def read_input(path, section_paths=None):
     path = Path(path).expanduser().resolve()
     section_paths = section_paths or {}
@@ -186,17 +263,46 @@ def read_input(path, section_paths=None):
         files, single_section = _zip(path, section_paths)
     elif path.is_file() and path.suffix.lower() == ".json":
         kind, single_section = "section", True
-        raw = _read_file(path, path.parent)
+        raw = _normalize_section_bytes(_read_file(path, path.parent), path.name)
         files = {_section_name(path.name, raw, section_paths): raw}
+    elif path.is_file() and path.suffix.lower() in (".html", ".htm"):
+        from netscape import encode as _encode_html_section, section_from_html
+        kind, single_section = "section", True
+        text = _read_file(path, path.parent).decode("utf-8", "replace")
+        raw = _encode_html_section(section_from_html(text, title=path.stem))
+        files = {_section_name(path.stem + ".json", raw, section_paths): raw}
     else:
-        raise ValueError("Input must be an existing package directory, ZIP, or section JSON: " + str(path))
+        raise ValueError("Input must be an existing package directory, ZIP, section JSON, or bookmark HTML: " + str(path))
     if not files:
         raise ValueError("No protocol JSON or .canvas files in export; the saved index was retained")
     if sum(name.endswith(".canvas") for name in files) > 1:
         raise ValueError("An export must have at most one .canvas entry")
     file_hashes = hashes(files)
+    file_stats = {}
+    if kind == "directory":
+        for name in files:
+            file_entry = path / name
+            try:
+                st = file_entry.stat()
+                file_stats[name] = [st.st_mtime_ns, st.st_size]
+            except OSError:
+                pass
+        for loose in path.glob("*.json"):
+            if not loose.is_symlink() and loose.is_file():
+                try:
+                    st = loose.stat()
+                    file_stats[loose.relative_to(path).as_posix()] = [st.st_mtime_ns, st.st_size]
+                except OSError:
+                    pass
+    elif kind in ("zip", "section"):
+        try:
+            st = path.stat()
+            file_stats[path.name] = [st.st_mtime_ns, st.st_size]
+        except OSError:
+            pass
     return {"path": str(path), "kind": kind, "single_section": single_section,
-            "files": files, "hashes": file_hashes, "fingerprint": fingerprint(file_hashes)}
+            "files": files, "hashes": file_hashes, "stats": file_stats,
+            "fingerprint": fingerprint(file_hashes)}
 
 
 def git_directory(path):

package/src/source_manager.py

@@ -14,7 +14,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 
 from bookmark_index import _dump, _inside, _string
-from source_inputs import fingerprint, git_busy, git_directory, hashes, read_input
+from source_inputs import fingerprint, git_busy, git_directory, hashes, read_input, scan_stats
 
 
 SCHEMA = (
@@ -35,10 +35,11 @@ class SourceManager:
         self.index, self.connection = index, index.connection
         self.clock, self.debounce, self.deletion_grace = clock, debounce, deletion_grace
         self.storage = index.db_path.with_name(index.db_path.name + ".sources") if index.db_path else None
-        with index.transaction():
-            for statement in SCHEMA:
-                self.connection.execute(statement)
-            self._recover_retired_snapshots()
+        if not index.readonly:
+            with index.transaction():
+                for statement in SCHEMA:
+                    self.connection.execute(statement)
+                self._recover_retired_snapshots()
 
     def _recover_retired_snapshots(self):
         """Recover a removal interrupted before commit, using surviving DB references."""
@@ -307,7 +308,7 @@ class SourceManager:
             self.connection.execute("INSERT OR IGNORE INTO source_aliases VALUES(?,?)", (source_id, prepared["path"]))
             record = {"mode": mode, "completeness": completeness, "input_path": prepared["path"],
                 "input_kind": prepared["kind"], "version_id": version_id, "snapshot_path": str(snapshot_path),
-                "observed_hashes": prepared["hashes"], "checked_at": self._now(),
+                "observed_hashes": prepared["hashes"], "observed_stats": prepared.get("stats"), "checked_at": self._now(),
                 "state": "snapshot" if mode == "snapshot" else "current", "error": None,
                 "pending_files": [], "pending_fingerprint": None, "pending_since": None,
                 "retained_missing_files": summary["retained_missing_files"], "legacy": False}
@@ -344,6 +345,39 @@ class SourceManager:
             "SELECT COUNT(*) FROM %s WHERE source_id=?" % table, (source_id,)).fetchone()[0]
             for table in ("files", "sections", "items", "nodes", "edges", "memberships")}
 
+    def _reference_impact(self, source_id):
+        """Find durable research scopes and Wiki revisions that name a source."""
+        from settings import Settings
+        settings = Settings().load()
+        research_root = Path(settings["output"]["directory"])
+        wiki_root = Path(settings["wiki"]["directory"])
+        research_ids, pages = set(), set()
+        if research_root.is_dir() and not research_root.is_symlink():
+            for state_path in research_root.glob("*/state.json"):
+                if state_path.is_symlink():
+                    continue
+                try:
+                    state = json.loads(state_path.read_text(encoding="utf-8"))
+                except (OSError, ValueError):
+                    continue
+                if source_id in state.get("source_scope", {}).get("source_ids", []):
+                    research_ids.add(state.get("research_id") or state_path.parent.name)
+        revisions = wiki_root / "revisions"
+        if revisions.is_dir() and not revisions.is_symlink():
+            for revision_path in revisions.glob("*/*.json"):
+                if revision_path.is_symlink():
+                    continue
+                try:
+                    record = json.loads(revision_path.read_text(encoding="utf-8"))
+                except (OSError, ValueError):
+                    continue
+                if any(ref.get("source_id") == source_id
+                       for evidence in record.get("evidence", [])
+                       for citation in evidence.get("citations", [])
+                       for ref in citation.get("bookmark_refs", []) if isinstance(ref, dict)):
+                    pages.add(record.get("page_id") or revision_path.parent.name)
+        return {"affected_pages": sorted(pages), "affected_research_ids": sorted(research_ids)}
+
     def _snapshot_directory(self, source_id):
         if self.storage is None:
             return None
@@ -400,9 +434,11 @@ class SourceManager:
             snapshot = self._snapshot_directory(source_id)
             aliases = [row[0] for row in self.connection.execute(
                 "SELECT input_path FROM source_aliases WHERE source_id=? ORDER BY input_path", (source_id,))]
+            impact = self._reference_impact(source_id)
             if not confirm:
                 return {"source_id": source_id, "status": "confirmation_required", "would_remove": counts,
                         "aliases": aliases, "snapshot_directory": str(snapshot) if snapshot else None,
+                        **impact,
                         "impact": "Research tasks that froze this source keep their saved inventory but stop resolving "
                                   "its items; Wiki pages citing those claims require review. Re-run with confirm=true "
                                   "(CLI: --confirm) to delete."}
@@ -411,7 +447,7 @@ class SourceManager:
                 self.connection.execute("DELETE FROM %s WHERE source_id=?" % table, (source_id,))
             self.connection.execute("DELETE FROM sources WHERE source_id=?", (source_id,))
         return {"source_id": source_id, "status": "removed", "removed": counts,
-                "aliases": aliases, **cleanup}
+                "aliases": aliases, **impact, **cleanup}
 
     def merge(self, source_id, into_source_id, confirm=False):
         """Drop a duplicate source whose content version equals the target's.
@@ -432,9 +468,11 @@ class SourceManager:
                                  "instead of merging")
             aliases = [row[0] for row in self.connection.execute(
                 "SELECT input_path FROM source_aliases WHERE source_id=? ORDER BY input_path", (source_id,))]
+            impact = self._reference_impact(source_id)
             if not confirm:
                 return {"source_id": source_id, "into_source_id": into_source_id, "status": "confirmation_required",
                         "version_id": record["version_id"], "moved_aliases": aliases,
+                        **impact,
                         "impact": "The duplicate is deleted and its paths re-point at into_source_id; saved research "
                                   "inventories that froze the duplicate stop resolving. Re-run with confirm=true "
                                   "(CLI: --confirm) to merge."}
@@ -442,13 +480,18 @@ class SourceManager:
                 self.connection.execute("INSERT OR IGNORE INTO source_aliases VALUES(?,?)", (into_source_id, alias))
             self.connection.execute("INSERT OR IGNORE INTO source_aliases VALUES(?,?)",
                                     (into_source_id, target.get("input_path")))
+            self.connection.execute(
+                "INSERT INTO source_redirects(old_source_id,new_source_id,created_at,reason) VALUES(?,?,?,?) "
+                "ON CONFLICT(old_source_id) DO UPDATE SET new_source_id=excluded.new_source_id, "
+                "created_at=excluded.created_at, reason=excluded.reason",
+                (source_id, into_source_id, self._now(), "merged_duplicate"))
             for table in ("items", "sections", "files", "nodes", "edges", "memberships",
                           "source_snapshots", "source_aliases", "source_records"):
                 self.connection.execute("DELETE FROM %s WHERE source_id=?" % table, (source_id,))
             self.connection.execute("DELETE FROM sources WHERE source_id=?", (source_id,))
         return {"source_id": source_id, "into_source_id": into_source_id, "status": "merged",
                 "version_id": record.get("version_id"), "moved_aliases": aliases,
-                **cleanup}
+                **impact, **cleanup}
 
     def _observation(self, source_id, record, state, **fields):
         record.update(state=state, checked_at=self._now(), **fields)
@@ -458,7 +501,7 @@ class SourceManager:
                 "pending_files": record.get("pending_files", []), "error": record.get("error"),
                 "version_id": record.get("version_id"), "checked_at": record["checked_at"]}
 
-    def refresh(self, source_id, force=True):
+    def refresh(self, source_id, force=True, verify_hashes=False):
         record = self._record(source_id)
         if record["mode"] == "snapshot":
             return {"source_id": source_id, "mode": "snapshot", "state": "snapshot", "performed": False,
@@ -467,17 +510,33 @@ class SourceManager:
             with self.index.transaction():
                 record = self._record(source_id)
                 if record["mode"] == "snapshot":
-                    return self.refresh(source_id, force)
+                    return self.refresh(source_id, force, verify_hashes=verify_hashes)
                 if git_busy(record["input_path"]):
                     return self._observation(source_id, record, "pending", error="Git is updating the source",
                         pending_fingerprint=None, pending_since=None)
+
+                # Pre-screen via mtime and size if not forcing full verification and not awaiting debounce
+                if not verify_hashes and not record.get("pending_files") and record.get("observed_stats") and record.get("observed_hashes"):
+                    try:
+                        current_stats = scan_stats(record["input_path"])
+                        if current_stats == record["observed_stats"]:
+                            return self._observation(source_id, record, "current",
+                                observed_hashes=record["observed_hashes"],
+                                observed_stats=record["observed_stats"],
+                                pending_files=[],
+                                pending_fingerprint=None, pending_since=None, error=None)
+                    except (OSError, ValueError):
+                        pass
+
                 prepared = read_input(record["input_path"])
                 before = record["observed_hashes"] or {row[0]: row[1] for row in self.connection.execute(
                     "SELECT file_path,sha256 FROM files WHERE source_id=?", (source_id,))}
                 after = prepared["hashes"]
                 pending = sorted(name for name in before.keys() | after.keys() if before.get(name) != after.get(name))
                 if not pending:
-                    return self._observation(source_id, record, "current", observed_hashes=after, pending_files=[],
+                    return self._observation(source_id, record, "current", observed_hashes=after,
+                        observed_stats=prepared.get("stats"),
+                        pending_files=[],
                         pending_fingerprint=None, pending_since=None, error=None)
                 deletion = bool(before.keys() - after.keys()) and record["completeness"] == "complete"
                 delay = self.deletion_grace if deletion else self.debounce
@@ -507,7 +566,7 @@ class SourceManager:
     def source_status(self, source_id):
         """Read lifecycle state without recounting items or loading section bodies."""
         record = self._record(source_id)
-        visible = {key: value for key, value in record.items() if key not in ("observed_hashes", "pending_fingerprint", "pending_since")}
+        visible = {key: value for key, value in record.items() if key not in ("observed_hashes", "observed_stats", "pending_fingerprint", "pending_since")}
         checked = record.get("checked_at")
         if record["mode"] == "live" and record["state"] == "current" and (not checked or
                 self.clock() - datetime.fromisoformat(checked).timestamp() > 10):

package/src/source_versions.py

@@ -5,6 +5,8 @@ from contextlib import closing
 from datetime import datetime, timezone
 from pathlib import Path
 
+import storage
+
 
 class SourceVersions:
     def __init__(self, path):
@@ -18,9 +20,15 @@ class SourceVersions:
         except (ValueError, TypeError, AttributeError):
             return None
 
-    def _connect(self):
+    def _connect(self, readonly=False):
         if self.path.is_symlink():
             raise ValueError("Source version index must not be a symlink")
+        if readonly:
+            if not self.path.is_file():
+                raise sqlite3.OperationalError("Source version index is not built yet: %s" % self.path)
+            connection = storage.connect(self.path, readonly=True)
+            connection.row_factory = sqlite3.Row
+            return storage.mark_query_only(connection)
         self.path.parent.mkdir(parents=True, exist_ok=True)
         connection = sqlite3.connect(str(self.path), timeout=10)
         connection.row_factory = sqlite3.Row
@@ -69,11 +77,21 @@ class SourceVersions:
                                "DO UPDATE SET attempted_at=excluded.attempted_at,status=excluded.status",
                                (SourceArchive._key(url), datetime.now(timezone.utc).isoformat(), status))
 
+    def _open_read(self):
+        """Reads keep working when the observations directory is read-only."""
+        try:
+            connection = self._connect()
+        except (sqlite3.DatabaseError, OSError) as error:
+            if not storage.is_readonly_failure(error) or not self.path.is_file():
+                raise
+            return self._connect(readonly=True)
+        return connection
+
     def inspect(self, url, sha256, acquired_at, published_at, scope_id=None, source_id=None):
         from archive import SourceArchive
         if not self.path.is_file():
             return {"available": False, "latest": None, "newer": None, "last_attempt": None}
-        with closing(self._connect()) as connection:
+        with closing(self._open_read()) as connection:
             key = SourceArchive._key(url)
             baseline = connection.execute("SELECT seq FROM snapshots WHERE scope_id=? AND source_id=? AND sha256=?",
                                           (scope_id, source_id, sha256)).fetchone()

package/src/source_watcher.py

@@ -9,13 +9,14 @@ from source_manager import SourceManager
 
 
 class SourceWatcher:
-    def __init__(self, db_path, interval=1.0, debounce=2.0, deletion_grace=5.0, on_change=None):
+    def __init__(self, db_path, interval=1.0, debounce=2.0, deletion_grace=5.0, on_change=None, verify_hashes=False):
         for value in (interval, debounce, deletion_grace):
             if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
                 raise ValueError("Watcher intervals must be finite positive seconds")
         self.db_path = Path(db_path)
         self.interval, self.debounce, self.deletion_grace = interval, debounce, deletion_grace
         self.on_change = on_change
+        self.verify_hashes = bool(verify_hashes)
         self.stop_event, self.ready = threading.Event(), threading.Event()
         self.thread = None
         self.last_error = None
@@ -35,6 +36,7 @@ class SourceWatcher:
         return {"running": bool(self.thread and self.thread.is_alive() and not self.stop_event.is_set()),
                 "mechanism": "poll", "interval_seconds": self.interval,
                 "debounce_seconds": self.debounce, "deletion_grace_seconds": self.deletion_grace,
+                "verify_hashes": self.verify_hashes,
                 "error": self.last_error, "lifetime": "host_process"}
 
     def run(self):
@@ -50,7 +52,7 @@ class SourceWatcher:
                         for source_id in manager.live_sources():
                             if self.stop_event.is_set():
                                 break
-                            result = manager.refresh(source_id, force=False)
+                            result = manager.refresh(source_id, force=False, verify_hashes=self.verify_hashes)
                             signature = (result["state"], result.get("version_id"),
                                          tuple(result.get("pending_files", [])), result.get("error"))
                             if self.on_change is not None and signature != previous.get(source_id):

package/src/storage.py

@@ -0,0 +1,60 @@
+"""One open policy for every local SQLite store.
+
+Read paths must keep working when the data directory is read-only: a shared
+index, a read-only mount, or a host sandbox that only allows the workspace.
+Writers keep WAL and schema initialization; readers fall back to ``mode=ro``
+instead of failing or silently creating sidecar files next to the index.
+"""
+import sqlite3
+from pathlib import Path
+
+_READONLY_HINTS = (
+    "readonly",
+    "read-only",
+    "unable to open database",
+    "operation not permitted",
+    "permission denied",
+    "no authorization to access",
+)
+
+
+def is_readonly_failure(error):
+    """True when the filesystem refused a write, rather than the query being wrong."""
+    if isinstance(error, (PermissionError, NotADirectoryError)):
+        return True
+    message = str(error).casefold()
+    return any(hint in message for hint in _READONLY_HINTS)
+
+
+def connect(path, readonly=False, timeout=10.0):
+    """Open ``path`` read-only (URI, ``mode=ro``) or read-write."""
+    path = Path(path)
+    if not readonly:
+        return sqlite3.connect(str(path), timeout=timeout)
+    try:
+        connection = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True, timeout=timeout)
+        connection.execute("SELECT count(*) FROM sqlite_master").fetchone()  # open now, not on first use
+        return connection
+    except sqlite3.OperationalError as error:
+        # A WAL database with no ``-shm`` beside it cannot be opened read-only on
+        # a read-only directory. The sidecars are absent exactly when no writer is
+        # active, so reading the main file as immutable is safe here.
+        if "unable to open database file" not in str(error).casefold():
+            raise
+        return sqlite3.connect(path.as_uri() + "?mode=ro&immutable=1", uri=True, timeout=timeout)
+
+
+def mark_query_only(connection):
+    """Tell shared helpers that this connection must not attempt self-healing writes."""
+    try:
+        connection.execute("PRAGMA query_only=ON")
+    except sqlite3.DatabaseError:
+        pass
+    return connection
+
+
+def is_query_only(connection):
+    try:
+        return bool(connection.execute("PRAGMA query_only").fetchone()[0])
+    except (sqlite3.DatabaseError, TypeError):
+        return False

package/src/url_classify.py

@@ -0,0 +1,45 @@
+"""Cheap URL classification used by fetch routing and duplicate reports."""
+from urllib.parse import urlsplit
+
+# Base domains whose page is a search session, not a document: fetching the page
+# only returns the same result list a fresh search would produce.
+SEARCH_DOMAINS = (
+    "bing.com", "google.com", "google.com.hk", "duckduckgo.com",
+    "search.yahoo.com", "search.brave.com", "baidu.com", "yandex.com",
+    "yandex.ru", "sogou.com", "so.com", "startpage.com", "ecosia.org",
+    "search.naver.com",
+)
+
+
+def host_of(url):
+    try:
+        return (urlsplit(url).hostname or "").casefold()
+    except (ValueError, TypeError):
+        return ""
+
+
+def _matches(host, domain):
+    return host == domain or host.endswith("." + domain)
+
+
+def is_search_result_url(url):
+    """True for search-engine result pages (query pages), not documents."""
+    if not isinstance(url, str) or not url:
+        return False
+    host = host_of(url)
+    if not host or not any(_matches(host, domain) for domain in SEARCH_DOMAINS):
+        return False
+    parts = urlsplit(url)
+    path = (parts.path or "").casefold()
+    query = parts.query or ""
+    if _matches(host, "google.com") or _matches(host, "google.com.hk"):
+        return path in ("/search", "/url", "/webhp") or "q=" in query
+    if _matches(host, "bing.com"):
+        return path.startswith("/search") or "q=" in query
+    if _matches(host, "baidu.com"):
+        return path.startswith("/s") or "wd=" in query
+    if _matches(host, "duckduckgo.com"):
+        return True
+    if _matches(host, "yandex.com") or _matches(host, "yandex.ru"):
+        return "text=" in query or "q=" in query
+    return path in ("/search", "/s", "/web") or "q=" in query