bookmark-research 0.5.0-beta.1

package/src/settings.py

@@ -0,0 +1,247 @@
+"""Persistent user preferences, stored separately from plugins and canvas packages."""
+
+import copy
+import errno
+import json
+import os
+import re
+import tempfile
+import time
+from contextlib import contextmanager
+from pathlib import Path
+
+
+class Settings:
+    """Load on each operation so MCP clients can change preferences without restart."""
+
+    PROVIDERS = ("exa", "parallel", "tavily", "jina")
+
+    def __init__(self, path=None):
+        config_home = self._xdg_home("XDG_CONFIG_HOME", Path.home() / ".config")
+        selected = path or os.environ.get("BOOKMARK_RESEARCH_CONFIG") or config_home / "bookmark-research/settings.json"
+        self.path = Path(selected).expanduser().resolve()
+
+    @staticmethod
+    def _xdg_home(variable, fallback):
+        value = os.environ.get(variable)
+        # XDG base directories must be absolute; empty or relative values use
+        # the specification's home-directory defaults, never the process cwd.
+        return Path(value) if value and Path(value).is_absolute() else fallback
+
+    @staticmethod
+    def data_directory():
+        selected = os.environ.get("BOOKMARK_RESEARCH_DATA_DIR")
+        if selected:
+            return Path(selected).expanduser().resolve()
+        return (Settings._xdg_home("XDG_DATA_HOME", Path.home() / ".local/share") / "bookmark-research").resolve()
+
+    @staticmethod
+    def external_path(value, purpose):
+        path = Path(value).expanduser()
+        if not path.is_absolute():
+            raise ValueError(purpose + " must be an absolute path (or start with ~)")
+        path = path.resolve()
+        plugin = Path(__file__).resolve().parents[1]
+        for parent in (path, *path.parents):
+            if (parent == plugin or (parent / ".codex-plugin/plugin.json").is_file()
+                    or (parent / ".claude-plugin/plugin.json").is_file()):
+                raise ValueError(purpose + " must be outside plugin directories")
+            if ((parent / "永久栏目").is_dir() or (parent / "临时栏目").is_dir()
+                    or any(parent.glob("*.canvas"))):
+                raise ValueError(purpose + " must be outside canvas package directories")
+        return path
+
+    @classmethod
+    def defaults(cls):
+        return {"schema_version": 1, "timeout_seconds": 30,
+                "search": {"providers": ["exa", "parallel"],
+                           "fallback_providers": ["tavily", "jina"], "limit_per_target": 5},
+                "fetch": {"provider": "exa", "providers": ["exa", "parallel", "jina"], "max_characters": 12000},
+                "archive": {"enabled": True, "directory": str(cls.data_directory() / "knowledge")},
+                "research": {"depth": "auto", "prefer_host_workflows": True, "response_language": "auto",
+                             "methods": ["comparative_analysis", "fact_check", "benchmark_review"]},
+                "readiness": {"mode": "cached", "ttl_seconds": 900, "timeout_seconds": 10},
+                "professional_research": {"enabled": False, "provider": None,
+                    "openai": {"model": "o4-mini-deep-research", "max_tool_calls": 24},
+                    "parallel": {"processor": "pro"}},
+                "wiki": {"directory": str(cls.data_directory() / "wiki")}}
+
+    @staticmethod
+    def _merge(base, patch):
+        if not isinstance(patch, dict):
+            raise ValueError("Settings must be a JSON object")
+        merged = copy.deepcopy(base)
+        for key, value in patch.items():
+            if isinstance(merged.get(key), dict) and isinstance(value, dict):
+                merged[key] = Settings._merge(merged[key], value)
+            else:
+                merged[key] = copy.deepcopy(value)
+        return merged
+
+    @classmethod
+    def _validate(cls, value):
+        defaults = cls.defaults()
+        if set(value) - set(defaults):
+            raise ValueError("Unknown settings fields; credentials belong in the environment or private credential file")
+        for section in ("search", "fetch", "archive", "research", "readiness", "professional_research", "wiki"):
+            if not isinstance(value.get(section), dict) or set(value[section]) != set(defaults[section]):
+                raise ValueError("Invalid settings fields in " + section)
+        for label, number, minimum, maximum in (
+                ("schema_version", value["schema_version"], 1, 1),
+                ("timeout_seconds", value["timeout_seconds"], 1, 60),
+                ("search.limit_per_target", value["search"]["limit_per_target"], 1, 20),
+                ("fetch.max_characters", value["fetch"]["max_characters"], 100, 100000),
+                ("readiness.ttl_seconds", value["readiness"]["ttl_seconds"], 60, 86400),
+                ("readiness.timeout_seconds", value["readiness"]["timeout_seconds"], 1, 30)):
+            if type(number) is not int or not minimum <= number <= maximum:
+                raise ValueError("%s must be an integer between %s and %s" % (label, minimum, maximum))
+        for section, field, minimum in (("search", "providers", 1), ("fetch", "providers", 1),
+                                        ("search", "fallback_providers", 0)):
+            providers = value[section][field]
+            if (not isinstance(providers, list) or not minimum <= len(providers) <= len(cls.PROVIDERS)
+                    or any(p not in cls.PROVIDERS for p in providers) or len(set(providers)) != len(providers)):
+                raise ValueError(section + "." + field + " must select supported providers without duplicates")
+        if value["fetch"]["provider"] not in cls.PROVIDERS:
+            raise ValueError("fetch.provider must select a supported provider")
+        if type(value["archive"]["enabled"]) is not bool:
+            raise ValueError("archive.enabled must be a boolean")
+        directory = value["archive"]["directory"]
+        if not isinstance(directory, str) or not directory.strip() or "\x00" in directory:
+            raise ValueError("archive.directory must be a nonempty path")
+        value["archive"]["directory"] = str(cls.external_path(directory, "Archive directory"))
+        research = value["research"]
+        if research["depth"] not in ("auto", "quick", "agentic", "deep"):
+            raise ValueError("research.depth must be auto, quick, agentic or deep")
+        if type(research["prefer_host_workflows"]) is not bool:
+            raise ValueError("research.prefer_host_workflows must be a boolean")
+        if research["response_language"] not in ("auto", "en", "zh"):
+            raise ValueError("research.response_language must be auto, en or zh; per-request instructions override it")
+        if value["readiness"]["mode"] not in ("cached", "always", "manual"):
+            raise ValueError("readiness.mode must be cached, always or manual")
+        methods = research["methods"]
+        allowed_methods = ("comparative_analysis", "fact_check", "benchmark_review", "wiki_synthesis")
+        if (not isinstance(methods, list) or not 1 <= len(methods) <= len(allowed_methods)
+                or any(not isinstance(method, str) or method not in allowed_methods for method in methods)
+                or len(set(methods)) != len(methods)):
+            raise ValueError("research.methods must select distinct supported analysis methods")
+        services = value["professional_research"]
+        if type(services["enabled"]) is not bool or services["provider"] not in (None, "openai", "parallel"):
+            raise ValueError("professional_research requires enabled boolean and an openai/parallel provider or null")
+        for provider in ("openai", "parallel"):
+            if (not isinstance(services[provider], dict)
+                    or set(services[provider]) != set(defaults["professional_research"][provider])):
+                raise ValueError("Invalid professional_research fields for " + provider)
+        if services["openai"]["model"] not in ("o3-deep-research", "o4-mini-deep-research"):
+            raise ValueError("OpenAI research model must be o3-deep-research or o4-mini-deep-research")
+        limit = services["openai"]["max_tool_calls"]
+        if type(limit) is not int or not 1 <= limit <= 1000:
+            raise ValueError("professional_research.openai.max_tool_calls must be between 1 and 1000")
+        processor = services["parallel"]["processor"]
+        if not isinstance(processor, str) or not re.fullmatch(r"[a-z][a-z0-9-]{0,63}", processor):
+            raise ValueError("professional_research.parallel.processor must be a processor name")
+        wiki_directory = value["wiki"]["directory"]
+        if not isinstance(wiki_directory, str) or not wiki_directory.strip() or "\x00" in wiki_directory:
+            raise ValueError("wiki.directory must be a nonempty path")
+        value["wiki"]["directory"] = str(cls.external_path(wiki_directory, "Wiki directory"))
+        return value
+
+    @staticmethod
+    def fetch_providers(preferences):
+        return list(dict.fromkeys([preferences["fetch"]["provider"]] + preferences["fetch"]["providers"]))
+
+    @staticmethod
+    def fetch_call_count(provider, urls):
+        # Reader has one HTTP request per URL; the other providers batch URLs
+        # in one MCP tool call. Reserve this before any network work begins.
+        return len(urls) if provider == "jina" else 1
+
+    @staticmethod
+    def _unique_object(pairs):
+        result = {}
+        for key, value in pairs:
+            if key in result:
+                raise ValueError("Duplicate settings field")
+            result[key] = value
+        return result
+
+    def _read(self):
+        if not self.path.exists():
+            return {}
+        value = json.loads(self.path.read_text(encoding="utf-8"), object_pairs_hook=self._unique_object)
+        if not isinstance(value, dict):
+            raise ValueError("Settings file must contain a JSON object")
+        return value
+
+    def load(self):
+        stored = self._read()
+        merged = self._merge(self.defaults(), stored)
+        search = stored.get("search", {})
+        if isinstance(search, dict) and "providers" in search and "fallback_providers" not in search:
+            # An earlier explicit provider list must not silently gain services
+            # on upgrade. Users can opt into a separate fallback list.
+            merged["search"]["fallback_providers"] = []
+        return self._validate(merged)
+
+    def describe(self):
+        return {"config_path": str(self.path), "config_exists": self.path.is_file(), "settings": self.load()}
+
+    @staticmethod
+    @contextmanager
+    def _update_lock(path):
+        # Keep the lock inode across updates: replacing or deleting it would let
+        # another CLI/MCP process lock a different file and lose an update.
+        lock_path = path.with_name("." + path.name + ".lock")
+        descriptor = os.open(lock_path, os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0), 0o600)
+        with os.fdopen(descriptor, "r+b") as lock:
+            if os.name == "nt":
+                import msvcrt
+                if lock.seek(0, os.SEEK_END) == 0:
+                    lock.write(b"\x00")
+                    lock.flush()
+                lock.seek(0)
+                acquire = lambda: msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
+                release = lambda: msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1)
+            else:
+                import fcntl
+                acquire = lambda: fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                release = lambda: fcntl.flock(lock.fileno(), fcntl.LOCK_UN)
+            deadline = time.monotonic() + 10
+            while True:
+                try:
+                    acquire()
+                    break
+                except OSError as error:
+                    if error.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
+                        raise
+                    if time.monotonic() >= deadline:
+                        raise TimeoutError("Settings are being updated by another process") from error
+                    time.sleep(0.05)
+            try:
+                yield
+            finally:
+                release()
+
+    def update(self, changes):
+        if not isinstance(changes, dict) or not changes:
+            raise ValueError("Provide at least one settings change")
+        stored = self._merge(self._read(), changes)
+        self._validate(self._merge(self.defaults(), stored))
+        path = self.external_path(str(self.path), "Settings file")
+        path.parent.mkdir(parents=True, exist_ok=True)
+        # Persist only overrides; inherited defaults still follow the user's environment.
+        with self._update_lock(path):
+            stored = self._merge(self._read(), changes)
+            self._validate(self._merge(self.defaults(), stored))
+            temporary = None
+            try:
+                with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="", dir=path.parent,
+                                                 prefix=".settings-", delete=False) as stream:
+                    temporary = Path(stream.name)
+                    stream.write(json.dumps(stored, ensure_ascii=False, allow_nan=False, indent=2) + "\n")
+                    stream.flush()
+                    os.fsync(stream.fileno())
+                temporary.replace(path)
+            finally:
+                if temporary is not None:
+                    temporary.unlink(missing_ok=True)
+            return self.describe()

package/src/source_inputs.py

@@ -0,0 +1,201 @@
+"""Read directory, ZIP and single-section exports without changing their files."""
+
+import hashlib
+import json
+import stat
+import zipfile
+from pathlib import Path, PurePosixPath
+
+from bookmark_index import BookmarkIndex, _dump, _reject_constant, _unique_object
+
+
+PROTOCOL_DIRS = ("永久栏目", "临时栏目")
+MAX_FILE_BYTES = 64 * 1024 * 1024
+MAX_TOTAL_BYTES = 256 * 1024 * 1024
+MAX_ENTRIES = 10000
+
+
+def hashes(files):
+    return {name: hashlib.sha256(raw).hexdigest() for name, raw in sorted(files.items())}
+
+
+def fingerprint(file_hashes):
+    return hashlib.sha256(_dump(file_hashes).encode("utf-8")).hexdigest()
+
+
+def json_object(raw):
+    value = json.loads(raw.decode("utf-8-sig"), object_pairs_hook=_unique_object, parse_constant=_reject_constant)
+    if not isinstance(value, dict):
+        raise ValueError("An export file must contain one JSON object")
+    return value
+
+
+def _safe_name(name):
+    parts = PurePosixPath(name).parts
+    if (not name or "\x00" in name or "\\" in name or name.startswith("/")
+            or any(part in (".", "..") or ":" in part for part in name.rstrip("/").split("/"))):
+        raise ValueError("Unsafe archive path: " + name[:200])
+    return parts
+
+
+def _section_name(name, raw, section_paths):
+    obj = json_object(raw)
+    prefix = "永久栏目" if obj.get("sectionType") == "permanent" else "临时栏目"
+    relative = prefix + "/" + Path(name).name
+    section_id, section = BookmarkIndex._section(relative, obj)
+    BookmarkIndex._items(section_id, section, obj)
+    # A card downloaded under a new filename still updates its known card.
+    return section_paths.get(section_id, relative)
+
+
+def _read_file(path, root):
+    if path.is_symlink() or root not in path.resolve().parents:
+        raise ValueError("Export contains an external symlink: " + str(path))
+    if path.stat().st_size > MAX_FILE_BYTES:
+        raise ValueError("Export file exceeds the 64 MiB limit: " + path.name)
+    raw = path.read_bytes()
+    if len(raw) > MAX_FILE_BYTES:
+        raise ValueError("Export file exceeds the 64 MiB limit: " + path.name)
+    return raw
+
+
+def _directory(path):
+    paths = list(path.glob("*.canvas"))
+    for loose in path.glob("*.json"):
+        raw = _read_file(loose, path)
+        try:
+            obj = json_object(raw)
+        except ValueError:
+            continue
+        if obj.get("format") == "bookmark-canvas-section":
+            raise ValueError("Loose section JSON at package root; import the card file directly or use protocol directories")
+    for directory in PROTOCOL_DIRS:
+        branch = path / directory
+        if branch.is_symlink():
+            raise ValueError("Export contains a symlinked protocol directory")
+        for entry in branch.rglob("*"):
+            if entry.is_symlink():
+                raise ValueError("Export contains an external symlink: " + str(entry))
+            if entry.is_file() and entry.suffix == ".json":
+                paths.append(entry)
+    if len(paths) > MAX_ENTRIES:
+        raise ValueError("Export contains too many protocol files")
+    files, total = {}, 0
+    for entry in sorted(paths):
+        raw = _read_file(entry, path)
+        total += len(raw)
+        if total > MAX_TOTAL_BYTES:
+            raise ValueError("Export exceeds the 256 MiB limit")
+        files[entry.relative_to(path).as_posix()] = raw
+    return files
+
+
+def _zip(path, section_paths):
+    try:
+        with zipfile.ZipFile(path) as archive:
+            entries = archive.infolist()
+            if len(entries) > MAX_ENTRIES:
+                raise ValueError("Archive contains too many entries")
+            selected, roots, seen, loose = [], set(), set(), []
+            total = 0
+            for entry in entries:
+                parts = _safe_name(entry.filename)
+                normalized = "/".join(parts)
+                if normalized in seen:
+                    raise ValueError("Duplicate archive entry: " + normalized)
+                seen.add(normalized)
+                mode = entry.external_attr >> 16
+                if stat.S_ISLNK(mode) or (stat.S_IFMT(mode) and not (stat.S_ISREG(mode) or stat.S_ISDIR(mode))):
+                    raise ValueError("Archive links and special files are unsupported")
+                if entry.flag_bits & 1:
+                    raise ValueError("Encrypted archives are unsupported")
+                if entry.is_dir() or parts[0] == "__MACOSX" or parts[-1].startswith("."):
+                    continue
+                suffix = PurePosixPath(normalized).suffix
+                root = None
+                if suffix == ".canvas":
+                    root = parts[:-1]
+                elif suffix == ".json":
+                    for position, part in enumerate(parts[:-1]):
+                        if part in PROTOCOL_DIRS:
+                            root = parts[:position]
+                            break
+                    if root is None:
+                        loose.append(entry)
+                if root is not None:
+                    roots.add(root)
+                    selected.append((entry, "/".join(parts[len(root):])))
+                # Bound all decompressed sizes, including ignored files.
+                total += entry.file_size
+                if entry.file_size > MAX_FILE_BYTES or total > MAX_TOTAL_BYTES:
+                    raise ValueError("Archive exceeds export size limits")
+            if len(roots) > 1:
+                raise ValueError("Archive contains multiple package roots; import each canvas as a separate source")
+            if selected:
+                for entry in loose:
+                    try:
+                        obj = json_object(archive.read(entry))
+                    except ValueError:
+                        continue
+                    if obj.get("format") == "bookmark-canvas-section":
+                        raise ValueError("Archive mixes a package with loose card JSON; import the inputs separately")
+            if not selected and len(loose) == 1:
+                raw = archive.read(loose[0])
+                return {_section_name(loose[0].filename, raw, section_paths): raw}, True
+            files = {name: archive.read(entry) for entry, name in selected}
+            return files, False
+    except (zipfile.BadZipFile, NotImplementedError, RuntimeError) as exc:
+        raise ValueError("Cannot read ZIP export: " + str(exc)) from exc
+
+
+def read_input(path, section_paths=None):
+    path = Path(path).expanduser().resolve()
+    section_paths = section_paths or {}
+    single_section = False
+    if path.is_dir():
+        kind, files = "directory", _directory(path)
+    elif path.is_file() and path.suffix.lower() == ".zip":
+        kind = "zip"
+        files, single_section = _zip(path, section_paths)
+    elif path.is_file() and path.suffix.lower() == ".json":
+        kind, single_section = "section", True
+        raw = _read_file(path, path.parent)
+        files = {_section_name(path.name, raw, section_paths): raw}
+    else:
+        raise ValueError("Input must be an existing package directory, ZIP, or section JSON: " + str(path))
+    if not files:
+        raise ValueError("No protocol JSON or .canvas files in export; the saved index was retained")
+    if sum(name.endswith(".canvas") for name in files) > 1:
+        raise ValueError("An export must have at most one .canvas entry")
+    file_hashes = hashes(files)
+    return {"path": str(path), "kind": kind, "single_section": single_section,
+            "files": files, "hashes": file_hashes, "fingerprint": fingerprint(file_hashes)}
+
+
+def git_directory(path):
+    """Locate local Git metadata, including worktrees; never invoke Git or fetch."""
+    path = Path(path)
+    for parent in (path, *path.parents):
+        marker = parent / ".git"
+        if marker.is_dir():
+            return marker
+        if marker.is_file():
+            if marker.stat().st_size > 4096:
+                raise ValueError("Invalid Git worktree marker")
+            text = marker.read_text(encoding="utf-8").strip()
+            if not text.startswith("gitdir: "):
+                raise ValueError("Invalid Git worktree marker")
+            return (parent / text[8:]).resolve()
+    return None
+
+
+def git_busy(path):
+    directory = git_directory(path)
+    if directory is None:
+        return False
+    directories = [directory]
+    common = directory / "commondir"
+    if common.is_file() and common.stat().st_size <= 4096:
+        directories.append((directory / common.read_text(encoding="utf-8").strip()).resolve())
+    return any((root / name).exists() for root in directories
+               for name in ("index.lock", "HEAD.lock", "packed-refs.lock", "shallow.lock"))