@bcmyguest/openmem 0.0.0-stage → 0.6.3

package/indexer/index_config.py

@@ -0,0 +1,290 @@
+"""Load and validate the project indexer config (config/index.yml).
+
+Lookup, first hit wins: --config PATH, $HINDSIGHT_INDEX_CONFIG, ~/.config/hindsight/index.yml, then the shipped default.
+A file found this way replaces the shipped default entirely. A key the file omits takes its built-in value (DEFAULT_DATA).
+"""
+
+import fnmatch
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+
+ENV_VAR = "HINDSIGHT_INDEX_CONFIG"
+DEFAULT_PATH = Path(__file__).resolve().parent.parent / "config" / "index.yml"
+USER_PATH = "~/.config/hindsight/index.yml"
+TOP_LEVEL_KEYS = (
+    "roots",
+    "repos",
+    "vaults",
+    "notes",
+    "papers",
+    "bookmarks",
+    "feeds",
+    "exclude",
+    "banks",
+    "max_file_kb",
+    "skip_dirs",
+    "poll_seconds",
+    "vault_quiet_seconds",
+    "relist_seconds",
+)
+VAULT_KEYS = ("auto", "paths")
+PATH_LIST_KEYS = ("paths",)
+FEEDS_KEYS = ("out_dir", "rss", "openalex_queries")
+ROOT_KEYS = ("path", "depth")
+DEFAULT_SKIP_DIRS = [".cache", ".codegraph", ".decision", ".git", ".next", ".tasks", ".venv", "build", "dist", "node_modules", "target", "vendor"]
+DEFAULT_DATA = {
+    "roots": [{"path": "~", "depth": 3}],
+    "repos": ["~/.config/nvim"],
+    "vaults": {"auto": True, "paths": []},
+    "notes": {"paths": []},
+    "papers": {"paths": []},
+    "bookmarks": {"paths": []},
+    "feeds": {"out_dir": "~/.local/share/personal-memory/feeds", "rss": [], "openalex_queries": []},
+    "exclude": [],
+    "banks": {"~/Documents/sample-project": "coding-agent::documents-sample-project"},
+    "max_file_kb": 128,
+    "skip_dirs": DEFAULT_SKIP_DIRS,
+    "poll_seconds": 60,
+    "vault_quiet_seconds": 30,
+    "relist_seconds": 600,
+}
+
+
+class ConfigError(ValueError):
+    """A config file that cannot be read or fails validation. The message names the file and the key."""
+
+
+@dataclass(frozen=True)
+class Root:
+    path: Path
+    depth: int
+
+
+@dataclass(frozen=True)
+class Config:
+    roots: tuple[Root, ...]
+    repos: tuple[Path, ...]
+    vault_auto: bool
+    vault_paths: tuple[Path, ...]
+    notes_paths: tuple[Path, ...]
+    papers_paths: tuple[Path, ...]
+    bookmarks_paths: tuple[Path, ...]
+    feeds_out_dir: Path | None
+    feeds_rss: tuple[object, ...]
+    feeds_openalex_queries: tuple[object, ...]
+    exclude: tuple[str, ...]
+    banks: tuple[tuple[Path, str], ...]  # longest prefix first
+    max_file_kb: int
+    skip_dirs: frozenset[str]
+    poll_seconds: int
+    vault_quiet_seconds: int
+    relist_seconds: int
+    source: str = field(default="built-in", compare=False)
+
+    @property
+    def max_file_bytes(self) -> int:
+        return self.max_file_kb * 1000
+
+    def bank_for(self, path: Path) -> str | None:
+        for prefix, bank in self.banks:
+            if path == prefix or prefix in path.parents:
+                return bank
+        return None
+
+    def is_excluded(self, path: Path) -> bool:
+        return any(fnmatch.fnmatchcase(str(path), pattern) for pattern in self.exclude)
+
+    def markdown_sources(self) -> tuple[tuple[str, Path], ...]:
+        """(kind, path) for watched markdown trees: notes, papers, bookmarks, feed cache."""
+        out: list[tuple[str, Path]] = []
+        for kind, paths in (
+            ("notes", self.notes_paths),
+            ("papers", self.papers_paths),
+            ("bookmarks", self.bookmarks_paths),
+        ):
+            for path in paths:
+                out.append((kind, path))
+        if self.feeds_out_dir is not None:
+            out.append(("feed", self.feeds_out_dir))
+        return tuple(out)
+
+
+def _kind(value: object) -> str:
+    if value is None:
+        return "nothing (null)"
+    if isinstance(value, bool):
+        return "a boolean"
+    if isinstance(value, int):
+        return "an integer"
+    if isinstance(value, float):
+        return "a number"
+    if isinstance(value, str):
+        return "a string"
+    if isinstance(value, list):
+        return "a list"
+    if isinstance(value, dict):
+        return "a mapping"
+    return type(value).__name__
+
+
+def _int(source: str, key: str, value: object, minimum: int) -> int:
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ConfigError(f"{source}: {key}: expected an integer, got {_kind(value)}")
+    if value < minimum:
+        raise ConfigError(f"{source}: {key}: must be at least {minimum}, got {value}")
+    return value
+
+
+def _bool(source: str, key: str, value: object) -> bool:
+    if not isinstance(value, bool):
+        raise ConfigError(f"{source}: {key}: expected true or false, got {_kind(value)}")
+    return value
+
+
+def _text(source: str, key: str, value: object) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise ConfigError(f"{source}: {key}: expected a non-empty string, got {_kind(value)}")
+    return value
+
+
+def _list(source: str, key: str, value: object) -> list:
+    if not isinstance(value, list):
+        raise ConfigError(f"{source}: {key}: expected a list, got {_kind(value)}")
+    return value
+
+
+def _mapping(source: str, key: str, value: object, allowed: tuple[str, ...]) -> dict:
+    if not isinstance(value, dict):
+        raise ConfigError(f"{source}: {key}: expected a mapping, got {_kind(value)}")
+    unknown = [str(name) for name in value if name not in allowed]
+    if unknown:
+        raise ConfigError(f"{source}: {key}: unknown key(s) {', '.join(unknown)}; allowed: {', '.join(allowed)}")
+    return value
+
+
+def _abs_path(source: str, key: str, text: str) -> Path:
+    path = Path(text).expanduser()
+    if not path.is_absolute():
+        raise ConfigError(f"{source}: {key}: expected an absolute path or one starting with ~, got {text!r}")
+    return path
+
+
+def _roots(source: str, value: object) -> tuple[Root, ...]:
+    roots = []
+    for index, item in enumerate(_list(source, "roots", value)):
+        key = f"roots[{index}]"
+        entry = _mapping(source, key, item, ROOT_KEYS)
+        if "path" not in entry:
+            raise ConfigError(f"{source}: {key}: missing required key 'path'")
+        # A bare ~ is YAML null, so null means home.
+        text = "~" if entry["path"] is None else _text(source, f"{key}.path", entry["path"])
+        depth = _int(source, f"{key}.depth", entry.get("depth", 3), 1)
+        roots.append(Root(_abs_path(source, f"{key}.path", text), depth))
+    return tuple(roots)
+
+
+def _paths(source: str, key: str, value: object) -> tuple[Path, ...]:
+    return tuple(_abs_path(source, f"{key}[{index}]", _text(source, f"{key}[{index}]", item)) for index, item in enumerate(_list(source, key, value)))
+
+
+def _banks(source: str, value: object) -> tuple[tuple[Path, str], ...]:
+    if not isinstance(value, dict):
+        raise ConfigError(f"{source}: banks: expected a mapping of path to bank id, got {_kind(value)}")
+    pairs = []
+    for prefix, bank in value.items():
+        path = _abs_path(source, f"banks[{prefix}]", _text(source, "banks key", prefix))
+        pairs.append((path, _text(source, f"banks[{prefix}]", bank)))
+    return tuple(sorted(pairs, key=lambda pair: len(pair[0].parts), reverse=True))
+
+
+def _exclude(source: str, value: object) -> tuple[str, ...]:
+    patterns = []
+    for index, item in enumerate(_list(source, "exclude", value)):
+        pattern = _text(source, f"exclude[{index}]", item)
+        if not pattern.startswith(("/", "~")):
+            raise ConfigError(f"{source}: exclude[{index}]: expected a glob on an absolute path, got {pattern!r}")
+        patterns.append(os.path.expanduser(pattern))
+    return tuple(patterns)
+
+
+def _skip_dirs(source: str, value: object) -> frozenset[str]:
+    names = []
+    for index, item in enumerate(_list(source, "skip_dirs", value)):
+        name = _text(source, f"skip_dirs[{index}]", item)
+        if "/" in name:
+            raise ConfigError(f"{source}: skip_dirs[{index}]: expected a directory name, not a path, got {name!r}")
+        names.append(name)
+    return frozenset(names)
+
+
+def parse(data: object, source: str = "built-in") -> Config:
+    """Validate parsed YAML. None (an empty file) and omitted keys take the DEFAULT_DATA values."""
+    top = _mapping(source, "top level", DEFAULT_DATA if data is None else data, TOP_LEVEL_KEYS)
+    merged = {**DEFAULT_DATA, **top}
+    vaults = _mapping(source, "vaults", merged["vaults"], VAULT_KEYS)
+    notes = _mapping(source, "notes", merged["notes"], PATH_LIST_KEYS)
+    papers = _mapping(source, "papers", merged["papers"], PATH_LIST_KEYS)
+    bookmarks = _mapping(source, "bookmarks", merged["bookmarks"], PATH_LIST_KEYS)
+    feeds_raw = _mapping(source, "feeds", merged.get("feeds") or DEFAULT_DATA["feeds"], FEEDS_KEYS)
+    feeds_out = _paths(source, "feeds.out_dir", [feeds_raw.get("out_dir") or DEFAULT_DATA["feeds"]["out_dir"]])[0]
+    feeds_rss = tuple(_list(source, "feeds.rss", feeds_raw.get("rss") or []))
+    feeds_oa = tuple(_list(source, "feeds.openalex_queries", feeds_raw.get("openalex_queries") or []))
+    has_feeds = bool(feeds_rss or feeds_oa)
+    banks = merged["banks"]
+    return Config(
+        roots=_roots(source, merged["roots"]),
+        repos=_paths(source, "repos", merged["repos"]),
+        vault_auto=_bool(source, "vaults.auto", vaults.get("auto", True)),
+        vault_paths=_paths(source, "vaults.paths", vaults.get("paths", [])),
+        notes_paths=_paths(source, "notes.paths", notes.get("paths", [])),
+        papers_paths=_paths(source, "papers.paths", papers.get("paths", [])),
+        bookmarks_paths=_paths(source, "bookmarks.paths", bookmarks.get("paths", [])),
+        feeds_out_dir=feeds_out if has_feeds else None,
+        feeds_rss=feeds_rss,
+        feeds_openalex_queries=feeds_oa,
+        exclude=_exclude(source, merged["exclude"]),
+        banks=_banks(source, banks),
+        max_file_kb=_int(source, "max_file_kb", merged["max_file_kb"], 1),
+        skip_dirs=_skip_dirs(source, merged["skip_dirs"]),
+        poll_seconds=_int(source, "poll_seconds", merged["poll_seconds"], 1),
+        vault_quiet_seconds=_int(source, "vault_quiet_seconds", merged["vault_quiet_seconds"], 0),
+        relist_seconds=_int(source, "relist_seconds", merged["relist_seconds"], 1),
+        source=source,
+    )
+
+
+def read(path: Path) -> object:
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as error:
+        raise ConfigError(f"{path}: cannot read config: {error.strerror or error}") from error
+    try:
+        return yaml.safe_load(text)
+    except yaml.YAMLError as error:
+        raise ConfigError(f"{path}: invalid YAML: {error}") from error
+
+
+def _user_path() -> Path | None:
+    """The per-machine file, or None when absent or not visible (for example, inside a masked directory)."""
+    user = Path(USER_PATH).expanduser()
+    try:
+        return user if user.is_file() else None
+    except OSError:
+        return None
+
+
+def load(explicit: str | Path | None = None, environ: Mapping[str, str] = os.environ) -> Config:
+    if explicit:
+        path = Path(explicit).expanduser()
+    elif environ.get(ENV_VAR):
+        path = Path(environ[ENV_VAR]).expanduser()
+    else:
+        path = _user_path() or DEFAULT_PATH
+    if not path.is_file():
+        raise ConfigError(f"{path}: config file not found")
+    return parse(read(path), source=str(path))

package/indexer/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+filterwarnings =
+    ignore::DeprecationWarning

package/indexer/sinks/__init__.py

@@ -0,0 +1,148 @@
+"""Memory sinks: where the indexer writes documents.
+
+`bin/hindsight-project-index` produces documents (stable id, text, tags, metadata) and hands them to a sink. A sink
+is any object satisfying the `Sink` protocol below. The indexer holds no provider knowledge beyond choosing a sink
+with `--sink NAME` (env `MEMORY_SINK`, default `hindsight`).
+
+Import name: `indexer/` is the import root (PYTHONPATH=/opt/indexer in the image, `indexer/` on sys.path in the
+tests), so this package imports as `sinks`, next to `index_config` and `code_outline`.
+
+Adding a sink: drop `indexer/sinks/<name>.py` that defines `create(url=None, **cfg) -> Sink`. Nothing else is
+edited: `get_sink` finds the module by name, and the Dockerfile copies the whole directory. `<name>` is the
+provider id (lowercase letters, digits, underscore) and is what `--sink` and `MEMORY_SINK` accept.
+
+Namespace ("bank") mapping. The indexer says `bank` for the logical grouping a document belongs to (a repository's
+bank, an Obsidian vault's bank; for example `coding-agent::proj` or `obsidian::my-vault`). Hindsight uses it
+verbatim as the bank id. A provider without banks maps it to its nearest scoping concept and documents the rule in
+its own module docstring:
+
+    mem0    bank -> user_id (agent_id may hold a fixed indexer name)
+    cognee  bank -> dataset name
+
+The bank is an opaque, non-empty string: sinks must quote or sanitize it for their own API, and must not assume
+it is URL-safe. `doc_id` is a stable string (`project-index:v1:<hash>`); the same id always names the same
+logical document within a bank.
+
+Errors. HTTP error replies raise `SinkError` (or a subclass); `SinkError.status` is the HTTP status. A server
+that cannot be reached raises `SinkError` or `OSError` (urllib's URLError is an OSError). `delete` of a document
+that is already gone is NOT an error. `health()` never raises.
+
+Optional `flush(bank: str) -> None`. A sink whose provider ingests asynchronously (for example Cognee, where
+cognify builds the graph after add) may define `flush`. The indexer calls `flush(bank)` with an explicit bank,
+once per bank per run, whenever a batch retained or deleted at least one document for that bank OR a previous
+flush for that bank is still pending (it failed, or the run died before reaching it). The pending state is
+persisted in the indexer state file, so a fresh process retries it even when nothing changed. A sink must
+therefore treat `flush(bank)` as "process this bank now", even if its own in-memory bookkeeping says the bank is
+clean. `flush(None)` is never called, and a dry run never flushes. `flush` is NOT part of the `Sink` protocol
+(so sinks without it still satisfy `isinstance`); callers go through `flush_sink`, which does nothing for a sink
+that has no callable `flush`. A flush error is handled like a retain error: it raises `SinkError` (or `OSError`)
+and fails that source, and the pending marker stays so the next run flushes again.
+
+URL. `get_sink(name, url=...)` passes the API base URL; the CLI resolves it from `--url`, then `HINDSIGHT_URL`
+for the hindsight sink, or `MEMORY_API_URL` for every other sink. A sink reads any credentials from its own env
+vars inside `create`, never from the document path.
+"""
+
+import importlib
+import pkgutil
+import re
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import Any, Protocol, runtime_checkable
+
+DEFAULT_SINK = "hindsight"
+NAME = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+class SinkError(RuntimeError):
+    """A sink request failed. `status` is the HTTP status, or None for a connection-level failure."""
+
+    def __init__(self, message: str, status: int | None = None):
+        super().__init__(message)
+        self.status = status
+
+
+class UnknownSinkError(ValueError):
+    """The requested sink name is not registered."""
+
+
+@dataclass(frozen=True)
+class Capabilities:
+    """What a sink can do beyond the basics.
+
+    async_ingest: retain() may return before the provider has made the document searchable (for example Cognee
+    add -> cognify). Callers must not read their own writes immediately.
+    """
+
+    async_ingest: bool = False
+
+
+@runtime_checkable
+class Sink(Protocol):
+    capabilities: Capabilities
+
+    def health(self) -> bool:
+        """True when the provider answers its health endpoint. Never raises."""
+        ...
+
+    def prepare(self, bank: str) -> None:
+        """Make `bank` ready for retain (create it, set retain strategy). Called before a batch of retains for
+        that bank; must be idempotent and may be a no-op."""
+        ...
+
+    def retain(
+        self,
+        bank: str,
+        doc_id: str,
+        content: str,
+        *,
+        tags: list[str] | None = None,
+        metadata: Mapping[str, str] | None = None,
+        context: str | None = None,
+        timestamp: str | None = None,
+    ) -> None:
+        """Upsert one document: retaining an existing `doc_id` replaces it, never duplicates it.
+
+        timestamp None means "no event time"; the sink maps that to the provider's own convention.
+        """
+        ...
+
+    def delete(self, bank: str, doc_id: str) -> None:
+        """Remove a document. A document that does not exist is not an error."""
+        ...
+
+
+def flush_sink(sink: Any, bank: str) -> None:
+    """Call the optional `sink.flush(bank)` when the sink defines it; otherwise do nothing."""
+    flush = getattr(sink, "flush", None)
+    if callable(flush):
+        flush(bank)
+
+
+_registry: dict[str, Callable[..., Sink]] = {}
+
+
+def register_sink(name: str, factory: Callable[..., Sink]) -> None:
+    """Register a factory under `name` for this process, taking precedence over a module of the same name."""
+    if not NAME.match(name):
+        raise ValueError(f"invalid sink name {name!r}: use lowercase letters, digits and underscore")
+    _registry[name] = factory
+
+
+def registered_sinks() -> list[str]:
+    """Sorted sink names: explicit registrations plus every public module in this package."""
+    names = {module.name for module in pkgutil.iter_modules(__path__) if NAME.match(module.name)}
+    return sorted(names | set(_registry))
+
+
+def get_sink(name: str, **cfg: Any) -> Sink:
+    """Build the sink `name` with `cfg` (normally `url=`). Raises UnknownSinkError listing the valid names."""
+    if name in _registry:
+        return _registry[name](**cfg)
+    if name not in registered_sinks():
+        raise UnknownSinkError(f"unknown sink {name!r}; registered sinks: {', '.join(registered_sinks()) or 'none'}")
+    module = importlib.import_module(f"{__name__}.{name}")
+    factory = getattr(module, "create", None)
+    if not callable(factory):
+        raise UnknownSinkError(f"sink module {module.__name__} defines no create(url=None, **cfg) factory")
+    return factory(**cfg)

package/indexer/sinks/_http.py

@@ -0,0 +1,44 @@
+"""Shared HTTP plumbing for sinks (stdlib only). The leading underscore keeps this module out of the sink registry.
+
+`open_url` is the one place a sink opens a URL. It differs from `urllib.request.urlopen` in two ways that every
+REST sink wants:
+
+- It follows no redirect at all. A 3xx reply surfaces as `urllib.error.HTTPError` (carrying the status), so a sink
+  never silently re-issues a write as a GET, and an API key in the request can never be handed to another host.
+- `http.client.HTTPException` (BadStatusLine, IncompleteRead, LineTooLong, ...) becomes `OSError`. A server that
+  speaks garbage is then "unreachable", which is the failure the sink contract already allows (`SinkError` or
+  `OSError`), and `health()` can catch `OSError` alone.
+"""
+
+import http.client
+import urllib.request
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    """Refuse every redirect: `redirect_request` returns None, so urllib raises the 3xx as an HTTPError."""
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None
+
+
+OPENER = urllib.request.build_opener(NoRedirect)
+
+
+def open_url(request: urllib.request.Request, timeout: float):
+    """Open `request` without following redirects. Returns the response (a context manager).
+
+    Raises `urllib.error.HTTPError` for any non-2xx status (3xx included), `urllib.error.URLError` or another
+    `OSError` when the server cannot be reached, and `OSError` for a malformed HTTP reply.
+    """
+    try:
+        return OPENER.open(request, timeout=timeout)
+    except http.client.HTTPException as error:
+        raise OSError(f"{type(error).__name__}: {error}") from error
+
+
+def read_all(response) -> bytes:
+    """`response.read()` for a response or an HTTPError, with `http.client.HTTPException` turned into `OSError`."""
+    try:
+        return response.read()
+    except http.client.HTTPException as error:
+        raise OSError(f"{type(error).__name__}: {error}") from error