arkaos 4.0.2 → 4.1.1

package/core/knowledge/embedder.py

@@ -1,29 +1,136 @@
 """Embedding wrapper — local embeddings via fastembed.
 
 Graceful degradation: if fastembed is not installed, returns None
-and the vector store falls back to keyword matching.
+and the vector store falls back to keyword matching (which the store
+now labels as ``retrieval: "keyword-degraded"`` — see vector_store.py).
+
+Model selection (PR-3 v4.1 — RAG honesty):
+    1. ``ARKA_EMBED_MODEL`` env var
+    2. ``knowledge.embedModel`` in ``~/.arkaos/config.json``
+    3. default ``BAAI/bge-small-en-v1.5`` (backward compatible, 384 dims)
+
+Recommendation for PT-heavy corpora:
+    ``sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2``
+    (384 dims — same dimension as the default, so switching does NOT
+    require re-creating the vec0 table). ``intfloat/multilingual-e5-small``
+    is NOT in fastembed's supported model list (fastembed only ships the
+    ``intfloat/multilingual-e5-large`` variant, 1024 dims, which WOULD
+    force a re-index); the MiniLM multilingual model is the supported
+    drop-in. Verify with ``TextEmbedding.list_supported_models()``.
+
+Dimensions are derived from the model (``embedding_dims()``), never
+hardcoded at call sites. The vector store guards against a stored vec0
+table whose dimension differs from the configured model — it logs and
+keeps the stored dimension instead of corrupting the table.
 """
 
+import json
+import logging
+import os
+from pathlib import Path
 from typing import Optional
 
-# Lazy import — fastembed is optional
-_model = None
-_model_name = "BAAI/bge-small-en-v1.5"  # 384 dims, fast, good quality
+logger = logging.getLogger(__name__)
+
+DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"  # 384 dims, fast, good quality
+RECOMMENDED_MULTILINGUAL_MODEL = (
+    "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+)
+
+# Known dims for models we document — used when fastembed is missing so
+# dimension resolution stays deterministic in degraded environments.
+_KNOWN_DIMS: dict[str, int] = {
+    "BAAI/bge-small-en-v1.5": 384,
+    "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2": 384,
+    "sentence-transformers/paraphrase-multilingual-mpnet-base-v2": 768,
+    "intfloat/multilingual-e5-large": 1024,
+}
+
+# Legacy constant (dims of the default model). Prefer embedding_dims().
 EMBEDDING_DIMS = 384
 
+# Lazy singletons — fastembed is optional.
+_model = None
+_loaded_model_name: Optional[str] = None
+
+
+def get_model_name() -> str:
+    """Resolve the configured embedding model name (env > config > default)."""
+    env_name = os.environ.get("ARKA_EMBED_MODEL", "").strip()
+    if env_name:
+        return env_name
+    config_path = Path.home() / ".arkaos" / "config.json"
+    try:
+        data = json.loads(config_path.read_text(encoding="utf-8"))
+        name = (data.get("knowledge") or {}).get("embedModel", "")
+        if isinstance(name, str) and name.strip():
+            return name.strip()
+    except (OSError, json.JSONDecodeError, AttributeError):
+        pass
+    return DEFAULT_MODEL
+
+
+def embedding_dims(model_name: Optional[str] = None) -> int:
+    """Dimension of the active embedding model — derived, not hardcoded.
+
+    Prefers the actually-loaded model (which may have fallen back to the
+    default when a configured name failed to load), then fastembed's own
+    supported-models metadata, then the documented known-dims map.
+    """
+    name = model_name or _loaded_model_name or get_model_name()
+    dim = _dims_from_fastembed(name)
+    if dim is not None:
+        return dim
+    return _KNOWN_DIMS.get(name, EMBEDDING_DIMS)
+
+
+def _dims_from_fastembed(name: str) -> Optional[int]:
+    try:
+        from fastembed import TextEmbedding
+        for entry in TextEmbedding.list_supported_models():
+            info = entry if isinstance(entry, dict) else vars(entry)
+            if info.get("model") == name and info.get("dim"):
+                return int(info["dim"])
+    except Exception:  # noqa: BLE001 — degraded envs must never raise here
+        pass
+    return None
+
 
 def get_model():
-    """Get or create the embedding model (lazy singleton)."""
-    global _model
-    if _model is None:
+    """Get or create the embedding model (lazy singleton, config-aware)."""
+    global _model, _loaded_model_name
+    name = get_model_name()
+    if _model is not None and _loaded_model_name == name:
+        return _model
+    try:
+        from fastembed import TextEmbedding
+    except ImportError:
+        return None
+    try:
+        _model = TextEmbedding(name)
+        _loaded_model_name = name
+    except Exception as exc:  # noqa: BLE001 — unsupported model name, etc.
+        logger.warning(
+            "embed model %r failed to load (%s) — falling back to %s",
+            name, exc, DEFAULT_MODEL,
+        )
+        if name == DEFAULT_MODEL:
+            return None
         try:
-            from fastembed import TextEmbedding
-            _model = TextEmbedding(_model_name)
-        except ImportError:
+            _model = TextEmbedding(DEFAULT_MODEL)
+            _loaded_model_name = DEFAULT_MODEL
+        except Exception:  # noqa: BLE001
             return None
     return _model
 
 
+def reset_model_cache() -> None:
+    """Drop the cached model so the next call re-resolves configuration."""
+    global _model, _loaded_model_name
+    _model = None
+    _loaded_model_name = None
+
+
 def embed(text: str) -> Optional[list[float]]:
     """Embed a single text. Returns None if fastembed unavailable."""
     model = get_model()
@@ -46,7 +153,7 @@ def embed_batch(texts: list[str]) -> Optional[list[list[float]]]:
 def is_available() -> bool:
     """Check if embedding model is available."""
     try:
-        from fastembed import TextEmbedding
+        from fastembed import TextEmbedding  # noqa: F401
         return True
     except ImportError:
         return False

package/core/knowledge/vector_store.py

@@ -1,19 +1,29 @@
 """Vector store — SQLite-vec backed semantic search.
 
 Stores document chunks with embeddings for fast similarity search.
-Graceful degradation: works without sqlite-vec (brute-force fallback).
+Graceful degradation: works without sqlite-vec (keyword fallback) — but
+HONESTLY (PR-3 v4.1): degraded results carry ``retrieval:
+"keyword-degraded"`` and ``score: None`` instead of a fake similarity
+score, and the first degraded search per process emits a visible
+stderr warning explaining why semantic search is unavailable.
 """
 
 import json
+import logging
+import re
 import sqlite3
+import sys
 import time
 from pathlib import Path
 from typing import Any, Optional
 
-from core.knowledge.embedder import embed, embed_batch, EMBEDDING_DIMS
+from core.knowledge.embedder import embed, embed_batch, embedding_dims
 
+logger = logging.getLogger(__name__)
 
 _VEC_UNAVAILABLE_REASON: str = ""  # set by _load_vec on failure
+_DEGRADED_WARNED: bool = False  # one-time per-process degradation warning
+_VEC_DIMS_RE = re.compile(r"float\[(\d+)\]")
 
 
 def _load_vec(db: sqlite3.Connection) -> bool:
@@ -61,6 +71,32 @@ def vec_unavailable_reason() -> str:
     return _VEC_UNAVAILABLE_REASON
 
 
+def _parse_vec_dims(create_sql: str | None) -> Optional[int]:
+    """Extract the float[N] dimension from a vec0 CREATE statement."""
+    if not create_sql:
+        return None
+    match = _VEC_DIMS_RE.search(create_sql)
+    return int(match.group(1)) if match else None
+
+
+def _warn_degraded_once(reason: str) -> None:
+    """Emit the one-time visible degradation warning (stderr + log)."""
+    global _DEGRADED_WARNED
+    if _DEGRADED_WARNED:
+        return
+    _DEGRADED_WARNED = True
+    msg = (
+        f"[arka:kb-degraded] semantic search unavailable ({reason}) — "
+        "results are keyword matches, NOT similarity-ranked. "
+        "Fix: pip install fastembed sqlite-vec (or: npx arkaos doctor)."
+    )
+    try:
+        sys.stderr.write(msg + "\n")
+    except OSError:
+        pass
+    logger.warning(msg)
+
+
 class VectorStore:
     """SQLite-vec backed vector store for knowledge retrieval.
 
@@ -108,15 +144,40 @@ class VectorStore:
             CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(file_hash);
         """)
         self._migrate_vss_to_vec()
+        self._vec_dims = embedding_dims()  # derived from the active model
         if self._vec_available:
             try:
-                self._db.execute(
-                    f"CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(embedding float[{EMBEDDING_DIMS}])"
-                )
+                stored = self._stored_vec_dims()
+                if stored is not None and stored != self._vec_dims:
+                    # An existing index was built with a different model
+                    # dimension. NEVER corrupt it — keep the stored dim
+                    # and let dimension guards route mismatched vectors
+                    # to the degraded path until the user re-indexes.
+                    logger.warning(
+                        "vec_chunks dimension %d differs from configured "
+                        "embed model dimension %d — keeping the stored "
+                        "index. Re-index (/arka index) to switch models.",
+                        stored, self._vec_dims,
+                    )
+                    self._vec_dims = stored
+                else:
+                    self._db.execute(
+                        f"CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(embedding float[{self._vec_dims}])"
+                    )
             except Exception:
                 self._vec_available = False
         self._db.commit()
 
+    def _stored_vec_dims(self) -> Optional[int]:
+        """Dimension of an existing vec_chunks table, if any."""
+        try:
+            row = self._db.execute(
+                "SELECT sql FROM sqlite_master WHERE name = 'vec_chunks'"
+            ).fetchone()
+        except sqlite3.Error:
+            return None
+        return _parse_vec_dims(row["sql"] if row else None)
+
     def _migrate_vss_to_vec(self) -> None:
         """Drop legacy sqlite-vss tables if present.
 
@@ -176,7 +237,15 @@ class VectorStore:
                     (text, heading, source, file_hash, meta_json, emb_blob),
                 )
 
-                if self._vec_available and emb_blob:
+                # Dimension guard: a vector produced by a differently-sized
+                # model must never be inserted into the vec0 table (it
+                # would poison distance queries). The raw embedding is
+                # still stored on the chunk row for a later re-index.
+                if (
+                    self._vec_available
+                    and emb_blob
+                    and len(embeddings[i]) == self._vec_dims
+                ):
                     self._db.execute(
                         "INSERT INTO vec_chunks (rowid, embedding) VALUES (?, ?)",
                         (cursor.lastrowid, emb_blob),
@@ -199,13 +268,24 @@ class VectorStore:
         query_emb = embed(query)
 
         if query_emb and self._vec_available:
+            if len(query_emb) != self._vec_dims:
+                return self._keyword_search(
+                    query, top_k,
+                    reason="embed model dimension differs from the stored "
+                           "index — re-index to switch models",
+                )
             try:
                 return self._vec_search(query_emb, top_k)
             except Exception:
-                return self._keyword_search(query, top_k)
+                return self._keyword_search(query, top_k, reason="vector query failed")
 
-        # Fallback: keyword search
-        return self._keyword_search(query, top_k)
+        # Fallback: keyword search — labeled, never presented as semantic.
+        return self._keyword_search(query, top_k, reason=self._degradation_reason(query_emb))
+
+    def _degradation_reason(self, query_emb: Optional[list[float]]) -> str:
+        if query_emb is None:
+            return "fastembed missing or embedding failed"
+        return vec_unavailable_reason() or "sqlite-vec unavailable"
 
     def _vec_search(self, query_emb: list[float], top_k: int) -> list[dict]:
         """Vector similarity search via sqlite-vec."""
@@ -224,19 +304,31 @@ class VectorStore:
                 "heading": r["heading"],
                 "source": r["source"],
                 "score": 1.0 / (1.0 + r["distance"]),  # Convert distance to similarity
+                "retrieval": "semantic",
                 "metadata": json.loads(r["metadata"]),
             }
             for r in rows
         ]
 
-    def _keyword_search(self, query: str, top_k: int) -> list[dict]:
-        """Fallback keyword search when VSS unavailable."""
-        words = query.lower().split()
+    def _keyword_search(self, query: str, top_k: int, reason: str = "") -> list[dict]:
+        """Fallback keyword search when semantic retrieval is unavailable.
+
+        HONESTY CONTRACT (PR-3 v4.1): these results are substring matches,
+        not similarity-ranked. ``score`` is None (there is no similarity)
+        and ``retrieval`` is ``"keyword-degraded"`` so every consumer can
+        label them. The previous behaviour — a hardcoded fake ``0.5``
+        score — silently impersonated semantic search.
+        """
+        words = query.lower().split()[:5]  # Max 5 keywords
         if not words:
             return []
 
+        _warn_degraded_once(reason or "semantic retrieval unavailable")
+
+        # Placeholders and params MUST come from the same capped list —
+        # previously a 6+ word query produced more `?` than bindings.
         conditions = " OR ".join(["lower(text) LIKE ?" for _ in words])
-        params = [f"%{w}%" for w in words[:5]]  # Max 5 keywords
+        params = [f"%{w}%" for w in words]
 
         rows = self._db.execute(
             f"SELECT text, heading, source, metadata FROM chunks WHERE {conditions} LIMIT ?",
@@ -248,7 +340,8 @@ class VectorStore:
                 "text": r["text"],
                 "heading": r["heading"],
                 "source": r["source"],
-                "score": 0.5,  # No real score for keyword search
+                "score": None,  # keyword match — NO similarity score exists
+                "retrieval": "keyword-degraded",
                 "metadata": json.loads(r["metadata"]),
             }
             for r in rows
@@ -274,12 +367,16 @@ class VectorStore:
 
     def get_stats(self) -> dict:
         """Get store statistics."""
+        from core.knowledge.embedder import is_available as _embedder_available
+
         total = self._db.execute("SELECT COUNT(*) as cnt FROM chunks").fetchone()["cnt"]
         sources = self._db.execute("SELECT COUNT(DISTINCT source) as cnt FROM chunks").fetchone()["cnt"]
+        semantic = self._vec_available and _embedder_available()
         return {
             "total_chunks": total,
             "total_files": sources,
             "vec_available": self._vec_available,
+            "retrieval_mode": "semantic" if semantic else "keyword-degraded",
             "db_path": self._db_path,
         }
 

package/core/obsidian/relator.py

@@ -36,6 +36,13 @@ _STOPWORDS = frozenset({
 })
 
 
+# Jaccard between full documents rarely exceeds ~0.3 even for clearly
+# related notes, while cosine similarity of embeddings sits comfortably
+# above 0.5. A single threshold calibrated for cosine silently disabled
+# the relator on every machine without fastembed (v4.1.1 fix).
+_KEYWORD_MIN_SIMILARITY = 0.15
+
+
 def find_related(
     content: str,
     vault: Path,
@@ -48,8 +55,9 @@ def find_related(
     candidates = _collect_notes(vault, exclude)
     if not candidates:
         return []
-    scored = _score_candidates(content, candidates)
-    filtered = [r for r in scored if r.score >= min_similarity]
+    scored, method = _score_candidates(content, candidates)
+    threshold = min_similarity if method == "semantic" else _KEYWORD_MIN_SIMILARITY
+    filtered = [r for r in scored if r.score >= threshold]
     filtered.sort(key=lambda r: r.score, reverse=True)
     return filtered[:top_n]
 
@@ -70,22 +78,24 @@ def _collect_notes(vault: Path, exclude: Optional[Path]) -> list[tuple[Path, str
 
 def _score_candidates(
     content: str, candidates: list[tuple[Path, str]]
-) -> list[RelatedNote]:
+) -> tuple[list[RelatedNote], str]:
+    """Score candidates, returning (results, method) so the caller can
+    apply a threshold appropriate to the scoring space."""
     try:
         return _semantic_score(content, candidates)
     except Exception:
-        return _keyword_score(content, candidates)
+        return _keyword_score(content, candidates), "keyword"
 
 
 def _semantic_score(
     content: str, candidates: list[tuple[Path, str]]
-) -> list[RelatedNote]:
+) -> tuple[list[RelatedNote], str]:
     from core.knowledge.embedder import embed, is_available
     if not is_available():
-        return _keyword_score(content, candidates)
+        return _keyword_score(content, candidates), "keyword"
     query_emb = embed(content)
     if query_emb is None:
-        return _keyword_score(content, candidates)
+        return _keyword_score(content, candidates), "keyword"
     results = []
     for path, text in candidates:
         note_emb = embed(text[:2000])
@@ -96,7 +106,7 @@ def _semantic_score(
             path=path, title=_title_from_path(path),
             score=score, excerpt=text[:160],
         ))
-    return results
+    return results, "semantic"
 
 
 def _keyword_score(

package/core/runtime/base.py

@@ -145,6 +145,35 @@ class RuntimeAdapter(ABC):
         }
         return feature_map.get(feature, False)
 
+    def capabilities(self) -> dict[str, bool]:
+        """Honest, static support matrix for this adapter (PR-6 v4.1.0).
+
+        Single source of truth for the multi-runtime story — consumed by
+        ``python -m core.runtime.capabilities_cli`` and wiki/08.
+
+        Keys (all bool):
+        - ``agent_dispatch``: the runtime can dispatch ArkaOS agents /
+          subagents natively (Claude Code Agent tool). False when
+          ``dispatch_agent`` only returns a descriptor for a runtime that
+          cannot actually spawn ArkaOS agents.
+        - ``headless``: the adapter implements a real one-shot
+          ``headless_complete`` against the runtime's CLI (static — use
+          ``headless_supported()`` for binary-on-PATH detection).
+        - ``file_ops``: the runtime performs file operations natively
+          when driven interactively (single-agent use counts).
+        - ``hooks``: the runtime exposes lifecycle hooks that ArkaOS
+          enforcement (PreToolUse/Stop/...) can attach to.
+
+        Default: conservative all-False except hooks, which mirrors the
+        runtime config — unknown adapters must not overclaim.
+        """
+        return {
+            "agent_dispatch": False,
+            "headless": False,
+            "file_ops": False,
+            "hooks": self.get_config().supports_hooks,
+        }
+
     def headless_supported(self) -> bool:
         """Whether this adapter can perform a headless CLI completion.
 

package/core/runtime/capabilities_cli.py

@@ -0,0 +1,74 @@
+"""Runtime capabilities matrix CLI (PR-6 v4.1.0 — multi-runtime honesty).
+
+Single source of truth for what each runtime adapter actually supports.
+Prints the static ``capabilities()`` matrix of all registered adapters,
+plus live binary detection for the headless CLIs.
+
+Usage:
+    python -m core.runtime.capabilities_cli          # table
+    python -m core.runtime.capabilities_cli --json   # machine-readable
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from core.runtime.registry import RUNTIME_ADAPTERS
+
+_CAP_KEYS = ("agent_dispatch", "headless", "file_ops", "hooks")
+
+
+def build_matrix() -> dict[str, dict]:
+    """Collect capabilities + live headless availability per runtime."""
+    matrix: dict[str, dict] = {}
+    for runtime_id, adapter_cls in RUNTIME_ADAPTERS.items():
+        adapter = adapter_cls()
+        caps = adapter.capabilities()
+        row = {key: bool(caps.get(key, False)) for key in _CAP_KEYS}
+        try:
+            row["headless_available"] = bool(adapter.headless_supported())
+        except Exception:  # noqa: BLE001 — detection must never crash
+            row["headless_available"] = False
+        row["name"] = adapter.get_config().name
+        matrix[runtime_id] = row
+    return matrix
+
+
+def _render_table(matrix: dict[str, dict]) -> str:
+    header = (
+        f"{'runtime':<14}{'agent_dispatch':<16}{'headless':<10}"
+        f"{'file_ops':<10}{'hooks':<7}{'binary on PATH'}"
+    )
+    lines = [header, "-" * len(header)]
+    for runtime_id, row in matrix.items():
+        def mark(key: str) -> str:
+            return "yes" if row.get(key) else "no"
+        lines.append(
+            f"{runtime_id:<14}{mark('agent_dispatch'):<16}"
+            f"{mark('headless'):<10}{mark('file_ops'):<10}"
+            f"{mark('hooks'):<7}{mark('headless_available')}"
+        )
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m core.runtime.capabilities_cli",
+        description="Print the multi-runtime support matrix.",
+    )
+    parser.add_argument(
+        "--json", action="store_true", help="Emit JSON instead of a table"
+    )
+    args = parser.parse_args(argv)
+    matrix = build_matrix()
+    if args.json:
+        print(json.dumps(matrix, indent=2))
+    else:
+        print(_render_table(matrix))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())