ctxora 6.2.0

package/src/memory/episodic.py

@@ -0,0 +1,257 @@
+"""Persistent, locally ranked memory tiers for the MCP harness."""
+
+from __future__ import annotations
+
+import math
+import os
+import re
+import sqlite3
+import time
+from collections import Counter
+from enum import Enum
+
+from retrieval.embeddings import EmbeddingEngine
+
+_TOKEN_RE = re.compile(r"\w+", re.UNICODE)
+
+
+class MemoryType(Enum):
+    EPISODIC = "episodic"
+    SEMANTIC = "semantic"
+    PROCEDURAL = "procedural"
+
+
+class MemoryStore:
+    _instance = None
+
+    @classmethod
+    def instance(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def __init__(self, db_path: str | None = None):
+        self.db_path = (
+            db_path
+            or os.environ.get("CTXORA_MEMORY_DB")
+            or os.environ.get("MCP_HARNESS_MEMORY_DB")
+            or os.path.expanduser("~/.ctxora/memory.sqlite3")
+        )
+        if self.db_path != ":memory:":
+            directory = os.path.dirname(os.path.abspath(self.db_path))
+            os.makedirs(directory, exist_ok=True)
+        self._conn = sqlite3.connect(self.db_path, timeout=5)
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA busy_timeout=5000")
+        self._conn.execute("""
+            CREATE TABLE IF NOT EXISTS memories_v2 (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                workspace_id TEXT NOT NULL DEFAULT 'legacy_global',
+                type TEXT NOT NULL,
+                key TEXT NOT NULL,
+                value TEXT NOT NULL,
+                tags TEXT NOT NULL DEFAULT '',
+                scope TEXT NOT NULL DEFAULT 'workspace',
+                source TEXT NOT NULL DEFAULT 'user',
+                confidence REAL NOT NULL DEFAULT 1.0,
+                created_by TEXT NOT NULL DEFAULT 'user',
+                expires_at REAL,
+                supersedes TEXT NOT NULL DEFAULT '',
+                content_hash TEXT NOT NULL DEFAULT '',
+                created_at REAL NOT NULL,
+                updated_at REAL NOT NULL,
+                last_access REAL NOT NULL,
+                hits INTEGER NOT NULL DEFAULT 0,
+                UNIQUE(workspace_id, type, key)
+            )
+        """)
+        if self._table_exists("memories"):
+            self._conn.execute("""
+                INSERT OR IGNORE INTO memories_v2 (
+                    workspace_id, type, key, value, tags, created_at,
+                    updated_at, last_access, hits
+                )
+                SELECT 'legacy_global', type, key, value, tags, created_at,
+                       updated_at, last_access, hits
+                FROM memories
+            """)
+        self._conn.commit()
+
+    def _table_exists(self, name: str) -> bool:
+        return self._conn.execute(
+            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,)
+        ).fetchone() is not None
+
+    @staticmethod
+    def _document(row: sqlite3.Row) -> str:
+        return f"{row['key']} {row['tags']} {row['value']}"
+
+    @staticmethod
+    def _lexical_score(query: str, document: str) -> float:
+        query_counts = Counter(_TOKEN_RE.findall(query.lower()))
+        document_counts = Counter(_TOKEN_RE.findall(document.lower()))
+        if not query_counts or not document_counts:
+            return 0.0
+        numerator = sum(
+            query_counts[token] * document_counts[token]
+            for token in query_counts.keys() & document_counts.keys()
+        )
+        query_norm = math.sqrt(sum(count * count for count in query_counts.values()))
+        document_norm = math.sqrt(
+            sum(count * count for count in document_counts.values())
+        )
+        return numerator / (query_norm * document_norm)
+
+    @staticmethod
+    def _semantic_scores(query: str, documents: list[str]) -> list[float]:
+        if not documents:
+            return []
+        engine = EmbeddingEngine()
+        vectors = engine.rebuild(documents)
+        query_vector = engine.embed_text(query)
+        return [
+            max(0.0, sum(value * query_value for value, query_value in zip(vector, query_vector)))
+            for vector in vectors
+        ]
+
+    @staticmethod
+    def _entry(row: sqlite3.Row) -> dict:
+        return {
+            "workspace_id": row["workspace_id"],
+            "type": row["type"],
+            "key": row["key"],
+            "value": row["value"],
+            "tags": row["tags"],
+            "time": row["created_at"],
+            "scope": row["scope"],
+            "source": row["source"],
+            "confidence": row["confidence"],
+            "expires_at": row["expires_at"],
+            "content_hash": row["content_hash"],
+        }
+
+    def search(
+        self,
+        query: str,
+        mtype: MemoryType | None = None,
+        top_k: int = 5,
+        min_sim: float = 0.12,
+        workspace_id: str = "legacy_global",
+    ) -> list[dict]:
+        if top_k <= 0:
+            return []
+        sql = "SELECT * FROM memories_v2 WHERE (workspace_id = ? OR scope = 'global') AND (expires_at IS NULL OR expires_at > ?)"
+        params: tuple = (workspace_id, time.time())
+        if mtype is not None:
+            sql += " AND type = ?"
+            params += (mtype.value,)
+        rows = self._conn.execute(sql, params).fetchall()
+        documents = [self._document(row) for row in rows]
+        semantic_scores = self._semantic_scores(query, documents)
+        ranked = []
+        for row, document, semantic_score in zip(rows, documents, semantic_scores):
+            score = 0.8 * self._lexical_score(query, document) + 0.2 * semantic_score
+            if score >= min_sim:
+                ranked.append((score, row))
+        ranked.sort(key=lambda item: item[0], reverse=True)
+        selected = ranked[:top_k]
+        if selected:
+            now = time.time()
+            self._conn.executemany(
+                "UPDATE memories_v2 SET hits = hits + 1, last_access = ? WHERE id = ?",
+                [(now, row["id"]) for _, row in selected],
+            )
+            self._conn.commit()
+        return [self._entry(row) for _, row in selected]
+
+    def save(self, mtype: MemoryType, key: str, value: str, tags: str, workspace_id: str = "legacy_global", scope: str = "workspace", source: str = "user", confidence: float = 1.0, created_by: str = "user", expires_at: float | None = None, supersedes: str = "") -> dict:
+        now = time.time()
+        import hashlib
+        content_hash = hashlib.sha256(value.encode("utf-8")).hexdigest()
+        self._conn.execute("""
+            INSERT INTO memories_v2 (workspace_id, type, key, value, tags, scope, source, confidence, created_by, expires_at, supersedes, content_hash, created_at, updated_at, last_access)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(workspace_id, type, key) DO UPDATE SET
+                value = excluded.value,
+                tags = excluded.tags,
+                scope = excluded.scope,
+                source = excluded.source,
+                confidence = excluded.confidence,
+                expires_at = excluded.expires_at,
+                supersedes = excluded.supersedes,
+                content_hash = excluded.content_hash,
+                updated_at = excluded.updated_at,
+                last_access = excluded.last_access
+        """, (workspace_id, mtype.value, key, value, tags, scope, source, confidence, created_by, expires_at, supersedes, content_hash, now, now, now))
+        self._conn.commit()
+        row = self._conn.execute(
+            "SELECT * FROM memories_v2 WHERE workspace_id = ? AND type = ? AND key = ?",
+            (workspace_id, mtype.value, key),
+        ).fetchone()
+        return self._entry(row)
+
+    def delete(self, mtype: MemoryType, key: str, workspace_id: str = "legacy_global") -> bool:
+        cursor = self._conn.execute(
+            "DELETE FROM memories_v2 WHERE workspace_id = ? AND type = ? AND key = ?", (workspace_id, mtype.value, key)
+        )
+        self._conn.commit()
+        return cursor.rowcount > 0
+
+    def list_keys(self, mtype: MemoryType | None = None, limit: int = 30, workspace_id: str = "legacy_global") -> list[str]:
+        sql = "SELECT key FROM memories_v2 WHERE workspace_id = ?"
+        params: tuple = (workspace_id,)
+        if mtype is not None:
+            sql += " AND type = ?"
+            params += (mtype.value,)
+        sql += " ORDER BY last_access DESC LIMIT ?"
+        return [row["key"] for row in self._conn.execute(sql, (*params, limit))]
+
+    def evict_lru(self, keep_top: int) -> int:
+        keep_top = max(keep_top, 0)
+        before = self._conn.execute("SELECT COUNT(*) FROM memories_v2").fetchone()[0]
+        self._conn.execute("""
+            DELETE FROM memories_v2
+            WHERE id NOT IN (
+                SELECT id FROM memories_v2 ORDER BY last_access DESC, id DESC LIMIT ?
+            )
+        """, (keep_top,))
+        self._conn.commit()
+        return before - self._conn.execute("SELECT COUNT(*) FROM memories_v2").fetchone()[0]
+
+    def stats(self) -> dict:
+        summary = {"total": 0}
+        rows = self._conn.execute("""
+            SELECT type, COUNT(*) AS count, COALESCE(SUM(hits), 0) AS hits,
+                   COALESCE(SUM((LENGTH(key) + LENGTH(value) + LENGTH(tags)) / 4), 0) AS tokens
+            FROM memories_v2 GROUP BY type
+        """).fetchall()
+        for row in rows:
+            summary[row["type"]] = {
+                "count": row["count"],
+                "tokens": row["tokens"],
+                "hits": row["hits"],
+            }
+            summary["total"] += row["count"]
+        return summary
+
+
+class EpisodicMemory:
+    def __init__(self, store): self.store = store
+
+    def save(self, key, value, tags):
+        return self.store.save(MemoryType.EPISODIC, key, value, tags)
+
+
+class SemanticMemory:
+    def __init__(self, store): self.store = store
+
+    def save(self, key, value, tags):
+        return self.store.save(MemoryType.SEMANTIC, key, value, tags)
+
+
+class ProceduralMemory:
+    def __init__(self, store): self.store = store
+
+    def save(self, key, value, tags):
+        return self.store.save(MemoryType.PROCEDURAL, key, value, tags)

package/src/memory/vector_store.py

@@ -0,0 +1,104 @@
+"""
+vector_store.py — In-memory numpy vector store
+===============================================
+No LanceDB, no PyArrow, no fixed dimension constraint.
+Stores chunk embeddings in RAM and performs brute-force cosine search.
+Fast enough for typical workspaces (<10k chunks).
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+class VectorStore:
+    """
+    In-memory cosine-similarity vector store.
+    Replaces lancedb to remove the fixed 384-dim Hugging Face constraint.
+    """
+
+    def __init__(self) -> None:
+        self._ids: list[str] = []
+        self._paths: list[str] = []
+        self._matrix: np.ndarray | None = None  # shape (N, D)
+
+    def clear(self) -> None:
+        self._ids.clear()
+        self._paths.clear()
+        self._matrix = None
+
+    def rebuild(self, chunks: list) -> None:
+        """Replace the full index so every vector shares one embedding basis."""
+        self.clear()
+        self.upsert(chunks)
+
+    # ── Helpers ──────────────────────────────────────────────────────────────
+
+    def _index_of(self, id_: str) -> int | None:
+        try:
+            return self._ids.index(id_)
+        except ValueError:
+            return None
+
+    # ── Public API ───────────────────────────────────────────────────────────
+
+    def upsert(self, chunks: list) -> None:
+        """Insert or overwrite chunk embeddings."""
+        for c in chunks:
+            if not c.embedding:
+                continue
+            vec = np.array(c.embedding, dtype=np.float32)
+            norm = np.linalg.norm(vec)
+            if norm > 0:
+                vec /= norm  # pre-normalise for cosine via dot
+
+            idx = self._index_of(c.id)
+            if idx is not None:
+                # Update in-place
+                self._matrix[idx] = vec  # type: ignore[index]
+                self._paths[idx] = c.path
+            else:
+                # Append
+                self._ids.append(c.id)
+                self._paths.append(c.path)
+                if self._matrix is None:
+                    self._matrix = vec.reshape(1, -1)
+                else:
+                    # Pad/trim to current dim if needed
+                    d = self._matrix.shape[1]
+                    if vec.shape[0] < d:
+                        vec = np.pad(vec, (0, d - vec.shape[0]))
+                    elif vec.shape[0] > d:
+                        vec = vec[:d]
+                    self._matrix = np.vstack([self._matrix, vec.reshape(1, -1)])
+
+    def search(self, query_vec: list[float], top_k: int = 50) -> list[dict]:
+        """Return top-k chunks by cosine similarity."""
+        if self._matrix is None or len(self._ids) == 0:
+            return []
+
+        q = np.array(query_vec, dtype=np.float32)
+        # Match dimension
+        d = self._matrix.shape[1]
+        if q.shape[0] < d:
+            q = np.pad(q, (0, d - q.shape[0]))
+        elif q.shape[0] > d:
+            q = q[:d]
+
+        norm = np.linalg.norm(q)
+        if norm > 0:
+            q /= norm
+
+        # Brute-force cosine (dot of normalised vectors)
+        scores = self._matrix @ q  # shape (N,)
+        n = min(top_k, len(self._ids))
+        top_indices = np.argpartition(scores, -n)[-n:]
+        top_indices = top_indices[np.argsort(scores[top_indices])[::-1]]
+
+        return [
+            {"id": self._ids[i], "score": float(scores[i])}
+            for i in top_indices
+        ]
+
+    def count(self) -> int:
+        return len(self._ids)

package/src/retrieval/bm25.py

@@ -0,0 +1,23 @@
+from rank_bm25 import BM25Okapi
+
+from retrieval.tokenize import tokenize
+
+
+class BM25Index:
+    def __init__(self):
+        self.bm25 = None
+        self.chunks = []
+
+    def build(self, chunks: list):
+        self.chunks = chunks
+        if not chunks:
+            self.bm25 = None
+            return
+        tokenized_corpus = [tokenize(c.content) for c in chunks]
+        self.bm25 = BM25Okapi(tokenized_corpus)
+
+    def score(self, query: str) -> list[float]:
+        if not self.bm25:
+            return []
+        tokenized_query = tokenize(query)
+        return self.bm25.get_scores(tokenized_query).tolist()

package/src/retrieval/cache.py

@@ -0,0 +1,76 @@
+"""TTL cache for immutable retrieval selections."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from copy import deepcopy
+from pathlib import Path
+
+
+class RetrievalCache:
+    def __init__(self, ttl: int = 600):
+        self.ttl = ttl
+        self.cache: dict[str, tuple[list, float]] = {}
+
+    @staticmethod
+    def path_fingerprint(path: str) -> str:
+        """Fingerprint a requested source path so changed files cannot hit cache."""
+        source = Path(path)
+        digest = hashlib.sha256()
+
+        if not source.exists():
+            return "missing"
+
+        files = [source] if source.is_file() else sorted(
+            candidate for candidate in source.rglob("*") if candidate.is_file()
+        )
+        for file_path in files:
+            stat = file_path.stat()
+            digest.update(str(file_path.resolve()).encode())
+            digest.update(f"{stat.st_mtime_ns}:{stat.st_size}".encode())
+        return digest.hexdigest()
+
+    def _hash(
+        self,
+        query: str,
+        paths: list[str],
+        options: dict | None = None,
+    ) -> str:
+        payload = {
+            "query": query,
+            "paths": sorted((path, self.path_fingerprint(path)) for path in paths),
+            "options": options or {},
+        }
+        encoded = json.dumps(payload, sort_keys=True, ensure_ascii=False)
+        return hashlib.sha256(encoded.encode()).hexdigest()
+
+    def get(
+        self,
+        query: str,
+        paths: list[str],
+        options: dict | None = None,
+    ) -> list | None:
+        key = self._hash(query, paths, options)
+        entry = self.cache.get(key)
+        if entry is None:
+            return None
+
+        data, timestamp = entry
+        if time.time() - timestamp >= self.ttl:
+            del self.cache[key]
+            return None
+        return deepcopy(data)
+
+    def set(
+        self,
+        query: str,
+        paths: list[str],
+        chunks: list,
+        options: dict | None = None,
+    ) -> None:
+        self.cache[self._hash(query, paths, options)] = (deepcopy(chunks), time.time())
+
+    def invalidate_all(self) -> None:
+        self.cache.clear()

package/src/retrieval/embeddings.py

@@ -0,0 +1,75 @@
+"""Local TF-IDF + LSA embedding engine."""
+
+from __future__ import annotations
+
+import numpy as np
+from sklearn.decomposition import TruncatedSVD
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import Normalizer
+
+_DIM = 128
+
+
+class EmbeddingEngine:
+    """Local LSA-based embedding engine (TF-IDF → SVD → L2-normalise)."""
+
+    def __init__(self, dim: int = _DIM):
+        self.dim = dim
+        self._corpus: list[str] = []
+        self._pipe: Pipeline | None = None
+
+    def _fit(self, texts: list[str]) -> None:
+        n_components = min(self.dim, len(texts) - 1) if len(texts) > 1 else 1
+        self._pipe = Pipeline([
+            ("tfidf", TfidfVectorizer(
+                analyzer="word",
+                ngram_range=(1, 2),
+                max_features=20_000,
+                sublinear_tf=True,
+            )),
+            ("svd", TruncatedSVD(n_components=n_components, random_state=42)),
+            ("norm", Normalizer(copy=False)),
+        ])
+        self._pipe.fit(texts)
+
+    def _transform(self, texts: list[str]) -> np.ndarray:
+        if self._pipe is None:
+            return np.zeros((len(texts), self.dim), dtype=np.float32)
+
+        vecs = self._pipe.transform(texts).astype(np.float32)
+        if vecs.shape[1] < self.dim:
+            pad = np.zeros(
+                (vecs.shape[0], self.dim - vecs.shape[1]), dtype=np.float32)
+            vecs = np.hstack([vecs, pad])
+        return vecs[:, : self.dim]
+
+    def rebuild(self, texts: list[str]) -> list[list[float]]:
+        """Fit once against the complete corpus and return aligned vectors.
+
+        LSA components change whenever the corpus changes. Rebuilding all vectors
+        together prevents the vector store from comparing embeddings from two
+        incompatible bases.
+        """
+        self._corpus = list(dict.fromkeys(texts))
+        self._pipe = None
+        if len(self._corpus) >= 2:
+            self._fit(self._corpus)
+        return self._transform(texts).tolist()
+
+    def embed_text(self, text: str) -> list[float]:
+        """Embed a query with the existing corpus model without refitting it."""
+        return self._transform([text])[0].tolist()
+
+    def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """Compatibility wrapper for callers indexing a complete corpus."""
+        if not texts:
+            return []
+        return self.rebuild(texts)
+
+    def similarity(self, a: str, b: str) -> float:
+        if self._pipe is None:
+            self.rebuild([a, b])
+        vecs = self._transform([a, b])
+        dot = float(np.dot(vecs[0], vecs[1]))
+        return max(-1.0, min(1.0, dot))

package/src/retrieval/graph.py

@@ -0,0 +1,45 @@
+import networkx as nx
+
+
+class DependencyGraph:
+    def __init__(self):
+        self.graph = nx.DiGraph()
+
+    def build(self, chunks: list):
+        self.graph.clear()
+        for c in chunks:
+            self.graph.add_node(c.id, chunk=c)
+
+        # Basic heuristic to build graph: look for symbol usage
+        # In a real app, you would use tree-sitter references
+        symbols = {c.symbol: c.id for c in chunks if c.symbol != "module"}
+
+        for c in chunks:
+            for sym, t_id in symbols.items():
+                if sym in c.content and t_id != c.id:
+                    self.graph.add_edge(c.id, t_id)  # c depends on sym
+
+    def symbol_score(self, query: str, chunk) -> float:
+        score = 0.0
+        if chunk.symbol and chunk.symbol.lower() in query.lower():
+            score += 0.5
+        return score
+
+    def expand(self, chunks: list, max_extra: int = 4) -> list:
+        expanded = set()
+        for c in chunks:
+            if c.id in self.graph:
+                neighbors = list(self.graph.successors(c.id)) + \
+                    list(self.graph.predecessors(c.id))
+                for n in neighbors:
+                    expanded.add(n)
+
+        new_chunks = []
+        for n_id in expanded:
+            if len(new_chunks) >= max_extra:
+                break
+            chunk_data = self.graph.nodes[n_id].get("chunk")
+            if chunk_data and chunk_data not in chunks:
+                new_chunks.append(chunk_data)
+
+        return new_chunks

package/src/retrieval/reranker.py

@@ -0,0 +1,78 @@
+"""
+reranker.py — Local cross-score reranker
+=========================================
+No Hugging Face, no model downloads.
+
+Score = BM25 term-overlap + keyword exact-match bonus + length-penalty.
+This approximates cross-encoder relevance for code/documentation retrieval
+without any network dependency.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+
+from retrieval.tokenize import tokenize
+
+
+def _tokenize(text: str) -> list[str]:
+    """Lower-case word tokeniser (strips punctuation)."""
+    return tokenize(text)
+
+
+def _bm25_score(
+    query_tokens: list[str],
+    doc_tokens: list[str],
+    avg_dl: float,
+    k1: float = 1.5,
+    b: float = 0.75,
+) -> float:
+    """Single-document BM25 score against a query."""
+    doc_freq = Counter(doc_tokens)
+    dl = len(doc_tokens)
+    score = 0.0
+    for term in set(query_tokens):
+        tf = doc_freq.get(term, 0)
+        if tf == 0:
+            continue
+        idf = math.log(1 + 1)  # single-doc approx; always 1 query doc
+        tf_norm = (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * dl / max(avg_dl, 1)))
+        score += idf * tf_norm
+    return score
+
+
+def _keyword_bonus(query_tokens: set[str], doc_tokens: list[str]) -> float:
+    """Fraction of distinct query tokens found in doc (0–1)."""
+    if not query_tokens:
+        return 0.0
+    found = sum(1 for t in query_tokens if t in doc_tokens)
+    return found / len(query_tokens)
+
+
+class Reranker:
+    """
+    Local reranker: BM25 term-overlap + keyword coverage bonus.
+    Replaces sentence-transformers CrossEncoder with a zero-download solution.
+    """
+
+    def rerank(self, query: str, chunks: list, top_k: int = 12) -> list:
+        if not chunks:
+            return []
+
+        q_tokens = _tokenize(query)
+        q_set = set(q_tokens)
+
+        # Average document length for BM25 normalisation
+        doc_token_lists = [_tokenize(c.content) for c in chunks]
+        avg_dl = sum(len(d) for d in doc_token_lists) / max(len(doc_token_lists), 1)
+
+        for chunk, doc_tokens in zip(chunks, doc_token_lists):
+            bm25 = _bm25_score(q_tokens, doc_tokens, avg_dl)
+            bonus = _keyword_bonus(q_set, doc_tokens)
+            # Blend: BM25 dominates, keyword coverage as tiebreaker
+            prior = float(getattr(chunk, "_score", 0.0))
+            chunk._score = prior * 0.6 + bm25 * 0.3 + bonus * 0.1  # type: ignore[attr-defined]
+
+        chunks.sort(key=lambda x: getattr(x, "_score", 0.0), reverse=True)
+        return chunks[:top_k]

package/src/retrieval/tokenize.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+import re
+import unicodedata
+
+
+def tokenize(text: str) -> list[str]:
+    text = unicodedata.normalize("NFC", text)
+    text = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", text).replace("_", " ")
+    words = re.findall(r"[^\W_]+", text.casefold(), flags=re.UNICODE)
+    return words + [word[index:index + 3] for word in words for index in range(max(0, len(word) - 2))]