@tyroneross/build-loop 0.30.3

package/scripts/semantic_index/__init__.py

@@ -0,0 +1,482 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2025-2026 Tyrone Ross, Jr <46267523+tyroneross@users.noreply.github.com>
+# SPDX-License-Identifier: Apache-2.0
+"""Local SQLite semantic-facts index.
+
+The canonical memory store remains file-backed. This module provides the
+local, rebuildable semantic index used when Postgres is absent or deliberately
+disabled. It is stdlib-only so fresh package installs can bootstrap memory
+without database credentials or optional Python packages.
+"""
+from __future__ import annotations
+
+import json
+import re
+import sqlite3
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+HERE = Path(__file__).resolve().parent
+SCRIPTS_DIR = HERE.parent
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+try:
+    from _paths import memory_indexes_dir  # type: ignore  # noqa: E402
+except Exception:  # noqa: BLE001
+    memory_indexes_dir = None  # type: ignore[assignment]
+
+DB_FILENAME = "semantic_facts.sqlite"
+SCHEMA_VERSION = "1.0.0"
+GLOBAL_PROJECT_KEY = "__GLOBAL__"
+TOKEN_RE = re.compile(r"[A-Za-z0-9_.:/-]+")
+
+
+def now_iso() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def default_db_path() -> Path:
+    if memory_indexes_dir is None:
+        return Path.home() / "dev" / "git-folder" / "build-loop-memory" / "indexes" / DB_FILENAME
+    return memory_indexes_dir() / DB_FILENAME
+
+
+def _db_path(db_path: str | Path | None = None) -> Path:
+    return Path(db_path).expanduser().resolve() if db_path else default_db_path().expanduser().resolve()
+
+
+def _project_key(project: str | None) -> str:
+    return project if project else GLOBAL_PROJECT_KEY
+
+
+def _json(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=False, sort_keys=True)
+
+
+def _parse_iso(value: Any) -> float | None:
+    if not value:
+        return None
+    if isinstance(value, (int, float)):
+        return float(value) / 1000 if value > 10_000_000_000 else float(value)
+    if isinstance(value, str):
+        try:
+            return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
+        except ValueError:
+            return None
+    return None
+
+
+def connect(db_path: str | Path | None = None) -> sqlite3.Connection:
+    path = _db_path(db_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(path))
+    conn.row_factory = sqlite3.Row
+    init(conn)
+    return conn
+
+
+def init(conn_or_path: sqlite3.Connection | str | Path | None = None) -> Path | None:
+    owns_conn = not isinstance(conn_or_path, sqlite3.Connection)
+    if owns_conn:
+        path = _db_path(conn_or_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        conn = sqlite3.connect(str(path))
+    else:
+        conn = conn_or_path
+    if conn is None:
+        raise RuntimeError("sqlite connection unavailable")
+    try:
+        conn.execute("PRAGMA journal_mode=WAL;")
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS semantic_facts (
+              subject TEXT NOT NULL,
+              project_key TEXT NOT NULL,
+              predicate TEXT NOT NULL DEFAULT '',
+              object TEXT NOT NULL DEFAULT '',
+              confidence REAL,
+              status TEXT NOT NULL DEFAULT 'active',
+              metadata_json TEXT NOT NULL DEFAULT '{}',
+              project TEXT,
+              tool TEXT,
+              task_category TEXT,
+              files_touched_json TEXT NOT NULL DEFAULT '[]',
+              confidence_source TEXT,
+              domain TEXT,
+              source_prefix TEXT,
+              embedding_json TEXT,
+              last_synced TEXT NOT NULL,
+              schema_version TEXT NOT NULL DEFAULT '1.0.0',
+              PRIMARY KEY(subject, project_key)
+            );
+            """
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_semantic_project ON semantic_facts(project_key, status, last_synced DESC);"
+        )
+        conn.execute(
+            "CREATE INDEX IF NOT EXISTS idx_semantic_domain ON semantic_facts(domain, status, last_synced DESC);"
+        )
+        conn.commit()
+    finally:
+        if owns_conn:
+            conn.close()
+    return None if isinstance(conn_or_path, sqlite3.Connection) else _db_path(conn_or_path)
+
+
+def _embed_text_for_fact(
+    subject: str, predicate: str, object_text: str
+) -> str:
+    """Canonical text fed to the write-path embedder.
+
+    Mirrors the haystack used by the keyword scorer (``subject + predicate +
+    object``) so a query that lexically matches the row is in the same
+    semantic neighborhood as the stored vector. The keyword scorer also
+    includes ``project / domain / tool`` but those are categorical labels —
+    embedding them inflates norm without adding semantic signal, so we
+    keep the embed text tight.
+    """
+    parts = [subject or "", predicate or "", object_text or ""]
+    return " ".join(p for p in parts if p).strip()
+
+
+def _safe_embed_for_write(
+    text: str,
+    embed_fn: Any,
+) -> list[float] | None:
+    """Best-effort embed for the write path. Returns None on any failure.
+
+    Contract: writes MUST NEVER fail because the embed backend is down.
+    A NULL ``embedding_json`` row stays keyword-discoverable; the backfill
+    capability can populate it later when a backend is available.
+
+    ``embed_fn=None`` lazily resolves ``embed_backend.embed`` so missing
+    optional MLX/Ollama deps don't fail at module import time.
+    """
+    if not text:
+        return None
+    if embed_fn is None:
+        try:
+            from embed_backend import embed as _embed  # type: ignore  # noqa: PLC0415
+        except Exception:  # noqa: BLE001
+            return None
+        embed_fn = _embed
+    try:
+        vec = embed_fn(text)
+    except Exception:  # noqa: BLE001
+        return None
+    if not isinstance(vec, list) or not vec:
+        return None
+    try:
+        return [float(x) for x in vec]
+    except (TypeError, ValueError):
+        return None
+
+
+def upsert_fact(
+    *,
+    subject: str,
+    predicate: str,
+    object_text: str,
+    project: str | None,
+    confidence: float | None = None,
+    status: str = "active",
+    metadata: dict[str, Any] | None = None,
+    tool: str | None = None,
+    task_category: str | None = None,
+    files_touched: list[str] | None = None,
+    confidence_source: str | None = None,
+    domain: str | None = None,
+    source_prefix: str | None = None,
+    embedding: list[float] | None = None,
+    embed_fn: Any = None,
+    auto_embed: bool = True,
+    db_path: str | Path | None = None,
+) -> None:
+    """Insert/update a semantic fact.
+
+    Auto-embed contract (NEW — fixes P1 dormant-production bug):
+      - When ``embedding`` is explicitly supplied, it is stored as-is
+        (callers can pre-compute or opt into a specific vector).
+      - When ``embedding`` is None AND ``auto_embed=True`` (default):
+        synthesize the embed text from ``subject + predicate + object``
+        and call ``embed_fn`` (or ``embed_backend.embed`` when None).
+        Failure (backend down, import error, raise) → store NULL. Writes
+        NEVER fail because of an embed-backend outage.
+      - ``auto_embed=False``: byte-equivalent to the pre-P1 behavior —
+        used by the backfill script (which controls the embedder itself)
+        and by tests that want to assert NULL-embedding state.
+
+    Why opt-in-via-default rather than required: the failure mode this
+    fixes is "production rows never get embeddings" — opting in by
+    default is the only thing that makes hybrid actually fire against
+    live data. Making it optional preserves the legacy contract for
+    callers that intentionally don't want write-time embed cost.
+    """
+    if not subject.strip():
+        raise ValueError("subject is required")
+    metadata = dict(metadata or {})
+    metadata.setdefault("last_synced", now_iso())
+    metadata.setdefault("schema_version", SCHEMA_VERSION)
+    files = [str(item) for item in (files_touched or [])]
+
+    # Auto-embed on write when caller didn't supply a vector. Best-effort;
+    # backend down → store NULL, fall back to keyword recall, never raise.
+    if embedding is None and auto_embed:
+        text = _embed_text_for_fact(subject, predicate, object_text)
+        embedding = _safe_embed_for_write(text, embed_fn)
+
+    conn = connect(db_path)
+    try:
+        conn.execute(
+            """
+            INSERT INTO semantic_facts (
+              subject, project_key, predicate, object, confidence, status,
+              metadata_json, project, tool, task_category, files_touched_json,
+              confidence_source, domain, source_prefix, embedding_json,
+              last_synced, schema_version
+            )
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(subject, project_key) DO UPDATE SET
+              predicate=excluded.predicate,
+              object=excluded.object,
+              confidence=excluded.confidence,
+              status=excluded.status,
+              metadata_json=excluded.metadata_json,
+              project=excluded.project,
+              tool=excluded.tool,
+              task_category=excluded.task_category,
+              files_touched_json=excluded.files_touched_json,
+              confidence_source=excluded.confidence_source,
+              domain=excluded.domain,
+              source_prefix=excluded.source_prefix,
+              embedding_json=excluded.embedding_json,
+              last_synced=excluded.last_synced,
+              schema_version=excluded.schema_version;
+            """,
+            (
+                subject,
+                _project_key(project),
+                predicate or "",
+                object_text or "",
+                confidence,
+                status or "active",
+                _json(metadata),
+                project,
+                tool,
+                task_category,
+                _json(files),
+                confidence_source,
+                domain,
+                source_prefix,
+                _json(embedding) if embedding else None,
+                metadata["last_synced"],
+                SCHEMA_VERSION,
+            ),
+        )
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def upsert_lesson(
+    *,
+    lesson: dict[str, Any],
+    project: str | None,
+    subject_prefix: str = "lesson:nav:",
+    confidence: float | None = None,
+    confidence_source: str | None = None,
+    embedding: list[float] | None = None,
+    embed_fn: Any = None,
+    auto_embed: bool = True,
+    db_path: str | Path | None = None,
+    tool: str = "navgator",
+    domain: str = "architecture",
+) -> None:
+    lesson_id = str(lesson.get("id", "") or "").strip()
+    if not lesson_id:
+        raise ValueError("lesson id is required")
+    context = lesson.get("context") or {}
+    files_touched: list[str] = []
+    if isinstance(context, dict) and isinstance(context.get("files_affected"), list):
+        files_touched = [str(item) for item in context["files_affected"]]
+    promoted = bool(lesson.get("promoted", False))
+    metadata = {
+        "lesson_id": lesson_id,
+        "promoted": promoted,
+        "navgator_lesson": lesson,
+    }
+    upsert_fact(
+        subject=f"{subject_prefix}{lesson_id}",
+        predicate=str(lesson.get("category", "") or "uncategorized"),
+        object_text=str(lesson.get("pattern", "") or ""),
+        project=project,
+        confidence=confidence,
+        metadata=metadata,
+        tool=tool,
+        task_category="research",
+        files_touched=files_touched,
+        confidence_source=confidence_source,
+        domain=domain,
+        source_prefix=subject_prefix,
+        embedding=embedding,
+        embed_fn=embed_fn,
+        auto_embed=auto_embed,
+        db_path=db_path,
+    )
+
+
+def _tokens(query: str) -> list[str]:
+    return [m.group(0).lower() for m in TOKEN_RE.finditer(query or "")]
+
+
+def _score(row: sqlite3.Row, tokens: list[str]) -> int:
+    if not tokens:
+        return 1
+    haystack = " ".join(
+        str(row[key] or "") for key in ("subject", "predicate", "object", "project", "domain", "tool")
+    ).lower()
+    return sum(1 for token in tokens if token in haystack)
+
+
+VALID_RECALL_MODES = ("hybrid", "keyword")
+
+
+def query_facts(
+    *,
+    query: str = "",
+    limit: int = 10,
+    project: str | None = None,
+    db_path: str | Path | None = None,
+    mode: str = "hybrid",
+    embed_fn: Any = None,
+) -> list[dict[str, Any]]:
+    """Recall facts. Default mode is **hybrid** (keyword candidates → embedding rerank).
+
+    Pipeline:
+      1. Keyword scorer (token match count, overfetched to ``limit * 20``).
+      2. If ``mode='hybrid'`` AND an embed backend is available AND any
+         candidate has a persisted ``embedding_json`` — rerank by adding
+         cosine(query, candidate) to the keyword score.
+      3. Otherwise return the pure-keyword ranking.
+
+    Graceful fallback (never raises):
+      - ``mode='keyword'``: legacy behavior, byte-equivalent ranking.
+      - Embed backend down or no candidate has an embedding: degrades to
+        keyword ranking, returns successfully.
+
+    ``embed_fn`` is injectable for tests/callers that want a deterministic
+    embedder. When None, ``embed_backend.embed`` is used lazily.
+    """
+    if mode not in VALID_RECALL_MODES:
+        raise ValueError(f"unknown mode {mode!r}; valid: {VALID_RECALL_MODES}")
+    path = _db_path(db_path)
+    if not path.exists():
+        return []
+    conn = connect(path)
+    try:
+        where = ["status = 'active'"]
+        params: list[Any] = []
+        if project:
+            where.append("project = ?")
+            params.append(project)
+        rows = conn.execute(
+            f"""
+            SELECT rowid, subject, predicate, object, project, confidence,
+                   last_synced, metadata_json, tool, domain, confidence_source,
+                   embedding_json
+            FROM semantic_facts
+            WHERE {' AND '.join(where)}
+            ORDER BY last_synced DESC, rowid DESC
+            LIMIT ?
+            """,
+            [*params, max(limit * 20, limit)],
+        ).fetchall()
+    finally:
+        conn.close()
+    tokens = _tokens(query)
+    candidates: list[tuple[float, sqlite3.Row]] = []
+    for row in rows:
+        score = _score(row, tokens)
+        if tokens and score <= 0:
+            # Hybrid mode admits embedding-only matches: include the row
+            # with a zero keyword score so a strong cosine match can still
+            # surface synonyms / paraphrases. Keyword mode preserves the
+            # legacy "must have at least one token hit" behavior.
+            if mode != "hybrid":
+                continue
+            if not _row_has_embedding(row):
+                continue
+        candidates.append((float(score), row))
+
+    use_hybrid = mode == "hybrid" and bool(query and query.strip())
+    if use_hybrid:
+        from .hybrid import (  # noqa: PLC0415
+            _safe_embed_query,
+            has_any_embedding,
+            rerank_candidates,
+        )
+
+        if has_any_embedding(candidates):
+            query_emb = _safe_embed_query(query, embed_fn)
+            if query_emb is not None:
+                ranked = rerank_candidates(candidates, query_emb)
+            else:
+                ranked = _keyword_sort(candidates)
+        else:
+            ranked = _keyword_sort(candidates)
+    else:
+        ranked = _keyword_sort(candidates)
+
+    out: list[dict[str, Any]] = []
+    for _, row in ranked[:limit]:
+        out.append({
+            "_kind": "semantic",
+            "_recency_ts": _parse_iso(row["last_synced"]),
+            "id": f"sqlite:{row['rowid']}",
+            "subject": row["subject"],
+            "predicate": row["predicate"],
+            "object": row["object"],
+            "project": row["project"],
+            "confidence": row["confidence"],
+            "last_accessed": row["last_synced"],
+            "backend": "sqlite",
+            "tool": row["tool"],
+            "domain": row["domain"],
+            "confidence_source": row["confidence_source"],
+        })
+    return out
+
+
+def _row_has_embedding(row: sqlite3.Row) -> bool:
+    try:
+        raw = row["embedding_json"]
+    except (IndexError, KeyError):
+        return False
+    return bool(raw)
+
+
+def _keyword_sort(
+    candidates: list[tuple[float, sqlite3.Row]],
+) -> list[tuple[float, sqlite3.Row]]:
+    """Legacy keyword ranking, byte-equivalent to pre-hybrid behavior."""
+    return sorted(
+        candidates,
+        key=lambda item: (item[0], _parse_iso(item[1]["last_synced"]) or 0),
+        reverse=True,
+    )
+
+
+def stats(db_path: str | Path | None = None) -> dict[str, Any]:
+    path = _db_path(db_path)
+    if not path.exists():
+        return {"db_path": str(path), "exists": False, "rows": 0}
+    conn = connect(path)
+    try:
+        rows = conn.execute("SELECT COUNT(*) FROM semantic_facts").fetchone()[0]
+    finally:
+        conn.close()
+    return {"db_path": str(path), "exists": True, "rows": int(rows)}

package/scripts/semantic_index/_bench_hybrid.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2025-2026 Tyrone Ross, Jr <46267523+tyroneross@users.noreply.github.com>
+# SPDX-License-Identifier: Apache-2.0
+"""Quick latency bench for the SQLite hybrid recall path.
+
+Not a unit test — just a measurement harness so the P1 run report can
+record real numbers (cold vs warm; keyword vs hybrid). Run directly:
+
+    PYTHONPATH=scripts python3 scripts/semantic_index/_bench_hybrid.py
+"""
+from __future__ import annotations
+
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+SCRIPTS = HERE.parent
+if str(SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS))
+
+from semantic_index import query_facts, upsert_fact  # noqa: E402
+
+
+_VEC_DIM = 1024
+
+
+def _det_vec(seed: int) -> list[float]:
+    """Deterministic 1024-dim vector seeded from an int."""
+    rng_state = seed
+    out: list[float] = []
+    for _ in range(_VEC_DIM):
+        rng_state = (rng_state * 1103515245 + 12345) & 0x7FFFFFFF
+        out.append((rng_state % 1000) / 1000.0 - 0.5)
+    return out
+
+
+def _embed_det(text: str) -> list[float]:
+    return _det_vec(abs(hash(text)) & 0xFFFFFF)
+
+
+def _seed(db: Path, n: int) -> None:
+    """Seed with explicit embeddings (legacy path) — used for read benches."""
+    for i in range(n):
+        upsert_fact(
+            subject=f"fact:{i}",
+            predicate="describes",
+            object_text=f"row {i} adapter boundary lesson token{i % 50}",
+            project="bench",
+            embedding=_det_vec(i),
+            db_path=db,
+        )
+
+
+def _bench_write_path(db: Path, n: int) -> dict[str, float]:
+    """Measure auto-embed-on-write latency delta vs explicit-embedding write.
+
+    Both paths persist the same vector to the same column; the only
+    difference is whether ``upsert_fact`` calls ``embed_fn`` internally
+    (auto path) or the caller pre-computed it (explicit path). The delta
+    is the write-time cost of the f1 dormancy fix.
+
+    Uses a stub embed_fn so the bench reflects the in-process overhead
+    of the new seam, not the (much larger) MLX/Ollama cold-load.
+    """
+    from semantic_index import _embed_text_for_fact  # noqa: PLC0415
+
+    # Path A: explicit embedding (legacy).
+    def write_explicit():
+        for i in range(n):
+            upsert_fact(
+                subject=f"explicit:{i}",
+                predicate="describes",
+                object_text=f"row {i} content",
+                project="bench-write",
+                embedding=_det_vec(i),
+                db_path=db,
+            )
+
+    # Path B: auto-embed via injected embed_fn (production path post-f1).
+    def write_auto():
+        for i in range(n):
+            upsert_fact(
+                subject=f"auto:{i}",
+                predicate="describes",
+                object_text=f"row {i} content",
+                project="bench-write",
+                db_path=db,
+                embed_fn=_embed_det,
+            )
+
+    t0 = time.perf_counter()
+    write_explicit()
+    explicit_ms = (time.perf_counter() - t0) * 1000
+    t1 = time.perf_counter()
+    write_auto()
+    auto_ms = (time.perf_counter() - t1) * 1000
+    return {
+        "explicit_total_ms": round(explicit_ms, 2),
+        "auto_total_ms": round(auto_ms, 2),
+        "explicit_per_row_ms": round(explicit_ms / n, 3),
+        "auto_per_row_ms": round(auto_ms / n, 3),
+        "delta_per_row_ms": round((auto_ms - explicit_ms) / n, 3),
+        "rows": n,
+    }
+
+
+def _bench(fn, runs: int) -> dict[str, float]:
+    """Return ms timings (min/median/mean across runs)."""
+    samples: list[float] = []
+    for _ in range(runs):
+        t0 = time.perf_counter()
+        fn()
+        samples.append((time.perf_counter() - t0) * 1000)
+    samples.sort()
+    return {
+        "min_ms": round(samples[0], 2),
+        "median_ms": round(samples[len(samples) // 2], 2),
+        "mean_ms": round(sum(samples) / len(samples), 2),
+        "max_ms": round(samples[-1], 2),
+    }
+
+
+def main() -> int:
+    rows_n = 200
+    runs_n = 20
+    with tempfile.TemporaryDirectory() as td:
+        db = Path(td) / "bench.sqlite"
+        t0 = time.perf_counter()
+        _seed(db, rows_n)
+        seed_ms = (time.perf_counter() - t0) * 1000
+        print(f"[seed] {rows_n} rows in {seed_ms:.1f}ms")
+
+        def run_keyword():
+            return query_facts(
+                query="adapter token17",
+                project="bench",
+                db_path=db,
+                mode="keyword",
+            )
+
+        def run_hybrid():
+            return query_facts(
+                query="adapter token17",
+                project="bench",
+                db_path=db,
+                mode="hybrid",
+                embed_fn=_embed_det,
+            )
+
+        # First call (cold), then warm samples.
+        cold_kw = _bench(run_keyword, runs=1)
+        warm_kw = _bench(run_keyword, runs=runs_n)
+        cold_hy = _bench(run_hybrid, runs=1)
+        warm_hy = _bench(run_hybrid, runs=runs_n)
+
+        print(f"[keyword] cold={cold_kw['min_ms']}ms  warm={warm_kw}")
+        print(f"[hybrid]  cold={cold_hy['min_ms']}ms  warm={warm_hy}")
+
+        # Write-path bench on a separate DB so seeded rows don't pollute
+        # the read bench above.
+        write_db = Path(td) / "bench-write.sqlite"
+        wp = _bench_write_path(write_db, n=100)
+        print(
+            f"[write]   explicit={wp['explicit_per_row_ms']}ms/row  "
+            f"auto={wp['auto_per_row_ms']}ms/row  "
+            f"delta=+{wp['delta_per_row_ms']}ms/row "
+            f"(n={wp['rows']})"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())