ltcai 11.5.2 → 11.6.0

package/scripts/bench_vector_index.py

@@ -1,295 +0,0 @@
-#!/usr/bin/env python3
-"""Vector-index backend benchmark: latency *and* the recall it costs.
-
-A speed number for an approximate index is only half a measurement. This
-script reports both halves for every backend — p50/p95 query latency and
-recall@k against the exact brute-force scan, which is the ground truth by
-construction — so "hnsw is 20x faster" always arrives next to "and returns
-0.97 of the exact top-10".
-
-Deliberately **not** part of CI: it builds a 10 000-vector corpus, it depends
-on an optional compiled extra for one of the three backends, and its numbers
-are machine-specific. It is a local instrument, run by hand, whose output is
-transcribed into docs/PERFORMANCE.md with the machine it came from.
-
-Honesty notes:
-
-* The corpus is synthetic — deterministic pseudo-text from a fixed vocabulary
-  embedded with the built-in hash embedder. Real embeddings cluster very
-  differently, so absolute recall here is an upper bound for a well-separated
-  corpus, not a promise about your notes.
-* Rows are written straight into ``vector_embeddings`` rather than through the
-  ingest pipeline. This measures the *search* path; it is not an ingestion
-  benchmark, and PERFORMANCE.md keeps those numbers separate.
-* A backend whose optional dependency is missing is reported as "not
-  measured", never as a blank or a zero.
-* The candidate cap is lifted by default (``--max-candidates 0``). With the
-  product default of 10 000 the exact scan silently becomes "the newest 10 000
-  rows" on a larger index, and recall@k would then be measured against a
-  baseline that never looked at most of the corpus — which reads as "the ANN
-  backend is inaccurate" when the truth is that the two searched different
-  candidate sets. The ``trunc`` column reports whether any query still hit a
-  cap.
-* Latency and memory are measured in separate passes: tracemalloc roughly
-  triples Python allocation cost, so timing under it would measure the
-  profiler. The ``peak MB`` column comes from one extra traced query.
-
-Usage::
-
-    .venv/bin/python scripts/bench_vector_index.py                 # 10k vectors
-    .venv/bin/python scripts/bench_vector_index.py --nodes 1000     # quick
-    .venv/bin/python scripts/bench_vector_index.py --json           # machine-readable
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import random
-import statistics
-import sys
-import tempfile
-import time
-import tracemalloc
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-ROOT = Path(__file__).resolve().parent.parent
-if str(ROOT) not in sys.path:
-    sys.path.insert(0, str(ROOT))
-
-from lattice_brain.graph.retrieval_vector import (  # noqa: E402
-    VECTOR_MAX_CANDIDATES_ENV,
-)
-from lattice_brain.graph.store import KnowledgeGraphStore  # noqa: E402
-from lattice_brain.graph.vector_index import (  # noqa: E402
-    VECTOR_INDEX_ENV,
-    hnswlib_available,
-    sidecar_paths,
-)
-from lattice_brain.utils import utc_now_iso  # noqa: E402
-
-VOCABULARY = [
-    "retrieval", "embedding", "vector", "index", "graph", "ingestion",
-    "provenance", "workspace", "chunk", "similarity", "cosine", "quantize",
-    "sqlite", "postgres", "backup", "restore", "agent", "proposal", "review",
-    "latency", "throughput", "recall", "precision", "fusion", "lexical",
-    "지식", "그래프", "검색", "임베딩", "인덱스", "문서", "요약", "회의",
-]
-BACKENDS = ("brute", "quantized", "hnsw")
-
-
-def _document(rng: random.Random, index: int) -> str:
-    words = [rng.choice(VOCABULARY) for _ in range(rng.randint(12, 60))]
-    return f"note {index} " + " ".join(words)
-
-
-def build_corpus(store: KnowledgeGraphStore, count: int, seed: int) -> float:
-    """Write ``count`` synthetic nodes + embeddings. Returns wall seconds."""
-    rng = random.Random(seed)  # noqa: S311 — synthetic corpus, not a secret
-    model = store._embedding_model
-    started = time.perf_counter()
-    now = utc_now_iso()
-    with store._connect() as conn:
-        for index in range(count):
-            node_id = f"note:{index:06d}"
-            text = _document(rng, index)
-            conn.execute(
-                "INSERT OR REPLACE INTO nodes(id, type, title, summary, "
-                "metadata_json, raw_json, created_at, updated_at) "
-                "VALUES (?, 'Document', ?, ?, '{}', '{}', ?, ?)",
-                (node_id, f"Note {index}", text[:300], now, now),
-            )
-            conn.execute(
-                "INSERT OR REPLACE INTO vector_embeddings(item_id, item_type, "
-                "source_node, text_hash, embedding, embedding_dim, "
-                "embedding_model, metadata_json, indexed_at) "
-                "VALUES (?, 'node', ?, ?, ?, ?, ?, '{}', ?)",
-                (
-                    node_id,
-                    node_id,
-                    f"hash-{index}",
-                    model.encode(model.embed(text)),
-                    int(model.dim),
-                    model.model_id,
-                    now,
-                ),
-            )
-    return time.perf_counter() - started
-
-
-def _queries(rng: random.Random, count: int) -> List[str]:
-    return [
-        " ".join(rng.choice(VOCABULARY) for _ in range(rng.randint(2, 5)))
-        for _ in range(count)
-    ]
-
-
-def _percentiles(samples: List[float]) -> Dict[str, float]:
-    ordered = sorted(samples)
-    return {
-        "p50_ms": round(statistics.median(ordered) * 1000, 2),
-        "p95_ms": round(ordered[max(0, int(len(ordered) * 0.95) - 1)] * 1000, 2),
-        "mean_ms": round(statistics.fmean(ordered) * 1000, 2),
-    }
-
-
-def measure(
-    store: KnowledgeGraphStore,
-    backend: str,
-    queries: List[str],
-    *,
-    top_k: int,
-    baseline: Optional[Dict[str, List[str]]],
-) -> Dict[str, Any]:
-    """Run every query against ``backend``; return latency + recall + hits."""
-    os.environ[VECTOR_INDEX_ENV] = backend
-    selection = store._vector_index_selection()
-    if not selection.honored:
-        return {"backend": backend, "measured": False, "detail": selection.detail}
-
-    # Warm the sidecar/graph so the reported percentiles are steady-state.
-    warm_started = time.perf_counter()
-    store.vector_search(queries[0], limit=top_k)
-    warm_ms = round((time.perf_counter() - warm_started) * 1000, 2)
-
-    # Latency first, *untraced*: tracemalloc roughly triples Python allocation
-    # cost, and a timing table where one column was measured under a profiler
-    # would compare backends against the profiler rather than each other.
-    hits: Dict[str, List[str]] = {}
-    samples: List[float] = []
-    truncated = False
-    for query in queries:
-        started = time.perf_counter()
-        payload = store.vector_search(query, limit=top_k)
-        samples.append(time.perf_counter() - started)
-        hits[query] = [match["id"] for match in payload["matches"]]
-        truncated = truncated or bool(payload["recall"]["truncated"])
-
-    # Memory in its own pass — one query is enough to see the peak.
-    tracemalloc.start()
-    store.vector_search(queries[0], limit=top_k)
-    peak = tracemalloc.get_traced_memory()[1]
-    tracemalloc.stop()
-
-    report: Dict[str, Any] = {
-        "backend": backend,
-        "measured": True,
-        "engine": selection.backend,
-        "approx": selection.approx,
-        "first_query_ms": warm_ms,
-        "candidates_truncated": truncated,
-        "peak_mb": round(peak / (1024 * 1024), 2),
-        **_percentiles(samples),
-    }
-    if baseline is None:
-        report[f"recall_at_{top_k}"] = 1.0  # this run *is* the ground truth
-    else:
-        scores = [
-            len(set(hits[query]) & set(baseline[query])) / max(1, len(baseline[query]))
-            for query in queries
-        ]
-        report[f"recall_at_{top_k}"] = round(statistics.fmean(scores), 4)
-    # The product-level number: hybrid_search on the same backend, because
-    # that is what a user's question actually runs through.
-    hybrid: List[float] = []
-    for query in queries:
-        started = time.perf_counter()
-        store.hybrid_search(query, top_k=10)
-        hybrid.append(time.perf_counter() - started)
-    hybrid_stats = _percentiles(hybrid)
-    report["hybrid_p50_ms"] = hybrid_stats["p50_ms"]
-    report["hybrid_p95_ms"] = hybrid_stats["p95_ms"]
-    report["_hits"] = hits
-    return report
-
-
-def render(report: Dict[str, Any]) -> str:
-    lines = [
-        f"vectors: {report['nodes']}  queries: {report['queries']}  "
-        f"top_k: {report['top_k']}  max_candidates: {report['max_candidates']}"
-        f"  corpus build: {report['build_seconds']}s",
-        "",
-        f"{'backend':<12}{'p50 ms':>10}{'p95 ms':>10}{'first ms':>10}"
-        f"{'peak MB':>10}{'recall@k':>10}{'trunc':>8}{'hybrid p50':>12}",
-        "-" * 82,
-    ]
-    recall_key = "recall_at_%d" % report["top_k"]
-    for row in report["backends"]:
-        name = row["backend"]
-        if not row["measured"]:
-            lines.append(f"{name:<12}  not measured — {row['detail']}")
-            continue
-        lines.append(
-            f"{name:<12}{row['p50_ms']:>10}{row['p95_ms']:>10}"
-            f"{row['first_query_ms']:>10}{row['peak_mb']:>10}"
-            f"{row[recall_key]:>10}"
-            f"{('yes' if row['candidates_truncated'] else 'no'):>8}"
-            f"{row['hybrid_p50_ms']:>12}"
-        )
-    return "\n".join(lines)
-
-
-def main(argv: Optional[List[str]] = None) -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--nodes", type=int, default=10_000)
-    parser.add_argument("--queries", type=int, default=50)
-    parser.add_argument("--top-k", type=int, default=10)
-    parser.add_argument("--seed", type=int, default=42)
-    parser.add_argument(
-        "--max-candidates",
-        type=int,
-        default=0,
-        help="candidate cap for every backend (0 = uncapped, comparable recall)",
-    )
-    parser.add_argument("--db-path", default=None, help="keep the database around")
-    parser.add_argument("--json", action="store_true")
-    args = parser.parse_args(argv)
-
-    os.environ[VECTOR_MAX_CANDIDATES_ENV] = str(args.max_candidates)
-    workdir = Path(args.db_path) if args.db_path else Path(tempfile.mkdtemp())
-    workdir.mkdir(parents=True, exist_ok=True)
-    store = KnowledgeGraphStore(workdir / "bench.sqlite", workdir / "blobs")
-    build_seconds = build_corpus(store, args.nodes, args.seed)
-    queries = _queries(
-        random.Random(args.seed + 1),  # noqa: S311 — synthetic queries
-        args.queries,
-    )
-
-    rows: List[Dict[str, Any]] = []
-    baseline: Optional[Dict[str, List[str]]] = None
-    for backend in BACKENDS:
-        if backend == "hnsw" and not hnswlib_available():
-            rows.append(
-                {
-                    "backend": backend,
-                    "measured": False,
-                    "detail": "hnswlib is not installed (pip install 'ltcai[hnsw]')",
-                }
-            )
-            continue
-        row = measure(store, backend, queries, top_k=args.top_k, baseline=baseline)
-        if baseline is None and row["measured"]:
-            baseline = row["_hits"]
-        row.pop("_hits", None)
-        rows.append(row)
-
-    os.environ.pop(VECTOR_INDEX_ENV, None)
-    report = {
-        "nodes": args.nodes,
-        "queries": args.queries,
-        "top_k": args.top_k,
-        "max_candidates": args.max_candidates,
-        "seed": args.seed,
-        "build_seconds": round(build_seconds, 2),
-        "db_path": str(store.db_path),
-        "sidecar": str(sidecar_paths(store.db_path)[0]),
-        "backends": rows,
-    }
-    print(json.dumps(report, indent=2) if args.json else render(report))
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

package/scripts/funnel_soft_gate.py

@@ -1,192 +0,0 @@
-#!/usr/bin/env python3
-"""Funnel soft gate (review Wave 3.3) — funnel metrics as an advisory verdict.
-
-Reads the same ``funnel_metrics.json`` production writes and grades it through
-``FunnelMetricsService.snapshot()`` — no rate logic is duplicated here. Two
-funnel-health rates carry WARN thresholds:
-
-* ``code_only_rate``    — WARN above 0.05 (goal: >95% of recognized file
-  requests deliver a real artifact, not a code/prose-only answer);
-* ``needs_review_rate`` — WARN above 0.30 (verifier fail-closed is healthy,
-  but a third of runs ending NEEDS_REVIEW means verification is starving).
-
-Rates whose denominator is still zero come back ``None`` from the snapshot;
-they are reported as "no data" and never warn — a fresh install stays green.
-``real_file_rate``, ``approval_resume_rate`` and TTFV are shown as
-informational context only.
-
-Exit codes
-==========
-* default         — advisory: ALWAYS ``0``; warnings are printed, not enforced;
-* ``--strict``    — CI opt-in: ``1`` when any WARN fired ("no data" never
-  fails, even under --strict);
-* bad flags       — argparse's usual ``2`` (the only real script error).
-
-Usage
-=====
-    .venv/bin/python scripts/funnel_soft_gate.py                 # advisory report
-    .venv/bin/python scripts/funnel_soft_gate.py --strict        # CI gate opt-in
-    .venv/bin/python scripts/funnel_soft_gate.py --json          # machine output
-    .venv/bin/python scripts/funnel_soft_gate.py --path /tmp/m.json
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import sys
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
-
-from latticeai.services.funnel_metrics import FunnelMetricsService  # noqa: E402
-
-# Advisory thresholds (overridable per run via flags). WARN fires strictly
-# ABOVE the threshold, so a rate sitting exactly on the goal boundary passes.
-CODE_ONLY_WARN_THRESHOLD = 0.05
-NEEDS_REVIEW_WARN_THRESHOLD = 0.30
-
-
-def default_metrics_path() -> Path:
-    """The exact file production writes: ``Config.from_env`` data-dir pattern
-    (``LATTICEAI_DATA_DIR`` env or ``~/.ltcai``) + the ``funnel_metrics.json``
-    filename from ``latticeai/runtime/persistence_runtime.py``."""
-    data_dir = Path(os.environ.get("LATTICEAI_DATA_DIR") or (Path.home() / ".ltcai"))
-    return data_dir / "funnel_metrics.json"
-
-
-def evaluate_snapshot(
-    snapshot: Dict[str, Any],
-    *,
-    code_only_warn: float = CODE_ONLY_WARN_THRESHOLD,
-    needs_review_warn: float = NEEDS_REVIEW_WARN_THRESHOLD,
-) -> Dict[str, Any]:
-    """Grade a ``FunnelMetricsService.snapshot()`` payload into verdicts."""
-    rates = snapshot.get("rates") or {}
-    checks: List[Dict[str, Any]] = []
-
-    def check(name: str, threshold: float, goal: str) -> None:
-        rate: Optional[float] = rates.get(name)
-        if rate is None:
-            verdict, detail = "no_data", "no denominator recorded yet"
-        elif rate > threshold:
-            verdict, detail = "warn", f"{rate} > {threshold} — {goal}"
-        else:
-            verdict, detail = "ok", f"{rate} <= {threshold}"
-        checks.append({
-            "name": name, "rate": rate, "threshold": threshold,
-            "verdict": verdict, "detail": detail,
-        })
-
-    check("code_only_rate", code_only_warn,
-          "goal: >95% of file requests deliver real files")
-    check("needs_review_rate", needs_review_warn,
-          "goal: <30% of agent runs end NEEDS_REVIEW")
-
-    warnings = [c["name"] for c in checks if c["verdict"] == "warn"]
-    if warnings:
-        status = "warn"
-    elif all(c["verdict"] == "no_data" for c in checks):
-        status = "no_data"
-    else:
-        status = "ok"
-    return {
-        "status": status,
-        "warnings": warnings,
-        "checks": checks,
-        # Informational context — never gated.
-        "info": {
-            "real_file_rate": rates.get("real_file_rate"),
-            "approval_resume_rate": rates.get("approval_resume_rate"),
-            "ttfv_seconds": snapshot.get("ttfv_seconds"),
-            "counters": snapshot.get("counters") or {},
-        },
-    }
-
-
-def format_report(payload: Dict[str, Any]) -> str:
-    lines = [
-        f"Funnel soft gate (advisory) — {payload['path']}",
-        "=" * 72,
-    ]
-    for check in payload["checks"]:
-        rate = "no data" if check["rate"] is None else str(check["rate"])
-        verdict = {"warn": "WARN", "ok": "ok", "no_data": "-"}[check["verdict"]]
-        lines.append(
-            f"  {check['name']:<20} {rate:<8} {verdict:<5} {check['detail']}"
-        )
-    info = payload["info"]
-    lines.append("-" * 72)
-    lines.append(
-        f"  info: real_file_rate={info['real_file_rate']} "
-        f"approval_resume_rate={info['approval_resume_rate']} "
-        f"ttfv_seconds={info['ttfv_seconds']}"
-    )
-    lines.append("=" * 72)
-    if payload["status"] == "warn":
-        mode = (
-            "STRICT — exiting 1" if payload["strict"]
-            else "advisory — exit 0; opt into enforcement with --strict"
-        )
-        lines.append(f"verdict: WARN ({', '.join(payload['warnings'])}) [{mode}]")
-    elif payload["status"] == "no_data":
-        lines.append("verdict: no data yet — nothing to grade (exit 0)")
-    else:
-        lines.append("verdict: OK — funnel rates within goals")
-    return "\n".join(lines)
-
-
-def main(argv: List[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(
-        description="Advisory funnel-metrics gate (warn on funnel-health regressions)"
-    )
-    parser.add_argument(
-        "--path",
-        help="metrics JSON path (default: the production data-dir funnel_metrics.json)",
-    )
-    parser.add_argument(
-        "--strict", action="store_true",
-        help="exit 1 when any WARN fired (CI opt-in; 'no data' still exits 0)",
-    )
-    parser.add_argument(
-        "--json", dest="json_out", action="store_true",
-        help="print the machine-readable verdict JSON to stdout",
-    )
-    parser.add_argument(
-        "--code-only-warn", type=float, default=CODE_ONLY_WARN_THRESHOLD,
-        help=f"code_only_rate WARN threshold (default {CODE_ONLY_WARN_THRESHOLD})",
-    )
-    parser.add_argument(
-        "--needs-review-warn", type=float, default=NEEDS_REVIEW_WARN_THRESHOLD,
-        help=f"needs_review_rate WARN threshold (default {NEEDS_REVIEW_WARN_THRESHOLD})",
-    )
-    args = parser.parse_args(argv)
-
-    path = Path(args.path) if args.path else default_metrics_path()
-    # A missing or corrupt file yields the service's zeroed state → all rates
-    # None → "no data" (never a crash, never a failure).
-    snapshot = FunnelMetricsService(path).snapshot()
-    result = evaluate_snapshot(
-        snapshot,
-        code_only_warn=args.code_only_warn,
-        needs_review_warn=args.needs_review_warn,
-    )
-    exit_code = 1 if (args.strict and result["status"] == "warn") else 0
-    payload = {
-        "mode": "funnel-soft-gate",
-        "path": str(path),
-        "strict": bool(args.strict),
-        "exit_code": exit_code,
-        **result,
-    }
-    if args.json_out:
-        print(json.dumps(payload, ensure_ascii=False, indent=2))
-    else:
-        print(format_report(payload))
-    return exit_code
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())