@kernlang/agon-dedup 0.2.0 → 0.2.1

package/classifier.py

@@ -23,6 +23,7 @@ Exit codes:
 from __future__ import annotations
 
 import json
+import os
 import sys
 
 
@@ -30,6 +31,20 @@ MIN_CONFIDENCE = 0.10
 MARGIN = 0.05  # top class must beat #2 by this much to commit; otherwise 'other'
 MODEL = "sentence-transformers/all-MiniLM-L6-v2"
 
+
+def _cache_dir() -> str:
+    """Persistent model cache. fastembed's default is a tmpdir that the OS
+    prunes between runs — a half-pruned snapshot then fails with NO_SUCHFILE
+    on every call. Pin the cache under ~/.agon (or FASTEMBED_CACHE_PATH)."""
+    explicit = os.environ.get("FASTEMBED_CACHE_PATH", "").strip()
+    if explicit:
+        return explicit
+    agon_home = os.environ.get("AGON_HOME", "").strip() or os.path.join(
+        os.path.expanduser("~"), ".agon")
+    path = os.path.join(agon_home, "cache", "fastembed")
+    os.makedirs(path, exist_ok=True)
+    return path
+
 # Few-shot examples per label. We embed concrete task phrasings rather than
 # abstract definitions because all-MiniLM scores concrete-vs-concrete cosine
 # much higher than concrete-vs-abstract. Empirically this lifts true-positive
@@ -89,7 +104,7 @@ def _classify(text: str) -> dict:
               file=sys.stderr)
         sys.exit(2)
 
-    embedder = TextEmbedding(MODEL)
+    embedder = TextEmbedding(MODEL, cache_dir=_cache_dir())
     label_keys = list(LABELS.keys())
     label_descriptions = [LABELS[k] for k in label_keys]
     all_texts = [text] + label_descriptions

package/embedder.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+Embedding sidecar — raw sentence embeddings for the RAG retriever.
+
+Same model + runtime as sidecar.py (MiniLM via fastembed/ONNX, ~30MB,
+fully offline after the first model download), but instead of clustering
+it returns the raw vectors so the TS side can build a persistent
+cosine-similarity index.
+
+Protocol:
+  stdin  — JSONL, one per line: {"id": "<chunkId>", "text": "<chunk text>"}
+  stdout — single JSON:
+    {
+      "model": "sentence-transformers/all-MiniLM-L6-v2",
+      "dims": 384,
+      "vectors": [{"id": "<chunkId>", "vector": [..384 floats..]}, ...]
+    }
+  Vector order matches input order. Vectors are L2-normalized so cosine
+  similarity reduces to a dot product on the consumer side.
+
+Exit codes:
+  0  — success
+  1  — bad input (malformed JSON, no items)
+  2  — fastembed not installed (caller should fall back / surface a hint)
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+
+MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+
+
+def _cache_dir() -> str:
+    """Persistent model cache. fastembed's default is a tmpdir that the OS
+    prunes between runs — a half-pruned snapshot then fails with NO_SUCHFILE
+    on every call. Pin the cache under ~/.agon (or FASTEMBED_CACHE_PATH)."""
+    explicit = os.environ.get("FASTEMBED_CACHE_PATH", "").strip()
+    if explicit:
+        return explicit
+    agon_home = os.environ.get("AGON_HOME", "").strip() or os.path.join(
+        os.path.expanduser("~"), ".agon")
+    path = os.path.join(agon_home, "cache", "fastembed")
+    os.makedirs(path, exist_ok=True)
+    return path
+
+
+def _read_input() -> list[dict[str, str]]:
+    items: list[dict[str, str]] = []
+    for line_num, raw_line in enumerate(sys.stdin, 1):
+        line = raw_line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError as err:
+            print(f"embedder-sidecar: line {line_num} is not valid JSON: {err}",
+                  file=sys.stderr)
+            sys.exit(1)
+        if not isinstance(obj, dict) or "id" not in obj or "text" not in obj:
+            print(f"embedder-sidecar: line {line_num} missing 'id' or 'text'",
+                  file=sys.stderr)
+            sys.exit(1)
+        items.append({"id": str(obj["id"]), "text": str(obj["text"])})
+    return items
+
+
+def main() -> int:
+    items = _read_input()
+    if not items:
+        print("embedder-sidecar: no input items", file=sys.stderr)
+        return 1
+
+    try:
+        import numpy as np
+        from fastembed import TextEmbedding
+    except ImportError:
+        print("embedder-sidecar: fastembed not installed — install via "
+              "`pip install fastembed numpy` (see requirements.txt)",
+              file=sys.stderr)
+        return 2
+
+    embedder = TextEmbedding(MODEL, cache_dir=_cache_dir())
+    texts = [item["text"] for item in items]
+    embs = np.array(list(embedder.embed(texts)), dtype=np.float32)
+
+    # L2-normalize so the consumer can use a plain dot product as cosine.
+    norms = np.linalg.norm(embs, axis=1, keepdims=True)
+    norms[norms == 0] = 1.0
+    embs = embs / norms
+
+    vectors = [
+        {"id": item["id"], "vector": [round(float(x), 7) for x in emb]}
+        for item, emb in zip(items, embs)
+    ]
+    print(json.dumps({"model": MODEL, "dims": int(embs.shape[1]), "vectors": vectors}))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/history-search.py

@@ -34,6 +34,7 @@ Exit codes:
 from __future__ import annotations
 
 import json
+import os
 import sys
 from typing import Any
 
@@ -43,6 +44,20 @@ DEFAULT_TOP_K = 10
 MIN_SIMILARITY = 0.15  # below this, the match is effectively noise — drop it
 
 
+def _cache_dir() -> str:
+    """Persistent model cache. fastembed's default is a tmpdir that the OS
+    prunes between runs — a half-pruned snapshot then fails with NO_SUCHFILE
+    on every call. Pin the cache under ~/.agon (or FASTEMBED_CACHE_PATH)."""
+    explicit = os.environ.get("FASTEMBED_CACHE_PATH", "").strip()
+    if explicit:
+        return explicit
+    agon_home = os.environ.get("AGON_HOME", "").strip() or os.path.join(
+        os.path.expanduser("~"), ".agon")
+    path = os.path.join(agon_home, "cache", "fastembed")
+    os.makedirs(path, exist_ok=True)
+    return path
+
+
 def _read_input() -> dict[str, Any]:
     raw = sys.stdin.read().strip()
     if not raw:
@@ -102,7 +117,7 @@ def _rank(payload: dict[str, Any]) -> list[dict[str, Any]]:
     items: list[dict[str, str]] = payload["items"]
     top_k: int = payload["top_k"]
 
-    embedder = TextEmbedding(MODEL)
+    embedder = TextEmbedding(MODEL, cache_dir=_cache_dir())
     texts = [query] + [item["text"] for item in items]
     embs = np.array(list(embedder.embed(texts)))
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kernlang/agon-dedup",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Python sidecars for Agon AI — semantic embeddings (fastembed/MiniLM) and tree-sitter syntax validation. Bridged from KERN via stdin/stdout JSON. Ships .py files that the @kernlang/agon-core bridges spawn at runtime.",
   "type": "module",
   "files": [

package/sidecar.py

@@ -32,6 +32,7 @@ Exit codes:
 from __future__ import annotations
 
 import json
+import os
 import sys
 from typing import Any
 
@@ -40,6 +41,20 @@ THRESHOLD = 0.55
 MODEL = "sentence-transformers/all-MiniLM-L6-v2"
 
 
+def _cache_dir() -> str:
+    """Persistent model cache. fastembed's default is a tmpdir that the OS
+    prunes between runs — a half-pruned snapshot then fails with NO_SUCHFILE
+    on every call. Pin the cache under ~/.agon (or FASTEMBED_CACHE_PATH)."""
+    explicit = os.environ.get("FASTEMBED_CACHE_PATH", "").strip()
+    if explicit:
+        return explicit
+    agon_home = os.environ.get("AGON_HOME", "").strip() or os.path.join(
+        os.path.expanduser("~"), ".agon")
+    path = os.path.join(agon_home, "cache", "fastembed")
+    os.makedirs(path, exist_ok=True)
+    return path
+
+
 def _read_input() -> list[dict[str, str]]:
     items: list[dict[str, str]] = []
     for line_num, raw_line in enumerate(sys.stdin, 1):
@@ -77,7 +92,7 @@ def _cluster(items: list[dict[str, str]]) -> list[dict[str, Any]]:
               file=sys.stderr)
         sys.exit(2)
 
-    embedder = TextEmbedding(MODEL)
+    embedder = TextEmbedding(MODEL, cache_dir=_cache_dir())
     texts = [item["text"] for item in items]
     embs = np.array(list(embedder.embed(texts)))