@geravant/sinain 1.12.0 → 1.14.0

package/sinain-memory/embed_client.py

@@ -0,0 +1,117 @@
+"""Embedding client — calls sinain-core's /embed endpoint for vector operations.
+
+Provides semantic similarity for:
+- Write path: dedup before asserting facts (knowledge_integrator.py)
+- Read path: semantic retrieval (graph_query.py)
+
+Falls back gracefully if sinain-core is not running or model not loaded.
+"""
+
+import base64
+import json
+import struct
+import urllib.request
+from functools import lru_cache
+
+SINAIN_CORE_URL = "http://localhost:9500"
+EMBED_TIMEOUT_S = 5
+SIMILARITY_THRESHOLD = 0.78  # calibrated: catches rephrased facts, rejects different facts
+
+
+def embed(texts: list[str]) -> list[list[float]] | None:
+    """Embed texts via sinain-core /embed endpoint. Returns None if unavailable."""
+    try:
+        data = json.dumps({"texts": texts}).encode()
+        req = urllib.request.Request(
+            f"{SINAIN_CORE_URL}/embed",
+            data=data,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=EMBED_TIMEOUT_S) as resp:
+            result = json.loads(resp.read())
+            # Decode base64 float32 arrays
+            embeddings = []
+            for b64 in result["embeddings"]:
+                raw = base64.b64decode(b64)
+                floats = list(struct.unpack(f"{len(raw)//4}f", raw))
+                embeddings.append(floats)
+            return embeddings
+    except Exception:
+        return None
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    """Cosine similarity between two vectors."""
+    dot = sum(x * y for x, y in zip(a, b))
+    return dot  # vectors are pre-normalized by the model
+
+
+def find_duplicates_batch(
+    new_texts: list[str],
+    existing_texts: list[str],
+    threshold: float = SIMILARITY_THRESHOLD,
+) -> dict[int, int]:
+    """Find duplicates for multiple new texts against existing texts in one batch.
+
+    Returns {new_index: existing_index} for texts with similarity >= threshold.
+    Single HTTP call for all texts — avoids per-fact round trips.
+    """
+    if not existing_texts or not new_texts:
+        return {}
+
+    all_texts = new_texts + existing_texts
+    embeddings = embed(all_texts)
+    if embeddings is None:
+        return {}
+
+    n_new = len(new_texts)
+    result = {}
+
+    for i in range(n_new):
+        best_idx = None
+        best_sim = threshold
+        for j in range(n_new, len(embeddings)):
+            sim = cosine(embeddings[i], embeddings[j])
+            if sim > best_sim:
+                best_sim = sim
+                best_idx = j - n_new
+        if best_idx is not None:
+            result[i] = best_idx
+
+    return result
+
+
+def find_duplicate(
+    new_text: str,
+    existing_texts: list[str],
+    threshold: float = SIMILARITY_THRESHOLD,
+) -> int | None:
+    """Find the index of the most similar existing text, or None if no match."""
+    result = find_duplicates_batch([new_text], existing_texts, threshold)
+    return result.get(0)
+
+
+def rank_by_similarity(
+    query: str,
+    texts: list[str],
+) -> list[tuple[int, float]] | None:
+    """Rank texts by semantic similarity to query. Returns [(index, score), ...] descending.
+
+    Returns None if embedding service unavailable (caller should fall back to keyword).
+    """
+    if not texts:
+        return []
+
+    all_texts = [query] + texts
+    embeddings = embed(all_texts)
+    if embeddings is None:
+        return None
+
+    query_emb = embeddings[0]
+    scored = []
+    for i, emb in enumerate(embeddings[1:]):
+        scored.append((i, cosine(query_emb, emb)))
+
+    scored.sort(key=lambda x: x[1], reverse=True)
+    return scored

package/sinain-memory/graph_query.py

@@ -129,32 +129,278 @@ def query_top_facts(db_path: str, limit: int = 30) -> list[dict]:
         return []
 
 
+def query_facts_fts(db_path: str, query: str, max_facts: int = 10) -> list[dict]:
+    """Full-text search on fact values via FTS5 index.
+
+    Returns facts whose value field matches the query keywords.
+    Falls back to LIKE search if FTS5 is not available.
+    """
+    if not Path(db_path).exists():
+        return []
+
+    try:
+        from triplestore import TripleStore
+        store = TripleStore(db_path)
+
+        # Try FTS5 first
+        try:
+            rows = store._conn.execute(
+                """SELECT DISTINCT t.entity_id
+                   FROM triples_fts fts
+                   JOIN triples t ON fts.rowid = t.id
+                   WHERE triples_fts MATCH ?
+                     AND t.attribute = 'value'
+                     AND NOT t.retracted
+                   LIMIT ?""",
+                (query, max_facts),
+            ).fetchall()
+        except Exception:
+            # FTS5 not available — fall back to LIKE search
+            keywords = [w.lower() for w in query.split() if len(w) > 2]
+            if not keywords:
+                store.close()
+                return []
+            # Match any keyword in value
+            conditions = " OR ".join(["LOWER(value) LIKE ?"] * len(keywords))
+            params = [f"%{k}%" for k in keywords] + [max_facts]
+            rows = store._conn.execute(
+                f"""SELECT DISTINCT entity_id
+                    FROM triples
+                    WHERE attribute = 'value'
+                      AND NOT retracted
+                      AND ({conditions})
+                    LIMIT ?""",
+                params,
+            ).fetchall()
+
+        entity_ids = [r["entity_id"] for r in rows]
+        if not entity_ids:
+            store.close()
+            return []
+
+        # Fetch full attributes for matched entities
+        facts = []
+        for eid in entity_ids:
+            attrs = store.entity(eid)
+            fact = {"entity_id": eid, "entity": eid.split(":")[-1].rsplit("-", 1)[0] if ":" in eid else eid}
+            for attr, values in attrs.items():
+                if attr == "tag":
+                    continue
+                fact[attr] = values[0] if len(values) == 1 else values
+            facts.append(fact)
+
+        store.close()
+        return facts[:max_facts]
+    except Exception:
+        return []
+
+
+def query_facts_by_entity_graph(
+    db_path: str,
+    entity_name: str,
+    max_facts: int = 10,
+) -> list[dict]:
+    """Find facts about an entity via VAET backref traversal.
+
+    Uses the entity graph layer: entity:* nodes linked to fact:* nodes
+    via 'about' ref edges. Also follows 'mentions' ref edges for
+    cross-entity context.
+    """
+    if not Path(db_path).exists():
+        return []
+
+    try:
+        from triplestore import TripleStore
+        store = TripleStore(db_path)
+
+        entity_node_id = f"entity:{entity_name.lower().replace(' ', '-')}"
+        if not store.entity(entity_node_id):
+            store.close()
+            return []
+
+        # Get all facts linked to this entity via "about" ref edge
+        fact_refs = store.backrefs(entity_node_id, attribute="about")
+        # Also get facts that "mention" this entity
+        mention_refs = store.backrefs(entity_node_id, attribute="mentions")
+        all_refs = fact_refs + mention_refs
+
+        # Load fact details
+        seen = set()
+        facts = []
+        for fact_eid, _ in all_refs:
+            if fact_eid in seen or not fact_eid.startswith("fact:"):
+                continue
+            seen.add(fact_eid)
+            attrs = store.entity(fact_eid)
+            if attrs and "value" in attrs:
+                fact = {"entity_id": fact_eid}
+                for attr, values in attrs.items():
+                    if attr == "tag":
+                        continue
+                    fact[attr] = values[0] if len(values) == 1 else values
+                facts.append(fact)
+
+        store.close()
+        return facts[:max_facts]
+    except Exception:
+        return []
+
+
+def query_facts_hybrid(
+    db_path: str,
+    query: str,
+    max_facts: int = 10,
+) -> list[dict]:
+    """Hybrid retrieval with Reciprocal Rank Fusion (Graphiti pattern).
+
+    Runs three independent retrieval methods, fuses via RRF, then
+    expands top results with 1-hop graph neighbors.
+    """
+    import re
+    keywords = [w.lower() for w in re.findall(r"[a-zA-Z][a-zA-Z0-9-]+", query) if len(w) > 2]
+
+    # Entity graph pre-filter: find facts linked to mentioned entities via backrefs.
+    # Used to BOOST relevant facts in RRF, not as a separate tier (avoids dilution).
+    graph_fact_ids: set[str] = set()
+    for kw in keywords:
+        for f in query_facts_by_entity_graph(db_path, kw, max_facts=50):
+            eid = f.get("entity_id", "")
+            if eid:
+                graph_fact_ids.add(eid)
+
+    # Run three retrieval methods independently
+    candidate_limit = max_facts * 3
+    fts_results = query_facts_fts(db_path, query, max_facts=candidate_limit)
+    tag_results = query_facts_by_entities(db_path, keywords, max_facts=candidate_limit) if keywords else []
+    top_results = query_top_facts(db_path, limit=candidate_limit)
+
+    # Build ranked lists by entity_id
+    def _ranked_ids(facts: list[dict]) -> list[str]:
+        seen = set()
+        out = []
+        for f in facts:
+            eid = f.get("entity_id", "")
+            if eid and eid not in seen:
+                seen.add(eid)
+                out.append(eid)
+        return out
+
+    fts_ranked = _ranked_ids(fts_results)
+    tag_ranked = _ranked_ids(tag_results)
+    top_ranked = _ranked_ids(top_results)
+
+    # Reciprocal Rank Fusion: RRF(d) = Σ 1/(k + rank_i(d))
+    K = 60  # standard RRF constant
+    rrf_scores: dict[str, float] = {}
+    for ranked_list in [fts_ranked, tag_ranked, top_ranked]:
+        for rank, eid in enumerate(ranked_list):
+            rrf_scores[eid] = rrf_scores.get(eid, 0.0) + 1.0 / (K + rank)
+
+    # Graph boost: facts linked to mentioned entities via backrefs get priority
+    if graph_fact_ids:
+        for eid in rrf_scores:
+            if eid in graph_fact_ids:
+                rrf_scores[eid] += 0.02  # significant boost — graph-linked facts rank higher
+
+    # Apply confidence decay as secondary signal (fresh facts rank above stale ones)
+    from triplestore import decayed_confidence
+    for facts_list in [fts_results, tag_results, top_results]:
+        for f in facts_list:
+            eid = f.get("entity_id", "")
+            if eid in rrf_scores:
+                conf = 0.5
+                created = ""
+                try:
+                    conf = float(f.get("confidence", 0.5))
+                    created = str(f.get("first_seen", ""))
+                except (ValueError, TypeError):
+                    pass
+                if created:
+                    effective = decayed_confidence(conf, created)
+                    rrf_scores[eid] += effective * 0.01  # small boost, preserves RRF rank
+
+    # Sort by RRF score descending
+    sorted_ids = sorted(rrf_scores, key=rrf_scores.get, reverse=True)
+
+    # Build fact lookup from all candidates
+    fact_map: dict[str, dict] = {}
+    for facts in [fts_results, tag_results, top_results]:
+        for f in facts:
+            eid = f.get("entity_id", "")
+            if eid and eid not in fact_map:
+                fact_map[eid] = f
+
+    # Return top RRF candidates. Embedding re-ranking is done by the caller
+    # (sinain-core Node.js) to avoid deadlock — the Python subprocess can't call
+    # back to sinain-core's /embed endpoint while sinain-core is blocked waiting
+    # for the subprocess.
+    results = [fact_map[eid] for eid in sorted_ids[:max_facts] if eid in fact_map]
+
+    # Expand top results with 1-hop graph neighbors
+    if results and len(results) < max_facts:
+        seen_ids = {f.get("entity_id", "") for f in results}
+        try:
+            from triplestore import TripleStore
+            store = TripleStore(db_path)
+            for fact in list(results):
+                eid = fact.get("entity_id", "")
+                if not eid:
+                    continue
+                neighbors = store.neighbors(eid, depth=1)
+                for nid, nattrs in neighbors.items():
+                    if nid not in seen_ids and len(results) < max_facts:
+                        seen_ids.add(nid)
+                        nfact = {"entity_id": nid, "entity": nid.split(":")[-1].rsplit("-", 1)[0] if ":" in nid else nid}
+                        for attr, values in nattrs.items():
+                            if attr != "tag":
+                                nfact[attr] = values[0] if len(values) == 1 else values
+                        results.append(nfact)
+            store.close()
+        except Exception:
+            pass
+
+    return results[:max_facts]
+
+
 def format_facts_text(facts: list[dict], max_chars: int = 500) -> str:
-    """Format facts as human-readable text for escalation message injection."""
+    """Format facts grouped by entity for better cross-fact reasoning.
+
+    Groups related facts under entity headers so the QA model sees
+    connected context (e.g., all Citibank facts together).
+    """
     if not facts:
         return ""
 
-    lines = []
-    total = 0
+    # Group by entity name (strip fact: prefix and hash suffix)
+    from collections import OrderedDict
+    groups: OrderedDict[str, list[dict]] = OrderedDict()
     for f in facts:
-        value = f.get("value", "")
-        conf = f.get("confidence", "?")
-        count = f.get("reinforce_count", "1")
-        domain = f.get("domain", "")
-
-        line = f"- {value} (confidence: {conf}, confirmed {count}x)"
-        if domain:
-            line = f"- [{domain}] {value} (confidence: {conf}, confirmed {count}x)"
+        entity = f.get("entity", "")
+        if isinstance(entity, list):
+            entity = entity[0] if entity else ""
+        if not entity:
+            eid = str(f.get("entity_id", ""))
+            entity = eid.split(":")[-1].rsplit("-", 1)[0] if ":" in eid else eid
+        groups.setdefault(str(entity), []).append(f)
 
-        if total + len(line) > max_chars:
-            break
-        lines.append(line)
-        total += len(line)
+    lines = []
+    total = 0
+    for entity, group_facts in groups.items():
+        for f in group_facts:
+            value = f.get("value", "")
+            conf = f.get("confidence", "?")
+            count = f.get("reinforce_count", "1")
+
+            line = f"- [{entity}] {value} (conf: {conf}, {count}x)"
+            if total + len(line) > max_chars:
+                return "\n".join(lines)
+            lines.append(line)
+            total += len(line)
 
     return "\n".join(lines)
 
 
-def format_facts_compact(facts: list[dict], max_chars: int = 400) -> str:
+def format_facts_compact(facts: list[dict], max_chars: int = 1200) -> str:
     """Encode facts for efficient escalation context injection.
 
     Compact format: domain/entity: value (conf, Nx)
@@ -167,7 +413,7 @@ def format_facts_compact(facts: list[dict], max_chars: int = 400) -> str:
     total = 0
     for f in facts:
         entity = f.get("entityId", "").split(":")[-1][:20]
-        value = f.get("value", "")[:60]
+        value = f.get("value", "")
         conf = f.get("confidence", "?")
         count = f.get("reinforce_count", "1")
         domain = f.get("domain", "")
@@ -227,7 +473,12 @@ def main() -> None:
         facts = query_top_facts(args.db, limit=args.top)
     elif args.entities:
         entities = json.loads(args.entities)
-        facts = query_facts_by_entities(args.db, entities, max_facts=args.max_facts)
+        # Use hybrid retrieval (FTS5 + tags + entity graph + RRF) for best results
+        query_text = " ".join(entities)
+        facts = query_facts_hybrid(args.db, query_text, max_facts=args.max_facts)
+        # Fallback to tag-only if hybrid returns nothing
+        if not facts:
+            facts = query_facts_by_entities(args.db, entities, max_facts=args.max_facts)
     else:
         facts = query_top_facts(args.db, limit=args.max_facts)