@geravant/sinain 1.19.0 → 1.22.0

package/sinain-memory/concept_export.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python3
+"""Concept Export — package an entity + its neighborhood as a portable bundle.
+
+Produces a sinain-concept/v1 envelope that can be transferred to another
+machine and re-imported with concept_import.py to reconstruct the same
+entity page (including the LLM-rendered view, if bundled).
+
+The reproducibility invariant we honor:
+    "On a new machine: import the bundle → open the same URL → see the same page."
+
+For that to hold:
+  1. Entity IDs are content-addressed slugs → stable across machines.
+  2. Triples are exported verbatim (created_at, retracted) for round-trip.
+  3. Optionally bundle the rendered_page JSON so the receiver gets a
+     cache hit on first view (deterministic visual identity).
+
+We do NOT bundle embeddings — same model on both ends → same vectors,
+so receiver recomputes for ~1.5KB/fact saved.
+
+Usage:
+    python3 concept_export.py --db <kg.db> --root entity:foo \\
+        [--depth 1] [--include-retracted] [--include-page] [--web-db <web.db>] \\
+        [--redact private,creditcard,apikey,...]
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import sys
+import time
+from pathlib import Path
+
+# ---------------------------------------------------------------------------
+# Redaction (MIRROR OF sense_client/privacy.py — keep patterns in sync until
+# we extract a shared sinain-memory/redaction.py module).
+# ---------------------------------------------------------------------------
+REDACT_RULES_VERSION = "1.2"
+
+_REDACT_PATTERNS: list[tuple[re.Pattern, str, str]] = [
+    # (regex, replacement, rule-name)
+    (re.compile(r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b"), "[REDACTED:card]", "creditcard"),
+    (re.compile(r"\b(?:sk-|pk-|api[_-]?key[=:]\s*)[A-Za-z0-9_\-]{20,}\b"), "[REDACTED:apikey]", "apikey"),
+    (re.compile(r"Bearer\s+[A-Za-z0-9_\-\.]{20,}"), "[REDACTED:bearer]", "bearer"),
+    (re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"), "[REDACTED:awskey]", "awskey"),
+    (re.compile(r"(?:password|passwd|pwd)\s*[:=]\s*\S+", re.IGNORECASE), "[REDACTED:password]", "password"),
+    (re.compile(r"\bghp_[A-Za-z0-9]{36}\b"), "[REDACTED:github_pat]", "github_pat"),
+    (re.compile(r"\bghs_[A-Za-z0-9]{36}\b"), "[REDACTED:github_srv]", "github_srv"),
+    (re.compile(r"\bxox[bpoa]-[0-9A-Za-z\-]+"), "[REDACTED:slack]", "slack"),
+    (re.compile(r"\bya29\.[0-9A-Za-z\-_]+"), "[REDACTED:google_oauth]", "google_oauth"),
+    (re.compile(r"\beyJ[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+"), "[REDACTED:jwt]", "jwt"),
+    (re.compile(r"(?:secret|token|key)\s*[:=]\s*[A-Za-z0-9_\-\.]{10,}", re.IGNORECASE), "[REDACTED:secret]", "secret"),
+    (re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), "[REDACTED:ssn]", "ssn"),
+    (re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), "[REDACTED:privkey]", "privkey"),
+]
+_PRIVATE_TAG = re.compile(r"<private>.*?</private>", re.DOTALL)
+
+
+def apply_redactions(text: str, enabled_rules: set[str]) -> tuple[str, list[str]]:
+    """Run enabled redaction rules over *text*. Returns (redacted_text, applied)."""
+    applied: list[str] = []
+    if "private" in enabled_rules:
+        new_text = _PRIVATE_TAG.sub("[REDACTED:private]", text)
+        if new_text != text:
+            applied.append("private")
+            text = new_text
+    for pattern, replacement, name in _REDACT_PATTERNS:
+        if name in enabled_rules:
+            new_text = pattern.sub(replacement, text)
+            if new_text != text:
+                applied.append(name)
+                text = new_text
+    return text, applied
+
+
+# ---------------------------------------------------------------------------
+# Export
+# ---------------------------------------------------------------------------
+
+def collect_neighborhood(store, root_entity: str, depth: int,
+                         include_retracted: bool) -> list[str]:
+    """BFS from root, following both incoming and outgoing refs.
+
+    Outgoing: triples WHERE entity_id = X AND value_type = 'ref' → recurse to value.
+    Incoming: triples WHERE value = X AND value_type = 'ref' → recurse to entity_id.
+
+    Returns deterministically-ordered list of entity_ids reachable within depth.
+    """
+    visited: dict[str, int] = {root_entity: 0}
+    queue: list[tuple[str, int]] = [(root_entity, 0)]
+    retracted_filter = "" if include_retracted else "AND retracted = 0"
+
+    while queue:
+        eid, d = queue.pop(0)
+        if d >= depth:
+            continue
+
+        # Outgoing refs
+        rows_out = store._conn.execute(
+            f"""SELECT DISTINCT value FROM triples
+                WHERE entity_id = ? AND value_type = 'ref' {retracted_filter}""",
+            (eid,),
+        ).fetchall()
+        for r in rows_out:
+            ref = r["value"]
+            if ref and ref not in visited:
+                visited[ref] = d + 1
+                queue.append((ref, d + 1))
+
+        # Incoming refs
+        rows_in = store._conn.execute(
+            f"""SELECT DISTINCT entity_id FROM triples
+                WHERE value = ? AND value_type = 'ref' {retracted_filter}""",
+            (eid,),
+        ).fetchall()
+        for r in rows_in:
+            ref = r["entity_id"]
+            if ref and ref not in visited:
+                visited[ref] = d + 1
+                queue.append((ref, d + 1))
+
+    return sorted(visited.keys())  # deterministic ordering for stable checksum
+
+
+def serialize_entity(store, entity_id: str, include_retracted: bool,
+                     redact_rules: set[str]) -> tuple[dict, list[str], int]:
+    """Pull all triples for entity_id, apply redactions to string values.
+
+    Returns ({id, type, triples: [...]}, applied_rules, redacted_count).
+    """
+    where = "" if include_retracted else "AND retracted = 0"
+    rows = store._conn.execute(
+        f"""SELECT attribute, value, value_type, tx_id, created_at, retracted, valid_to
+            FROM triples WHERE entity_id = ? {where}
+            ORDER BY tx_id, attribute, value""",
+        (entity_id,),
+    ).fetchall()
+
+    triples = []
+    all_applied: list[str] = []
+    redacted_count = 0
+    for r in rows:
+        value = r["value"]
+        if r["value_type"] == "string" and redact_rules and value:
+            new_value, applied = apply_redactions(value, redact_rules)
+            if applied:
+                all_applied.extend(applied)
+                redacted_count += 1
+                value = new_value
+        triples.append({
+            "attribute": r["attribute"],
+            "value": value,
+            "value_type": r["value_type"],
+            "tx_id": r["tx_id"],
+            "created_at": r["created_at"],
+            "retracted": int(r["retracted"]),
+            "valid_to": r["valid_to"],
+        })
+
+    type_prefix = entity_id.split(":", 1)[0] if ":" in entity_id else "unknown"
+    return ({"id": entity_id, "type": type_prefix, "triples": triples},
+            all_applied, redacted_count)
+
+
+def fetch_cached_page(web_db_path: str, root_entity: str,
+                     redact_rules: set[str]) -> dict | None:
+    """Pull the most recent cached rendered_page for root_entity, if any.
+    Apply redactions over the page summary + bullet text too — the LLM may
+    have woven sensitive content into its synthesis.
+    """
+    import sqlite3
+    if not Path(web_db_path).exists():
+        return None
+    try:
+        conn = sqlite3.connect(web_db_path)
+        conn.row_factory = sqlite3.Row
+        row = conn.execute(
+            """SELECT page_json, generated_at, tokens_in, tokens_out, cost_usd
+               FROM page_cache WHERE entity_id = ?
+               ORDER BY generated_at DESC LIMIT 1""",
+            (root_entity,),
+        ).fetchone()
+        conn.close()
+        if not row:
+            return None
+        page = json.loads(row["page_json"])
+        # Redact rendered_page content.
+        if redact_rules:
+            if page.get("summary"):
+                new_summary, _ = apply_redactions(page["summary"], redact_rules)
+                page["summary"] = new_summary
+            for sec in page.get("sections", []) or []:
+                for b in sec.get("bullets", []) or []:
+                    if b.get("text"):
+                        new_text, _ = apply_redactions(b["text"], redact_rules)
+                        b["text"] = new_text
+        page["generated_at"] = row["generated_at"]
+        page.setdefault("rendered_with", {})
+        if row["tokens_in"]: page["rendered_with"]["tokens_in"] = row["tokens_in"]
+        if row["tokens_out"]: page["rendered_with"]["tokens_out"] = row["tokens_out"]
+        return page
+    except Exception as e:
+        sys.stderr.write(f"fetch_cached_page failed: {e}\n")
+        return None
+
+
+def export_concept(db_path: str, root_entity: str, depth: int = 1,
+                   include_retracted: bool = False,
+                   include_page: bool = True,
+                   web_db_path: str | None = None,
+                   redact_rules: set[str] | None = None) -> dict:
+    from triplestore import TripleStore
+
+    if redact_rules is None:
+        redact_rules = {"private", "creditcard", "apikey", "bearer", "awskey",
+                        "password", "secret"}
+
+    store = TripleStore(db_path)
+    entity_ids = collect_neighborhood(store, root_entity, depth, include_retracted)
+
+    entities = []
+    all_applied: list[str] = []
+    total_redacted = 0
+    triple_count = 0
+    fact_count = 0
+
+    for eid in entity_ids:
+        entity_obj, applied, n_redacted = serialize_entity(
+            store, eid, include_retracted, redact_rules,
+        )
+        entities.append(entity_obj)
+        all_applied.extend(applied)
+        total_redacted += n_redacted
+        triple_count += len(entity_obj["triples"])
+        if eid.startswith("fact:"):
+            fact_count += 1
+
+    store.close()
+
+    rendered_page = None
+    if include_page and web_db_path:
+        rendered_page = fetch_cached_page(web_db_path, root_entity, redact_rules)
+
+    envelope = {
+        "format": "sinain-concept/v1",
+        "exported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "exporter": {
+            "tool": "sinain-core",
+            "tool_version": "1.14.0",
+            "schema_version": "triplestore/v3",
+            "embedding_model": "all-MiniLM-L6-v2",
+        },
+        "root_entity": root_entity,
+        "depth": depth,
+        "stats": {
+            "entities": len(entity_ids),
+            "facts": fact_count,
+            "triples": triple_count,
+        },
+        "entities": entities,
+        "rendered_page": rendered_page,
+        "redactions": {
+            "applied": sorted(set(all_applied)),
+            "rules_version": REDACT_RULES_VERSION,
+            "redacted_count": total_redacted,
+        },
+    }
+
+    # Compute checksum over canonical JSON of (root_entity + entities) — this is
+    # what the receiver should validate against on import.
+    canonical = json.dumps(
+        {"root_entity": root_entity, "entities": entities},
+        sort_keys=True, ensure_ascii=False, separators=(",", ":"),
+    )
+    envelope["checksum"] = "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+    return envelope
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Concept Export")
+    parser.add_argument("--db", required=True)
+    parser.add_argument("--root", required=True, help="Root entity id (e.g. entity:citibank)")
+    parser.add_argument("--depth", type=int, default=1)
+    parser.add_argument("--include-retracted", action="store_true")
+    parser.add_argument("--include-page", action="store_true",
+                        help="Bundle the cached rendered_page if available")
+    parser.add_argument("--web-db", default=None,
+                        help="Path to web.db for page cache lookup")
+    parser.add_argument("--redact", default="private,creditcard,apikey,bearer,awskey,password,secret",
+                        help="Comma-separated redaction rule names")
+    args = parser.parse_args()
+
+    if not Path(args.db).exists():
+        print(json.dumps({"error": f"db not found: {args.db}"}))
+        sys.exit(1)
+
+    rules = {r.strip() for r in args.redact.split(",") if r.strip()}
+    envelope = export_concept(
+        args.db, args.root, depth=args.depth,
+        include_retracted=args.include_retracted,
+        include_page=args.include_page,
+        web_db_path=args.web_db,
+        redact_rules=rules,
+    )
+    print(json.dumps(envelope, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

package/sinain-memory/concept_import.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""Concept Import — replay a sinain-concept/v1 bundle into the local triplestore.
+
+Designed for idempotency: re-importing the same bundle in `merge` mode is a
+no-op (existing triples skip-as-duplicate). The receiver re-issues tx_ids
+locally but preserves the source-tx grouping, so the digest atomicity that
+knowledge_integrator.py relies on survives the round-trip.
+
+Usage:
+    python3 concept_import.py --db <kg.db> --bundle <bundle.json> \\
+        [--web-db <web.db>] [--conflict skip|merge|overwrite]
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sqlite3
+import sys
+import time
+from pathlib import Path
+
+
+def verify_envelope(envelope: dict) -> tuple[bool, str]:
+    """Validate format and checksum. Returns (ok, error_msg)."""
+    fmt = envelope.get("format")
+    if fmt != "sinain-concept/v1":
+        return False, f"unsupported format: {fmt!r} (need sinain-concept/v1)"
+    if "root_entity" not in envelope or "entities" not in envelope:
+        return False, "envelope missing root_entity or entities"
+    expected = envelope.get("checksum")
+    if expected:
+        canonical = json.dumps(
+            {"root_entity": envelope["root_entity"], "entities": envelope["entities"]},
+            sort_keys=True, ensure_ascii=False, separators=(",", ":"),
+        )
+        actual = "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+        if actual != expected:
+            return False, f"checksum mismatch (expected {expected[:23]}..., got {actual[:23]}...)"
+    return True, ""
+
+
+def bundle_sha(envelope: dict) -> str:
+    """Compute envelope-level sha for idempotency tracking (web.db.concept_imports)."""
+    body = json.dumps(envelope, sort_keys=True, ensure_ascii=False).encode("utf-8")
+    return hashlib.sha256(body).hexdigest()
+
+
+def import_bundle(db_path: str, envelope: dict, conflict: str = "merge",
+                  web_db_path: str | None = None) -> dict:
+    """Replay envelope into the local knowledge graph.
+
+    conflict modes:
+      - skip:      existing (entity, attribute, value) wins; imported dropped.
+      - merge:     duplicate triples skipped; new (attribute, value) inserted as new.
+      - overwrite: retract conflicting active local triples, then assert imported.
+    """
+    from triplestore import TripleStore
+
+    store = TripleStore(db_path)
+
+    # Group imported triples by their source_tx so we preserve atomicity.
+    # source_tx_id → list of (entity_id, attribute, value, value_type, retracted, valid_to, original_created_at)
+    by_source_tx: dict[int, list[tuple]] = {}
+    for ent in envelope.get("entities", []):
+        eid = ent.get("id")
+        for t in ent.get("triples", []):
+            stx = int(t.get("tx_id") or 0)
+            by_source_tx.setdefault(stx, []).append((
+                eid,
+                t["attribute"],
+                t["value"],
+                t.get("value_type", "string"),
+                int(t.get("retracted", 0)),
+                t.get("valid_to"),
+                t.get("created_at"),
+            ))
+
+    inserted = 0
+    skipped_dup = 0
+    overwritten = 0
+    tx_mapping: dict[int, int] = {}  # source_tx → new_tx
+
+    for source_tx in sorted(by_source_tx.keys()):
+        triples = by_source_tx[source_tx]
+
+        # Begin one local tx per source tx (preserves grouping)
+        new_tx = store.begin_tx(
+            source="concept-import",
+            metadata={"source_tx": source_tx, "bundle_root": envelope.get("root_entity")},
+        )
+        tx_mapping[source_tx] = new_tx
+
+        for (eid, attr, value, value_type, retracted, valid_to, created_at) in triples:
+            # Imported retracted triples → preserve retracted state. (They're
+            # part of the bundle's audit trail.)
+            if retracted:
+                # Insert as retracted; valid_to stays as-is.
+                # We use a direct INSERT to preserve the retracted flag — assert_triple
+                # always sets retracted=0.
+                store._conn.execute(
+                    """INSERT INTO triples
+                       (tx_id, entity_id, attribute, value, value_type, retracted, retracted_tx, valid_to, created_at)
+                       VALUES (?, ?, ?, ?, ?, 1, ?, ?, ?)""",
+                    (new_tx, eid, attr, value, value_type, new_tx, valid_to, created_at or _iso_now()),
+                )
+                store._conn.execute(
+                    "INSERT OR IGNORE INTO entity_types (entity_id, entity_type) VALUES (?, ?)",
+                    (eid, eid.split(":", 1)[0] if ":" in eid else "unknown"),
+                )
+                inserted += 1
+                continue
+
+            # Active triple — apply conflict mode for (entity, attribute, value)
+            existing = store._conn.execute(
+                """SELECT id FROM triples WHERE entity_id = ? AND attribute = ?
+                       AND value = ? AND retracted = 0 LIMIT 1""",
+                (eid, attr, value),
+            ).fetchone()
+
+            if existing:
+                # Exact triple already present
+                if conflict == "skip" or conflict == "merge":
+                    skipped_dup += 1
+                    continue
+                if conflict == "overwrite":
+                    # Retract conflicting then insert imported
+                    store.retract_triple(new_tx, eid, attr, value)
+                    overwritten += 1
+
+            # Insert
+            store.assert_triple(new_tx, eid, attr, value, value_type=value_type)
+            # Patch created_at to preserve source timeline (assert_triple uses now()).
+            if created_at:
+                store._conn.execute(
+                    """UPDATE triples SET created_at = ?
+                       WHERE tx_id = ? AND entity_id = ? AND attribute = ? AND value = ?
+                       AND id = (SELECT MAX(id) FROM triples WHERE tx_id = ? AND entity_id = ? AND attribute = ? AND value = ?)""",
+                    (created_at, new_tx, eid, attr, value, new_tx, eid, attr, value),
+                )
+            inserted += 1
+
+        store._conn.commit()
+
+    store.close()
+
+    # Page cache reuse
+    page_cached = False
+    if web_db_path and envelope.get("rendered_page"):
+        page = envelope["rendered_page"]
+        # Map source tx_watermark to local — find max new_tx for facts in the bundle
+        local_watermark = max(tx_mapping.values()) if tx_mapping else 0
+        page["tx_watermark"] = local_watermark
+        try:
+            conn = sqlite3.connect(web_db_path)
+            conn.execute(
+                """INSERT OR REPLACE INTO page_cache
+                   (entity_id, tx_watermark, page_json, generated_at, tokens_in, tokens_out, cost_usd)
+                   VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    envelope["root_entity"],
+                    local_watermark,
+                    json.dumps(page, ensure_ascii=False),
+                    int(time.time() * 1000),
+                    (page.get("rendered_with") or {}).get("tokens_in"),
+                    (page.get("rendered_with") or {}).get("tokens_out"),
+                    (page.get("rendered_with") or {}).get("cost_usd"),
+                ),
+            )
+            conn.commit()
+            conn.close()
+            page_cached = True
+        except Exception as e:
+            sys.stderr.write(f"page cache reuse failed: {e}\n")
+
+    # Audit row in concept_imports
+    if web_db_path:
+        try:
+            sha = bundle_sha(envelope)
+            conn = sqlite3.connect(web_db_path)
+            conn.execute(
+                """INSERT INTO concept_imports
+                   (imported_at, root_entity, source_tool, source_version, envelope_format,
+                    bundle_sha256, conflict_mode, triples_count, redactions_seen, notes)
+                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                (
+                    int(time.time() * 1000),
+                    envelope["root_entity"],
+                    (envelope.get("exporter") or {}).get("tool"),
+                    (envelope.get("exporter") or {}).get("tool_version"),
+                    envelope.get("format"),
+                    sha,
+                    conflict,
+                    inserted,
+                    json.dumps((envelope.get("redactions") or {}).get("applied", [])),
+                    None,
+                ),
+            )
+            conn.commit()
+            conn.close()
+        except Exception as e:
+            sys.stderr.write(f"concept_imports log failed: {e}\n")
+
+    return {
+        "ok": True,
+        "imported": True,
+        "root_entity": envelope.get("root_entity"),
+        "stats": {
+            "entities_seen": len(envelope.get("entities", [])),
+            "triples_inserted": inserted,
+            "triples_skipped_duplicate": skipped_dup,
+            "triples_overwritten": overwritten,
+            "tx_mapping_count": len(tx_mapping),
+        },
+        "rendered_page_cached": page_cached,
+        "view_url": f"/knowledge/ui/entity/{envelope.get('root_entity', '').replace(':', '%3A')}",
+    }
+
+
+def _iso_now() -> str:
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Concept Import")
+    parser.add_argument("--db", required=True)
+    parser.add_argument("--bundle", required=True, help="Path to .sinain-concept.json (or - for stdin)")
+    parser.add_argument("--web-db", default=None)
+    parser.add_argument("--conflict", choices=["skip", "merge", "overwrite"], default="merge")
+    args = parser.parse_args()
+
+    if not Path(args.db).exists():
+        # Auto-create empty knowledge DB on first import — receiver may have nothing yet.
+        Path(args.db).parent.mkdir(parents=True, exist_ok=True)
+        from triplestore import TripleStore
+        TripleStore(args.db).close()
+
+    if args.bundle == "-":
+        envelope = json.load(sys.stdin)
+    else:
+        envelope = json.loads(Path(args.bundle).read_text(encoding="utf-8"))
+
+    ok, err = verify_envelope(envelope)
+    if not ok:
+        print(json.dumps({"ok": False, "error": err}))
+        sys.exit(1)
+
+    result = import_bundle(args.db, envelope, conflict=args.conflict,
+                          web_db_path=args.web_db)
+    print(json.dumps(result, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()