@geravant/sinain 1.0.1

package/sinain-memory/triple_migrate.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+"""One-shot historical data migration into the EAV triple store.
+
+Migrates ~3 weeks of pre-triplestore data (playbook logs, daily memories,
+playbook patterns, active modules) into the triple store so that triple_query
+can surface historical context.
+
+Idempotent: checks for a `migration:v1` stamp entity before running.
+No embeddings: those accumulate organically at runtime.
+
+Usage:
+    python3 triple_migrate.py --memory-dir memory/ --modules-dir modules/ [--dry-run]
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Ensure sibling imports work when invoked from workspace root
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from common import (
+    _read_jsonl,
+    list_daily_memory_files,
+    read_effective_playbook,
+    read_file_safe,
+    output_json,
+)
+from triple_extractor import TripleExtractor
+from triplestore import TripleStore
+
+MIGRATION_ENTITY = "migration:v1"
+
+
+def _assert_triples(store: TripleStore, tx_id: int, triples: list) -> int:
+    """Assert a batch of Triple objects into the store. Returns count.
+
+    Skips triples with None/empty entity_id or value (legacy data tolerance).
+    """
+    count = 0
+    for t in triples:
+        if t.value is None or t.entity_id is None:
+            continue
+        store.assert_triple(tx_id, t.entity_id, t.attribute, str(t.value), t.value_type)
+        count += 1
+    return count
+
+
+def migrate_playbook(extractor: TripleExtractor, store: TripleStore, memory_dir: str) -> int:
+    """Migrate the effective playbook into patterns. Returns triple count."""
+    text = read_effective_playbook(memory_dir)
+    if not text.strip():
+        print("[migrate] no playbook found, skipping", file=sys.stderr)
+        return 0
+    triples = extractor.extract_playbook(text)
+    if not triples:
+        return 0
+    tx = store.begin_tx("migration:playbook")
+    return _assert_triples(store, tx, triples)
+
+
+def migrate_modules(
+    extractor: TripleExtractor, store: TripleStore, modules_dir: str
+) -> tuple[int, int]:
+    """Migrate active modules. Returns (module_count, triple_count)."""
+    registry_path = Path(modules_dir) / "module-registry.json"
+    if not registry_path.exists():
+        print("[migrate] no module-registry.json, skipping modules", file=sys.stderr)
+        return 0, 0
+
+    registry = json.loads(registry_path.read_text(encoding="utf-8"))
+    modules = registry.get("modules", {})
+
+    module_count = 0
+    triple_count = 0
+
+    for mod_id, entry in modules.items():
+        if entry.get("status") != "active":
+            continue
+
+        manifest_path = Path(modules_dir) / mod_id / "manifest.json"
+        patterns_path = Path(modules_dir) / mod_id / "patterns.md"
+
+        manifest = {}
+        if manifest_path.exists():
+            try:
+                manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+            except json.JSONDecodeError:
+                print(f"[migrate] bad manifest for {mod_id}, using empty", file=sys.stderr)
+
+        patterns_text = read_file_safe(str(patterns_path))
+
+        triples = extractor.extract_module(mod_id, manifest, patterns_text)
+        if triples:
+            tx = store.begin_tx(f"migration:module:{mod_id}")
+            _assert_triples(store, tx, triples)
+            triple_count += len(triples)
+            module_count += 1
+
+    return module_count, triple_count
+
+
+def migrate_playbook_logs(
+    extractor: TripleExtractor, store: TripleStore, memory_dir: str
+) -> tuple[int, int]:
+    """Migrate all playbook-log JSONL files. Returns (file_count, triple_count)."""
+    log_dir = Path(memory_dir) / "playbook-logs"
+    if not log_dir.is_dir():
+        print("[migrate] no playbook-logs/ directory, skipping", file=sys.stderr)
+        return 0, 0
+
+    file_count = 0
+    triple_count = 0
+
+    for jsonl_file in sorted(log_dir.glob("*.jsonl")):
+        entries = _read_jsonl(jsonl_file)
+        if not entries:
+            continue
+
+        day_triples = []
+        for entry in entries:
+            # Skip idle entries with no signals
+            if entry.get("idle", False) and not entry.get("signals"):
+                continue
+
+            ts = entry.get("ts", "")
+            if not ts:
+                continue
+
+            # Normalize legacy string signals → dict format
+            raw_signals = entry.get("signals", [])
+            if raw_signals and isinstance(raw_signals[0], str):
+                entry["signals"] = [
+                    {"description": s, "priority": "medium"} for s in raw_signals
+                ]
+
+            day_triples.extend(extractor.extract_signal(entry, ts))
+
+        if day_triples:
+            tx = store.begin_tx(f"migration:logs:{jsonl_file.stem}")
+            _assert_triples(store, tx, day_triples)
+            triple_count += len(day_triples)
+            file_count += 1
+            print(f"  logs/{jsonl_file.name}: {len(day_triples)} triples", file=sys.stderr)
+
+    return file_count, triple_count
+
+
+def migrate_daily_memories(
+    extractor: TripleExtractor, store: TripleStore, memory_dir: str
+) -> tuple[int, int]:
+    """Migrate YYYY-MM-DD.md daily memory files. Returns (file_count, triple_count)."""
+    files = list_daily_memory_files(memory_dir)
+    if not files:
+        print("[migrate] no daily memory files, skipping", file=sys.stderr)
+        return 0, 0
+
+    file_count = 0
+    triple_count = 0
+
+    for filepath in files:
+        text = read_file_safe(filepath)
+        if not text.strip():
+            continue
+
+        date = Path(filepath).stem  # YYYY-MM-DD
+        entity_id = f"observation:{date}"
+        truncated = text[:2000]
+
+        tx = store.begin_tx(f"migration:memory:{date}")
+
+        # Core observation entity
+        store.assert_triple(tx, entity_id, "text", truncated)
+        store.assert_triple(tx, entity_id, "source", "daily_memory")
+        count = 2
+
+        # Extract and link concepts
+        concept_triples = extractor.extract_concepts(truncated)
+        for ct in concept_triples:
+            store.assert_triple(tx, ct.entity_id, ct.attribute, ct.value, ct.value_type)
+            count += 1
+            if ct.entity_id.startswith("concept:"):
+                store.assert_triple(tx, entity_id, "related_to", ct.entity_id, "ref")
+                count += 1
+
+        triple_count += count
+        file_count += 1
+        print(f"  memory/{date}.md: {count} triples", file=sys.stderr)
+
+    return file_count, triple_count
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Migrate historical data to triple store")
+    parser.add_argument("--memory-dir", required=True, help="Path to memory/ directory")
+    parser.add_argument("--modules-dir", default=None, help="Path to modules/ directory")
+    parser.add_argument("--dry-run", action="store_true", help="Print plan without writing")
+    args = parser.parse_args()
+
+    memory_dir = args.memory_dir
+    modules_dir = args.modules_dir or str(Path(memory_dir).parent / "modules")
+
+    db_path = str(Path(memory_dir) / "triplestore.db")
+    if args.dry_run:
+        print(f"[dry-run] would migrate into {db_path}", file=sys.stderr)
+        print(f"  playbook: {read_effective_playbook(memory_dir)[:80]}...", file=sys.stderr)
+        log_dir = Path(memory_dir) / "playbook-logs"
+        if log_dir.is_dir():
+            jsonl_files = list(log_dir.glob("*.jsonl"))
+            print(f"  log files: {len(jsonl_files)}", file=sys.stderr)
+        mem_files = list_daily_memory_files(memory_dir)
+        print(f"  daily memories: {len(mem_files)}", file=sys.stderr)
+        output_json({"dryRun": True, "dbPath": db_path})
+        return
+
+    store = TripleStore(db_path)
+    extractor = TripleExtractor(store)
+
+    # 1. Idempotency guard
+    existing = store.entity(MIGRATION_ENTITY)
+    if existing:
+        print("[migrate] already migrated — migration:v1 entity exists", file=sys.stderr)
+        output_json({"alreadyMigrated": True, **existing})
+        store.close()
+        return
+
+    print("[migrate] starting historical data migration...", file=sys.stderr)
+
+    # 2. Playbook
+    pb_triples = migrate_playbook(extractor, store, memory_dir)
+    print(f"[migrate] playbook: {pb_triples} triples", file=sys.stderr)
+
+    # 3. Modules
+    mod_count, mod_triples = migrate_modules(extractor, store, modules_dir)
+    print(f"[migrate] modules: {mod_count} modules, {mod_triples} triples", file=sys.stderr)
+
+    # 4. Playbook logs
+    log_files, log_triples = migrate_playbook_logs(extractor, store, memory_dir)
+    print(f"[migrate] logs: {log_files} files, {log_triples} triples", file=sys.stderr)
+
+    # 5. Daily memories
+    mem_files, mem_triples = migrate_daily_memories(extractor, store, memory_dir)
+    print(f"[migrate] memories: {mem_files} files, {mem_triples} triples", file=sys.stderr)
+
+    # 6. Stamp
+    total_triples = pb_triples + mod_triples + log_triples + mem_triples
+    stats = store.stats()
+    stamp_tx = store.begin_tx("migration:stamp")
+    now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    store.assert_triple(stamp_tx, MIGRATION_ENTITY, "completed_at", now)
+    store.assert_triple(stamp_tx, MIGRATION_ENTITY, "playbook_triples", str(pb_triples))
+    store.assert_triple(stamp_tx, MIGRATION_ENTITY, "module_count", str(mod_count))
+    store.assert_triple(stamp_tx, MIGRATION_ENTITY, "log_files", str(log_files))
+    store.assert_triple(stamp_tx, MIGRATION_ENTITY, "memory_files", str(mem_files))
+    store.assert_triple(stamp_tx, MIGRATION_ENTITY, "total_triples", str(total_triples))
+
+    store.close()
+
+    # 7. Output
+    output_json({
+        "migrated": {
+            "playbook": pb_triples,
+            "modules": mod_count,
+            "logs": log_files,
+            "dailyMemory": mem_files,
+        },
+        "totalTriples": stats["triples"],
+        "totalEntities": stats["entities"],
+    })
+
+
+if __name__ == "__main__":
+    main()

package/sinain-memory/triple_query.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""Triple Query — read-only utilities for querying the triple store.
+
+Importable module + CLI for generating context from the knowledge graph.
+
+Usage (CLI):
+    python3 triple_query.py --memory-dir memory/ --context "OCR pipeline" --max-chars 1500
+
+Usage (import):
+    from triple_query import get_related_context
+    context = get_related_context("memory/", ["OCR pipeline"], max_chars=1500)
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from triplestore import TripleStore
+from common import output_json
+
+
+def _db_path(memory_dir: str) -> str:
+    return str(Path(memory_dir) / "triplestore.db")
+
+
+def build_entity_text(store: TripleStore, entity_id: str) -> str:
+    """Build a readable text representation of an entity."""
+    attrs = store.entity(entity_id)
+    if not attrs:
+        return ""
+
+    etype = entity_id.split(":")[0] if ":" in entity_id else "unknown"
+    parts = [f"[{etype}] {entity_id}"]
+
+    for attr, vals in sorted(attrs.items()):
+        if attr == "related_to":
+            parts.append(f"  links: {', '.join(vals)}")
+        elif attr == "belongs_to":
+            parts.append(f"  module: {', '.join(vals)}")
+        elif len(vals) == 1:
+            parts.append(f"  {attr}: {vals[0]}")
+        else:
+            parts.append(f"  {attr}: {', '.join(vals)}")
+
+    return "\n".join(parts)
+
+
+def get_related_concepts(
+    memory_dir: str, keywords: list[str]
+) -> str:
+    """Find concepts matching keywords, then follow backrefs to related entities.
+
+    Returns formatted markdown suitable for injection into LLM context.
+    """
+    db = _db_path(memory_dir)
+    if not Path(db).exists():
+        return ""
+
+    store = TripleStore(db)
+    try:
+        parts: list[str] = []
+        seen_entities: set[str] = set()
+
+        for keyword in keywords:
+            kw_lower = keyword.lower().strip()
+            if not kw_lower:
+                continue
+
+            # AVET: look up concepts by name
+            all_concepts = store.entities_with_attr("name")
+            for eid, name in all_concepts:
+                if not eid.startswith("concept:"):
+                    continue
+                if kw_lower in name.lower():
+                    if eid in seen_entities:
+                        continue
+                    seen_entities.add(eid)
+
+                    # VAET: find what references this concept
+                    refs = store.backrefs(eid)
+                    if refs:
+                        ref_parts = []
+                        for ref_eid, ref_attr in refs[:10]:  # cap
+                            if ref_eid in seen_entities:
+                                continue
+                            seen_entities.add(ref_eid)
+                            text = build_entity_text(store, ref_eid)
+                            if text:
+                                ref_parts.append(text)
+                        if ref_parts:
+                            parts.append(f"### {name}\n" + "\n".join(ref_parts))
+
+        return "\n\n".join(parts) if parts else ""
+    finally:
+        store.close()
+
+
+def get_related_context(
+    memory_dir: str,
+    seed_texts: list[str],
+    max_chars: int = 1500,
+) -> str:
+    """Build a context block from the knowledge graph for the given seed texts.
+
+    Phase 1: keyword matching against entity attributes.
+    Phase 2 (when embedder available): adds vector search as primary channel.
+    """
+    db = _db_path(memory_dir)
+    if not Path(db).exists():
+        return ""
+
+    store = TripleStore(db)
+    try:
+        context_parts: list[str] = []
+        total_chars = 0
+        seen: set[str] = set()
+
+        # Phase 2: try vector search first
+        try:
+            from embedder import Embedder
+            embedder = Embedder(db)
+            for text in seed_texts:
+                vecs = embedder.embed([text])
+                if vecs and vecs[0]:
+                    results = embedder.vector_search(vecs[0], top_k=5)
+                    for eid, score in results:
+                        if eid in seen:
+                            continue
+                        seen.add(eid)
+                        ent_text = build_entity_text(store, eid)
+                        if ent_text and total_chars + len(ent_text) < max_chars:
+                            context_parts.append(f"{ent_text} (relevance: {score:.2f})")
+                            total_chars += len(ent_text) + 20
+        except (ImportError, Exception):
+            pass  # Phase 2 not available, fall through to keyword
+
+        # Phase 1: keyword matching
+        if total_chars < max_chars // 2:
+            # Extract keywords from seed texts
+            keywords: set[str] = set()
+            for text in seed_texts:
+                for word in text.lower().split():
+                    word = word.strip(".,!?;:'\"()[]{}").strip()
+                    if len(word) > 3:
+                        keywords.add(word)
+
+            # Search patterns and concepts by keyword
+            all_text_triples = store.entities_with_attr("text")
+            all_name_triples = store.entities_with_attr("name")
+
+            for eid, val in all_text_triples + all_name_triples:
+                if eid in seen:
+                    continue
+                val_lower = val.lower()
+                if any(kw in val_lower for kw in keywords):
+                    seen.add(eid)
+                    ent_text = build_entity_text(store, eid)
+                    if ent_text and total_chars + len(ent_text) < max_chars:
+                        context_parts.append(ent_text)
+                        total_chars += len(ent_text)
+                    if total_chars >= max_chars:
+                        break
+
+        return "\n\n".join(context_parts) if context_parts else ""
+    finally:
+        store.close()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Triple Store Query CLI")
+    parser.add_argument("--memory-dir", required=True, help="Path to memory/ directory")
+    parser.add_argument("--context", required=True, help="Query text for context generation")
+    parser.add_argument("--max-chars", type=int, default=1500, help="Maximum context chars")
+    args = parser.parse_args()
+
+    context = get_related_context(args.memory_dir, [args.context], max_chars=args.max_chars)
+    output_json({"context": context})
+
+
+if __name__ == "__main__":
+    main()