@geravant/sinain 1.11.0 → 1.13.0

package/sinain-memory/session_distiller.py

@@ -28,7 +28,7 @@ from common import (
 
 SYSTEM_PROMPT = """\
 You are a session distiller for a personal AI overlay system (sinain).
-Your job: analyze a session transcript and extract structured knowledge.
+Your job: analyze a session transcript and extract ALL knowledge worth remembering.
 
 The transcript contains feed items from sinain-core:
 - audio: transcribed speech from the user's environment
@@ -37,24 +37,42 @@ The transcript contains feed items from sinain-core:
 - system: system events and status messages
 
 Extract:
-1. whatHappened: 2-3 sentences summarizing what was accomplished in this session
-2. patterns: up to 5 reusable patterns discovered (things that worked, techniques used)
-3. antiPatterns: up to 3 things that failed and why
-4. preferences: up to 3 user preferences or workflow habits observed
-5. entities: key domains, tools, technologies, or topics worked with (for graph linking)
-6. toolInsights: tool usage insights (e.g., "grep before read reduces misses")
-
-Focus on ACTIONABLE knowledge that would help a future agent in similar contexts.
-Skip trivial observations. If the session was idle or empty, say so briefly.
+1. whatHappened: 2-3 sentences summarizing what occurred in this session
+2. facts: up to 15 concrete factual claims. Each must be a self-contained sentence. \
+IMPORTANT — spread across these dimensions (do not let one theme dominate):
+   - WHO: people mentioned, their roles, backgrounds, relationships to each other
+   - WHAT: specific claims, properties, descriptions of things discussed
+   - HOW MUCH: any numbers, quantities, dates, durations, counts stated
+   - WHAT CHANGED: decisions made, agreements reached, state changes
+   - WHAT'S NEXT: commitments, action items, plans, deadlines
+   If you have 5+ facts about one dimension and 0 about another that was discussed, \
+you are missing something. Breadth over depth.
+   Good: "The CTO of Al-Futaim previously worked at Citibank for 17 years as Director of IT in Singapore"
+   Good: "Citibank has 2400 IntelliJ subscriptions and heavy TeamCity usage"
+   Good: "The meeting is 45 minutes, scheduled for Tuesday"
+   Bad: "client-understanding-key: True"
+   Bad: five variations of "Al-Futaim is moving to the cloud"
+3. decisions: up to 5 decisions or agreements made (who decided what, with any deadline)
+4. entities: named things discussed or interacted with — as objects with name \
+(lowercase-hyphenated slug) and type (freeform — person, org, tool, file, concept, \
+service, framework, error, whatever fits the context).
+   Examples: {"name": "citibank", "type": "org"}, {"name": "auth-module", "type": "file"}, \
+{"name": "react-native", "type": "framework"}
+5. patterns: up to 3 reusable techniques or workflows (if any — skip if none)
+6. preferences: up to 3 user preferences or habits observed
+
+If existing entities are provided, reference them by name to enable reinforcement.
+Focus on CONCRETE, SPECIFIC knowledge. Skip vague observations.
+If the session was idle or empty, say so briefly.
 
 Respond with ONLY a JSON object:
 {
   "whatHappened": "string",
-  "patterns": ["string", ...],
-  "antiPatterns": ["string", ...],
-  "preferences": ["string", ...],
-  "entities": ["string", ...],
-  "toolInsights": ["string", ...],
+  "facts": ["self-contained factual sentence", ...],
+  "decisions": ["decision sentence with who/what/when", ...],
+  "entities": [{"name": "citibank", "type": "org"}, {"name": "artom", "type": "person"}, ...],
+  "patterns": ["reusable technique or workflow", ...],
+  "preferences": ["user preference or habit", ...],
   "isEmpty": false
 }"""
 
@@ -95,6 +113,7 @@ def main() -> None:
     parser.add_argument("--memory-dir", required=True, help="Path to memory/ directory")
     parser.add_argument("--transcript", required=True, help="JSON array of feed items")
     parser.add_argument("--session-meta", default="{}", help="JSON session metadata")
+    parser.add_argument("--existing-entities", default="", help="Compact summary of existing knowledge graph entities")
     args = parser.parse_args()
 
     # Parse inputs
@@ -111,11 +130,11 @@ def main() -> None:
     if not items or len(items) < 2:
         output_json({
             "whatHappened": "Empty or trivial session",
+            "facts": [],
+            "decisions": [],
+            "entities": [],
             "patterns": [],
-            "antiPatterns": [],
             "preferences": [],
-            "entities": [],
-            "toolInsights": [],
             "isEmpty": True,
         })
         return
@@ -130,11 +149,16 @@ def main() -> None:
         lines = [l for l in playbook.splitlines() if l.strip() and not l.startswith("<!--")]
         playbook_summary = f"\n\n## Current Playbook (for reference — don't repeat known patterns)\n{chr(10).join(lines[:30])}"
 
+    # Include existing entities for retrieve-before-extract (Mem0 pattern)
+    existing_section = ""
+    if args.existing_entities and args.existing_entities.strip():
+        existing_section = f"\n\n## Existing Knowledge (reinforce or update these if the session confirms/changes them)\n{args.existing_entities}"
+
     user_prompt = f"""## Session Transcript ({len(items)} items)
 {transcript_text}
 
 ## Session Metadata
-{json.dumps(meta, indent=2)}{playbook_summary}"""
+{json.dumps(meta, indent=2)}{playbook_summary}{existing_section}"""
 
     try:
         raw = call_llm_with_fallback(

package/sinain-memory/triplestore.py

@@ -79,6 +79,40 @@ CREATE INDEX IF NOT EXISTS idx_avet
     ON triples(attribute, value, entity_id, tx_id);
 """
 
+_FTS_SQL = """
+-- Full-text search on fact values (for hybrid retrieval)
+CREATE VIRTUAL TABLE IF NOT EXISTS triples_fts
+    USING fts5(entity_id, value, content=triples, content_rowid=id);
+
+-- Triggers to keep FTS in sync with triples table
+CREATE TRIGGER IF NOT EXISTS triples_ai AFTER INSERT ON triples BEGIN
+    INSERT INTO triples_fts(rowid, entity_id, value) VALUES (new.id, new.entity_id, new.value);
+END;
+
+CREATE TRIGGER IF NOT EXISTS triples_ad AFTER DELETE ON triples BEGIN
+    INSERT INTO triples_fts(triples_fts, rowid, entity_id, value) VALUES ('delete', old.id, old.entity_id, old.value);
+END;
+
+CREATE TRIGGER IF NOT EXISTS triples_au AFTER UPDATE ON triples BEGIN
+    INSERT INTO triples_fts(triples_fts, rowid, entity_id, value) VALUES ('delete', old.id, old.entity_id, old.value);
+    INSERT INTO triples_fts(rowid, entity_id, value) VALUES (new.id, new.entity_id, new.value);
+END;
+"""
+
+_TOUCHED_SQL = """
+-- Track which entities are modified per transaction (for fast novelty checks)
+CREATE TABLE IF NOT EXISTS touched_entities (
+    tx_id INTEGER NOT NULL,
+    entity_id TEXT NOT NULL,
+    PRIMARY KEY (tx_id, entity_id)
+);
+
+CREATE TRIGGER IF NOT EXISTS track_touched AFTER INSERT ON triples BEGIN
+    INSERT OR IGNORE INTO touched_entities (tx_id, entity_id)
+    VALUES (new.tx_id, new.entity_id);
+END;
+"""
+
 
 def _now_iso() -> str:
     return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
@@ -119,6 +153,14 @@ class TripleStore:
         self._conn.execute("PRAGMA journal_mode=WAL")
         self._conn.execute("PRAGMA busy_timeout=10000")
         self._conn.executescript(_SCHEMA_SQL)
+        try:
+            self._conn.executescript(_FTS_SQL)
+        except sqlite3.OperationalError:
+            pass  # FTS5 not available on this Python build — degrade gracefully
+        try:
+            self._conn.executescript(_TOUCHED_SQL)
+        except sqlite3.OperationalError:
+            pass
         self._migrate()
         self._conn.commit()
 
@@ -409,6 +451,24 @@ class TripleStore:
             ).fetchall()
         return [dict(r) for r in rows]
 
+    # ----- Touched entities (fast novelty check) -----
+
+    def was_touched(self, entity_id: str, since_tx: int) -> bool:
+        """Check if entity was modified since a given transaction. O(1) via index."""
+        row = self._conn.execute(
+            "SELECT 1 FROM touched_entities WHERE entity_id = ? AND tx_id > ? LIMIT 1",
+            (entity_id, since_tx),
+        ).fetchone()
+        return row is not None
+
+    def touched_entities_since(self, since_tx: int) -> list[str]:
+        """Return entity_ids modified since a transaction."""
+        rows = self._conn.execute(
+            "SELECT DISTINCT entity_id FROM touched_entities WHERE tx_id > ?",
+            (since_tx,),
+        ).fetchall()
+        return [r[0] for r in rows]
+
     # ----- Stats -----
 
     def stats(self) -> dict: