@cohortapp/agent-sdk 2.3.0

package/scripts/rag-indexer.py

@@ -0,0 +1,629 @@
+#!/usr/bin/env python3
+"""
+rag-indexer.py — SQLite FTS5 indexer for per-user ring-fenced RAG (Phase 2)
+
+Indexes all content sources into a SQLite FTS5 database for fast full-text search
+with per-user access control (ring-fencing).
+
+Sources indexed:
+  - memory/interactions/**/*.jsonl  (Slack DMs, channels, emails — each line = one doc)
+  - knowledge/sources/*.yaml        (public)
+  - knowledge/decisions/*.yaml       (leadership+)
+  - knowledge/syntheses/*.yaml       (public, except strategy-state.yaml = ceo)
+  - state/queues/*.yaml              (leadership+)
+
+Each document gets a user_scope:
+  - "public"       — visible to all authenticated users
+  - "leadership"   — visible to leadership + CEO
+  - "ceo"          — CEO only
+  - "{user-slug}"  — private to that specific user (DMs, emails)
+
+Incremental indexing: tracks file modification times in state/rag/index-state.json.
+Only re-indexes files that have changed since last run.
+
+Usage:
+    python3 scripts/rag-indexer.py                  # incremental index
+    python3 scripts/rag-indexer.py --full            # full re-index (drop + rebuild)
+    python3 scripts/rag-indexer.py --stats           # show index statistics
+    python3 scripts/rag-indexer.py --db /path/to/db  # custom DB path
+
+Author: Northwind
+Date: 2026-04-03
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import sys
+import time
+from pathlib import Path
+
+# ── Constants ──────────────────────────────────────────────────────────────────
+
+BASE_DIR = Path(os.environ.get("AGENT_DIR", str(Path(__file__).resolve().parent.parent)))  # agent repo root
+DEFAULT_DB_PATH = BASE_DIR / "state" / "rag" / "search.db"
+INDEX_STATE_PATH = BASE_DIR / "state" / "rag" / "index-state.json"
+
+
+# ── Database Schema ────────────────────────────────────────────────────────────
+
+SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS documents (
+    id TEXT PRIMARY KEY,
+    source_type TEXT NOT NULL,
+    source_path TEXT NOT NULL,
+    user_scope TEXT NOT NULL,
+    content TEXT NOT NULL,
+    title TEXT,
+    timestamp TEXT,
+    metadata_json TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_user_scope ON documents(user_scope);
+CREATE INDEX IF NOT EXISTS idx_source_type ON documents(source_type);
+CREATE INDEX IF NOT EXISTS idx_source_path ON documents(source_path);
+CREATE INDEX IF NOT EXISTS idx_timestamp ON documents(timestamp);
+"""
+
+FTS_SQL = """
+CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
+    content,
+    title,
+    content='documents',
+    content_rowid='rowid'
+);
+"""
+
+# Triggers to keep FTS in sync with documents table
+TRIGGER_SQL = """
+CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents BEGIN
+    INSERT INTO documents_fts(rowid, content, title)
+    VALUES (new.rowid, new.content, new.title);
+END;
+
+CREATE TRIGGER IF NOT EXISTS documents_ad AFTER DELETE ON documents BEGIN
+    INSERT INTO documents_fts(documents_fts, rowid, content, title)
+    VALUES ('delete', old.rowid, old.content, old.title);
+END;
+
+CREATE TRIGGER IF NOT EXISTS documents_au AFTER UPDATE ON documents BEGIN
+    INSERT INTO documents_fts(documents_fts, rowid, content, title)
+    VALUES ('delete', old.rowid, old.content, old.title);
+    INSERT INTO documents_fts(rowid, content, title)
+    VALUES (new.rowid, new.content, new.title);
+END;
+"""
+
+
+# ── Database Initialization ───────────────────────────────────────────────────
+
+def init_db(db_path):
+    """Create or open the SQLite database with FTS5 schema."""
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(db_path))
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+
+    # Create main table and indexes
+    conn.executescript(SCHEMA_SQL)
+
+    # Create FTS5 virtual table
+    conn.executescript(FTS_SQL)
+
+    # Create sync triggers
+    conn.executescript(TRIGGER_SQL)
+
+    conn.commit()
+    return conn
+
+
+def drop_and_rebuild(db_path):
+    """Drop all tables and recreate from scratch."""
+    if db_path.exists():
+        os.remove(str(db_path))
+    # Also clear the index state
+    if INDEX_STATE_PATH.exists():
+        os.remove(str(INDEX_STATE_PATH))
+    return init_db(db_path)
+
+
+# ── Index State Management ────────────────────────────────────────────────────
+
+def load_index_state():
+    """Load the incremental indexing state (file modification times)."""
+    if INDEX_STATE_PATH.exists():
+        with open(INDEX_STATE_PATH, "r") as f:
+            return json.load(f)
+    return {"files": {}}
+
+
+def save_index_state(state):
+    """Save the incremental indexing state."""
+    INDEX_STATE_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with open(INDEX_STATE_PATH, "w") as f:
+        json.dump(state, f, indent=2)
+
+
+# ── Source Classification ─────────────────────────────────────────────────────
+
+def classify_source(rel_path):
+    """
+    Determine source_type and user_scope from a file's relative path.
+
+    Returns (source_type, user_scope) where:
+      source_type: "slack_dm", "slack_channel", "email", "knowledge", "decision",
+                   "synthesis", "queue"
+      user_scope:  "public", "leadership", "ceo", or a user slug for private data
+    """
+    parts = Path(rel_path).parts
+
+    # memory/interactions/slack/dm-{user-slug}/**
+    if (len(parts) >= 4 and parts[0] == "memory" and parts[1] == "interactions"
+            and parts[2] == "slack"):
+        dir_name = parts[3]
+        if dir_name.startswith("dm-"):
+            user_slug = dir_name[3:]  # "dm-alex-chen" -> "alex-chen"
+            return "slack_dm", user_slug
+        elif dir_name == "dm":
+            # memory/interactions/slack/dm/{something} — treat as private
+            if len(parts) >= 5:
+                return "slack_dm", parts[4]
+            return "slack_dm", "unknown"
+        else:
+            # Channel (ceo-office, channels, etc.)
+            return "slack_channel", "leadership"
+
+    # memory/interactions/slack/{user-slug}/** (alternate DM format like alex-chen/)
+    if (len(parts) >= 4 and parts[0] == "memory" and parts[1] == "interactions"
+            and parts[2] == "slack"):
+        dir_name = parts[3]
+        # Check if it looks like a user slug (contains a hyphen, isn't a known channel)
+        known_channels = {"ceo-office", "channels", "dm", "general", "engineering"}
+        if dir_name not in known_channels and "-" in dir_name:
+            return "slack_dm", dir_name
+
+    # memory/interactions/email/{user-slug}/**
+    if (len(parts) >= 4 and parts[0] == "memory" and parts[1] == "interactions"
+            and parts[2] == "email"):
+        user_slug = parts[3]
+        return "email", user_slug
+
+    # knowledge/sources/*.yaml — public
+    if (len(parts) >= 2 and parts[0] == "knowledge" and parts[1] == "sources"):
+        return "knowledge", "public"
+
+    # knowledge/decisions/**/*.yaml — leadership+
+    if (len(parts) >= 2 and parts[0] == "knowledge" and parts[1] == "decisions"):
+        return "decision", "leadership"
+
+    # knowledge/syntheses/*.yaml — public, except strategy-state = ceo
+    if (len(parts) >= 2 and parts[0] == "knowledge" and parts[1] == "syntheses"):
+        filename = parts[-1]
+        if filename == "strategy-state.yaml":
+            return "synthesis", "ceo"
+        return "synthesis", "public"
+
+    # state/queues/*.yaml — leadership+
+    if (len(parts) >= 2 and parts[0] == "state" and parts[1] == "queues"):
+        return "queue", "leadership"
+
+    # Fallback
+    return "unknown", "public"
+
+
+# ── Document ID Generation ────────────────────────────────────────────────────
+
+def make_doc_id(source_path, line_num=None):
+    """
+    Generate a deterministic document ID.
+    For JSONL: source_path:line_number
+    For YAML/other: hash of source_path
+    """
+    if line_num is not None:
+        return f"{source_path}:{line_num}"
+    return hashlib.sha256(source_path.encode()).hexdigest()[:16]
+
+
+# ── Content Parsing ───────────────────────────────────────────────────────────
+
+def parse_jsonl_file(filepath, rel_path, source_type, user_scope):
+    """
+    Parse a JSONL file into individual documents.
+    Each line becomes one document.
+    """
+    documents = []
+    try:
+        with open(filepath, "r") as f:
+            for line_num, line in enumerate(f, 1):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    record = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+
+                # Extract content from various field names
+                content = (record.get("content")
+                           or record.get("text")
+                           or record.get("content_preview")
+                           or "")
+
+                if not content:
+                    continue
+
+                # Build a rich text representation
+                sender = record.get("sender") or record.get("from") or ""
+                channel = record.get("channel") or ""
+                direction = record.get("direction") or ""
+                action = record.get("action_taken") or ""
+                context = record.get("context") or ""
+
+                # Build title from available fields
+                title_parts = []
+                if sender:
+                    title_parts.append(f"From: {sender}")
+                if channel:
+                    title_parts.append(f"Channel: {channel}")
+                if direction:
+                    title_parts.append(f"({direction})")
+                title = " | ".join(title_parts) if title_parts else None
+
+                # Include action_taken and context in searchable content
+                full_content = content
+                if action:
+                    full_content += f"\nAction: {action}"
+                if context:
+                    full_content += f"\nContext: {context}"
+
+                # Extract timestamp
+                timestamp = (record.get("timestamp")
+                             or record.get("processed_at")
+                             or record.get("ts")
+                             or "")
+
+                # Metadata
+                metadata = {
+                    "sender": sender,
+                    "channel": channel,
+                    "direction": direction,
+                    "classification": record.get("classification", ""),
+                    "ref": record.get("ref", ""),
+                    "message_ts": record.get("message_ts", ""),
+                }
+
+                doc_id = make_doc_id(rel_path, line_num)
+                documents.append({
+                    "id": doc_id,
+                    "source_type": source_type,
+                    "source_path": rel_path,
+                    "user_scope": user_scope,
+                    "content": full_content,
+                    "title": title,
+                    "timestamp": timestamp,
+                    "metadata_json": json.dumps(metadata),
+                })
+
+    except Exception as e:
+        print(f"  Warning: Failed to parse {filepath}: {e}", file=sys.stderr)
+
+    return documents
+
+
+def parse_yaml_file(filepath, rel_path, source_type, user_scope):
+    """
+    Parse a YAML file into a single document.
+    Uses the full file content as the searchable text.
+    For simplicity and no-dependency constraint, we read the raw text.
+    """
+    documents = []
+    try:
+        with open(filepath, "r") as f:
+            content = f.read()
+
+        if not content.strip():
+            return documents
+
+        # Extract a title from the first comment line or YAML key
+        title = None
+        for line in content.split("\n"):
+            line = line.strip()
+            if line.startswith("# ") and not line.startswith("# =="):
+                title = line[2:].strip()
+                break
+            elif line and not line.startswith("#") and ":" in line:
+                title = line.split(":")[0].strip()
+                break
+
+        # For large YAML files, split into sections by top-level keys
+        # But keep it simple: one document per file
+        doc_id = make_doc_id(rel_path)
+        documents.append({
+            "id": doc_id,
+            "source_type": source_type,
+            "source_path": rel_path,
+            "user_scope": user_scope,
+            "content": content[:50000],  # cap at 50K chars
+            "title": title,
+            "timestamp": "",
+            "metadata_json": json.dumps({"filename": Path(filepath).name}),
+        })
+
+    except Exception as e:
+        print(f"  Warning: Failed to parse {filepath}: {e}", file=sys.stderr)
+
+    return documents
+
+
+def parse_markdown_file(filepath, rel_path, source_type, user_scope):
+    """Parse a Markdown file into a single document."""
+    documents = []
+    try:
+        with open(filepath, "r") as f:
+            content = f.read()
+
+        if not content.strip():
+            return documents
+
+        # Extract title from first heading
+        title = None
+        for line in content.split("\n"):
+            line = line.strip()
+            if line.startswith("# "):
+                title = line[2:].strip()
+                break
+
+        doc_id = make_doc_id(rel_path)
+        documents.append({
+            "id": doc_id,
+            "source_type": source_type,
+            "source_path": rel_path,
+            "user_scope": user_scope,
+            "content": content[:50000],
+            "title": title,
+            "timestamp": "",
+            "metadata_json": json.dumps({"filename": Path(filepath).name}),
+        })
+
+    except Exception as e:
+        print(f"  Warning: Failed to parse {filepath}: {e}", file=sys.stderr)
+
+    return documents
+
+
+# ── File Discovery ────────────────────────────────────────────────────────────
+
+def discover_files():
+    """
+    Walk the configured source directories and return a list of
+    (absolute_path, relative_path) tuples for indexable files.
+    """
+    source_dirs = [
+        "memory/interactions",
+        "knowledge/sources",
+        "knowledge/decisions",
+        "knowledge/syntheses",
+        "state/queues",
+    ]
+
+    indexable_extensions = {".jsonl", ".yaml", ".yml", ".md"}
+    files = []
+
+    for src_dir in source_dirs:
+        abs_dir = BASE_DIR / src_dir
+        if not abs_dir.exists():
+            continue
+
+        for root, _dirs, filenames in os.walk(str(abs_dir)):
+            for fname in filenames:
+                ext = os.path.splitext(fname)[1].lower()
+                if ext not in indexable_extensions:
+                    continue
+
+                abs_path = os.path.join(root, fname)
+                rel_path = os.path.relpath(abs_path, BASE_DIR)
+                files.append((abs_path, rel_path))
+
+    return files
+
+
+# ── Indexing Pipeline ─────────────────────────────────────────────────────────
+
+def index_file(filepath, rel_path):
+    """
+    Parse a file into documents based on its type and path.
+    Returns a list of document dicts.
+    """
+    source_type, user_scope = classify_source(rel_path)
+    ext = os.path.splitext(filepath)[1].lower()
+
+    if ext == ".jsonl":
+        return parse_jsonl_file(filepath, rel_path, source_type, user_scope)
+    elif ext in (".yaml", ".yml"):
+        return parse_yaml_file(filepath, rel_path, source_type, user_scope)
+    elif ext == ".md":
+        return parse_markdown_file(filepath, rel_path, source_type, user_scope)
+    else:
+        return []
+
+
+def upsert_documents(conn, documents):
+    """Insert or replace documents in the database."""
+    if not documents:
+        return 0
+
+    # We need to handle FTS sync carefully:
+    # 1. Delete existing docs by ID (triggers will remove from FTS)
+    # 2. Insert new docs (triggers will add to FTS)
+    count = 0
+    for doc in documents:
+        # Delete existing (triggers handle FTS cleanup)
+        conn.execute("DELETE FROM documents WHERE id = ?", (doc["id"],))
+
+        # Insert new
+        conn.execute(
+            """INSERT INTO documents (id, source_type, source_path, user_scope,
+               content, title, timestamp, metadata_json)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+            (doc["id"], doc["source_type"], doc["source_path"], doc["user_scope"],
+             doc["content"], doc["title"], doc["timestamp"], doc["metadata_json"]),
+        )
+        count += 1
+
+    return count
+
+
+def remove_stale_documents(conn, source_path):
+    """Remove all documents from a given source path (before re-indexing it)."""
+    conn.execute("DELETE FROM documents WHERE source_path = ?", (source_path,))
+
+
+def run_index(db_path, full=False):
+    """
+    Run the indexing pipeline.
+    If full=True, drops and rebuilds the entire index.
+    Otherwise, only re-indexes changed files.
+    """
+    if full:
+        print("Full re-index: dropping and rebuilding database...")
+        conn = drop_and_rebuild(db_path)
+        state = {"files": {}}
+    else:
+        conn = init_db(db_path)
+        state = load_index_state()
+
+    files = discover_files()
+    print(f"Discovered {len(files)} indexable files")
+
+    indexed_count = 0
+    doc_count = 0
+    skipped_count = 0
+
+    for abs_path, rel_path in files:
+        try:
+            mtime = os.path.getmtime(abs_path)
+        except OSError:
+            continue
+
+        # Check if file has changed since last index
+        last_mtime = state["files"].get(rel_path, 0)
+        if not full and mtime <= last_mtime:
+            skipped_count += 1
+            continue
+
+        # Remove old documents from this file
+        remove_stale_documents(conn, rel_path)
+
+        # Parse and index
+        documents = index_file(abs_path, rel_path)
+        if documents:
+            count = upsert_documents(conn, documents)
+            doc_count += count
+
+        # Update state
+        state["files"][rel_path] = mtime
+        indexed_count += 1
+
+    conn.commit()
+    conn.close()
+
+    # Save index state
+    save_index_state(state)
+
+    print(f"Indexed {indexed_count} files ({doc_count} documents)")
+    if skipped_count > 0:
+        print(f"Skipped {skipped_count} unchanged files")
+
+    return indexed_count, doc_count
+
+
+# ── Statistics ────────────────────────────────────────────────────────────────
+
+def show_stats(db_path):
+    """Show index statistics."""
+    if not db_path.exists():
+        print("No index database found. Run indexer first.")
+        return
+
+    conn = sqlite3.connect(str(db_path))
+
+    # Total documents
+    total = conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0]
+    print(f"Total documents: {total}")
+
+    # By source type
+    print("\nBy source type:")
+    rows = conn.execute(
+        "SELECT source_type, COUNT(*) FROM documents GROUP BY source_type ORDER BY COUNT(*) DESC"
+    ).fetchall()
+    for source_type, count in rows:
+        print(f"  {source_type}: {count}")
+
+    # By user scope
+    print("\nBy user scope:")
+    rows = conn.execute(
+        "SELECT user_scope, COUNT(*) FROM documents GROUP BY user_scope ORDER BY COUNT(*) DESC"
+    ).fetchall()
+    for scope, count in rows:
+        print(f"  {scope}: {count}")
+
+    # By source path (top 10)
+    print("\nTop 10 source files:")
+    rows = conn.execute(
+        "SELECT source_path, COUNT(*) FROM documents GROUP BY source_path ORDER BY COUNT(*) DESC LIMIT 10"
+    ).fetchall()
+    for path, count in rows:
+        print(f"  {path}: {count}")
+
+    # Index state
+    state = load_index_state()
+    print(f"\nTracked files in index state: {len(state.get('files', {}))}")
+
+    conn.close()
+
+
+# ── Main ───────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="SQLite FTS5 indexer for per-user ring-fenced RAG (Phase 2). "
+                    "Indexes the agent repo's content sources into a full-text search database.",
+        epilog="Examples:\n"
+               "  %(prog)s                    # incremental index\n"
+               "  %(prog)s --full             # full re-index\n"
+               "  %(prog)s --stats            # show statistics\n"
+               "  %(prog)s --db /tmp/test.db  # custom DB path\n",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--full", action="store_true",
+        help="Full re-index: drop and rebuild the entire database"
+    )
+    parser.add_argument(
+        "--stats", action="store_true",
+        help="Show index statistics and exit"
+    )
+    parser.add_argument(
+        "--db", type=str, default=None,
+        help=f"Path to SQLite database (default: {DEFAULT_DB_PATH})"
+    )
+
+    args = parser.parse_args()
+
+    db_path = Path(args.db) if args.db else DEFAULT_DB_PATH
+
+    if args.stats:
+        show_stats(db_path)
+        return
+
+    start = time.time()
+    indexed, docs = run_index(db_path, full=args.full)
+    elapsed = time.time() - start
+    print(f"Completed in {elapsed:.2f}s")
+
+
+if __name__ == "__main__":
+    main()