@simbimbo/memory-ocmemog 0.1.4

package/brain/runtime/memory/retrieval.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from typing import Dict, List, Any, Iterable, Tuple
+
+from brain.runtime.instrumentation import emit_event
+from brain.runtime import state_store
+from brain.runtime.memory import memory_links, provenance, store, vector_index
+
+
+def _match_score(text: str, query: str) -> float:
+    if not text:
+        return 0.0
+    text_l = text.lower()
+    query_l = query.lower()
+    if query_l in text_l:
+        return 1.0
+    return 0.0
+
+
+MEMORY_BUCKETS: Tuple[str, ...] = (
+    "knowledge",
+    "reflections",
+    "directives",
+    "tasks",
+    "runbooks",
+    "lessons",
+)
+
+
+def _empty_results() -> Dict[str, List[Dict[str, Any]]]:
+    return {bucket: [] for bucket in MEMORY_BUCKETS}
+
+
+def retrieve(prompt: str, limit: int = 5, categories: Iterable[str] | None = None) -> Dict[str, List[Dict[str, Any]]]:
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_retrieval_start", status="ok")
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_retrieval_rank_start", status="ok")
+
+    conn = store.connect()
+    results = _empty_results()
+    selected_categories = tuple(dict.fromkeys(category for category in (categories or MEMORY_BUCKETS) if category in MEMORY_BUCKETS))
+
+    # reinforcement lookup (by memory_reference)
+    reinf_rows = conn.execute(
+        "SELECT memory_reference, reward_score, confidence FROM experiences",
+    ).fetchall()
+    reinforcement: Dict[str, Dict[str, float]] = {}
+    for row in reinf_rows:
+        reference = str(row[0] or "")
+        if not reference:
+            continue
+        current = reinforcement.setdefault(reference, {"reward_score": 0.0, "confidence": 0.0, "count": 0.0})
+        current["reward_score"] += float(row[1] or 0.0)
+        current["confidence"] += float(row[2] or 0.0)
+        current["count"] += 1.0
+    for current in reinforcement.values():
+        count = max(1.0, float(current.get("count") or 1.0))
+        current["reward_score"] = float(current.get("reward_score") or 0.0) / count
+        current["confidence"] = float(current.get("confidence") or 0.0) / count
+
+    def score_record(content: str, memory_ref: str, promo_conf: float) -> float:
+        keyword = _match_score(content, prompt)
+        reinf = reinforcement.get(memory_ref, {})
+        reinf_score = float(reinf.get("reward_score", 0.0)) * 0.5
+        promo_score = float(promo_conf) * 0.3
+        return round(keyword + reinf_score + promo_score, 3)
+
+    for table, key in [(bucket, bucket) for bucket in selected_categories]:
+        try:
+            rows = conn.execute(
+                f"SELECT id, content, confidence, metadata_json FROM {table} ORDER BY id DESC LIMIT ?",
+                (limit * 10,),
+            ).fetchall()
+        except Exception:
+            continue
+        for row in rows:
+            content = row["content"] if isinstance(row, dict) else row[1]
+            if not _match_score(content, prompt):
+                continue
+            mem_ref = f"{table}:{row[0]}"
+            promo_conf = row["confidence"] if isinstance(row, dict) else row[2]
+            metadata = provenance.fetch_reference(mem_ref)
+            results[key].append({
+                "content": content,
+                "score": score_record(content, mem_ref, promo_conf),
+                "memory_reference": mem_ref,
+                "links": memory_links.get_memory_links(mem_ref),
+                "provenance_preview": (metadata or {}).get("provenance_preview") or provenance.preview_from_metadata((metadata or {}).get("metadata")),
+            })
+
+        results[key] = sorted(results[key], key=lambda x: x["score"], reverse=True)[:limit]
+
+    if prompt.strip() and all(not results.get(bucket) for bucket in selected_categories):
+        semantic = vector_index.search_memory(prompt, limit=limit)
+        for item in semantic:
+            source_type = item.get("source_type") or "knowledge"
+            if source_type not in selected_categories:
+                continue
+            try:
+                row = conn.execute(
+                    f"SELECT id, content, confidence, metadata_json FROM {source_type} WHERE id=?",
+                    (int(item.get("source_id") or 0),),
+                ).fetchone()
+            except Exception:
+                continue
+            if not row:
+                continue
+            content = row["content"] if isinstance(row, dict) else row[1]
+            mem_ref = f"{source_type}:{row[0]}"
+            promo_conf = row["confidence"] if isinstance(row, dict) else row[2]
+            metadata = provenance.fetch_reference(mem_ref)
+            results[source_type].append({
+                "content": content,
+                "score": score_record(content, mem_ref, promo_conf),
+                "memory_reference": mem_ref,
+                "links": memory_links.get_memory_links(mem_ref),
+                "provenance_preview": (metadata or {}).get("provenance_preview") or provenance.preview_from_metadata((metadata or {}).get("metadata")),
+            })
+        for bucket in selected_categories:
+            results[bucket] = sorted(results[bucket], key=lambda x: x["score"], reverse=True)[:limit]
+
+    conn.close()
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_retrieval_rank_complete", status="ok")
+    emit_event(state_store.reports_dir() / "brain_memory.log.jsonl", "brain_memory_retrieval_complete", status="ok")
+    return results
+
+
+def retrieve_for_queries(
+    queries: Iterable[str],
+    *,
+    limit: int = 5,
+    categories: Iterable[str] | None = None,
+) -> Dict[str, List[Dict[str, Any]]]:
+    merged = _empty_results()
+    seen_refs = {bucket: set() for bucket in MEMORY_BUCKETS}
+    selected_categories = tuple(dict.fromkeys(category for category in (categories or MEMORY_BUCKETS) if category in MEMORY_BUCKETS))
+    normalized_queries = [query.strip() for query in queries if isinstance(query, str) and query.strip()]
+
+    if not normalized_queries:
+        return retrieve("", limit=limit, categories=selected_categories)
+
+    for query in normalized_queries:
+        partial = retrieve(query, limit=limit, categories=selected_categories)
+        for bucket in selected_categories:
+            for item in partial.get(bucket, []):
+                ref = item.get("memory_reference")
+                if ref in seen_refs[bucket]:
+                    continue
+                seen_refs[bucket].add(ref)
+                merged[bucket].append(item)
+
+    for bucket in selected_categories:
+        merged[bucket] = sorted(merged[bucket], key=lambda x: x["score"], reverse=True)[:limit]
+    return merged

package/brain/runtime/memory/semantic_search.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+from brain.runtime import state_store
+from brain.runtime.instrumentation import emit_event
+from brain.runtime.memory import embedding_engine, store, retrieval, freshness
+
+LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
+
+
+def _cosine_similarity(a: List[float], b: List[float]) -> float:
+    if not a or not b:
+        return 0.0
+    size = min(len(a), len(b))
+    if size == 0:
+        return 0.0
+    dot = sum(x * y for x, y in zip(a[:size], b[:size]))
+    mag_a = sum(x * x for x in a[:size]) ** 0.5
+    mag_b = sum(x * x for x in b[:size]) ** 0.5
+    if mag_a == 0 or mag_b == 0:
+        return 0.0
+    return dot / (mag_a * mag_b)
+
+
+def semantic_search(query: str, limit: int = 5) -> List[Dict[str, Any]]:
+    emit_event(LOGFILE, "brain_semantic_search_start", status="ok")
+    query_embedding = embedding_engine.generate_embedding(query)
+    conn = store.connect()
+    rows = conn.execute(
+        "SELECT id, source_type, source_id, embedding FROM vector_embeddings"
+    ).fetchall()
+    conn.close()
+
+    reinforcement = retrieval.retrieve(query, limit=limit * 2)
+    freshness_info = {item["memory_id"]: item for item in freshness.scan_freshness(limit=limit).get("advisories", [])}
+
+    results: List[Dict[str, Any]] = []
+    for row in rows:
+        try:
+            embedding = [float(x) for x in __import__("json").loads(row["embedding"])]
+        except Exception:
+            continue
+        similarity = _cosine_similarity(query_embedding or [], embedding)
+        memory_ref = f"{row['source_type']}:{row['source_id']}"
+        reinforcement_weight = 0.0
+        for bucket in reinforcement.values():
+            for item in bucket:
+                if item.get("memory_reference") == memory_ref:
+                    reinforcement_weight = item.get("score", 0.0)
+        freshness_score = freshness_info.get(int(row["source_id"],), {}).get("freshness_score", 0.0) if str(row["source_id"]).isdigit() else 0.0
+        combined = similarity + reinforcement_weight + freshness_score
+        results.append(
+            {
+                "memory_reference": memory_ref,
+                "score": round(combined, 6),
+                "similarity": round(similarity, 6),
+                "freshness": freshness_score,
+                "reinforcement_weight": reinforcement_weight,
+                "promotion_confidence": 0.0,
+            }
+        )
+
+    results.sort(key=lambda item: item["score"], reverse=True)
+    emit_event(LOGFILE, "brain_semantic_search_complete", status="ok", result_count=len(results[:limit]))
+    return results[:limit]

package/brain/runtime/memory/sentiment_memory.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import sqlite3
+import time
+from typing import Dict, List
+
+from brain.runtime import state_store
+from brain.runtime.instrumentation import emit_event
+from brain.runtime.memory import person_memory
+
+LOGFILE = state_store.reports_dir() / "brain_memory.log.jsonl"
+
+SENTIMENTS = {"positive", "neutral", "negative", "frustrated", "urgent", "excited"}
+
+
+def classify_sentiment(text: str) -> str:
+    lowered = (text or "").lower()
+    if "urgent" in lowered:
+        return "urgent"
+    if "frustrated" in lowered or "angry" in lowered:
+        return "frustrated"
+    if "excited" in lowered or "great" in lowered:
+        return "excited"
+    if "bad" in lowered:
+        return "negative"
+    if "good" in lowered:
+        return "positive"
+    return "neutral"
+
+
+def _connect() -> sqlite3.Connection:
+    path = state_store.data_dir() / "sentiment_memory.db"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(path))
+    conn.row_factory = sqlite3.Row
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS sentiment_memory (
+            person_id TEXT,
+            sentiment TEXT,
+            timestamp TEXT
+        )
+        """
+    )
+    return conn
+
+
+def update_person_sentiment_baseline(person_id: str, sentiment: str) -> None:
+    sentiment = sentiment if sentiment in SENTIMENTS else "neutral"
+    conn = _connect()
+    conn.execute(
+        "INSERT INTO sentiment_memory (person_id, sentiment, timestamp) VALUES (?, ?, ?)",
+        (person_id, sentiment, time.strftime("%Y-%m-%d %H:%M:%S")),
+    )
+    conn.commit()
+    conn.close()
+    emit_event(LOGFILE, "brain_person_sentiment_updated", status="ok", person_id=person_id)
+
+
+def list_sentiment(person_id: str, limit: int = 10) -> List[Dict[str, str]]:
+    conn = _connect()
+    rows = conn.execute(
+        "SELECT sentiment, timestamp FROM sentiment_memory WHERE person_id=? ORDER BY timestamp DESC LIMIT ?",
+        (person_id, limit),
+    ).fetchall()
+    conn.close()
+    return [dict(row) for row in rows]

package/brain/runtime/memory/store.py

@@ -0,0 +1,400 @@
+from __future__ import annotations
+
+import sqlite3
+import queue
+import threading
+from pathlib import Path
+from brain.runtime import state_store
+
+SCHEMA_VERSION = "v1"
+
+SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS memory_events (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  event_type TEXT NOT NULL,
+  source TEXT,
+  details_json TEXT DEFAULT '{}',
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS environment_cognition (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  source TEXT,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  metadata_json TEXT DEFAULT '{}',
+  content TEXT NOT NULL,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS experiences (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  task_id TEXT,
+  outcome TEXT,
+  reward_score REAL,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  memory_reference TEXT,
+  experience_type TEXT,
+  source_module TEXT,
+  metadata_json TEXT DEFAULT '{}',
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS directives (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  source TEXT,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  metadata_json TEXT DEFAULT '{}',
+  content TEXT NOT NULL,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS candidates (
+  candidate_id TEXT PRIMARY KEY,
+  source_event_id INTEGER,
+  distilled_summary TEXT,
+  verification_points TEXT,
+  confidence_score REAL,
+  status TEXT NOT NULL DEFAULT 'pending',
+  verification_status TEXT NOT NULL DEFAULT 'unverified',
+  metadata_json TEXT DEFAULT '{}',
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS promotions (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  candidate_id TEXT,
+  source TEXT,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  status TEXT NOT NULL DEFAULT 'promoted',
+  decision_reason TEXT,
+  metadata_json TEXT DEFAULT '{}',
+  content TEXT NOT NULL,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS demotions (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  memory_reference TEXT NOT NULL,
+  previous_confidence REAL,
+  new_confidence REAL,
+  reason TEXT,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS cold_storage (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  archived_at TEXT NOT NULL DEFAULT (datetime('now')),
+  source_table TEXT NOT NULL,
+  source_id INTEGER NOT NULL,
+  content TEXT NOT NULL,
+  metadata_json TEXT DEFAULT '{}',
+  reason TEXT,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS knowledge (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  source TEXT,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  metadata_json TEXT DEFAULT '{}',
+  content TEXT NOT NULL,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS runbooks (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  source TEXT,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  metadata_json TEXT DEFAULT '{}',
+  content TEXT NOT NULL,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS lessons (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  source TEXT,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  metadata_json TEXT DEFAULT '{}',
+  content TEXT NOT NULL,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS reflections (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  source TEXT,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  metadata_json TEXT DEFAULT '{}',
+  content TEXT NOT NULL,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS tasks (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  source TEXT,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  metadata_json TEXT DEFAULT '{}',
+  content TEXT NOT NULL,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS memory_index (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  source TEXT,
+  confidence REAL NOT NULL DEFAULT 1.0,
+  metadata_json TEXT DEFAULT '{}',
+  content TEXT NOT NULL,
+  schema_version TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS vector_embeddings (
+  id TEXT PRIMARY KEY,
+  source_type TEXT NOT NULL,
+  source_id TEXT NOT NULL,
+  embedding TEXT NOT NULL,
+  created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+CREATE TABLE IF NOT EXISTS artifacts (
+  artifact_id TEXT PRIMARY KEY,
+  artifact_type TEXT,
+  source_path TEXT,
+  content_hash TEXT,
+  metadata TEXT,
+  created_at TIMESTAMP DEFAULT (datetime('now'))
+);
+
+CREATE TABLE IF NOT EXISTS conversation_turns (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  conversation_id TEXT,
+  session_id TEXT,
+  thread_id TEXT,
+  message_id TEXT,
+  role TEXT NOT NULL,
+  content TEXT NOT NULL,
+  transcript_path TEXT,
+  transcript_offset INTEGER,
+  transcript_end_offset INTEGER,
+  source TEXT,
+  metadata_json TEXT DEFAULT '{}',
+  schema_version TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_conversation_turns_conversation ON conversation_turns(conversation_id, id DESC);
+CREATE INDEX IF NOT EXISTS idx_conversation_turns_session ON conversation_turns(session_id, id DESC);
+CREATE INDEX IF NOT EXISTS idx_conversation_turns_thread ON conversation_turns(thread_id, id DESC);
+CREATE INDEX IF NOT EXISTS idx_conversation_turns_message ON conversation_turns(message_id);
+
+CREATE TABLE IF NOT EXISTS conversation_checkpoints (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+  conversation_id TEXT,
+  session_id TEXT,
+  thread_id TEXT,
+  turn_start_id INTEGER,
+  turn_end_id INTEGER,
+  checkpoint_kind TEXT NOT NULL DEFAULT 'manual',
+  summary TEXT NOT NULL,
+  latest_user_ask TEXT,
+  last_assistant_commitment TEXT,
+  open_loops_json TEXT DEFAULT '[]',
+  pending_actions_json TEXT DEFAULT '[]',
+  parent_checkpoint_id INTEGER,
+  root_checkpoint_id INTEGER,
+  depth INTEGER NOT NULL DEFAULT 0,
+  metadata_json TEXT DEFAULT '{}',
+  schema_version TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_conversation_checkpoints_conversation ON conversation_checkpoints(conversation_id, id DESC);
+CREATE INDEX IF NOT EXISTS idx_conversation_checkpoints_session ON conversation_checkpoints(session_id, id DESC);
+CREATE INDEX IF NOT EXISTS idx_conversation_checkpoints_thread ON conversation_checkpoints(thread_id, id DESC);
+
+CREATE TABLE IF NOT EXISTS conversation_state (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  scope_type TEXT NOT NULL,
+  scope_id TEXT NOT NULL,
+  conversation_id TEXT,
+  session_id TEXT,
+  thread_id TEXT,
+  latest_user_turn_id INTEGER,
+  latest_assistant_turn_id INTEGER,
+  latest_user_ask TEXT,
+  last_assistant_commitment TEXT,
+  open_loops_json TEXT DEFAULT '[]',
+  pending_actions_json TEXT DEFAULT '[]',
+  unresolved_state_json TEXT DEFAULT '[]',
+  latest_checkpoint_id INTEGER,
+  metadata_json TEXT DEFAULT '{}',
+  updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+  schema_version TEXT NOT NULL,
+  UNIQUE(scope_type, scope_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_conversation_state_conversation ON conversation_state(conversation_id);
+CREATE INDEX IF NOT EXISTS idx_conversation_state_session ON conversation_state(session_id);
+CREATE INDEX IF NOT EXISTS idx_conversation_state_thread ON conversation_state(thread_id);
+"""
+
+_WRITE_QUEUE: "queue.Queue[tuple]" = queue.Queue()
+_WRITE_LOCK = threading.Lock()
+_WRITE_WORKER_STARTED = False
+
+
+def is_lock_error(exc: BaseException) -> bool:
+    message = str(exc).lower()
+    return isinstance(exc, sqlite3.OperationalError) and (
+        "database is locked" in message
+        or "database table is locked" in message
+        or "busy" in message
+    )
+
+
+def _write_worker() -> None:
+    while True:
+        fn, event, container = _WRITE_QUEUE.get()
+        try:
+            container["result"] = fn()
+        except Exception as exc:  # pragma: no cover
+            container["error"] = exc
+        finally:
+            event.set()
+            _WRITE_QUEUE.task_done()
+
+
+def _ensure_write_worker() -> None:
+    global _WRITE_WORKER_STARTED
+    with _WRITE_LOCK:
+        if _WRITE_WORKER_STARTED:
+            return
+        thread = threading.Thread(target=_write_worker, daemon=True)
+        thread.start()
+        _WRITE_WORKER_STARTED = True
+
+
+def submit_write(fn, timeout: float = 30.0):
+    _ensure_write_worker()
+    event = threading.Event()
+    container: dict = {}
+    _WRITE_QUEUE.put((fn, event, container))
+    if not event.wait(timeout=timeout):
+        raise TimeoutError("write queue timeout")
+    if "error" in container:
+        raise container["error"]
+    return container.get("result")
+
+
+def db_path() -> Path:
+    return state_store.memory_db_path()
+
+
+_SCHEMA_READY = False
+
+
+def connect(*, ensure_schema: bool = True) -> sqlite3.Connection:
+    global _SCHEMA_READY
+    if ensure_schema and not _SCHEMA_READY:
+        init_db()
+        _SCHEMA_READY = True
+    path = db_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(path), timeout=30)
+    conn.row_factory = sqlite3.Row
+
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=15000")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.execute("PRAGMA temp_store=MEMORY")
+    conn.execute("PRAGMA cache_size=-20000")
+    return conn
+
+
+def _ensure_column(conn: sqlite3.Connection, table: str, column: str, coltype: str) -> None:
+    cols = [row[1] for row in conn.execute(f"PRAGMA table_info({table})").fetchall()]
+    if column not in cols:
+        conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {coltype}")
+
+
+def init_db() -> None:
+    conn = connect(ensure_schema=False)
+    conn.executescript(SCHEMA_SQL)
+    _ensure_column(conn, "experiences", "experience_type", "TEXT")
+    _ensure_column(conn, "experiences", "source_module", "TEXT")
+    _ensure_column(conn, "experiences", "metadata_json", "TEXT DEFAULT '{}' ")
+    _ensure_column(conn, "candidates", "status", "TEXT NOT NULL DEFAULT 'pending'")
+    _ensure_column(conn, "candidates", "verification_status", "TEXT NOT NULL DEFAULT 'unverified'")
+    _ensure_column(conn, "candidates", "created_at", "TEXT")
+    _ensure_column(conn, "candidates", "updated_at", "TEXT")
+    conn.execute("UPDATE candidates SET created_at=datetime('now') WHERE created_at IS NULL")
+    conn.execute("UPDATE candidates SET updated_at=datetime('now') WHERE updated_at IS NULL")
+    _ensure_column(conn, "promotions", "candidate_id", "TEXT")
+    _ensure_column(conn, "promotions", "status", "TEXT NOT NULL DEFAULT 'promoted'")
+    _ensure_column(conn, "promotions", "decision_reason", "TEXT")
+    _ensure_column(conn, "conversation_turns", "conversation_id", "TEXT")
+    _ensure_column(conn, "conversation_turns", "session_id", "TEXT")
+    _ensure_column(conn, "conversation_turns", "thread_id", "TEXT")
+    _ensure_column(conn, "conversation_turns", "message_id", "TEXT")
+    _ensure_column(conn, "conversation_turns", "role", "TEXT NOT NULL DEFAULT 'unknown'")
+    _ensure_column(conn, "conversation_turns", "content", "TEXT")
+    _ensure_column(conn, "conversation_turns", "transcript_path", "TEXT")
+    _ensure_column(conn, "conversation_turns", "transcript_offset", "INTEGER")
+    _ensure_column(conn, "conversation_turns", "transcript_end_offset", "INTEGER")
+    _ensure_column(conn, "conversation_turns", "source", "TEXT")
+    _ensure_column(conn, "conversation_turns", "metadata_json", "TEXT DEFAULT '{}'")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_turns_conversation ON conversation_turns(conversation_id, id DESC)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_turns_session ON conversation_turns(session_id, id DESC)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_turns_thread ON conversation_turns(thread_id, id DESC)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_turns_message ON conversation_turns(message_id)")
+    _ensure_column(conn, "conversation_checkpoints", "conversation_id", "TEXT")
+    _ensure_column(conn, "conversation_checkpoints", "session_id", "TEXT")
+    _ensure_column(conn, "conversation_checkpoints", "thread_id", "TEXT")
+    _ensure_column(conn, "conversation_checkpoints", "turn_start_id", "INTEGER")
+    _ensure_column(conn, "conversation_checkpoints", "turn_end_id", "INTEGER")
+    _ensure_column(conn, "conversation_checkpoints", "checkpoint_kind", "TEXT NOT NULL DEFAULT 'manual'")
+    _ensure_column(conn, "conversation_checkpoints", "summary", "TEXT NOT NULL DEFAULT ''")
+    _ensure_column(conn, "conversation_checkpoints", "latest_user_ask", "TEXT")
+    _ensure_column(conn, "conversation_checkpoints", "last_assistant_commitment", "TEXT")
+    _ensure_column(conn, "conversation_checkpoints", "open_loops_json", "TEXT DEFAULT '[]'")
+    _ensure_column(conn, "conversation_checkpoints", "pending_actions_json", "TEXT DEFAULT '[]'")
+    _ensure_column(conn, "conversation_checkpoints", "parent_checkpoint_id", "INTEGER")
+    _ensure_column(conn, "conversation_checkpoints", "root_checkpoint_id", "INTEGER")
+    _ensure_column(conn, "conversation_checkpoints", "depth", "INTEGER NOT NULL DEFAULT 0")
+    _ensure_column(conn, "conversation_checkpoints", "metadata_json", "TEXT DEFAULT '{}'")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_checkpoints_conversation ON conversation_checkpoints(conversation_id, id DESC)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_checkpoints_session ON conversation_checkpoints(session_id, id DESC)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_checkpoints_thread ON conversation_checkpoints(thread_id, id DESC)")
+    _ensure_column(conn, "conversation_state", "scope_type", "TEXT")
+    _ensure_column(conn, "conversation_state", "scope_id", "TEXT")
+    _ensure_column(conn, "conversation_state", "conversation_id", "TEXT")
+    _ensure_column(conn, "conversation_state", "session_id", "TEXT")
+    _ensure_column(conn, "conversation_state", "thread_id", "TEXT")
+    _ensure_column(conn, "conversation_state", "latest_user_turn_id", "INTEGER")
+    _ensure_column(conn, "conversation_state", "latest_assistant_turn_id", "INTEGER")
+    _ensure_column(conn, "conversation_state", "latest_user_ask", "TEXT")
+    _ensure_column(conn, "conversation_state", "last_assistant_commitment", "TEXT")
+    _ensure_column(conn, "conversation_state", "open_loops_json", "TEXT DEFAULT '[]'")
+    _ensure_column(conn, "conversation_state", "pending_actions_json", "TEXT DEFAULT '[]'")
+    _ensure_column(conn, "conversation_state", "unresolved_state_json", "TEXT DEFAULT '[]'")
+    _ensure_column(conn, "conversation_state", "latest_checkpoint_id", "INTEGER")
+    _ensure_column(conn, "conversation_state", "metadata_json", "TEXT DEFAULT '{}'")
+    _ensure_column(conn, "conversation_state", "updated_at", "TEXT")
+    conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_conversation_state_scope ON conversation_state(scope_type, scope_id)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_state_conversation ON conversation_state(conversation_id)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_state_session ON conversation_state(session_id)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_conversation_state_thread ON conversation_state(thread_id)")
+    conn.commit()
+    conn.close()