arkaos 2.8.0 → 2.10.0

package/core/cognition/insights/__init__.py

@@ -0,0 +1,5 @@
+"""Cognitive Layer insights module — SQLite store for actionable insights."""
+
+from core.cognition.insights.store import InsightStore
+
+__all__ = ["InsightStore"]

package/core/cognition/insights/store.py

@@ -0,0 +1,155 @@
+"""SQLite CRUD store for actionable insights.
+
+Persists ActionableInsight instances with support for status-based retrieval,
+project filtering, presentation lifecycle, and dismissal analytics.
+"""
+
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+
+from core.cognition.memory.schemas import ActionableInsight
+
+
+class InsightStore:
+    """SQLite-backed store for actionable insights."""
+
+    def __init__(self, db_path: str) -> None:
+        """Connect to SQLite database and initialize tables."""
+        Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+        self._db_path = db_path
+        self._init_db()
+
+    def _conn(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(self._db_path)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL")
+        return conn
+
+    def _init_db(self) -> None:
+        with self._conn() as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS insights (
+                    id TEXT PRIMARY KEY,
+                    project TEXT NOT NULL,
+                    trigger_source TEXT NOT NULL,
+                    date_generated TEXT NOT NULL,
+                    category TEXT NOT NULL,
+                    severity TEXT NOT NULL,
+                    title TEXT NOT NULL,
+                    description TEXT NOT NULL,
+                    recommendation TEXT NOT NULL,
+                    context TEXT NOT NULL,
+                    status TEXT NOT NULL DEFAULT 'pending',
+                    presented_at TEXT
+                )
+            """)
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_insights_project ON insights (project)"
+            )
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_insights_status ON insights (status)"
+            )
+
+    def _row_to_insight(self, row: sqlite3.Row) -> ActionableInsight:
+        data = dict(row)
+        # Map trigger_source back to trigger (SQL keyword workaround)
+        data["trigger"] = data.pop("trigger_source")
+        # Parse datetime fields
+        data["date_generated"] = datetime.fromisoformat(data["date_generated"])
+        if data["presented_at"] is not None:
+            data["presented_at"] = datetime.fromisoformat(data["presented_at"])
+        return ActionableInsight(**data)
+
+    def save(self, insight: ActionableInsight) -> None:
+        """Insert or replace an ActionableInsight record."""
+        with self._conn() as conn:
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO insights
+                    (id, project, trigger_source, date_generated, category,
+                     severity, title, description, recommendation, context,
+                     status, presented_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    insight.id,
+                    insight.project,
+                    insight.trigger,
+                    insight.date_generated.isoformat(),
+                    insight.category,
+                    insight.severity,
+                    insight.title,
+                    insight.description,
+                    insight.recommendation,
+                    insight.context,
+                    insight.status,
+                    insight.presented_at.isoformat() if insight.presented_at else None,
+                ),
+            )
+
+    def get_pending(self, project: str) -> list[ActionableInsight]:
+        """Return pending insights for a specific project."""
+        with self._conn() as conn:
+            rows = conn.execute(
+                """
+                SELECT * FROM insights
+                WHERE project = ? AND status = 'pending'
+                ORDER BY date_generated ASC
+                """,
+                (project,),
+            ).fetchall()
+        return [self._row_to_insight(r) for r in rows]
+
+    def get_all_pending(self) -> list[ActionableInsight]:
+        """Return all pending insights across all projects."""
+        with self._conn() as conn:
+            rows = conn.execute(
+                "SELECT * FROM insights WHERE status = 'pending' ORDER BY date_generated ASC"
+            ).fetchall()
+        return [self._row_to_insight(r) for r in rows]
+
+    def get_by_project(self, project: str) -> list[ActionableInsight]:
+        """Return all insights for a project regardless of status."""
+        with self._conn() as conn:
+            rows = conn.execute(
+                "SELECT * FROM insights WHERE project = ? ORDER BY date_generated ASC",
+                (project,),
+            ).fetchall()
+        return [self._row_to_insight(r) for r in rows]
+
+    def update_status(self, insight_id: str, status: str) -> None:
+        """Update the status of a single insight."""
+        with self._conn() as conn:
+            conn.execute(
+                "UPDATE insights SET status = ? WHERE id = ?",
+                (status, insight_id),
+            )
+
+    def mark_presented(self, ids: list[str]) -> None:
+        """Mark insights as presented and record the timestamp."""
+        if not ids:
+            return
+        now = datetime.now(timezone.utc).isoformat()
+        placeholders = ",".join("?" * len(ids))
+        with self._conn() as conn:
+            conn.execute(
+                f"UPDATE insights SET status = 'presented', presented_at = ? WHERE id IN ({placeholders})",
+                [now, *ids],
+            )
+
+    def dismissed_counts(self, project: str) -> dict[str, int]:
+        """Return count of dismissed insights grouped by category for a project."""
+        with self._conn() as conn:
+            rows = conn.execute(
+                """
+                SELECT category, COUNT(*) as cnt FROM insights
+                WHERE project = ? AND status = 'dismissed'
+                GROUP BY category
+                """,
+                (project,),
+            ).fetchall()
+        return {r["category"]: r["cnt"] for r in rows}
+
+    def close(self) -> None:
+        """No-op — connections are opened per-operation."""

package/core/cognition/memory/__init__.py

@@ -0,0 +1,9 @@
+"""Cognitive Layer memory module — Pydantic schemas for capture, knowledge, and insights."""
+
+from core.cognition.memory.schemas import (
+    ActionableInsight,
+    KnowledgeEntry,
+    RawCapture,
+)
+
+__all__ = ["RawCapture", "KnowledgeEntry", "ActionableInsight"]

package/core/cognition/memory/obsidian.py

@@ -0,0 +1,73 @@
+"""ObsidianWriter — persists KnowledgeEntry objects as Obsidian markdown notes."""
+
+import re
+from pathlib import Path
+
+from core.cognition.memory.schemas import KnowledgeEntry
+
+
+CATEGORY_FOLDERS = {
+    "pattern": "Patterns",
+    "anti_pattern": "Anti-Patterns",
+    "solution": "Solutions",
+    "architecture": "Architecture",
+    "config": "Config",
+    "lesson": "Lessons",
+    "improvement": "Improvements",
+}
+
+
+def _slugify(title: str, max_len: int = 80) -> str:
+    """Convert title to a safe filename slug."""
+    slug = title.lower()
+    slug = re.sub(r"[^a-z0-9\s-]", "", slug)
+    slug = re.sub(r"[\s]+", "-", slug.strip())
+    slug = re.sub(r"-+", "-", slug)
+    return slug[:max_len].rstrip("-")
+
+
+def _format_frontmatter(entry: KnowledgeEntry) -> str:
+    """Build YAML frontmatter string from a KnowledgeEntry."""
+    tags_inline = "[" + ", ".join(entry.tags) + "]"
+    stacks_inline = "[" + ", ".join(entry.stacks) + "]"
+    created = entry.created_at.isoformat()
+    updated = entry.updated_at.isoformat()
+
+    lines = [
+        "---",
+        f"title: {entry.title}",
+        f"id: {entry.id}",
+        f"category: {entry.category}",
+        f"tags: {tags_inline}",
+        f"stacks: {stacks_inline}",
+        f"source_project: {entry.source_project}",
+        f"applicable_to: {entry.applicable_to}",
+        f"confidence: {entry.confidence}",
+        f"times_used: {entry.times_used}",
+        f"created_at: {created}",
+        f"updated_at: {updated}",
+        "---",
+    ]
+    return "\n".join(lines)
+
+
+class ObsidianWriter:
+    """Writes KnowledgeEntry objects as Obsidian-compatible markdown notes."""
+
+    def __init__(self, vault_base_path: str) -> None:
+        self._vault = Path(vault_base_path)
+
+    def write(self, entry: KnowledgeEntry) -> str:
+        """Persist a KnowledgeEntry as a markdown note. Returns the file path."""
+        folder_name = CATEGORY_FOLDERS.get(entry.category, "Knowledge")
+        folder = self._vault / folder_name
+        folder.mkdir(parents=True, exist_ok=True)
+
+        slug = _slugify(entry.title)
+        file_path = folder / f"{slug}.md"
+
+        frontmatter = _format_frontmatter(entry)
+        note = f"{frontmatter}\n\n{entry.content}\n"
+
+        file_path.write_text(note, encoding="utf-8")
+        return str(file_path)

package/core/cognition/memory/schemas.py

@@ -0,0 +1,141 @@
+"""Pydantic schemas for the ArkaOS Cognitive Layer memory system.
+
+Three core models:
+- RawCapture: Raw session captures during the day
+- KnowledgeEntry: Curated knowledge for dual-write (SQLite + Obsidian)
+- ActionableInsight: Insights for proactive presentation to the user
+"""
+
+import uuid
+from datetime import datetime, timezone
+from typing import Literal
+
+from pydantic import BaseModel, Field, field_validator
+
+
+# --- Allowed categories and statuses ---
+
+RAW_CAPTURE_CATEGORIES = ("decision", "solution", "pattern", "error", "config")
+
+KNOWLEDGE_ENTRY_CATEGORIES = (
+    "pattern",
+    "anti_pattern",
+    "solution",
+    "architecture",
+    "config",
+    "lesson",
+    "improvement",
+)
+
+ACTIONABLE_INSIGHT_CATEGORIES = ("business", "technical", "ux", "strategy")
+
+ACTIONABLE_INSIGHT_SEVERITIES = ("rethink", "improve", "consider")
+
+ACTIONABLE_INSIGHT_STATUSES = ("pending", "presented", "accepted", "dismissed")
+
+
+def _utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _new_uuid() -> str:
+    return str(uuid.uuid4())
+
+
+# --- Models ---
+
+class RawCapture(BaseModel):
+    """Raw session capture — recorded during active work sessions."""
+
+    id: str = Field(default_factory=_new_uuid)
+    timestamp: datetime = Field(default_factory=_utc_now)
+    session_id: str
+    project_path: str
+    project_name: str
+    category: str
+    content: str
+    context: dict = Field(default_factory=dict)
+
+    @field_validator("category")
+    @classmethod
+    def validate_category(cls, v: str) -> str:
+        if v not in RAW_CAPTURE_CATEGORIES:
+            raise ValueError(
+                f"Invalid category '{v}'. Must be one of: {RAW_CAPTURE_CATEGORIES}"
+            )
+        return v
+
+
+class KnowledgeEntry(BaseModel):
+    """Curated knowledge entry — written to SQLite and Obsidian on dual-write."""
+
+    id: str = Field(default_factory=_new_uuid)
+    title: str
+    category: str
+    tags: list[str] = Field(default_factory=list)
+    stacks: list[str] = Field(default_factory=list)
+    content: str
+    source_project: str
+    applicable_to: str = "any"
+    confidence: float = 0.5
+    times_used: int = 0
+    created_at: datetime = Field(default_factory=_utc_now)
+    updated_at: datetime = Field(default_factory=_utc_now)
+
+    @field_validator("category")
+    @classmethod
+    def validate_category(cls, v: str) -> str:
+        if v not in KNOWLEDGE_ENTRY_CATEGORIES:
+            raise ValueError(
+                f"Invalid category '{v}'. Must be one of: {KNOWLEDGE_ENTRY_CATEGORIES}"
+            )
+        return v
+
+    @field_validator("confidence")
+    @classmethod
+    def clamp_confidence(cls, v: float) -> float:
+        return max(0.0, min(1.0, v))
+
+
+class ActionableInsight(BaseModel):
+    """Actionable insight — generated by agents for proactive presentation."""
+
+    id: str = Field(default_factory=_new_uuid)
+    project: str
+    trigger: str
+    date_generated: datetime = Field(default_factory=_utc_now)
+    category: str
+    severity: str
+    title: str
+    description: str
+    recommendation: str
+    context: str
+    status: str = "pending"
+    presented_at: datetime | None = None
+
+    @field_validator("category")
+    @classmethod
+    def validate_category(cls, v: str) -> str:
+        if v not in ACTIONABLE_INSIGHT_CATEGORIES:
+            raise ValueError(
+                f"Invalid category '{v}'. Must be one of: {ACTIONABLE_INSIGHT_CATEGORIES}"
+            )
+        return v
+
+    @field_validator("severity")
+    @classmethod
+    def validate_severity(cls, v: str) -> str:
+        if v not in ACTIONABLE_INSIGHT_SEVERITIES:
+            raise ValueError(
+                f"Invalid severity '{v}'. Must be one of: {ACTIONABLE_INSIGHT_SEVERITIES}"
+            )
+        return v
+
+    @field_validator("status")
+    @classmethod
+    def validate_status(cls, v: str) -> str:
+        if v not in ACTIONABLE_INSIGHT_STATUSES:
+            raise ValueError(
+                f"Invalid status '{v}'. Must be one of: {ACTIONABLE_INSIGHT_STATUSES}"
+            )
+        return v

package/core/cognition/memory/vector.py

@@ -0,0 +1,223 @@
+"""VectorWriter — stores KnowledgeEntry objects in SQLite with optional embeddings.
+
+Graceful degradation: if fastembed is not installed, falls back to text-based
+keyword matching for search.
+"""
+
+import json
+import math
+import sqlite3
+import struct
+from typing import Optional
+
+from core.cognition.memory.schemas import KnowledgeEntry
+
+# Optional embedder — imported lazily to survive missing fastembed
+try:
+    from core.knowledge import embedder as _embedder
+except ImportError:
+    _embedder = None  # type: ignore[assignment]
+
+
+_CREATE_TABLE = """
+CREATE TABLE IF NOT EXISTS knowledge_entries (
+    id              TEXT PRIMARY KEY,
+    entry_id        TEXT UNIQUE NOT NULL,
+    title           TEXT NOT NULL,
+    category        TEXT NOT NULL,
+    tags            TEXT NOT NULL,
+    stacks          TEXT NOT NULL,
+    content         TEXT NOT NULL,
+    source_project  TEXT NOT NULL,
+    applicable_to   TEXT NOT NULL,
+    confidence      REAL NOT NULL,
+    times_used      INTEGER NOT NULL,
+    embedding       BLOB,
+    created_at      TEXT NOT NULL,
+    updated_at      TEXT NOT NULL
+)
+"""
+
+_CREATE_IDX_ENTRY_ID = "CREATE INDEX IF NOT EXISTS idx_entry_id ON knowledge_entries (entry_id)"
+_CREATE_IDX_CATEGORY = "CREATE INDEX IF NOT EXISTS idx_category ON knowledge_entries (category)"
+_CREATE_IDX_APPLICABLE = "CREATE INDEX IF NOT EXISTS idx_applicable_to ON knowledge_entries (applicable_to)"
+
+_UPSERT = """
+INSERT INTO knowledge_entries
+    (id, entry_id, title, category, tags, stacks, content, source_project,
+     applicable_to, confidence, times_used, embedding, created_at, updated_at)
+VALUES
+    (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT(entry_id) DO UPDATE SET
+    title          = excluded.title,
+    category       = excluded.category,
+    tags           = excluded.tags,
+    stacks         = excluded.stacks,
+    content        = excluded.content,
+    source_project = excluded.source_project,
+    applicable_to  = excluded.applicable_to,
+    confidence     = excluded.confidence,
+    times_used     = excluded.times_used,
+    embedding      = excluded.embedding,
+    updated_at     = excluded.updated_at
+"""
+
+
+def _pack_embedding(floats: list[float]) -> bytes:
+    """Pack a list of floats into a compact binary blob."""
+    return struct.pack(f"{len(floats)}f", *floats)
+
+
+def _unpack_embedding(blob: bytes) -> list[float]:
+    """Unpack a binary blob back into a list of floats."""
+    count = len(blob) // struct.calcsize("f")
+    return list(struct.unpack(f"{count}f", blob))
+
+
+def _cosine_similarity(a: list[float], b: list[float]) -> float:
+    """Compute cosine similarity between two vectors."""
+    dot = sum(x * y for x, y in zip(a, b))
+    norm_a = math.sqrt(sum(x * x for x in a))
+    norm_b = math.sqrt(sum(x * x for x in b))
+    if norm_a == 0.0 or norm_b == 0.0:
+        return 0.0
+    return dot / (norm_a * norm_b)
+
+
+def _text_score(query: str, row: dict) -> int:
+    """Count keyword matches in title, tags, and content."""
+    terms = query.lower().split()
+    haystack = " ".join([
+        row["title"],
+        " ".join(json.loads(row["tags"])),
+        row["content"],
+    ]).lower()
+    return sum(1 for term in terms if term in haystack)
+
+
+def _row_to_dict(row: sqlite3.Row) -> dict:
+    """Convert a sqlite3.Row to the public result dict."""
+    return {
+        "entry_id": row["entry_id"],
+        "title": row["title"],
+        "category": row["category"],
+        "tags": json.loads(row["tags"]),
+        "stacks": json.loads(row["stacks"]),
+        "content": row["content"],
+        "source_project": row["source_project"],
+        "applicable_to": row["applicable_to"],
+        "confidence": row["confidence"],
+        "times_used": row["times_used"],
+    }
+
+
+class VectorWriter:
+    """Writes KnowledgeEntry objects to SQLite, optionally with embeddings."""
+
+    def __init__(self, db_path: str) -> None:
+        self._db_path = db_path
+        self._conn = sqlite3.connect(db_path)
+        self._conn.row_factory = sqlite3.Row
+        self._bootstrap()
+
+    # --- Setup ---
+
+    def _bootstrap(self) -> None:
+        cur = self._conn.cursor()
+        cur.execute(_CREATE_TABLE)
+        cur.execute(_CREATE_IDX_ENTRY_ID)
+        cur.execute(_CREATE_IDX_CATEGORY)
+        cur.execute(_CREATE_IDX_APPLICABLE)
+        self._conn.commit()
+
+    # --- Public API ---
+
+    def write(self, entry: KnowledgeEntry) -> bool:
+        """UPSERT a KnowledgeEntry. Returns True on success."""
+        embedding_blob: Optional[bytes] = None
+        vector = self._embed(entry)
+        if vector is not None:
+            embedding_blob = _pack_embedding(vector)
+
+        self._conn.execute(
+            _UPSERT,
+            (
+                entry.id,
+                entry.id,              # entry_id mirrors id (stable natural key)
+                entry.title,
+                entry.category,
+                json.dumps(entry.tags),
+                json.dumps(entry.stacks),
+                entry.content,
+                entry.source_project,
+                entry.applicable_to,
+                entry.confidence,
+                entry.times_used,
+                embedding_blob,
+                entry.created_at.isoformat(),
+                entry.updated_at.isoformat(),
+            ),
+        )
+        self._conn.commit()
+        return True
+
+    def search(self, query: str, top_k: int = 5) -> list[dict]:
+        """Search entries. Semantic when embeddings available, text fallback otherwise."""
+        rows = self._conn.execute(
+            "SELECT * FROM knowledge_entries"
+        ).fetchall()
+
+        if not rows:
+            return []
+
+        use_semantic = self._can_embed() and any(r["embedding"] is not None for r in rows)
+
+        if use_semantic:
+            return self._semantic_search(query, rows, top_k)
+        return self._text_search(query, rows, top_k)
+
+    def close(self) -> None:
+        """Close the database connection."""
+        self._conn.close()
+
+    # --- Internal helpers ---
+
+    def _embed(self, entry: KnowledgeEntry) -> Optional[list[float]]:
+        """Try to embed the entry's title + content. Returns None on failure."""
+        if not self._can_embed():
+            return None
+        text = f"{entry.title}\n{entry.content}"
+        return _embedder.embed(text)  # type: ignore[union-attr]
+
+    def _can_embed(self) -> bool:
+        return _embedder is not None and _embedder.is_available()
+
+    def _semantic_search(
+        self, query: str, rows: list[sqlite3.Row], top_k: int
+    ) -> list[dict]:
+        query_vec = _embedder.embed(query)  # type: ignore[union-attr]
+        if query_vec is None:
+            return self._text_search(query, rows, top_k)
+
+        scored = []
+        for row in rows:
+            if row["embedding"] is None:
+                continue
+            entry_vec = _unpack_embedding(row["embedding"])
+            score = _cosine_similarity(query_vec, entry_vec)
+            scored.append((score, row))
+
+        scored.sort(key=lambda x: x[0], reverse=True)
+        return [_row_to_dict(r) for _, r in scored[:top_k]]
+
+    def _text_search(
+        self, query: str, rows: list[sqlite3.Row], top_k: int
+    ) -> list[dict]:
+        scored = []
+        for row in rows:
+            score = _text_score(query, dict(row))
+            if score > 0:
+                scored.append((score, row))
+
+        scored.sort(key=lambda x: x[0], reverse=True)
+        return [_row_to_dict(r) for _, r in scored[:top_k]]

package/core/cognition/memory/writer.py

@@ -0,0 +1,57 @@
+"""DualWriter — unified interface that writes KnowledgeEntry to Obsidian and Vector DB."""
+
+from dataclasses import dataclass, field
+
+from core.cognition.memory.obsidian import ObsidianWriter
+from core.cognition.memory.schemas import KnowledgeEntry
+from core.cognition.memory.vector import VectorWriter
+
+
+@dataclass
+class WriteResult:
+    """Result of a dual-write operation, with independent success/error per backend."""
+
+    obsidian_path: str | None = None
+    obsidian_error: str | None = None
+    vector_indexed: bool = False
+    vector_error: str | None = None
+
+
+class DualWriter:
+    """Writes KnowledgeEntry objects to both Obsidian and Vector DB in one call.
+
+    Each backend fails independently — if Obsidian fails, Vector still writes,
+    and vice versa. All results are captured in WriteResult.
+    """
+
+    def __init__(self, obsidian_base: str, vector_db_path: str) -> None:
+        self._obsidian = ObsidianWriter(vault_base_path=obsidian_base)
+        self._vector = VectorWriter(db_path=vector_db_path)
+
+    def write(self, entry: KnowledgeEntry) -> WriteResult:
+        """Write a single entry to both backends. Returns a WriteResult."""
+        result = WriteResult()
+
+        try:
+            result.obsidian_path = self._obsidian.write(entry)
+        except Exception as exc:  # noqa: BLE001
+            result.obsidian_error = str(exc)
+
+        try:
+            result.vector_indexed = self._vector.write(entry)
+        except Exception as exc:  # noqa: BLE001
+            result.vector_error = str(exc)
+
+        return result
+
+    def write_batch(self, entries: list[KnowledgeEntry]) -> list[WriteResult]:
+        """Write a list of entries. Returns one WriteResult per entry."""
+        return [self.write(entry) for entry in entries]
+
+    def search(self, query: str, top_k: int = 5) -> list[dict]:
+        """Search the vector store. Delegates directly to VectorWriter."""
+        return self._vector.search(query, top_k=top_k)
+
+    def close(self) -> None:
+        """Close the vector DB connection."""
+        self._vector.close()

package/core/cognition/research/__init__.py

@@ -0,0 +1,5 @@
+"""Research profiler — infers research topics from active project ecosystems."""
+
+from .profiler import ResearchProfiler, ResearchProfile, ResearchTopic
+
+__all__ = ["ResearchProfiler", "ResearchProfile", "ResearchTopic"]