arkaos 2.0.1 → 2.0.3

package/core/knowledge/indexer.py

@@ -0,0 +1,97 @@
+"""Knowledge indexer — walk directories and index markdown files.
+
+Supports incremental indexing (skips already-indexed files by hash).
+"""
+
+import hashlib
+from pathlib import Path
+from typing import Callable, Optional
+
+from core.knowledge.chunker import chunk_markdown
+from core.knowledge.vector_store import VectorStore
+
+
+def file_hash(path: Path) -> str:
+    """Compute SHA-256 hash of file content."""
+    return hashlib.sha256(path.read_bytes()).hexdigest()[:16]
+
+
+def index_directory(
+    directory: str | Path,
+    store: VectorStore,
+    pattern: str = "**/*.md",
+    on_progress: Optional[Callable[[int, int, str], None]] = None,
+    max_tokens: int = 512,
+    skip_indexed: bool = True,
+) -> dict:
+    """Index all markdown files in a directory.
+
+    Args:
+        directory: Root directory to scan.
+        store: VectorStore to index into.
+        pattern: Glob pattern for files.
+        on_progress: Callback(current, total, filename).
+        max_tokens: Max tokens per chunk.
+        skip_indexed: Skip files already indexed (by hash).
+
+    Returns:
+        Dict with: files_scanned, files_indexed, files_skipped, chunks_created.
+    """
+    root = Path(directory)
+    if not root.exists():
+        return {"files_scanned": 0, "files_indexed": 0, "files_skipped": 0, "chunks_created": 0}
+
+    files = sorted(root.glob(pattern))
+    # Skip hidden dirs (.obsidian, .git)
+    files = [f for f in files if not any(part.startswith(".") for part in f.relative_to(root).parts)]
+
+    total = len(files)
+    indexed = 0
+    skipped = 0
+    chunks_created = 0
+
+    for i, filepath in enumerate(files):
+        if on_progress:
+            on_progress(i + 1, total, filepath.name)
+
+        fhash = file_hash(filepath)
+
+        if skip_indexed and store.is_file_indexed(fhash):
+            skipped += 1
+            continue
+
+        try:
+            content = filepath.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            skipped += 1
+            continue
+
+        # Skip very small files
+        if len(content.split()) < 20:
+            skipped += 1
+            continue
+
+        # Remove old chunks for this file (re-index)
+        store.remove_file(str(filepath))
+
+        # Chunk and index
+        chunks = chunk_markdown(content, max_tokens=max_tokens, source=str(filepath))
+        if chunks:
+            texts = [c.text for c in chunks]
+            headings = [c.heading for c in chunks]
+            count = store.index_chunks(
+                texts=texts,
+                headings=headings,
+                source=str(filepath),
+                file_hash=fhash,
+                metadata={"relative_path": str(filepath.relative_to(root))},
+            )
+            chunks_created += count
+            indexed += 1
+
+    return {
+        "files_scanned": total,
+        "files_indexed": indexed,
+        "files_skipped": skipped,
+        "chunks_created": chunks_created,
+    }

package/core/knowledge/vector_store.py

@@ -0,0 +1,213 @@
+"""Vector store — SQLite-VSS backed semantic search.
+
+Stores document chunks with embeddings for fast similarity search.
+Graceful degradation: works without sqlite-vss (brute-force fallback).
+"""
+
+import json
+import sqlite3
+import time
+from pathlib import Path
+from typing import Any, Optional
+
+from core.knowledge.embedder import embed, embed_batch, EMBEDDING_DIMS
+
+
+def _load_vss(db: sqlite3.Connection) -> bool:
+    """Try to load sqlite-vss extension."""
+    try:
+        db.enable_load_extension(True)
+        import sqlite_vss
+        sqlite_vss.load(db)
+        return True
+    except (ImportError, Exception):
+        return False
+
+
+class VectorStore:
+    """SQLite-VSS backed vector store for knowledge retrieval."""
+
+    def __init__(self, db_path: str | Path = ":memory:") -> None:
+        self._db_path = str(db_path)
+        self._db = sqlite3.connect(self._db_path)
+        self._db.row_factory = sqlite3.Row
+        self._vss_available = _load_vss(self._db)
+        self._init_schema()
+
+    def _init_schema(self) -> None:
+        """Create tables if they don't exist."""
+        self._db.executescript("""
+            CREATE TABLE IF NOT EXISTS chunks (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                text TEXT NOT NULL,
+                heading TEXT DEFAULT '',
+                source TEXT DEFAULT '',
+                file_hash TEXT DEFAULT '',
+                metadata TEXT DEFAULT '{}',
+                created_at REAL DEFAULT (unixepoch('now')),
+                embedding BLOB
+            );
+            CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);
+            CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(file_hash);
+        """)
+        if self._vss_available:
+            try:
+                self._db.execute(
+                    f"CREATE VIRTUAL TABLE IF NOT EXISTS vss_chunks USING vss0(embedding({EMBEDDING_DIMS}))"
+                )
+            except Exception:
+                self._vss_available = False
+        self._db.commit()
+
+    def index_chunks(
+        self,
+        texts: list[str],
+        headings: list[str] | None = None,
+        source: str = "",
+        file_hash: str = "",
+        metadata: dict[str, Any] | None = None,
+    ) -> int:
+        """Index multiple text chunks with embeddings.
+
+        Returns number of chunks indexed.
+        """
+        if not texts:
+            return 0
+
+        embeddings = embed_batch(texts)
+        meta_json = json.dumps(metadata or {})
+        count = 0
+
+        for i, text in enumerate(texts):
+            heading = headings[i] if headings and i < len(headings) else ""
+            emb_blob = None
+
+            if embeddings and i < len(embeddings):
+                emb_blob = _vec_to_blob(embeddings[i])
+
+            cursor = self._db.execute(
+                "INSERT INTO chunks (text, heading, source, file_hash, metadata, embedding) VALUES (?, ?, ?, ?, ?, ?)",
+                (text, heading, source, file_hash, meta_json, emb_blob),
+            )
+
+            if self._vss_available and emb_blob:
+                self._db.execute(
+                    "INSERT INTO vss_chunks (rowid, embedding) VALUES (?, ?)",
+                    (cursor.lastrowid, emb_blob),
+                )
+            count += 1
+
+        self._db.commit()
+        return count
+
+    def search(self, query: str, top_k: int = 5) -> list[dict]:
+        """Search for similar chunks.
+
+        Returns list of dicts with: text, heading, source, score, metadata.
+        """
+        # Check if store has any data
+        total = self._db.execute("SELECT COUNT(*) as cnt FROM chunks").fetchone()["cnt"]
+        if total == 0:
+            return []
+
+        query_emb = embed(query)
+
+        if query_emb and self._vss_available:
+            try:
+                return self._vss_search(query_emb, top_k)
+            except Exception:
+                return self._keyword_search(query, top_k)
+
+        # Fallback: keyword search
+        return self._keyword_search(query, top_k)
+
+    def _vss_search(self, query_emb: list[float], top_k: int) -> list[dict]:
+        """Vector similarity search via sqlite-vss."""
+        query_blob = _vec_to_blob(query_emb)
+        rows = self._db.execute("""
+            SELECT c.text, c.heading, c.source, c.metadata, v.distance
+            FROM vss_chunks v
+            JOIN chunks c ON c.id = v.rowid
+            WHERE vss_search(v.embedding, vss_search_params(?, ?))
+        """, (query_blob, top_k)).fetchall()
+
+        return [
+            {
+                "text": r["text"],
+                "heading": r["heading"],
+                "source": r["source"],
+                "score": 1.0 - r["distance"],  # Convert distance to similarity
+                "metadata": json.loads(r["metadata"]),
+            }
+            for r in rows
+        ]
+
+    def _keyword_search(self, query: str, top_k: int) -> list[dict]:
+        """Fallback keyword search when VSS unavailable."""
+        words = query.lower().split()
+        if not words:
+            return []
+
+        conditions = " OR ".join(["lower(text) LIKE ?" for _ in words])
+        params = [f"%{w}%" for w in words[:5]]  # Max 5 keywords
+
+        rows = self._db.execute(
+            f"SELECT text, heading, source, metadata FROM chunks WHERE {conditions} LIMIT ?",
+            params + [top_k],
+        ).fetchall()
+
+        return [
+            {
+                "text": r["text"],
+                "heading": r["heading"],
+                "source": r["source"],
+                "score": 0.5,  # No real score for keyword search
+                "metadata": json.loads(r["metadata"]),
+            }
+            for r in rows
+        ]
+
+    def is_file_indexed(self, file_hash: str) -> bool:
+        """Check if a file has already been indexed."""
+        row = self._db.execute(
+            "SELECT COUNT(*) as cnt FROM chunks WHERE file_hash = ?", (file_hash,)
+        ).fetchone()
+        return row["cnt"] > 0
+
+    def remove_file(self, source: str) -> int:
+        """Remove all chunks from a source file."""
+        if self._vss_available:
+            rows = self._db.execute("SELECT id FROM chunks WHERE source = ?", (source,)).fetchall()
+            for r in rows:
+                self._db.execute("DELETE FROM vss_chunks WHERE rowid = ?", (r["id"],))
+        deleted = self._db.execute("DELETE FROM chunks WHERE source = ?", (source,)).rowcount
+        self._db.commit()
+        return deleted
+
+    def get_stats(self) -> dict:
+        """Get store statistics."""
+        total = self._db.execute("SELECT COUNT(*) as cnt FROM chunks").fetchone()["cnt"]
+        sources = self._db.execute("SELECT COUNT(DISTINCT source) as cnt FROM chunks").fetchone()["cnt"]
+        return {
+            "total_chunks": total,
+            "total_files": sources,
+            "vss_available": self._vss_available,
+            "db_path": self._db_path,
+        }
+
+    def clear(self) -> None:
+        """Remove all data."""
+        if self._vss_available:
+            self._db.execute("DELETE FROM vss_chunks")
+        self._db.execute("DELETE FROM chunks")
+        self._db.commit()
+
+    def close(self) -> None:
+        """Close database connection."""
+        self._db.close()
+
+
+def _vec_to_blob(vec: list[float]) -> bytes:
+    """Convert float vector to bytes for SQLite storage."""
+    import struct
+    return struct.pack(f"{len(vec)}f", *vec)

package/core/obsidian/__init__.py

@@ -0,0 +1,6 @@
+"""Obsidian vault integration — write workflow outputs to knowledge base."""
+
+from core.obsidian.writer import ObsidianWriter
+from core.obsidian.templates import build_frontmatter
+
+__all__ = ["ObsidianWriter", "build_frontmatter"]

package/core/obsidian/templates.py

@@ -0,0 +1,76 @@
+"""Obsidian frontmatter templates and conventions."""
+
+from datetime import date, datetime
+from typing import Any
+
+
+def build_frontmatter(
+    department: str = "",
+    agent: str = "",
+    workflow: str = "",
+    tags: list[str] | None = None,
+    extra: dict[str, Any] | None = None,
+) -> str:
+    """Build YAML frontmatter for an Obsidian note.
+
+    Returns:
+        String with opening/closing --- delimiters.
+    """
+    fields: dict[str, Any] = {}
+    fields["created"] = datetime.now().strftime("%Y-%m-%d %H:%M")
+    fields["source"] = "arkaos"
+
+    if department:
+        fields["department"] = department
+    if agent:
+        fields["agent"] = agent
+    if workflow:
+        fields["workflow"] = workflow
+
+    all_tags = ["arkaos"]
+    if department:
+        all_tags.append(f"dept/{department}")
+    if tags:
+        all_tags.extend(tags)
+    fields["tags"] = all_tags
+
+    if extra:
+        fields.update(extra)
+
+    lines = ["---"]
+    for key, value in fields.items():
+        if isinstance(value, list):
+            lines.append(f"{key}:")
+            for item in value:
+                lines.append(f"  - {item}")
+        elif isinstance(value, bool):
+            lines.append(f"{key}: {'true' if value else 'false'}")
+        else:
+            lines.append(f"{key}: {value}")
+    lines.append("---")
+
+    return "\n".join(lines)
+
+
+def resolve_template_vars(path: str, vars: dict[str, str] | None = None) -> str:
+    """Resolve template variables in an Obsidian path.
+
+    Supported variables:
+        {project}, {department}, {agent}, {date}, {number}, {name}
+    """
+    defaults = {
+        "date": date.today().isoformat(),
+        "project": "Default",
+        "department": "general",
+        "agent": "unknown",
+        "number": "001",
+        "name": "untitled",
+    }
+    if vars:
+        defaults.update(vars)
+
+    result = path
+    for key, value in defaults.items():
+        result = result.replace(f"{{{key}}}", value)
+
+    return result

package/core/obsidian/writer.py

@@ -0,0 +1,148 @@
+"""Obsidian vault writer — save workflow outputs to knowledge base.
+
+Resolves vault path from:
+1. Constructor argument
+2. knowledge/obsidian-config.json → vault_path
+3. ARKAOS_VAULT environment variable
+4. Fallback: ~/.arkaos/vault/
+"""
+
+import json
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Optional
+
+from core.obsidian.templates import build_frontmatter, resolve_template_vars
+
+
+class ObsidianWriter:
+    """Writes markdown files to an Obsidian vault with frontmatter."""
+
+    def __init__(self, vault_path: str | Path | None = None, arkaos_root: str | Path | None = None) -> None:
+        self._vault_path = self._resolve_vault_path(vault_path, arkaos_root)
+
+    def save(
+        self,
+        obsidian_path: str,
+        content: str,
+        department: str = "",
+        agent: str = "",
+        workflow: str = "",
+        tags: list[str] | None = None,
+        template_vars: dict[str, str] | None = None,
+        extra_frontmatter: dict[str, Any] | None = None,
+    ) -> Path:
+        """Save content to the Obsidian vault.
+
+        Args:
+            obsidian_path: Relative path within vault (may contain template vars).
+            content: Markdown content to save.
+            department: Source department.
+            agent: Agent that produced the output.
+            workflow: Workflow that generated this output.
+            tags: Additional tags for frontmatter.
+            template_vars: Variables to resolve in path ({project}, {date}, etc.).
+            extra_frontmatter: Additional frontmatter fields.
+
+        Returns:
+            Absolute path to the saved file.
+        """
+        # Resolve template variables in path
+        resolved_path = resolve_template_vars(obsidian_path, template_vars)
+
+        # Build full path
+        full_path = self._vault_path / resolved_path
+
+        # If path doesn't end with .md, treat as directory and add filename
+        if not full_path.suffix:
+            timestamp = datetime.now().strftime("%Y-%m-%d_%H%M")
+            filename = f"{department or 'output'}-{timestamp}.md"
+            full_path = full_path / filename
+        elif full_path.suffix != ".md":
+            full_path = full_path.with_suffix(".md")
+
+        # Handle duplicate filenames
+        if full_path.exists():
+            stem = full_path.stem
+            timestamp = datetime.now().strftime("%H%M%S")
+            full_path = full_path.with_stem(f"{stem}-{timestamp}")
+
+        # Create directories
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Build frontmatter
+        frontmatter = build_frontmatter(
+            department=department,
+            agent=agent,
+            workflow=workflow,
+            tags=tags,
+            extra=extra_frontmatter,
+        )
+
+        # Write file
+        file_content = f"{frontmatter}\n\n{content}"
+        full_path.write_text(file_content, encoding="utf-8")
+
+        return full_path
+
+    def ensure_vault(self) -> bool:
+        """Verify the vault directory exists."""
+        return self._vault_path.exists()
+
+    def list_outputs(self, department: str = "", limit: int = 50) -> list[Path]:
+        """List recent ArkaOS outputs in the vault."""
+        if not self._vault_path.exists():
+            return []
+
+        pattern = "**/*.md"
+        files = sorted(self._vault_path.glob(pattern), key=lambda p: p.stat().st_mtime, reverse=True)
+
+        results = []
+        for f in files:
+            if len(results) >= limit:
+                break
+            try:
+                head = f.read_text(encoding="utf-8")[:200]
+                if "source: arkaos" in head:
+                    if not department or f"department: {department}" in head:
+                        results.append(f)
+            except (OSError, UnicodeDecodeError):
+                continue
+
+        return results
+
+    @property
+    def vault_path(self) -> Path:
+        return self._vault_path
+
+    def _resolve_vault_path(self, explicit: str | Path | None, arkaos_root: str | Path | None) -> Path:
+        """Resolve vault path from multiple sources."""
+        # 1. Explicit argument
+        if explicit:
+            return Path(explicit)
+
+        # 2. Config file
+        if arkaos_root:
+            config_path = Path(arkaos_root) / "knowledge" / "obsidian-config.json"
+        else:
+            config_path = Path(__file__).resolve().parent.parent.parent / "knowledge" / "obsidian-config.json"
+
+        if config_path.exists():
+            try:
+                config = json.loads(config_path.read_text())
+                vault = config.get("vault_path", "")
+                if vault and Path(vault).exists():
+                    return Path(vault)
+            except (json.JSONDecodeError, OSError):
+                pass
+
+        # 3. Environment variable
+        env_vault = os.environ.get("ARKAOS_VAULT", "")
+        if env_vault and Path(env_vault).exists():
+            return Path(env_vault)
+
+        # 4. Fallback
+        fallback = Path.home() / ".arkaos" / "vault"
+        fallback.mkdir(parents=True, exist_ok=True)
+        return fallback

package/core/orchestration/__init__.py

@@ -0,0 +1,6 @@
+"""Orchestration protocol — coordination patterns for multi-agent work."""
+
+from core.orchestration.protocol import OrchestrationPattern, PhaseHandoff, OrchestrationPlan
+from core.orchestration.patterns import PATTERNS
+
+__all__ = ["OrchestrationPattern", "PhaseHandoff", "OrchestrationPlan", "PATTERNS"]