arkaos 2.0.2 → 2.1.0

package/VERSION

@@ -1 +1 @@
-2.0.2
+2.1.0

package/config/constitution.yaml

@@ -60,6 +60,8 @@ enforcement_levels:
 
   quality_gate:
     description: "Mandatory pre-delivery review. Nothing ships without APPROVED verdict."
+    trigger: "After the last execution phase, before delivery to user"
+    frequency: "Once per workflow execution, not per phase"
     agents:
       orchestrator:
         id: cqo-marta

package/config/hooks/user-prompt-submit-v2.sh

@@ -7,6 +7,17 @@
 
 input=$(cat)
 
+# ─── V1 Migration Detection ─────────────────────────────────────────────
+V1_PATHS=("$HOME/.claude/skills/arka-os" "$HOME/.claude/skills/arkaos")
+MIGRATION_MARKER="$HOME/.arkaos/migrated-from-v1"
+
+for v1_path in "${V1_PATHS[@]}"; do
+  if [ -d "$v1_path" ] && [ ! -f "$MIGRATION_MARKER" ]; then
+    echo "{\"additionalContext\": \"[MIGRATION] ArkaOS v1 detected at $v1_path. Run: npx arkaos migrate — This will backup v1, preserve your data, and install v2. See: https://github.com/andreagroferreira/arka-os#install\"}"
+    exit 0
+  fi
+done
+
 # ─── Performance Timing ──────────────────────────────────────────────────
 _HOOK_START_NS=$(date +%s%N 2>/dev/null || echo "0")
 _hook_ms() {

package/core/knowledge/__init__.py

@@ -0,0 +1,6 @@
+"""Knowledge system — vector store, chunking, embedding, and retrieval."""
+
+from core.knowledge.chunker import chunk_markdown
+from core.knowledge.vector_store import VectorStore
+
+__all__ = ["VectorStore", "chunk_markdown"]

package/core/knowledge/chunker.py

@@ -0,0 +1,121 @@
+"""Markdown chunker — split documents into embeddable chunks.
+
+Splits on paragraph boundaries, respects heading structure,
+and maintains overlap for context continuity.
+"""
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass
+class Chunk:
+    """A text chunk ready for embedding."""
+    text: str
+    heading: str = ""       # Current heading context
+    index: int = 0          # Position in document
+    source: str = ""        # Source file path
+
+    @property
+    def token_estimate(self) -> int:
+        return len(self.text.split())
+
+
+def chunk_markdown(
+    content: str,
+    max_tokens: int = 512,
+    overlap_tokens: int = 50,
+    source: str = "",
+) -> list[Chunk]:
+    """Split markdown content into chunks at paragraph boundaries.
+
+    Args:
+        content: Markdown text to chunk.
+        max_tokens: Maximum tokens per chunk.
+        overlap_tokens: Token overlap between consecutive chunks.
+        source: Source file path for metadata.
+
+    Returns:
+        List of Chunk objects.
+    """
+    # Strip frontmatter
+    body = content
+    if content.startswith("---"):
+        end = content.find("---", 3)
+        if end != -1:
+            body = content[end + 3:].strip()
+
+    # Split into paragraphs (double newline) preserving headings
+    blocks = re.split(r'\n\n+', body)
+    blocks = [b.strip() for b in blocks if b.strip()]
+
+    chunks: list[Chunk] = []
+    current_heading = ""
+    current_text = ""
+    current_tokens = 0
+
+    for block in blocks:
+        # Track headings
+        heading_match = re.match(r'^(#{1,6})\s+(.+)', block)
+        if heading_match:
+            current_heading = heading_match.group(2)
+
+        block_tokens = len(block.split())
+
+        # If single block exceeds max, split it
+        if block_tokens > max_tokens:
+            if current_text:
+                chunks.append(Chunk(
+                    text=current_text.strip(),
+                    heading=current_heading,
+                    index=len(chunks),
+                    source=source,
+                ))
+                current_text = ""
+                current_tokens = 0
+
+            # Split large block by sentences
+            sentences = re.split(r'(?<=[.!?])\s+', block)
+            for sentence in sentences:
+                sent_tokens = len(sentence.split())
+                if current_tokens + sent_tokens > max_tokens and current_text:
+                    chunks.append(Chunk(
+                        text=current_text.strip(),
+                        heading=current_heading,
+                        index=len(chunks),
+                        source=source,
+                    ))
+                    # Overlap: keep last few words
+                    words = current_text.split()
+                    current_text = " ".join(words[-overlap_tokens:]) + " " if len(words) > overlap_tokens else ""
+                    current_tokens = len(current_text.split())
+                current_text += sentence + " "
+                current_tokens += sent_tokens
+            continue
+
+        # Check if adding this block exceeds limit
+        if current_tokens + block_tokens > max_tokens and current_text:
+            chunks.append(Chunk(
+                text=current_text.strip(),
+                heading=current_heading,
+                index=len(chunks),
+                source=source,
+            ))
+            # Overlap
+            words = current_text.split()
+            current_text = " ".join(words[-overlap_tokens:]) + " " if len(words) > overlap_tokens else ""
+            current_tokens = len(current_text.split())
+
+        current_text += block + "\n\n"
+        current_tokens += block_tokens
+
+    # Final chunk
+    if current_text.strip():
+        chunks.append(Chunk(
+            text=current_text.strip(),
+            heading=current_heading,
+            index=len(chunks),
+            source=source,
+        ))
+
+    return chunks

package/core/knowledge/embedder.py

@@ -0,0 +1,52 @@
+"""Embedding wrapper — local embeddings via fastembed.
+
+Graceful degradation: if fastembed is not installed, returns None
+and the vector store falls back to keyword matching.
+"""
+
+from typing import Optional
+
+# Lazy import — fastembed is optional
+_model = None
+_model_name = "BAAI/bge-small-en-v1.5"  # 384 dims, fast, good quality
+EMBEDDING_DIMS = 384
+
+
+def get_model():
+    """Get or create the embedding model (lazy singleton)."""
+    global _model
+    if _model is None:
+        try:
+            from fastembed import TextEmbedding
+            _model = TextEmbedding(_model_name)
+        except ImportError:
+            return None
+    return _model
+
+
+def embed(text: str) -> Optional[list[float]]:
+    """Embed a single text. Returns None if fastembed unavailable."""
+    model = get_model()
+    if model is None:
+        return None
+    results = list(model.embed([text]))
+    return results[0].tolist() if results else None
+
+
+def embed_batch(texts: list[str]) -> Optional[list[list[float]]]:
+    """Embed multiple texts. Returns None if fastembed unavailable."""
+    if not texts:
+        return []
+    model = get_model()
+    if model is None:
+        return None
+    return [emb.tolist() for emb in model.embed(texts)]
+
+
+def is_available() -> bool:
+    """Check if embedding model is available."""
+    try:
+        from fastembed import TextEmbedding
+        return True
+    except ImportError:
+        return False

package/core/knowledge/indexer.py

@@ -0,0 +1,97 @@
+"""Knowledge indexer — walk directories and index markdown files.
+
+Supports incremental indexing (skips already-indexed files by hash).
+"""
+
+import hashlib
+from pathlib import Path
+from typing import Callable, Optional
+
+from core.knowledge.chunker import chunk_markdown
+from core.knowledge.vector_store import VectorStore
+
+
+def file_hash(path: Path) -> str:
+    """Compute SHA-256 hash of file content."""
+    return hashlib.sha256(path.read_bytes()).hexdigest()[:16]
+
+
+def index_directory(
+    directory: str | Path,
+    store: VectorStore,
+    pattern: str = "**/*.md",
+    on_progress: Optional[Callable[[int, int, str], None]] = None,
+    max_tokens: int = 512,
+    skip_indexed: bool = True,
+) -> dict:
+    """Index all markdown files in a directory.
+
+    Args:
+        directory: Root directory to scan.
+        store: VectorStore to index into.
+        pattern: Glob pattern for files.
+        on_progress: Callback(current, total, filename).
+        max_tokens: Max tokens per chunk.
+        skip_indexed: Skip files already indexed (by hash).
+
+    Returns:
+        Dict with: files_scanned, files_indexed, files_skipped, chunks_created.
+    """
+    root = Path(directory)
+    if not root.exists():
+        return {"files_scanned": 0, "files_indexed": 0, "files_skipped": 0, "chunks_created": 0}
+
+    files = sorted(root.glob(pattern))
+    # Skip hidden dirs (.obsidian, .git)
+    files = [f for f in files if not any(part.startswith(".") for part in f.relative_to(root).parts)]
+
+    total = len(files)
+    indexed = 0
+    skipped = 0
+    chunks_created = 0
+
+    for i, filepath in enumerate(files):
+        if on_progress:
+            on_progress(i + 1, total, filepath.name)
+
+        fhash = file_hash(filepath)
+
+        if skip_indexed and store.is_file_indexed(fhash):
+            skipped += 1
+            continue
+
+        try:
+            content = filepath.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            skipped += 1
+            continue
+
+        # Skip very small files
+        if len(content.split()) < 20:
+            skipped += 1
+            continue
+
+        # Remove old chunks for this file (re-index)
+        store.remove_file(str(filepath))
+
+        # Chunk and index
+        chunks = chunk_markdown(content, max_tokens=max_tokens, source=str(filepath))
+        if chunks:
+            texts = [c.text for c in chunks]
+            headings = [c.heading for c in chunks]
+            count = store.index_chunks(
+                texts=texts,
+                headings=headings,
+                source=str(filepath),
+                file_hash=fhash,
+                metadata={"relative_path": str(filepath.relative_to(root))},
+            )
+            chunks_created += count
+            indexed += 1
+
+    return {
+        "files_scanned": total,
+        "files_indexed": indexed,
+        "files_skipped": skipped,
+        "chunks_created": chunks_created,
+    }

package/core/knowledge/ingest.py

@@ -0,0 +1,270 @@
+"""Knowledge ingest engine — process YouTube, PDF, audio, web, markdown.
+
+Downloads, transcribes, extracts text, chunks, embeds, and indexes into
+the vector store. Reports progress via callback for real-time UI updates.
+"""
+
+import os
+import re
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable, Optional
+
+from core.knowledge.chunker import chunk_markdown
+from core.knowledge.vector_store import VectorStore
+
+
+@dataclass
+class IngestResult:
+    """Result of an ingest operation."""
+    source: str
+    source_type: str
+    text_length: int = 0
+    chunks_created: int = 0
+    title: str = ""
+    error: str = ""
+    success: bool = True
+
+
+ProgressCallback = Callable[[int, str], None]  # (percent, message)
+
+
+def detect_source_type(source: str) -> str:
+    """Auto-detect content type from URL or file extension."""
+    source_lower = source.lower()
+
+    # YouTube URLs
+    if any(domain in source_lower for domain in ["youtube.com", "youtu.be"]):
+        return "youtube"
+
+    # Web URLs
+    if source_lower.startswith(("http://", "https://")):
+        return "web"
+
+    # File extensions
+    ext = Path(source).suffix.lower()
+    if ext == ".pdf":
+        return "pdf"
+    if ext in (".mp3", ".wav", ".m4a", ".ogg", ".flac", ".webm"):
+        return "audio"
+    if ext in (".md", ".txt", ".rst"):
+        return "markdown"
+
+    return "unknown"
+
+
+class IngestEngine:
+    """Processes content from various sources into the vector store."""
+
+    def __init__(self, store: VectorStore, media_dir: str | Path = "") -> None:
+        self._store = store
+        self._media_dir = Path(media_dir) if media_dir else Path.home() / ".arkaos" / "media"
+        self._media_dir.mkdir(parents=True, exist_ok=True)
+
+    def ingest(
+        self,
+        source: str,
+        source_type: str = "",
+        on_progress: Optional[ProgressCallback] = None,
+        metadata: dict | None = None,
+    ) -> IngestResult:
+        """Ingest content from any supported source.
+
+        Args:
+            source: URL or file path.
+            source_type: youtube, pdf, audio, web, markdown. Auto-detected if empty.
+            on_progress: Callback(percent, message) for progress updates.
+            metadata: Extra metadata to attach to indexed chunks.
+        """
+        if not source_type:
+            source_type = detect_source_type(source)
+
+        progress = on_progress or (lambda p, m: None)
+        progress(0, f"Starting {source_type} ingest...")
+
+        processors = {
+            "youtube": self._process_youtube,
+            "pdf": self._process_pdf,
+            "audio": self._process_audio,
+            "web": self._process_web,
+            "markdown": self._process_markdown,
+        }
+
+        processor = processors.get(source_type)
+        if not processor:
+            return IngestResult(source=source, source_type=source_type, error=f"Unsupported type: {source_type}", success=False)
+
+        try:
+            text, title = processor(source, progress)
+        except Exception as e:
+            return IngestResult(source=source, source_type=source_type, error=str(e), success=False)
+
+        if not text or len(text.strip()) < 50:
+            return IngestResult(source=source, source_type=source_type, error="Extracted text too short", success=False)
+
+        # Chunk and index
+        progress(75, "Chunking content...")
+        chunks = chunk_markdown(text, max_tokens=512, source=source)
+
+        progress(85, f"Indexing {len(chunks)} chunks...")
+        texts = [c.text for c in chunks]
+        headings = [c.heading for c in chunks]
+        count = self._store.index_chunks(
+            texts=texts,
+            headings=headings,
+            source=source,
+            metadata={"type": source_type, "title": title, **(metadata or {})},
+        )
+
+        progress(100, f"Done — {count} chunks indexed")
+
+        return IngestResult(
+            source=source,
+            source_type=source_type,
+            text_length=len(text),
+            chunks_created=count,
+            title=title,
+            success=True,
+        )
+
+    def _process_youtube(self, url: str, progress: ProgressCallback) -> tuple[str, str]:
+        """Download YouTube video and transcribe audio."""
+        try:
+            import yt_dlp
+        except ImportError:
+            raise RuntimeError("yt-dlp not installed. Run: pip install yt-dlp")
+
+        progress(5, "Fetching video info...")
+
+        # Download audio only
+        audio_path = str(self._media_dir / "yt_audio.wav")
+        ydl_opts = {
+            "format": "bestaudio/best",
+            "outtmpl": str(self._media_dir / "yt_audio.%(ext)s"),
+            "postprocessors": [{
+                "key": "FFmpegExtractAudio",
+                "preferredcodec": "wav",
+                "preferredquality": "16",
+            }],
+            "quiet": True,
+            "no_warnings": True,
+        }
+
+        progress(10, "Downloading audio...")
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            info = ydl.extract_info(url, download=True)
+            title = info.get("title", "YouTube Video")
+
+        progress(35, "Transcribing audio...")
+        text = self._transcribe_audio(audio_path)
+
+        # Cleanup
+        try:
+            os.remove(audio_path)
+        except OSError:
+            pass
+
+        return text, title
+
+    def _process_pdf(self, path: str, progress: ProgressCallback) -> tuple[str, str]:
+        """Extract text from PDF."""
+        try:
+            import pdfplumber
+        except ImportError:
+            raise RuntimeError("pdfplumber not installed. Run: pip install pdfplumber")
+
+        progress(10, "Opening PDF...")
+        filepath = Path(path)
+        if not filepath.exists():
+            raise FileNotFoundError(f"PDF not found: {path}")
+
+        pages_text = []
+        with pdfplumber.open(filepath) as pdf:
+            total_pages = len(pdf.pages)
+            for i, page in enumerate(pdf.pages):
+                text = page.extract_text() or ""
+                pages_text.append(text)
+                pct = 10 + int((i / total_pages) * 60)
+                progress(pct, f"Extracting page {i + 1}/{total_pages}...")
+
+        title = filepath.stem.replace("-", " ").replace("_", " ")
+        return "\n\n".join(pages_text), title
+
+    def _process_audio(self, path: str, progress: ProgressCallback) -> tuple[str, str]:
+        """Transcribe audio file."""
+        progress(10, "Loading audio...")
+        filepath = Path(path)
+        if not filepath.exists():
+            raise FileNotFoundError(f"Audio not found: {path}")
+
+        progress(20, "Transcribing audio...")
+        text = self._transcribe_audio(str(filepath))
+        title = filepath.stem.replace("-", " ").replace("_", " ")
+        return text, title
+
+    def _process_web(self, url: str, progress: ProgressCallback) -> tuple[str, str]:
+        """Scrape web page content."""
+        try:
+            import requests
+            from bs4 import BeautifulSoup
+        except ImportError:
+            raise RuntimeError("beautifulsoup4 and requests not installed. Run: pip install beautifulsoup4 requests")
+
+        progress(10, "Fetching page...")
+        resp = requests.get(url, timeout=15, headers={
+            "User-Agent": "Mozilla/5.0 (ArkaOS Knowledge Indexer)"
+        })
+        resp.raise_for_status()
+
+        progress(40, "Parsing content...")
+        soup = BeautifulSoup(resp.text, "html.parser")
+
+        # Remove scripts, styles, nav, footer
+        for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
+            tag.decompose()
+
+        # Get title
+        title = soup.title.string if soup.title else url
+
+        # Get main content (article > main > body)
+        main = soup.find("article") or soup.find("main") or soup.find("body")
+        text = main.get_text(separator="\n\n", strip=True) if main else soup.get_text(separator="\n\n", strip=True)
+
+        # Clean up whitespace
+        text = re.sub(r'\n{3,}', '\n\n', text)
+
+        return text, title
+
+    def _process_markdown(self, path: str, progress: ProgressCallback) -> tuple[str, str]:
+        """Read markdown/text file directly."""
+        progress(10, "Reading file...")
+        filepath = Path(path)
+        if not filepath.exists():
+            raise FileNotFoundError(f"File not found: {path}")
+
+        text = filepath.read_text(encoding="utf-8")
+        title = filepath.stem.replace("-", " ").replace("_", " ")
+        return text, title
+
+    def _transcribe_audio(self, audio_path: str) -> str:
+        """Transcribe audio using faster-whisper (or fallback)."""
+        try:
+            from faster_whisper import WhisperModel
+            model = WhisperModel("base", device="cpu", compute_type="int8")
+            segments, _ = model.transcribe(audio_path, beam_size=5)
+            return " ".join(segment.text for segment in segments)
+        except ImportError:
+            pass
+
+        try:
+            import whisper
+            model = whisper.load_model("base")
+            result = model.transcribe(audio_path)
+            return result["text"]
+        except ImportError:
+            raise RuntimeError(
+                "No transcription engine available. Install one:\n"
+                "  pip install faster-whisper   (recommended, lighter)\n"
+                "  pip install openai-whisper   (original, heavier)"
+            )