arkaos 2.90.0 → 2.91.0

package/VERSION

@@ -1 +1 @@
-2.90.0
+2.91.0

package/core/knowledge/vector_store.py

@@ -13,24 +13,81 @@ from typing import Any, Optional
 from core.knowledge.embedder import embed, embed_batch, EMBEDDING_DIMS
 
 
+_VEC_UNAVAILABLE_REASON: str = ""  # set by _load_vec on failure
+
+
 def _load_vec(db: sqlite3.Connection) -> bool:
-    """Try to load sqlite-vec extension."""
+    """Try to load sqlite-vec extension.
+
+    PR73 v2.91.0 — failure reason is now captured in a module-level
+    string so the dashboard / /api/knowledge/stats can surface why
+    vector search is unavailable instead of just "Unavailable".
+    Common reasons:
+      - sqlite-vec not installed (pip install sqlite-vec)
+      - Python sqlite3 built without enable_load_extension (rare on
+        homebrew Python; common on system Python on some distros)
+    """
+    global _VEC_UNAVAILABLE_REASON
     try:
         db.enable_load_extension(True)
-        import sqlite_vec
+    except AttributeError:
+        _VEC_UNAVAILABLE_REASON = (
+            "Python sqlite3 was built without extension loading support. "
+            "On macOS, reinstall via Homebrew Python: brew install python."
+        )
+        return False
+    except sqlite3.OperationalError as exc:
+        _VEC_UNAVAILABLE_REASON = f"sqlite3 refused extension loading: {exc}"
+        return False
+    try:
+        import sqlite_vec  # noqa: PLC0415
+    except ImportError:
+        _VEC_UNAVAILABLE_REASON = (
+            "sqlite-vec package missing. Install with: "
+            "pip install sqlite-vec"
+        )
+        return False
+    try:
         sqlite_vec.load(db)
+        _VEC_UNAVAILABLE_REASON = ""
         return True
-    except (ImportError, Exception):
+    except Exception as exc:  # noqa: BLE001
+        _VEC_UNAVAILABLE_REASON = f"sqlite_vec.load failed: {exc}"
         return False
 
 
+def vec_unavailable_reason() -> str:
+    """Public accessor for the last vec-load failure reason."""
+    return _VEC_UNAVAILABLE_REASON
+
+
 class VectorStore:
-    """SQLite-vec backed vector store for knowledge retrieval."""
+    """SQLite-vec backed vector store for knowledge retrieval.
+
+    PR73 v2.91.0 — connection is opened with ``check_same_thread=False``
+    so background ingest workers (knowledge_ingest, knowledge_ingest_bulk
+    in scripts/dashboard-api.py) can reuse a long-lived FastAPI-scoped
+    store instance without hitting ``sqlite3.ProgrammingError: SQLite
+    objects created in a thread can only be used in that same thread``.
+    Concurrent writes are serialised via ``_write_lock``; SQLite's WAL
+    journal_mode lets readers continue while a writer holds the lock.
+    """
 
     def __init__(self, db_path: str | Path = ":memory:") -> None:
+        import threading
         self._db_path = str(db_path)
-        self._db = sqlite3.connect(self._db_path)
+        # check_same_thread=False — see class docstring.
+        self._db = sqlite3.connect(self._db_path, check_same_thread=False)
         self._db.row_factory = sqlite3.Row
+        # WAL gives concurrent readers + a single writer at the engine
+        # level. The Python lock below serialises our application-level
+        # writes per VectorStore instance.
+        try:
+            self._db.execute("PRAGMA journal_mode=WAL")
+        except sqlite3.OperationalError:
+            # in-memory DBs don't support WAL — harmless.
+            pass
+        self._write_lock = threading.Lock()
         self._vec_available = _load_vec(self._db)
         self._init_schema()
 
@@ -102,26 +159,31 @@ class VectorStore:
         meta_json = json.dumps(metadata or {})
         count = 0
 
-        for i, text in enumerate(texts):
-            heading = headings[i] if headings and i < len(headings) else ""
-            emb_blob = None
-
-            if embeddings and i < len(embeddings):
-                emb_blob = _vec_to_blob(embeddings[i])
-
-            cursor = self._db.execute(
-                "INSERT INTO chunks (text, heading, source, file_hash, metadata, embedding) VALUES (?, ?, ?, ?, ?, ?)",
-                (text, heading, source, file_hash, meta_json, emb_blob),
-            )
-
-            if self._vec_available and emb_blob:
-                self._db.execute(
-                    "INSERT INTO vec_chunks (rowid, embedding) VALUES (?, ?)",
-                    (cursor.lastrowid, emb_blob),
+        # PR73 v2.91.0 — serialise writes from background ingest threads.
+        # check_same_thread=False on the connection lets the threads use
+        # `self._db`; the lock ensures only one writer at a time so
+        # cursor lastrowid stays consistent.
+        with self._write_lock:
+            for i, text in enumerate(texts):
+                heading = headings[i] if headings and i < len(headings) else ""
+                emb_blob = None
+
+                if embeddings and i < len(embeddings):
+                    emb_blob = _vec_to_blob(embeddings[i])
+
+                cursor = self._db.execute(
+                    "INSERT INTO chunks (text, heading, source, file_hash, metadata, embedding) VALUES (?, ?, ?, ?, ?, ?)",
+                    (text, heading, source, file_hash, meta_json, emb_blob),
                 )
-            count += 1
 
-        self._db.commit()
+                if self._vec_available and emb_blob:
+                    self._db.execute(
+                        "INSERT INTO vec_chunks (rowid, embedding) VALUES (?, ?)",
+                        (cursor.lastrowid, emb_blob),
+                    )
+                count += 1
+
+            self._db.commit()
         return count
 
     def search(self, query: str, top_k: int = 5) -> list[dict]:
@@ -201,13 +263,14 @@ class VectorStore:
 
     def remove_file(self, source: str) -> int:
         """Remove all chunks from a source file."""
-        if self._vec_available:
-            rows = self._db.execute("SELECT id FROM chunks WHERE source = ?", (source,)).fetchall()
-            for r in rows:
-                self._db.execute("DELETE FROM vec_chunks WHERE rowid = ?", (r["id"],))
-        deleted = self._db.execute("DELETE FROM chunks WHERE source = ?", (source,)).rowcount
-        self._db.commit()
-        return deleted
+        with self._write_lock:
+            if self._vec_available:
+                rows = self._db.execute("SELECT id FROM chunks WHERE source = ?", (source,)).fetchall()
+                for r in rows:
+                    self._db.execute("DELETE FROM vec_chunks WHERE rowid = ?", (r["id"],))
+            deleted = self._db.execute("DELETE FROM chunks WHERE source = ?", (source,)).rowcount
+            self._db.commit()
+            return deleted
 
     def get_stats(self) -> dict:
         """Get store statistics."""

package/core/personas/obsidian_store.py

@@ -0,0 +1,276 @@
+"""Obsidian-backed persona store (PR73 v2.91.0).
+
+Reads personas from ``<vaultPath>/Personas/*.md`` and writes new
+personas back to the same folder. The vault path comes from
+``~/.arkaos/profile.json::vaultPath``; when that's empty / missing,
+the store gracefully degrades to a no-op (the JSON store keeps
+working).
+
+Per the operator's instruction: "personas podiamos ir buscar a lista
+aqui o que esta no obsidian acho que é o mais sensato e quando criamos
+guardamos no obsidian tambem". This module implements both directions.
+
+Schema (frontmatter):
+  type: persona
+  name: <full name>
+  source: <one-line summary>
+  title: <role label>
+  tagline: <essence>
+  mbti: <4-letter>
+  disc: { primary, secondary }
+  enneagram: { type, wing }
+  big_five: { openness, ... }
+  mental_models: [list]
+  expertise: [list]  # legacy alias for expertise_domains
+  expertise_domains: [list]
+  frameworks: [list]
+  key_quotes: [list]
+  communication: { tone, vocabulary_level, preferred_format, avoid }
+
+Unknown frontmatter keys are ignored. Body markdown is preserved
+when reading but not re-injected into the Persona model (the model
+doesn't carry free-form notes).
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Iterable
+
+from core.personas.schema import (
+    Persona,
+    PersonaBigFive,
+    PersonaCommunication,
+    PersonaDISC,
+    PersonaEnneagram,
+)
+from core.profile import ProfileManager
+
+
+_FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n?(.*)\Z", re.DOTALL)
+_PERSONAS_SUBDIR = "Personas"
+
+
+_UNSET = object()
+
+
+class ObsidianPersonaStore:
+    """Read / write personas as Markdown files in an Obsidian vault."""
+
+    def __init__(self, vault_path: Path | None | object = _UNSET) -> None:
+        # Distinguish "not passed" (auto-resolve from profile) from
+        # "passed as None" (explicitly no vault — tests use this).
+        if vault_path is _UNSET:
+            self._vault_path = self._resolve_vault_path()
+        else:
+            self._vault_path = vault_path  # type: ignore[assignment]
+
+    @property
+    def available(self) -> bool:
+        return bool(self._vault_path and self.personas_dir.exists())
+
+    @property
+    def personas_dir(self) -> Path:
+        return (self._vault_path or Path()) / _PERSONAS_SUBDIR
+
+    @staticmethod
+    def _resolve_vault_path() -> Path | None:
+        try:
+            profile = ProfileManager().read()
+        except Exception:  # noqa: BLE001
+            return None
+        if not profile.vaultPath:
+            return None
+        path = Path(profile.vaultPath).expanduser()
+        return path if path.exists() else None
+
+    # ─── Read ────────────────────────────────────────────────────────────
+
+    def list_all(self) -> list[Persona]:
+        """Return every parseable persona in <vault>/Personas/."""
+        if not self.available:
+            return []
+        out: list[Persona] = []
+        for md_path in sorted(self.personas_dir.glob("*.md")):
+            persona = self._read_file(md_path)
+            if persona is not None:
+                out.append(persona)
+        return out
+
+    def _read_file(self, md_path: Path) -> Persona | None:
+        try:
+            text = md_path.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            return None
+        fm = self._parse_frontmatter(text)
+        if not fm:
+            return None
+        # MOC files etc. can also live in this folder; skip anything not
+        # explicitly typed as persona.
+        if str(fm.get("type", "")).lower() != "persona":
+            return None
+        return self._frontmatter_to_persona(fm, md_path)
+
+    @staticmethod
+    def _parse_frontmatter(text: str) -> dict[str, Any] | None:
+        match = _FRONTMATTER_RE.match(text)
+        if not match:
+            return None
+        try:
+            import yaml  # noqa: PLC0415
+        except ImportError:
+            return None
+        try:
+            data = yaml.safe_load(match.group(1)) or {}
+        except Exception:  # noqa: BLE001 — yaml parsers raise many shapes
+            return None
+        return data if isinstance(data, dict) else None
+
+    @staticmethod
+    def _frontmatter_to_persona(fm: dict, md_path: Path) -> Persona:
+        name = str(fm.get("name") or md_path.stem)
+        disc_fm = fm.get("disc") if isinstance(fm.get("disc"), dict) else {}
+        enneagram_fm = fm.get("enneagram") if isinstance(fm.get("enneagram"), dict) else {}
+        big_five_fm = fm.get("big_five") if isinstance(fm.get("big_five"), dict) else {}
+        comm_fm = fm.get("communication") if isinstance(fm.get("communication"), dict) else {}
+        expertise = _as_str_list(fm.get("expertise_domains") or fm.get("expertise"))
+
+        return Persona(
+            id=ObsidianPersonaStore._slug_from_name(name),
+            name=name,
+            title=str(fm.get("title") or ""),
+            tagline=str(fm.get("tagline") or ""),
+            source=str(fm.get("source") or name),
+            disc=PersonaDISC(**_filter_known(PersonaDISC, disc_fm)),
+            enneagram=PersonaEnneagram(**_filter_known(PersonaEnneagram, enneagram_fm)),
+            big_five=PersonaBigFive(**_filter_known(PersonaBigFive, big_five_fm)),
+            mbti=str(fm.get("mbti") or "INTJ"),
+            mental_models=_as_str_list(fm.get("mental_models")),
+            expertise_domains=expertise,
+            frameworks=_as_str_list(fm.get("frameworks")),
+            key_quotes=_as_str_list(fm.get("key_quotes")),
+            communication=PersonaCommunication(
+                **_filter_known(PersonaCommunication, comm_fm),
+            ),
+            created_at=str(fm.get("created_at") or ""),
+            updated_at=str(fm.get("updated_at") or ""),
+        )
+
+    @staticmethod
+    def _slug_from_name(name: str) -> str:
+        return (
+            name.lower()
+            .replace(" ", "-")
+            .replace(".", "")
+            .replace("'", "")
+        ) or str(uuid.uuid4())
+
+    # ─── Write ───────────────────────────────────────────────────────────
+
+    def write(self, persona: Persona) -> Path | None:
+        """Persist a persona as <vault>/Personas/<name>.md.
+
+        Returns the path written, or None when the vault isn't
+        configured. Existing files are overwritten — the operator
+        is the source of truth.
+        """
+        if not self._vault_path:
+            return None
+        self.personas_dir.mkdir(parents=True, exist_ok=True)
+        target = self.personas_dir / f"{persona.name}.md"
+        body = self._render(persona)
+        tmp = target.with_suffix(target.suffix + ".tmp")
+        try:
+            tmp.write_text(body, encoding="utf-8")
+            tmp.replace(target)
+        except OSError:
+            try:
+                tmp.unlink()
+            except FileNotFoundError:
+                pass
+            return None
+        return target
+
+    @staticmethod
+    def _render(persona: Persona) -> str:
+        try:
+            import yaml  # noqa: PLC0415
+        except ImportError:
+            yaml = None  # type: ignore[assignment]
+        now = datetime.now(timezone.utc).isoformat()
+        frontmatter = {
+            "type": "persona",
+            "name": persona.name,
+            "source": persona.source or persona.name,
+            "title": persona.title,
+            "tagline": persona.tagline,
+            "mbti": persona.mbti,
+            "disc": {
+                "primary": persona.disc.primary,
+                "secondary": persona.disc.secondary,
+            },
+            "enneagram": {
+                "type": persona.enneagram.type,
+                "wing": persona.enneagram.wing,
+            },
+            "big_five": {
+                "openness": persona.big_five.openness,
+                "conscientiousness": persona.big_five.conscientiousness,
+                "extraversion": persona.big_five.extraversion,
+                "agreeableness": persona.big_five.agreeableness,
+                "neuroticism": persona.big_five.neuroticism,
+            },
+            "mental_models": list(persona.mental_models),
+            "expertise_domains": list(persona.expertise_domains),
+            "frameworks": list(persona.frameworks),
+            "key_quotes": list(persona.key_quotes),
+            "communication": {
+                "tone": persona.communication.tone,
+                "vocabulary_level": persona.communication.vocabulary_level,
+                "preferred_format": persona.communication.preferred_format,
+                "avoid": list(persona.communication.avoid),
+            },
+            "created_at": persona.created_at or now,
+            "updated_at": now,
+        }
+        if yaml is not None:
+            fm_block = yaml.safe_dump(
+                frontmatter,
+                sort_keys=False,
+                allow_unicode=True,
+                default_flow_style=False,
+            )
+        else:
+            fm_block = "\n".join(f"{k}: {v}" for k, v in frontmatter.items()) + "\n"
+        body = "# " + persona.name + "\n\n"
+        if persona.tagline:
+            body += f"> {persona.tagline}\n\n"
+        body += "## Source\n\n" + (persona.source or persona.name) + "\n\n"
+        if persona.mental_models:
+            body += "## Mental Models\n\n"
+            body += "\n".join(f"- {m}" for m in persona.mental_models) + "\n\n"
+        if persona.key_quotes:
+            body += "## Key Quotes\n\n"
+            body += "\n".join(f"> {q}" for q in persona.key_quotes) + "\n\n"
+        return f"---\n{fm_block}---\n\n{body}"
+
+
+# ─── Helpers ────────────────────────────────────────────────────────────
+
+
+def _as_str_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(item) for item in value if isinstance(item, (str, int, float))]
+
+
+def _filter_known(model: type, data: dict) -> dict:
+    """Drop keys the Pydantic model doesn't declare so unknown
+    frontmatter keys (e.g. older schema variants) don't crash."""
+    if not isinstance(data, dict):
+        return {}
+    known: Iterable[str] = model.model_fields.keys()  # type: ignore[attr-defined]
+    return {k: v for k, v in data.items() if k in known}

package/dashboard/app/pages/knowledge.vue

@@ -325,6 +325,13 @@ function formatScore(score: number): string {
   return `${(score * 100).toFixed(0)}%`
 }
 
+// PR73 v2.91.0 — `vec_available` is the canonical PR47-era flag from
+// the new VectorStore; `vss_available` is the legacy field name from
+// earlier sqlite-vss builds. Treat either as "active".
+const vectorSearchActive = computed(() =>
+  Boolean(stats.value?.vec_available || stats.value?.vss_available),
+)
+
 // PR71 v2.88.0 — delete all chunks from a given source.
 
 const deletingSource = ref<string | null>(null)
@@ -419,9 +426,9 @@ function escapeRegex(value: string): string {
         </template>
         <template #trailing>
           <UBadge
-            v-if="stats?.vss_available !== undefined"
-            :label="stats.vss_available ? 'VSS Active' : 'VSS Unavailable'"
-            :color="stats.vss_available ? 'success' : 'neutral'"
+            v-if="stats?.vec_available !== undefined || stats?.vss_available !== undefined"
+            :label="vectorSearchActive ? 'Vector Active' : 'Vector Off'"
+            :color="vectorSearchActive ? 'success' : 'warning'"
             variant="subtle"
           />
         </template>
@@ -768,12 +775,19 @@ function escapeRegex(value: string): string {
           </div>
           <div class="rounded-lg border border-default p-4 text-center">
             <UBadge
-              :label="stats?.vss_available ? 'Available' : 'Unavailable'"
-              :color="stats?.vss_available ? 'success' : 'neutral'"
+              :label="vectorSearchActive ? 'Active' : 'Unavailable'"
+              :color="vectorSearchActive ? 'success' : 'warning'"
               variant="subtle"
               size="sm"
             />
             <p class="text-xs text-muted mt-1">Vector Search</p>
+            <p
+              v-if="!vectorSearchActive && stats?.vec_unavailable_reason"
+              class="text-xs text-yellow-400 mt-2 text-left"
+              :title="stats.vec_unavailable_reason"
+            >
+              {{ stats.vec_unavailable_reason }}
+            </p>
           </div>
         </div>
 

package/dashboard/app/types/index.d.ts

@@ -128,6 +128,10 @@ export interface KnowledgeStats {
   total_chunks: number
   total_files: number
   vss_available?: boolean
+  // PR73 v2.91.0 — vec_available + reason surfaced by /api/knowledge/stats
+  vec_available?: boolean
+  vec_unavailable_reason?: string
+  indexed?: boolean
   areas?: {
     name: string
     chunks: number

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "arkaos",
-  "version": "2.90.0",
+  "version": "2.91.0",
   "description": "The Operating System for AI Agent Teams",
   "type": "module",
   "bin": {

package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "arkaos-core"
-version = "2.90.0"
+version = "2.91.0"
 description = "Core engine for ArkaOS — The Operating System for AI Agent Teams"
 readme = "README.md"
 license = {text = "MIT"}

package/scripts/dashboard-api.py

@@ -595,11 +595,29 @@ def task_detail(task_id: str):
 
 @app.get("/api/knowledge/stats")
 def knowledge_stats():
+    """PR73 v2.91.0 — vec_unavailable_reason surfaced so the dashboard
+    can show *why* vector search is offline instead of just a generic
+    "Unavailable" badge."""
     store = _get_vector_store()
     if not store:
-        return {"total_chunks": 0, "total_files": 0, "vss_available": False, "indexed": False}
+        return {
+            "total_chunks": 0,
+            "total_files": 0,
+            "vss_available": False,
+            "vec_available": False,
+            "vec_unavailable_reason": "Vector store could not be opened.",
+            "indexed": False,
+        }
     stats = store.get_stats()
     stats["indexed"] = stats["total_chunks"] > 0
+    if not stats.get("vec_available", False):
+        try:
+            from core.knowledge.vector_store import vec_unavailable_reason
+            stats["vec_unavailable_reason"] = vec_unavailable_reason()
+        except Exception:
+            stats["vec_unavailable_reason"] = "unknown"
+    else:
+        stats["vec_unavailable_reason"] = ""
     return stats
 
 
@@ -722,11 +740,47 @@ def _get_persona_manager():
 
 @app.get("/api/personas")
 def personas_list():
+    """PR73 v2.91.0 — merges JSON-store personas with the Obsidian
+    vault's ``<vaultPath>/Personas/*.md`` files. Obsidian is the
+    source of truth: when the same persona exists in both places
+    (matched by name/slug), the vault entry wins.
+    """
+    by_id: dict[str, dict] = {}
+
+    # JSON store first (in-memory copy is fast, vault read may be slow)
     mgr = _get_persona_manager()
-    if not mgr:
-        return {"personas": [], "total": 0}
-    personas = mgr.list_all()
-    return {"personas": [p.model_dump() for p in personas], "total": len(personas)}
+    if mgr:
+        for p in mgr.list_all():
+            payload = p.model_dump()
+            by_id[payload["id"]] = payload
+
+    # Obsidian vault — overwrites duplicates so the vault wins.
+    try:
+        from core.personas.obsidian_store import ObsidianPersonaStore
+        ob_store = ObsidianPersonaStore()
+        if ob_store.available:
+            for p in ob_store.list_all():
+                payload = p.model_dump()
+                payload["_source_store"] = "obsidian"
+                by_id[payload["id"]] = payload
+    except Exception:
+        pass
+
+    personas = list(by_id.values())
+    personas.sort(key=lambda p: p.get("name", "").lower())
+    return {
+        "personas": personas,
+        "total": len(personas),
+        "obsidian_available": _obsidian_store_available(),
+    }
+
+
+def _obsidian_store_available() -> bool:
+    try:
+        from core.personas.obsidian_store import ObsidianPersonaStore
+        return ObsidianPersonaStore().available
+    except Exception:
+        return False
 
 
 @app.get("/api/personas/{persona_id}")
@@ -772,7 +826,23 @@ def persona_create(body: dict):
     )
 
     mgr.create(persona)
-    return {"id": persona.id, "created": True}
+
+    # PR73 v2.91.0 — also write to the Obsidian vault so the persona
+    # survives outside the JSON store and is browsable in Obsidian.
+    # Best-effort: vault unavailable / write failure does not abort
+    # the JSON-side success.
+    obsidian_path: str | None = None
+    try:
+        from core.personas.obsidian_store import ObsidianPersonaStore
+        ob_store = ObsidianPersonaStore()
+        if ob_store.available or ob_store._vault_path is not None:
+            written = ob_store.write(persona)
+            if written is not None:
+                obsidian_path = str(written)
+    except Exception:
+        pass
+
+    return {"id": persona.id, "created": True, "obsidian_path": obsidian_path}
 
 
 @app.post("/api/personas/{persona_id}/clone")