@archznn/crewloop-skills 0.2.0 → 0.4.0

package/servers/obsidian-mcp/src/obsidian_mcp/indexer/store.py

@@ -0,0 +1,141 @@
+import json
+import sqlite3
+from pathlib import Path
+
+from obsidian_mcp.models import Chunk, GraphEdge
+
+
+class IndexStore:
+    def __init__(self, db_path: Path):
+        self.db_path = db_path
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._init_db()
+
+    def _connect(self):
+        return sqlite3.connect(self.db_path)
+
+    def _init_db(self):
+        with self._connect() as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS chunks (
+                    id TEXT PRIMARY KEY,
+                    note_path TEXT NOT NULL,
+                    text TEXT NOT NULL,
+                    embedding TEXT,
+                    start_line INTEGER,
+                    end_line INTEGER,
+                    mtime REAL
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS edges (
+                    source TEXT NOT NULL,
+                    target TEXT NOT NULL,
+                    relation TEXT NOT NULL,
+                    weight REAL,
+                    PRIMARY KEY (source, target, relation)
+                )
+            """)
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS notes_meta (
+                    path TEXT PRIMARY KEY,
+                    mtime REAL,
+                    hash TEXT
+                )
+            """)
+            conn.commit()
+
+    def get_note_meta(self, path: str) -> dict | None:
+        with self._connect() as conn:
+            row = conn.execute(
+                "SELECT mtime, hash FROM notes_meta WHERE path = ?", (path,)
+            ).fetchone()
+        if row is None:
+            return None
+        return {"mtime": row[0], "hash": row[1]}
+
+    def upsert_note_meta(self, path: str, mtime: float, hash: str):
+        with self._connect() as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO notes_meta (path, mtime, hash) VALUES (?, ?, ?)",
+                (path, mtime, hash),
+            )
+            conn.commit()
+
+    def delete_chunks_for_note(self, note_path: str):
+        with self._connect() as conn:
+            conn.execute("DELETE FROM chunks WHERE note_path = ?", (note_path,))
+            conn.commit()
+
+    def upsert_chunks(self, chunks: list[Chunk]):
+        with self._connect() as conn:
+            for chunk in chunks:
+                conn.execute(
+                    """
+                    INSERT OR REPLACE INTO chunks
+                    (id, note_path, text, embedding, start_line, end_line)
+                    VALUES (?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        chunk.id,
+                        chunk.note_path,
+                        chunk.text,
+                        json.dumps(chunk.embedding) if chunk.embedding is not None else None,
+                        chunk.start_line,
+                        chunk.end_line,
+                    ),
+                )
+            conn.commit()
+
+    def get_all_chunks(self) -> list[Chunk]:
+        with self._connect() as conn:
+            rows = conn.execute(
+                "SELECT id, note_path, text, embedding, start_line, end_line FROM chunks"
+            ).fetchall()
+        return [
+            Chunk(
+                id=r[0],
+                note_path=r[1],
+                text=r[2],
+                embedding=json.loads(r[3]) if r[3] else None,
+                start_line=r[4],
+                end_line=r[5],
+            )
+            for r in rows
+        ]
+
+    def delete_edges_for_note(self, note_path: str):
+        with self._connect() as conn:
+            conn.execute("DELETE FROM edges WHERE source = ?", (note_path,))
+            conn.commit()
+
+    def delete_backlinks(self):
+        with self._connect() as conn:
+            conn.execute("DELETE FROM edges WHERE relation = 'backlink'")
+            conn.commit()
+
+    def upsert_edges(self, edges: list[GraphEdge]):
+        with self._connect() as conn:
+            for edge in edges:
+                conn.execute(
+                    """
+                    INSERT OR REPLACE INTO edges (source, target, relation, weight)
+                    VALUES (?, ?, ?, ?)
+                    """,
+                    (edge.source, edge.target, edge.relation, edge.weight),
+                )
+            conn.commit()
+
+    def get_all_edges(self) -> list[GraphEdge]:
+        with self._connect() as conn:
+            rows = conn.execute(
+                "SELECT source, target, relation, weight FROM edges"
+            ).fetchall()
+        return [GraphEdge(source=r[0], target=r[1], relation=r[2], weight=r[3]) for r in rows]
+
+    def clear(self):
+        with self._connect() as conn:
+            conn.execute("DELETE FROM chunks")
+            conn.execute("DELETE FROM edges")
+            conn.execute("DELETE FROM notes_meta")
+            conn.commit()

package/servers/obsidian-mcp/src/obsidian_mcp/indexer/sync.py

@@ -0,0 +1,37 @@
+import logging
+from pathlib import Path
+
+from obsidian_mcp.config import Config
+from obsidian_mcp.indexer.indexer import Indexer
+from obsidian_mcp.models import Note
+from obsidian_mcp.vault.parser import parse_note
+from obsidian_mcp.vault.repository import VaultRepository
+
+logger = logging.getLogger(__name__)
+
+
+class BundleSync:
+    def __init__(self, config: Config, indexer: Indexer, vault: VaultRepository):
+        self.config = config
+        self.indexer = indexer
+        self.vault = vault
+
+    def _bundle_files(self) -> list[Path]:
+        root = self.config.bundle_path
+        files = []
+        for pattern in ["skills/**/*.md", "references/**/*.md", "README.md", "AGENTS.md"]:
+            files.extend(root.glob(pattern))
+        return files
+
+    def sync(self, force: bool = False) -> dict:
+        indexed = 0
+        for path in self._bundle_files():
+            rel = path.relative_to(self.config.bundle_path).as_posix()
+            try:
+                note = parse_note(rel, path)
+                self.indexer.index_note(note, force=force)
+                indexed += 1
+            except Exception as exc:
+                logger.warning("failed to index bundle file %s: %s", rel, exc)
+        self.indexer.compute_backlinks()
+        return {"indexed_bundle_files": indexed}

package/servers/obsidian-mcp/src/obsidian_mcp/learning/detector.py

@@ -0,0 +1,66 @@
+import hashlib
+import re
+
+from obsidian_mcp.config import Config
+
+
+class Learning:
+    def __init__(self, type: str, title: str, body: str, tags: list[str]):
+        self.type = type
+        self.title = title
+        self.body = body
+        self.tags = tags
+        self.id = hashlib.sha256(f"{type}:{title}".encode()).hexdigest()[:12]
+
+
+class LearningDetector:
+    def __init__(self, config: Config | None = None):
+        self.config = config or Config()
+        self._seen = set()
+
+    def _slug(self, title: str) -> str:
+        return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")
+
+    def detect(self, text: str) -> list[Learning]:
+        findings = []
+        concept = self._extract_concept(text)
+        if concept:
+            findings.append(concept)
+        decision = self._extract_decision(text)
+        if decision:
+            findings.append(decision)
+        return [f for f in findings if f.id not in self._seen]
+
+    def mark_seen(self, learnings: list[Learning]):
+        for learning in learnings:
+            self._seen.add(learning.id)
+
+    def _extract_concept(self, text: str) -> Learning | None:
+        match = re.search(
+            r"(?i)(?:novo\s+)?(?:conceito|concept)\s*[:\-]?\s*([A-Z][A-Za-z0-9\s\-_]{2,60})",
+            text,
+        )
+        if match:
+            title = match.group(1).strip()
+            return Learning(
+                type="concept",
+                title=title,
+                body=text.strip(),
+                tags=["concept", "auto-generated"],
+            )
+        return None
+
+    def _extract_decision(self, text: str) -> Learning | None:
+        match = re.search(
+            r"(?i)(?:decidimos|decision|decis[ãa]o)\s+(?:que|to|by)?\s*[:\-]?\s*(.+?)(?:\.|\n)",
+            text,
+        )
+        if match:
+            title = match.group(1).strip()[:80]
+            return Learning(
+                type="decision",
+                title=title,
+                body=text.strip(),
+                tags=["decision", "auto-generated"],
+            )
+        return None

package/servers/obsidian-mcp/src/obsidian_mcp/learning/note_generator.py

@@ -0,0 +1,40 @@
+import re
+from datetime import datetime, timezone
+
+from obsidian_mcp.config import Config
+from obsidian_mcp.learning.detector import Learning
+from obsidian_mcp.models import Note
+from obsidian_mcp.vault.repository import VaultRepository
+
+
+class NoteGenerator:
+    def __init__(self, config: Config, vault: VaultRepository):
+        self.config = config
+        self.vault = vault
+
+    def _slug(self, title: str) -> str:
+        return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")[:60]
+
+    def path_for(self, learning: Learning) -> str:
+        folder = "concepts" if learning.type == "concept" else "decisions"
+        return f"{folder}/{self._slug(learning.title)}.md"
+
+    def to_note(self, learning: Learning) -> Note:
+        now = datetime.now(timezone.utc).isoformat()
+        return Note(
+            path=self.path_for(learning),
+            title=learning.title,
+            content=learning.body,
+            frontmatter={
+                "type": learning.type,
+                "tags": learning.tags,
+                "created": now,
+                "auto_generated": True,
+            },
+            tags=learning.tags,
+        )
+
+    def generate_and_save(self, learning: Learning) -> Note:
+        note = self.to_note(learning)
+        self.vault.save(note)
+        return note

package/servers/obsidian-mcp/src/obsidian_mcp/main.py

@@ -0,0 +1,4 @@
+from obsidian_mcp.server import main
+
+if __name__ == "__main__":
+    main()

package/servers/obsidian-mcp/src/obsidian_mcp/models.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class Note:
+    path: str
+    title: str
+    content: str
+    frontmatter: dict[str, Any] = field(default_factory=dict)
+    links: list[str] = field(default_factory=list)
+    backlinks: list[str] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list)
+    ctime: datetime | None = None
+    mtime: datetime | None = None
+
+
+@dataclass
+class Chunk:
+    id: str
+    note_path: str
+    text: str
+    embedding: list[float] | None = None
+    start_line: int = 0
+    end_line: int = 0
+
+
+@dataclass
+class GraphEdge:
+    source: str
+    target: str
+    relation: str = "links"
+    weight: float = 1.0
+
+
+@dataclass
+class SearchResult:
+    note_path: str
+    score: float
+    snippet: str
+    matched_chunks: list[Chunk] = field(default_factory=list)

package/servers/obsidian-mcp/src/obsidian_mcp/privacy/filter.py

@@ -0,0 +1,68 @@
+import logging
+import re
+
+from obsidian_mcp.config import Config
+
+logger = logging.getLogger(__name__)
+
+
+class PrivacyFilter:
+    _RULES = {
+        "api_keys": [
+            r"\b(API_KEY|SECRET|TOKEN|PASSWORD)\s*[=:]\s*\S+",
+            r"\b(?:sk|ghp|gho|ghu|ghs|ghr|pat|np|openai|anthropic)-[A-Za-z0-9_\-]{10,}\b",
+        ],
+        "private_keys": [
+            r"\bPRIVATE_KEY\b",
+            r"-----BEGIN",
+        ],
+        "env_files": [
+            r"\.env",
+        ],
+        "emails": [
+            r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b",
+        ],
+        "credit_cards": [
+            r"\b\d{4}[ -]\d{4}[ -]\d{4}[ -]\d{4}\b",
+        ],
+    }
+
+    def __init__(self, config: Config | None = None):
+        self.config = config or Config()
+        self.privacy = self.config.privacy
+        self.patterns = self._compile_patterns()
+
+    def _compile_patterns(self) -> list[re.Pattern]:
+        raw_patterns: list[str] = []
+        if self.privacy.block_api_keys:
+            raw_patterns.extend(self._RULES["api_keys"])
+        if self.privacy.block_private_keys:
+            raw_patterns.extend(self._RULES["private_keys"])
+        if self.privacy.block_env_files:
+            raw_patterns.extend(self._RULES["env_files"])
+        if self.privacy.block_emails:
+            raw_patterns.extend(self._RULES["emails"])
+        if self.privacy.block_credit_cards:
+            raw_patterns.extend(self._RULES["credit_cards"])
+        if self.config.sensitive_patterns:
+            raw_patterns.extend(self.config.sensitive_patterns)
+        return [re.compile(p, re.IGNORECASE) for p in raw_patterns]
+
+    def _allowed(self, text: str) -> bool:
+        return any(allowed in text for allowed in self.privacy.allowed_strings)
+
+    def is_safe(self, text: str) -> bool:
+        if not self.privacy.enabled:
+            return True
+        if self._allowed(text):
+            return True
+        return not any(pattern.search(text) for pattern in self.patterns)
+
+    def validate(self, text: str) -> None:
+        if not self.privacy.enabled:
+            return
+        if self._allowed(text):
+            return
+        if not self.is_safe(text):
+            logger.warning("privacy filter blocked content")
+            raise ValueError("content blocked by privacy filter: sensitive data detected")

package/servers/obsidian-mcp/src/obsidian_mcp/rag/engine.py

@@ -0,0 +1,50 @@
+from obsidian_mcp.config import Config
+from obsidian_mcp.indexer.store import IndexStore
+from obsidian_mcp.models import SearchResult
+from obsidian_mcp.rag.graph_search import GraphSearch
+from obsidian_mcp.rag.text_search import TextSearch
+from obsidian_mcp.rag.vector_search import VectorSearch
+
+
+class RAGEngine:
+    def __init__(self, config: Config, store: IndexStore):
+        self.config = config
+        self.text_search = TextSearch(store)
+        self.vector_search = VectorSearch(store, config.embedding_model)
+        self.graph_search = GraphSearch(store)
+
+    @staticmethod
+    def _normalize_scores(results: list[SearchResult]) -> list[SearchResult]:
+        if len(results) < 2:
+            return results
+        scores = [r.score for r in results]
+        min_score = min(scores)
+        max_score = max(scores)
+        span = max_score - min_score
+        for result in results:
+            result.score = 1.0 if span == 0 else (result.score - min_score) / span
+        return results
+
+    def search(self, query: str, mode: str = "hybrid", limit: int = 10) -> list[SearchResult]:
+        results = []
+        if mode in ("text", "hybrid"):
+            results.extend(self._normalize_scores(self.text_search.search(query, limit=limit)))
+        if mode in ("vector", "hybrid"):
+            results.extend(self._normalize_scores(self.vector_search.search(query, limit=limit)))
+        if mode == "graph":
+            results.extend(self._normalize_scores(self.graph_search.search(query, limit=limit)))
+
+        by_note = {}
+        for result in results:
+            if result.note_path not in by_note:
+                by_note[result.note_path] = result
+            else:
+                by_note[result.note_path].score = max(
+                    by_note[result.note_path].score, result.score
+                )
+                by_note[result.note_path].matched_chunks.extend(result.matched_chunks)
+
+        return sorted(by_note.values(), key=lambda r: r.score, reverse=True)[:limit]
+
+    def related(self, note_path: str, depth: int = 1) -> list[SearchResult]:
+        return self.graph_search.related(note_path, depth=depth)

package/servers/obsidian-mcp/src/obsidian_mcp/rag/graph_search.py

@@ -0,0 +1,55 @@
+import logging
+from collections import deque
+
+from obsidian_mcp.indexer.store import IndexStore
+from obsidian_mcp.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+class GraphSearch:
+    def __init__(self, store: IndexStore):
+        self.store = store
+
+    def related(self, note_path: str, depth: int = 1) -> list[SearchResult]:
+        edges = self.store.get_all_edges()
+        logger.debug("graph related search: %s depth=%d edges=%d", note_path, depth, len(edges))
+        adjacency = {}
+        for edge in edges:
+            adjacency.setdefault(edge.source, []).append((edge.target, edge.weight))
+
+        visited = {note_path}
+        queue = deque([(note_path, 0, 1.0)])
+        scores = {}
+        while queue:
+            current, level, weight = queue.popleft()
+            for target, edge_weight in adjacency.get(current, []):
+                if target in visited:
+                    continue
+                score = weight * edge_weight * (1.0 / (level + 1))
+                scores[target] = max(scores.get(target, 0.0), score)
+                if level + 1 < depth:
+                    visited.add(target)
+                    queue.append((target, level + 1, score))
+        return [
+            SearchResult(note_path=path, score=score, snippet="", matched_chunks=[])
+            for path, score in sorted(scores.items(), key=lambda x: x[1], reverse=True)
+        ]
+
+    def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+        terms = [t.lower() for t in query.split() if t]
+        if not terms:
+            return []
+        edges = self.store.get_all_edges()
+        logger.debug("graph search: query=%r edges=%d", query, len(edges))
+        scores = {}
+        for edge in edges:
+            for node in (edge.source, edge.target):
+                node_lower = node.lower()
+                score = sum(1 for term in terms if term in node_lower)
+                if score:
+                    scores[node] = max(scores.get(node, 0.0), score * edge.weight)
+        return [
+            SearchResult(note_path=path, score=score, snippet="", matched_chunks=[])
+            for path, score in sorted(scores.items(), key=lambda x: x[1], reverse=True)[:limit]
+        ]

package/servers/obsidian-mcp/src/obsidian_mcp/rag/text_search.py

@@ -0,0 +1,37 @@
+import logging
+
+from obsidian_mcp.indexer.store import IndexStore
+from obsidian_mcp.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+class TextSearch:
+    def __init__(self, store: IndexStore):
+        self.store = store
+
+    def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+        query_lower = query.lower()
+        chunks = self.store.get_all_chunks()
+        logger.debug("text search over %d chunks", len(chunks))
+        scored = []
+        for chunk in chunks:
+            text_lower = chunk.text.lower()
+            score = 0.0
+            if query_lower in text_lower:
+                score = text_lower.count(query_lower) / max(len(text_lower.split()), 1)
+            if score > 0:
+                scored.append((score, chunk))
+        scored.sort(key=lambda x: x[0], reverse=True)
+        by_note = {}
+        for score, chunk in scored[:limit * 3]:
+            if chunk.note_path not in by_note:
+                by_note[chunk.note_path] = SearchResult(
+                    note_path=chunk.note_path,
+                    score=score,
+                    snippet=chunk.text[:300],
+                    matched_chunks=[chunk],
+                )
+            else:
+                by_note[chunk.note_path].matched_chunks.append(chunk)
+        return sorted(by_note.values(), key=lambda r: r.score, reverse=True)[:limit]

package/servers/obsidian-mcp/src/obsidian_mcp/rag/vector_search.py

@@ -0,0 +1,118 @@
+import logging
+import math
+
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+from obsidian_mcp.indexer.embeddings import EmbedderFactory
+from obsidian_mcp.indexer.store import IndexStore
+from obsidian_mcp.models import Chunk, SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+class TfidfIndex:
+    def __init__(self, max_features: int = 50000):
+        self.vectorizer = TfidfVectorizer(max_features=max_features)
+        self.matrix = None
+        self.doc_ids: list[str] = []
+
+    def fit(self, chunks: list[Chunk]) -> None:
+        if not chunks:
+            self.matrix = None
+            self.doc_ids = []
+            return
+        self.doc_ids = [chunk.id for chunk in chunks]
+        texts = [chunk.text for chunk in chunks]
+        self.matrix = self.vectorizer.fit_transform(texts)
+        logger.info("TF-IDF index fitted on %d chunks", len(chunks))
+
+    def query(self, query: str, top_k: int = 10) -> list[tuple[str, float]]:
+        if self.matrix is None or not self.doc_ids:
+            return []
+        qvec = self.vectorizer.transform([query])
+        scores = cosine_similarity(qvec, self.matrix).flatten()
+        ranked = np.argsort(scores)[::-1][:top_k]
+        return [(self.doc_ids[i], float(scores[i])) for i in ranked if scores[i] > 0]
+
+
+class VectorSearch:
+    def __init__(self, store: IndexStore, model_name: str):
+        self.store = store
+        self.model_name = model_name
+        self.embedder = EmbedderFactory.create(model_name)
+        self._tfidf_index: TfidfIndex | None = None
+        self._tfidf_chunk_count: int = 0
+
+    def _cosine_similarity(self, a: list[float], b: list[float]) -> float:
+        dot = sum(x * y for x, y in zip(a, b))
+        norm_a = math.sqrt(sum(x * x for x in a))
+        norm_b = math.sqrt(sum(x * x for x in b))
+        if norm_a == 0 or norm_b == 0:
+            return 0.0
+        return dot / (norm_a * norm_b)
+
+    def _embedding_search(
+        self, query: str, chunks: list[Chunk], limit: int
+    ) -> list[SearchResult]:
+        try:
+            query_embedding = self.embedder.encode([query])[0]
+        except Exception as exc:
+            logger.warning("vector search failed: %s", exc)
+            return []
+
+        scored = []
+        for chunk in chunks:
+            if not chunk.embedding:
+                continue
+            score = self._cosine_similarity(query_embedding, chunk.embedding)
+            if score > 0:
+                scored.append((score, chunk))
+        return self._rank_by_note(scored, limit)
+
+    def _ensure_tfidf_index(self, chunks: list[Chunk]) -> TfidfIndex:
+        if self._tfidf_index is None or self._tfidf_chunk_count != len(chunks):
+            self._tfidf_index = TfidfIndex()
+            self._tfidf_index.fit(chunks)
+            self._tfidf_chunk_count = len(chunks)
+        return self._tfidf_index
+
+    def _tfidf_search(self, query: str, chunks: list[Chunk], limit: int) -> list[SearchResult]:
+        if not chunks:
+            return []
+        index = self._ensure_tfidf_index(chunks)
+        id_to_chunk = {chunk.id: chunk for chunk in chunks}
+        results = index.query(query, top_k=limit * 3)
+        scored = [
+            (score, id_to_chunk[doc_id])
+            for doc_id, score in results
+            if doc_id in id_to_chunk
+        ]
+        return self._rank_by_note(scored, limit)
+
+    def _rank_by_note(
+        self, scored: list[tuple[float, object]], limit: int
+    ) -> list[SearchResult]:
+        scored.sort(key=lambda x: x[0], reverse=True)
+        by_note = {}
+        for score, chunk in scored[:limit * 3]:
+            if chunk.note_path not in by_note:
+                by_note[chunk.note_path] = SearchResult(
+                    note_path=chunk.note_path,
+                    score=score,
+                    snippet=chunk.text[:300],
+                    matched_chunks=[chunk],
+                )
+            else:
+                by_note[chunk.note_path].matched_chunks.append(chunk)
+        return sorted(by_note.values(), key=lambda r: r.score, reverse=True)[:limit]
+
+    def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+        chunks = self.store.get_all_chunks()
+        if not chunks:
+            logger.warning("vector search requested but no chunks are indexed")
+            return []
+        if self.embedder.uses_stored_embeddings():
+            return self._embedding_search(query, chunks, limit)
+        return self._tfidf_search(query, chunks, limit)