@archznn/crewloop-skills 0.1.0 → 0.3.0

package/servers/obsidian-mcp/src/obsidian_mcp/learning/note_generator.py

@@ -0,0 +1,40 @@
+import re
+from datetime import datetime, timezone
+
+from obsidian_mcp.config import Config
+from obsidian_mcp.learning.detector import Learning
+from obsidian_mcp.models import Note
+from obsidian_mcp.vault.repository import VaultRepository
+
+
+class NoteGenerator:
+    def __init__(self, config: Config, vault: VaultRepository):
+        self.config = config
+        self.vault = vault
+
+    def _slug(self, title: str) -> str:
+        return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")[:60]
+
+    def path_for(self, learning: Learning) -> str:
+        folder = "concepts" if learning.type == "concept" else "decisions"
+        return f"{folder}/{self._slug(learning.title)}.md"
+
+    def to_note(self, learning: Learning) -> Note:
+        now = datetime.now(timezone.utc).isoformat()
+        return Note(
+            path=self.path_for(learning),
+            title=learning.title,
+            content=learning.body,
+            frontmatter={
+                "type": learning.type,
+                "tags": learning.tags,
+                "created": now,
+                "auto_generated": True,
+            },
+            tags=learning.tags,
+        )
+
+    def generate_and_save(self, learning: Learning) -> Note:
+        note = self.to_note(learning)
+        self.vault.save(note)
+        return note

package/servers/obsidian-mcp/src/obsidian_mcp/main.py

@@ -0,0 +1,4 @@
+from obsidian_mcp.server import main
+
+if __name__ == "__main__":
+    main()

package/servers/obsidian-mcp/src/obsidian_mcp/models.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class Note:
+    path: str
+    title: str
+    content: str
+    frontmatter: dict[str, Any] = field(default_factory=dict)
+    links: list[str] = field(default_factory=list)
+    backlinks: list[str] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list)
+    ctime: datetime | None = None
+    mtime: datetime | None = None
+
+
+@dataclass
+class Chunk:
+    id: str
+    note_path: str
+    text: str
+    embedding: list[float] | None = None
+    start_line: int = 0
+    end_line: int = 0
+
+
+@dataclass
+class GraphEdge:
+    source: str
+    target: str
+    relation: str = "links"
+    weight: float = 1.0
+
+
+@dataclass
+class SearchResult:
+    note_path: str
+    score: float
+    snippet: str
+    matched_chunks: list[Chunk] = field(default_factory=list)

package/servers/obsidian-mcp/src/obsidian_mcp/privacy/filter.py

@@ -0,0 +1,68 @@
+import logging
+import re
+
+from obsidian_mcp.config import Config
+
+logger = logging.getLogger(__name__)
+
+
+class PrivacyFilter:
+    _RULES = {
+        "api_keys": [
+            r"\b(API_KEY|SECRET|TOKEN|PASSWORD)\s*[=:]\s*\S+",
+            r"\b(?:sk|ghp|gho|ghu|ghs|ghr|pat|np|openai|anthropic)-[A-Za-z0-9_\-]{10,}\b",
+        ],
+        "private_keys": [
+            r"\bPRIVATE_KEY\b",
+            r"-----BEGIN",
+        ],
+        "env_files": [
+            r"\.env",
+        ],
+        "emails": [
+            r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b",
+        ],
+        "credit_cards": [
+            r"\b\d{4}[ -]\d{4}[ -]\d{4}[ -]\d{4}\b",
+        ],
+    }
+
+    def __init__(self, config: Config | None = None):
+        self.config = config or Config()
+        self.privacy = self.config.privacy
+        self.patterns = self._compile_patterns()
+
+    def _compile_patterns(self) -> list[re.Pattern]:
+        raw_patterns: list[str] = []
+        if self.privacy.block_api_keys:
+            raw_patterns.extend(self._RULES["api_keys"])
+        if self.privacy.block_private_keys:
+            raw_patterns.extend(self._RULES["private_keys"])
+        if self.privacy.block_env_files:
+            raw_patterns.extend(self._RULES["env_files"])
+        if self.privacy.block_emails:
+            raw_patterns.extend(self._RULES["emails"])
+        if self.privacy.block_credit_cards:
+            raw_patterns.extend(self._RULES["credit_cards"])
+        if self.config.sensitive_patterns:
+            raw_patterns.extend(self.config.sensitive_patterns)
+        return [re.compile(p, re.IGNORECASE) for p in raw_patterns]
+
+    def _allowed(self, text: str) -> bool:
+        return any(allowed in text for allowed in self.privacy.allowed_strings)
+
+    def is_safe(self, text: str) -> bool:
+        if not self.privacy.enabled:
+            return True
+        if self._allowed(text):
+            return True
+        return not any(pattern.search(text) for pattern in self.patterns)
+
+    def validate(self, text: str) -> None:
+        if not self.privacy.enabled:
+            return
+        if self._allowed(text):
+            return
+        if not self.is_safe(text):
+            logger.warning("privacy filter blocked content")
+            raise ValueError("content blocked by privacy filter: sensitive data detected")

package/servers/obsidian-mcp/src/obsidian_mcp/rag/engine.py

@@ -0,0 +1,50 @@
+from obsidian_mcp.config import Config
+from obsidian_mcp.indexer.store import IndexStore
+from obsidian_mcp.models import SearchResult
+from obsidian_mcp.rag.graph_search import GraphSearch
+from obsidian_mcp.rag.text_search import TextSearch
+from obsidian_mcp.rag.vector_search import VectorSearch
+
+
+class RAGEngine:
+    def __init__(self, config: Config, store: IndexStore):
+        self.config = config
+        self.text_search = TextSearch(store)
+        self.vector_search = VectorSearch(store, config.embedding_model)
+        self.graph_search = GraphSearch(store)
+
+    @staticmethod
+    def _normalize_scores(results: list[SearchResult]) -> list[SearchResult]:
+        if len(results) < 2:
+            return results
+        scores = [r.score for r in results]
+        min_score = min(scores)
+        max_score = max(scores)
+        span = max_score - min_score
+        for result in results:
+            result.score = 1.0 if span == 0 else (result.score - min_score) / span
+        return results
+
+    def search(self, query: str, mode: str = "hybrid", limit: int = 10) -> list[SearchResult]:
+        results = []
+        if mode in ("text", "hybrid"):
+            results.extend(self._normalize_scores(self.text_search.search(query, limit=limit)))
+        if mode in ("vector", "hybrid"):
+            results.extend(self._normalize_scores(self.vector_search.search(query, limit=limit)))
+        if mode == "graph":
+            results.extend(self._normalize_scores(self.graph_search.search(query, limit=limit)))
+
+        by_note = {}
+        for result in results:
+            if result.note_path not in by_note:
+                by_note[result.note_path] = result
+            else:
+                by_note[result.note_path].score = max(
+                    by_note[result.note_path].score, result.score
+                )
+                by_note[result.note_path].matched_chunks.extend(result.matched_chunks)
+
+        return sorted(by_note.values(), key=lambda r: r.score, reverse=True)[:limit]
+
+    def related(self, note_path: str, depth: int = 1) -> list[SearchResult]:
+        return self.graph_search.related(note_path, depth=depth)

package/servers/obsidian-mcp/src/obsidian_mcp/rag/graph_search.py

@@ -0,0 +1,55 @@
+import logging
+from collections import deque
+
+from obsidian_mcp.indexer.store import IndexStore
+from obsidian_mcp.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+class GraphSearch:
+    def __init__(self, store: IndexStore):
+        self.store = store
+
+    def related(self, note_path: str, depth: int = 1) -> list[SearchResult]:
+        edges = self.store.get_all_edges()
+        logger.debug("graph related search: %s depth=%d edges=%d", note_path, depth, len(edges))
+        adjacency = {}
+        for edge in edges:
+            adjacency.setdefault(edge.source, []).append((edge.target, edge.weight))
+
+        visited = {note_path}
+        queue = deque([(note_path, 0, 1.0)])
+        scores = {}
+        while queue:
+            current, level, weight = queue.popleft()
+            for target, edge_weight in adjacency.get(current, []):
+                if target in visited:
+                    continue
+                score = weight * edge_weight * (1.0 / (level + 1))
+                scores[target] = max(scores.get(target, 0.0), score)
+                if level + 1 < depth:
+                    visited.add(target)
+                    queue.append((target, level + 1, score))
+        return [
+            SearchResult(note_path=path, score=score, snippet="", matched_chunks=[])
+            for path, score in sorted(scores.items(), key=lambda x: x[1], reverse=True)
+        ]
+
+    def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+        terms = [t.lower() for t in query.split() if t]
+        if not terms:
+            return []
+        edges = self.store.get_all_edges()
+        logger.debug("graph search: query=%r edges=%d", query, len(edges))
+        scores = {}
+        for edge in edges:
+            for node in (edge.source, edge.target):
+                node_lower = node.lower()
+                score = sum(1 for term in terms if term in node_lower)
+                if score:
+                    scores[node] = max(scores.get(node, 0.0), score * edge.weight)
+        return [
+            SearchResult(note_path=path, score=score, snippet="", matched_chunks=[])
+            for path, score in sorted(scores.items(), key=lambda x: x[1], reverse=True)[:limit]
+        ]

package/servers/obsidian-mcp/src/obsidian_mcp/rag/text_search.py

@@ -0,0 +1,37 @@
+import logging
+
+from obsidian_mcp.indexer.store import IndexStore
+from obsidian_mcp.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+class TextSearch:
+    def __init__(self, store: IndexStore):
+        self.store = store
+
+    def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+        query_lower = query.lower()
+        chunks = self.store.get_all_chunks()
+        logger.debug("text search over %d chunks", len(chunks))
+        scored = []
+        for chunk in chunks:
+            text_lower = chunk.text.lower()
+            score = 0.0
+            if query_lower in text_lower:
+                score = text_lower.count(query_lower) / max(len(text_lower.split()), 1)
+            if score > 0:
+                scored.append((score, chunk))
+        scored.sort(key=lambda x: x[0], reverse=True)
+        by_note = {}
+        for score, chunk in scored[:limit * 3]:
+            if chunk.note_path not in by_note:
+                by_note[chunk.note_path] = SearchResult(
+                    note_path=chunk.note_path,
+                    score=score,
+                    snippet=chunk.text[:300],
+                    matched_chunks=[chunk],
+                )
+            else:
+                by_note[chunk.note_path].matched_chunks.append(chunk)
+        return sorted(by_note.values(), key=lambda r: r.score, reverse=True)[:limit]

package/servers/obsidian-mcp/src/obsidian_mcp/rag/vector_search.py

@@ -0,0 +1,118 @@
+import logging
+import math
+
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+from obsidian_mcp.indexer.embeddings import EmbedderFactory
+from obsidian_mcp.indexer.store import IndexStore
+from obsidian_mcp.models import Chunk, SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+class TfidfIndex:
+    def __init__(self, max_features: int = 50000):
+        self.vectorizer = TfidfVectorizer(max_features=max_features)
+        self.matrix = None
+        self.doc_ids: list[str] = []
+
+    def fit(self, chunks: list[Chunk]) -> None:
+        if not chunks:
+            self.matrix = None
+            self.doc_ids = []
+            return
+        self.doc_ids = [chunk.id for chunk in chunks]
+        texts = [chunk.text for chunk in chunks]
+        self.matrix = self.vectorizer.fit_transform(texts)
+        logger.info("TF-IDF index fitted on %d chunks", len(chunks))
+
+    def query(self, query: str, top_k: int = 10) -> list[tuple[str, float]]:
+        if self.matrix is None or not self.doc_ids:
+            return []
+        qvec = self.vectorizer.transform([query])
+        scores = cosine_similarity(qvec, self.matrix).flatten()
+        ranked = np.argsort(scores)[::-1][:top_k]
+        return [(self.doc_ids[i], float(scores[i])) for i in ranked if scores[i] > 0]
+
+
+class VectorSearch:
+    def __init__(self, store: IndexStore, model_name: str):
+        self.store = store
+        self.model_name = model_name
+        self.embedder = EmbedderFactory.create(model_name)
+        self._tfidf_index: TfidfIndex | None = None
+        self._tfidf_chunk_count: int = 0
+
+    def _cosine_similarity(self, a: list[float], b: list[float]) -> float:
+        dot = sum(x * y for x, y in zip(a, b))
+        norm_a = math.sqrt(sum(x * x for x in a))
+        norm_b = math.sqrt(sum(x * x for x in b))
+        if norm_a == 0 or norm_b == 0:
+            return 0.0
+        return dot / (norm_a * norm_b)
+
+    def _embedding_search(
+        self, query: str, chunks: list[Chunk], limit: int
+    ) -> list[SearchResult]:
+        try:
+            query_embedding = self.embedder.encode([query])[0]
+        except Exception as exc:
+            logger.warning("vector search failed: %s", exc)
+            return []
+
+        scored = []
+        for chunk in chunks:
+            if not chunk.embedding:
+                continue
+            score = self._cosine_similarity(query_embedding, chunk.embedding)
+            if score > 0:
+                scored.append((score, chunk))
+        return self._rank_by_note(scored, limit)
+
+    def _ensure_tfidf_index(self, chunks: list[Chunk]) -> TfidfIndex:
+        if self._tfidf_index is None or self._tfidf_chunk_count != len(chunks):
+            self._tfidf_index = TfidfIndex()
+            self._tfidf_index.fit(chunks)
+            self._tfidf_chunk_count = len(chunks)
+        return self._tfidf_index
+
+    def _tfidf_search(self, query: str, chunks: list[Chunk], limit: int) -> list[SearchResult]:
+        if not chunks:
+            return []
+        index = self._ensure_tfidf_index(chunks)
+        id_to_chunk = {chunk.id: chunk for chunk in chunks}
+        results = index.query(query, top_k=limit * 3)
+        scored = [
+            (score, id_to_chunk[doc_id])
+            for doc_id, score in results
+            if doc_id in id_to_chunk
+        ]
+        return self._rank_by_note(scored, limit)
+
+    def _rank_by_note(
+        self, scored: list[tuple[float, object]], limit: int
+    ) -> list[SearchResult]:
+        scored.sort(key=lambda x: x[0], reverse=True)
+        by_note = {}
+        for score, chunk in scored[:limit * 3]:
+            if chunk.note_path not in by_note:
+                by_note[chunk.note_path] = SearchResult(
+                    note_path=chunk.note_path,
+                    score=score,
+                    snippet=chunk.text[:300],
+                    matched_chunks=[chunk],
+                )
+            else:
+                by_note[chunk.note_path].matched_chunks.append(chunk)
+        return sorted(by_note.values(), key=lambda r: r.score, reverse=True)[:limit]
+
+    def search(self, query: str, limit: int = 10) -> list[SearchResult]:
+        chunks = self.store.get_all_chunks()
+        if not chunks:
+            logger.warning("vector search requested but no chunks are indexed")
+            return []
+        if self.embedder.uses_stored_embeddings():
+            return self._embedding_search(query, chunks, limit)
+        return self._tfidf_search(query, chunks, limit)

package/servers/obsidian-mcp/src/obsidian_mcp/server.py

@@ -0,0 +1,61 @@
+import asyncio
+import json
+import logging
+import time
+
+from mcp import ErrorData, McpError
+from mcp.server import Server
+from mcp.server.stdio import stdio_server
+from mcp.types import TextContent, Tool
+
+from obsidian_mcp.config import Config
+from obsidian_mcp.tools.registry import TOOLS, dispatch_async
+
+logger = logging.getLogger(__name__)
+
+
+def _error_code_for(exc: Exception) -> int:
+    if isinstance(exc, (ValueError, FileExistsError)):
+        return -32600
+    if isinstance(exc, FileNotFoundError):
+        return -32602
+    return -32603
+
+
+async def serve(config: Config | None = None):
+    config = config or Config()
+    server = Server("obsidian-mcp")
+
+    @server.list_tools()
+    async def list_tools() -> list[Tool]:
+        return [
+            Tool(name=name, description=meta["description"], inputSchema=meta["input_schema"])
+            for name, meta in TOOLS.items()
+        ]
+
+    @server.call_tool()
+    async def call_tool(name: str, arguments: dict | None) -> list[TextContent]:
+        arguments = arguments or {}
+        start = time.perf_counter()
+        logger.info("tool start: %s", name)
+        try:
+            result = await dispatch_async(name, arguments, config)
+            if isinstance(result, str):
+                return [TextContent(type="text", text=result)]
+            return [TextContent(type="text", text=json.dumps(result, ensure_ascii=False, indent=2))]
+        except Exception as exc:
+            logger.exception("tool error: %s", name)
+            raise McpError(
+                ErrorData(code=_error_code_for(exc), message=str(exc))
+            ) from exc
+        finally:
+            elapsed = time.perf_counter() - start
+            logger.info("tool end: %s (%.3fs)", name, elapsed)
+
+    async with stdio_server() as (read_stream, write_stream):
+        await server.run(read_stream, write_stream, server.create_initialization_options())
+
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    asyncio.run(serve())

package/servers/obsidian-mcp/src/obsidian_mcp/tools/create.py

@@ -0,0 +1,43 @@
+import logging
+from pathlib import Path
+
+from obsidian_mcp.config import Config
+from obsidian_mcp.models import Note
+from obsidian_mcp.privacy.filter import PrivacyFilter
+from obsidian_mcp.vault.repository import VaultRepository
+
+logger = logging.getLogger(__name__)
+
+
+def handle_create_note(arguments: dict, config: Config) -> dict:
+    path = arguments.get("path")
+    content = arguments.get("content", "")
+    title = arguments.get("title")
+    tags = arguments.get("tags", [])
+    overwrite = bool(arguments.get("overwrite", False))
+    if not path:
+        raise ValueError("path is required")
+
+    if not path.endswith(".md"):
+        path = path + ".md"
+
+    PrivacyFilter(config).validate(path)
+    PrivacyFilter(config).validate(content)
+
+    vault = VaultRepository(config)
+    if vault.exists(path) and not overwrite:
+        raise FileExistsError(f"note already exists: {path}")
+
+    note = Note(
+        path=path,
+        title=title or _title_from_path(path),
+        content=content,
+        tags=tags,
+    )
+    vault.save(note)
+    logger.info("created note: %s", path)
+    return {"status": "created", "path": path}
+
+
+def _title_from_path(path: str) -> str:
+    return Path(path).stem.replace("-", " ").replace("_", " ").title()

package/servers/obsidian-mcp/src/obsidian_mcp/tools/delete.py

@@ -0,0 +1,16 @@
+import logging
+
+from obsidian_mcp.config import Config
+from obsidian_mcp.vault.repository import VaultRepository
+
+logger = logging.getLogger(__name__)
+
+
+def handle_delete_note(arguments: dict, config: Config) -> dict:
+    path = arguments.get("path")
+    if not path:
+        raise ValueError("path is required")
+    vault = VaultRepository(config)
+    vault.delete(path)
+    logger.info("deleted note: %s", path)
+    return {"status": "deleted", "path": path}

package/servers/obsidian-mcp/src/obsidian_mcp/tools/learn.py

@@ -0,0 +1,42 @@
+import logging
+
+from obsidian_mcp.config import Config
+from obsidian_mcp.indexer.indexer import Indexer
+from obsidian_mcp.indexer.store import IndexStore
+from obsidian_mcp.learning.detector import LearningDetector
+from obsidian_mcp.learning.note_generator import NoteGenerator
+from obsidian_mcp.privacy.filter import PrivacyFilter
+from obsidian_mcp.vault.repository import VaultRepository
+
+logger = logging.getLogger(__name__)
+
+
+def handle_learn_from_text(arguments: dict, config: Config) -> dict:
+    text = arguments.get("text", "")
+    if not text:
+        raise ValueError("text is required")
+
+    PrivacyFilter(config).validate(text)
+
+    detector = LearningDetector(config)
+    learnings = detector.detect(text)
+    if not learnings:
+        return {"status": "no_learning_detected"}
+
+    vault = VaultRepository(config)
+    generator = NoteGenerator(config, vault)
+    indexer = Indexer(config, vault, IndexStore(config.index_dir / "index.db"))
+
+    created = []
+    for learning in learnings:
+        if vault.exists(generator.path_for(learning)):
+            continue
+        note = generator.generate_and_save(learning)
+        indexer.index_note(note)
+        created.append(note.path)
+
+    if not created:
+        logger.info("no new learnings created from text")
+        return {"status": "duplicate", "created_notes": []}
+    logger.info("learned from text, created notes: %s", created)
+    return {"status": "learned", "created_notes": created}

package/servers/obsidian-mcp/src/obsidian_mcp/tools/list.py

@@ -0,0 +1,16 @@
+import logging
+
+from obsidian_mcp.config import Config
+from obsidian_mcp.vault.repository import VaultRepository
+
+logger = logging.getLogger(__name__)
+
+
+def handle_list_notes(arguments: dict, config: Config) -> str:
+    folder = arguments.get("folder")
+    vault = VaultRepository(config)
+    notes = vault.list_notes(folder)
+    logger.debug("listed %d notes in folder %s", len(notes), folder)
+    if not notes:
+        return "No notes found."
+    return "\n".join(f"- {n}" for n in notes)

package/servers/obsidian-mcp/src/obsidian_mcp/tools/read.py

@@ -0,0 +1,15 @@
+import logging
+
+from obsidian_mcp.config import Config
+from obsidian_mcp.vault.repository import VaultRepository
+
+logger = logging.getLogger(__name__)
+
+
+def handle_read_note(arguments: dict, config: Config) -> str:
+    path = arguments.get("path")
+    if not path:
+        raise ValueError("path is required")
+    vault = VaultRepository(config)
+    logger.debug("reading note: %s", path)
+    return vault.read_raw(path)