bluera-knowledge 0.9.21

package/python/crawl_worker.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+import sys
+import json
+import asyncio
+import os
+import ast
+from typing import List, Dict, Any
+
+# Suppress crawl4ai logging before import
+os.environ['CRAWL4AI_VERBOSE'] = '0'
+
+# Redirect stderr to suppress logging (crawl4ai uses console for progress)
+import io
+sys.stderr = io.StringIO()
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def fetch_headless(url: str):
+    """Fetch URL with headless browser (Playwright via crawl4ai)"""
+    browser_config = BrowserConfig(headless=True, verbose=False)
+    run_config = CrawlerRunConfig(
+        wait_for="js:() => document.readyState === 'complete'",
+        page_timeout=30000
+    )
+
+    async with AsyncWebCrawler(config=browser_config, verbose=False) as crawler:
+        result = await crawler.arun(url, config=run_config)
+
+        if not result.success:
+            raise Exception(f"Crawl failed: {result.error_message}")
+
+        return {
+            "html": result.html or '',
+            "markdown": result.markdown or result.cleaned_html or '',
+            "links": result.links.get("internal", []) if isinstance(result.links, dict) else []
+        }
+
+def is_exported(node: ast.AST) -> bool:
+    """Check if a function or class is exported (Python doesn't have explicit exports, check if starts with '_')"""
+    if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+        return not node.name.startswith('_')
+    return False
+
+def get_signature(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str:
+    """Extract function signature from AST node"""
+    args_list = []
+
+    for arg in node.args.args:
+        arg_str = arg.arg
+        if arg.annotation:
+            arg_str += f': {ast.unparse(arg.annotation)}'
+        args_list.append(arg_str)
+
+    return_annotation = ''
+    if node.returns:
+        return_annotation = f' -> {ast.unparse(node.returns)}'
+
+    return f"{node.name}({', '.join(args_list)}){return_annotation}"
+
+def extract_imports(tree: ast.AST) -> List[Dict[str, Any]]:
+    """Extract import statements from AST"""
+    imports = []
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                imports.append({
+                    'source': alias.name,
+                    'imported': alias.asname if alias.asname else alias.name
+                })
+        elif isinstance(node, ast.ImportFrom):
+            module = node.module if node.module else ''
+            for alias in node.names:
+                imports.append({
+                    'source': module,
+                    'imported': alias.name,
+                    'alias': alias.asname if alias.asname else None
+                })
+
+    return imports
+
+def extract_calls(node: ast.AST) -> List[str]:
+    """Extract function calls from a function/method body"""
+    calls = []
+
+    for child in ast.walk(node):
+        if isinstance(child, ast.Call):
+            if isinstance(child.func, ast.Name):
+                calls.append(child.func.id)
+            elif isinstance(child.func, ast.Attribute):
+                calls.append(child.func.attr)
+
+    return calls
+
+async def parse_python_ast(code: str, file_path: str) -> Dict[str, Any]:
+    """Parse Python code and return CodeNode array"""
+    try:
+        tree = ast.parse(code)
+        nodes = []
+
+        for node in tree.body:
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                nodes.append({
+                    'type': 'function',
+                    'name': node.name,
+                    'exported': is_exported(node),
+                    'startLine': node.lineno,
+                    'endLine': node.end_lineno if node.end_lineno else node.lineno,
+                    'async': isinstance(node, ast.AsyncFunctionDef),
+                    'signature': get_signature(node),
+                    'calls': extract_calls(node)
+                })
+
+            elif isinstance(node, ast.ClassDef):
+                methods = []
+                for item in node.body:
+                    if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                        methods.append({
+                            'name': item.name,
+                            'async': isinstance(item, ast.AsyncFunctionDef),
+                            'signature': get_signature(item),
+                            'startLine': item.lineno,
+                            'endLine': item.end_lineno if item.end_lineno else item.lineno,
+                            'calls': extract_calls(item)
+                        })
+
+                nodes.append({
+                    'type': 'class',
+                    'name': node.name,
+                    'exported': is_exported(node),
+                    'startLine': node.lineno,
+                    'endLine': node.end_lineno if node.end_lineno else node.lineno,
+                    'methods': methods
+                })
+
+        imports = extract_imports(tree)
+
+        return {
+            'nodes': nodes,
+            'imports': imports
+        }
+
+    except SyntaxError as e:
+        raise Exception(f"Python syntax error at line {e.lineno}: {e.msg}")
+    except Exception as e:
+        raise Exception(f"Failed to parse Python AST: {str(e)}")
+
+async def process_request(crawler, request):
+    """Process a single crawl request"""
+    try:
+        params = request.get('params', {})
+        url = params.get('url')
+
+        if not url:
+            raise ValueError('URL parameter is required')
+
+        # Perform async crawl
+        result = await crawler.arun(url=url)
+
+        if not result.success:
+            raise Exception(f"Crawl failed: {result.error_message}")
+
+        # Extract title from metadata (crawl4ai 0.7.8 stores title in metadata dict)
+        title = ''
+        if result.metadata and isinstance(result.metadata, dict):
+            title = result.metadata.get('title', '')
+
+        # Get markdown content (crawl4ai 0.7.8)
+        markdown = result.markdown or result.cleaned_html or ''
+
+        # Extract links - crawl4ai 0.7.8 returns dict with 'internal' and 'external' keys
+        # Each link is an object with href, text, title, etc. - extract just href strings
+        all_links = []
+        if isinstance(result.links, dict):
+            internal = result.links.get('internal', [])
+            external = result.links.get('external', [])
+            # Extract href from link objects (crawl4ai 0.7.8 returns objects, not strings)
+            for link in internal + external:
+                if isinstance(link, dict):
+                    all_links.append(link.get('href', ''))
+                elif isinstance(link, str):
+                    all_links.append(link)
+
+        response = {
+            'jsonrpc': '2.0',
+            'id': request.get('id'),
+            'result': {
+                'pages': [{
+                    'url': url,
+                    'title': title,
+                    'content': markdown,
+                    'html': result.html or '',
+                    'links': all_links,
+                    'crawledAt': '',  # crawl4ai 0.7.8 doesn't provide timestamp
+                }]
+            }
+        }
+        print(json.dumps(response), flush=True)
+    except Exception as e:
+        error_response = {
+            'jsonrpc': '2.0',
+            'id': request.get('id') if isinstance(request, dict) else None,
+            'error': {'code': -1, 'message': str(e)}
+        }
+        print(json.dumps(error_response), flush=True)
+
+async def main():
+    """Main async loop processing stdin requests"""
+    # Disable verbose logging in crawl4ai
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        for line in sys.stdin:
+            try:
+                request = json.loads(line.strip())
+                method = request.get('method')
+
+                if method == 'crawl':
+                    await process_request(crawler, request)
+                elif method == 'fetch_headless':
+                    # Handle headless fetch request
+                    try:
+                        params = request.get('params', {})
+                        url = params.get('url')
+                        if not url:
+                            raise ValueError('URL parameter is required')
+
+                        result = await fetch_headless(url)
+                        response = {
+                            'jsonrpc': '2.0',
+                            'id': request.get('id'),
+                            'result': result
+                        }
+                        print(json.dumps(response), flush=True)
+                    except Exception as e:
+                        error_response = {
+                            'jsonrpc': '2.0',
+                            'id': request.get('id'),
+                            'error': {'code': -1, 'message': str(e)}
+                        }
+                        print(json.dumps(error_response), flush=True)
+
+                elif method == 'parse_python':
+                    # Handle Python AST parsing request
+                    try:
+                        params = request.get('params', {})
+                        code = params.get('code')
+                        file_path = params.get('filePath', '<unknown>')
+
+                        if not code:
+                            raise ValueError('code parameter is required')
+
+                        result = await parse_python_ast(code, file_path)
+                        response = {
+                            'jsonrpc': '2.0',
+                            'id': request.get('id'),
+                            'result': result
+                        }
+                        print(json.dumps(response), flush=True)
+                    except Exception as e:
+                        error_response = {
+                            'jsonrpc': '2.0',
+                            'id': request.get('id'),
+                            'error': {'code': -1, 'message': str(e)}
+                        }
+                        print(json.dumps(error_response), flush=True)
+
+            except Exception as e:
+                error_response = {
+                    'jsonrpc': '2.0',
+                    'id': request.get('id') if isinstance(request, dict) else None,
+                    'error': {'code': -1, 'message': str(e)}
+                }
+                print(json.dumps(error_response), flush=True)
+
+if __name__ == '__main__':
+    asyncio.run(main())

package/python/requirements.txt

@@ -0,0 +1,2 @@
+crawl4ai==0.7.8
+playwright>=1.40.0

package/scripts/readme-version-updater.cjs

@@ -0,0 +1,18 @@
+/**
+ * Custom updater for commit-and-tag-version to update README.md version badge
+ */
+
+const { version } = require('../package.json');
+
+const versionRegex = /!\[Version\]\(https:\/\/img\.shields\.io\/badge\/version-[\d.]+(-[\w.]+)?-blue\)/;
+
+module.exports.readVersion = function () {
+  return version;
+};
+
+module.exports.writeVersion = function (contents, version) {
+  return contents.replace(
+    versionRegex,
+    `![Version](https://img.shields.io/badge/version-${version}-blue)`
+  );
+};

package/skills/advanced-workflows/SKILL.md

@@ -0,0 +1,273 @@
+---
+name: advanced-workflows
+description: Multi-tool orchestration patterns for complex Bluera Knowledge operations. Teaches progressive library exploration, adding libraries with job monitoring, handling large result sets, multi-store searches, and error recovery workflows.
+---
+
+# Advanced Bluera Knowledge Workflows
+
+Master complex multi-tool operations that combine multiple MCP tools for efficient knowledge retrieval and management.
+
+## Progressive Library Exploration
+
+When exploring a new library or codebase, use this pattern for efficient discovery:
+
+### Workflow: Find Relevant Code in Unknown Library
+
+```
+1. list_stores()
+   → See what's indexed, identify target store
+
+2. get_store_info(store)
+   → Get metadata: file paths, size, indexed files
+   → Understand scope before searching
+
+3. search(query, detail='minimal', stores=[target])
+   → Get high-level summaries of relevant code
+   → Review relevance scores (>0.7 = good match)
+
+4. get_full_context(result_ids[top_3])
+   → Deep dive on most relevant results only
+   → Get complete code with full context
+```
+
+**Example:**
+
+User: "How does Vue's computed properties work?"
+
+```
+list_stores()
+→ Found: vue, react, pydantic
+
+get_store_info('vue')
+→ Path: .bluera/bluera-knowledge/repos/vue/
+→ Files: 2,847 indexed
+
+search("computed properties", intent='find-implementation', detail='minimal', stores=['vue'])
+→ Result 1: packages/reactivity/src/computed.ts (score: 0.92)
+→ Result 2: packages/reactivity/__tests__/computed.spec.ts (score: 0.85)
+→ Result 3: packages/runtime-core/src/apiComputed.ts (score: 0.78)
+
+get_full_context(['result_1_id', 'result_2_id'])
+→ Full code for ComputedRefImpl class
+→ Complete API implementation
+
+Now explain with authoritative source code.
+```
+
+## Adding New Library with Job Monitoring
+
+When adding large libraries, monitor indexing progress to know when search is ready:
+
+### Workflow: Add Library and Wait for Index
+
+```
+1. create_store(url_or_path, name)
+   → Returns: job_id
+   → Background indexing starts
+
+2. check_job_status(job_id)
+   → Poll every 10-30 seconds
+   → Status: 'pending' | 'running' | 'completed' | 'failed'
+   → Progress: percentage, current file
+
+3. When status='completed':
+   list_stores()
+   → Verify store appears in list
+
+4. search(query, stores=[new_store], limit=5)
+   → Test search works
+   → Verify indexing quality
+```
+
+**Example:**
+
+```
+create_store('https://github.com/fastapi/fastapi', 'fastapi')
+→ job_id: 'job_abc123'
+→ Status: Indexing started in background
+
+# Poll for completion (typically 30-120 seconds for medium repos)
+check_job_status('job_abc123')
+→ Status: running
+→ Progress: 45% (processing src/fastapi/routing.py)
+
+# ... wait 30 seconds ...
+
+check_job_status('job_abc123')
+→ Status: completed
+→ Indexed: 487 files, 125k lines
+
+# Verify and test
+list_stores()
+→ fastapi: 487 files, vector + FTS indexed
+
+search("dependency injection", stores=['fastapi'], limit=3)
+→ Returns relevant FastAPI DI patterns
+→ Store is ready for use!
+```
+
+## Handling Large Result Sets
+
+When initial search returns many results, use progressive detail to avoid context overload:
+
+### Workflow: Progressive Detail Strategy
+
+```
+1. search(query, detail='minimal', limit=20)
+   → Get summaries only (~100 tokens/result)
+   → Review all 20 summaries quickly
+
+2. Filter by relevance score:
+   - Score > 0.8: Excellent match
+   - Score 0.6-0.8: Good match
+   - Score < 0.6: Possibly irrelevant
+
+3. For top 3-5 results (score > 0.7):
+   get_full_context(selected_ids)
+   → Fetch complete code only for relevant items
+   → Saves ~80% context vs fetching all upfront
+
+4. If nothing relevant:
+   search(refined_query, detail='contextual', limit=10)
+   → Try different query with more context
+   → Or broaden/narrow the search
+```
+
+**Example:**
+
+```
+# Initial broad search
+search("authentication middleware", detail='minimal', limit=20)
+→ 20 results, scores ranging 0.45-0.92
+→ Total context: ~2k tokens (minimal)
+
+# Filter by score
+Top results (>0.7):
+  - Result 3: auth/jwt.ts (score: 0.92)
+  - Result 7: middleware/authenticate.ts (score: 0.85)
+  - Result 12: auth/session.ts (score: 0.74)
+
+# Get full code for top 3 only
+get_full_context(['result_3', 'result_7', 'result_12'])
+→ Complete implementations for relevant files only
+→ Context: ~3k tokens (vs ~15k if we fetched all 20)
+
+# Found what we needed! If not, would refine query and retry.
+```
+
+## Multi-Store Search with Ranking
+
+When searching across multiple stores, use ranking to prioritize results:
+
+### Workflow: Cross-Library Search
+
+```
+1. search(query, limit=10)
+   → Searches ALL stores
+   → Returns mixed results ranked by relevance
+
+2. Review store distribution:
+   - If dominated by one store: might narrow to specific stores
+   - If balanced: good cross-library perspective
+
+3. For specific library focus:
+   search(query, stores=['lib1', 'lib2'], limit=15)
+   → Search only relevant libraries
+   → Get more results from target libraries
+```
+
+**Example:**
+
+User: "How do different frameworks handle routing?"
+
+```
+# Search all indexed frameworks
+search("routing implementation", intent='find-implementation', limit=15)
+→ Result mix:
+  - express (score: 0.91)
+  - fastapi (score: 0.89)
+  - hono (score: 0.87)
+  - vue-router (score: 0.82)
+  - ...
+
+# All stores represented, good comparative view!
+
+# If user wants deeper FastAPI focus:
+search("routing implementation", stores=['fastapi', 'starlette'], limit=20)
+→ More FastAPI/Starlette-specific results
+→ Deeper exploration of Python framework routing
+```
+
+## Error Recovery
+
+When operations fail, use these recovery patterns:
+
+### Workflow: Handle Indexing Failures
+
+```
+1. create_store() fails or job_status shows 'failed'
+   → Check error message
+   → Common issues:
+     - Git auth required (private repo)
+     - Invalid URL/path
+     - Disk space
+     - Network timeout
+
+2. Recovery actions:
+   - Auth issue: Provide credentials or use HTTPS
+   - Invalid path: Verify URL/path exists
+   - Disk space: delete_store() unused stores
+   - Network: Retry with smaller repo or use --shallow
+
+3. Verify recovery:
+   list_stores() → Check store appeared
+   search(test_query, stores=[new_store]) → Verify searchable
+```
+
+**Example:**
+
+```
+create_store('https://github.com/private/repo', 'my-repo')
+→ job_id: 'job_xyz'
+
+check_job_status('job_xyz')
+→ Status: failed
+→ Error: "Authentication required for private repository"
+
+# Recovery: Use authenticated URL or SSH
+create_store('git@github.com:private/repo.git', 'my-repo')
+→ job_id: 'job_xyz2'
+
+check_job_status('job_xyz2')
+→ Status: completed
+→ Success!
+```
+
+## Combining Workflows
+
+Real-world usage often combines these patterns:
+
+```
+User: "I need to understand how Express and Hono handle middleware differently"
+
+1. list_stores() → check if both indexed
+2. If not: create_store() for missing framework(s)
+3. check_job_status() → wait for indexing
+4. search("middleware implementation", stores=['express', 'hono'], detail='minimal')
+5. Review summaries, identify key files
+6. get_full_context() for 2-3 most relevant from each framework
+7. Compare implementations with full context
+```
+
+This multi-step workflow is efficient, targeted, and conserves context.
+
+## Best Practices
+
+1. **Always start with detail='minimal'** - Get summaries first, full context selectively
+2. **Monitor background jobs** - Don't search newly added stores until indexing completes
+3. **Use intent parameter** - Helps ranking ('find-implementation' vs 'find-pattern' vs 'find-usage')
+4. **Filter by stores when known** - Faster, more focused results
+5. **Check relevance scores** - >0.7 is usually a strong match, <0.5 might be noise
+6. **Progressive refinement** - Broad search → filter → narrow → full context
+
+These workflows reduce token usage, minimize tool calls, and get you to the right answer faster.

package/skills/atomic-commits/SKILL.md

@@ -0,0 +1,77 @@
+---
+name: atomic-commits
+description: Methodology for creating atomic git commits grouped by logical features. Use when committing changes, organizing commits, or deciding how to group changes into commits.
+---
+
+# Atomic Commit Methodology
+
+## Grouping Rules
+
+**Group together:**
+- Feature code + its documentation
+- Feature code + its tests
+- Feature code + its configuration
+- API changes + API documentation
+
+**Keep separate:**
+- Unrelated features
+- Independent bug fixes
+- Standalone doc improvements
+- Pure refactoring
+
+## Commit Message Format
+
+```
+<type>(<scope>): <description>
+
+[optional body explaining why]
+```
+
+**Types:** `feat`, `fix`, `docs`, `refactor`, `test`, `chore`, `perf`
+
+## Workflow
+
+### 1. Analyze
+```bash
+git diff HEAD                              # All changes
+git diff --cached                          # Staged only
+git ls-files --others --exclude-standard   # Untracked
+```
+
+### 2. Evaluate README Impact
+**STOP and evaluate each row - does this change warrant README updates?**
+
+| Category | Question | If YES, update |
+|----------|----------|----------------|
+| Features | New CLI command, Skill, MCP tool, or capability? | Features/Commands section |
+| Setup | Changed install steps, settings, or data paths? | Setup/Installation section |
+| Scripts | New/changed npm scripts? | package.json scripts table |
+| Behavior | Changed how existing features work? | Relevant feature section |
+| Deps | New package or common error scenario? | Technologies/Troubleshooting |
+
+**If any YES**: Update the README appropriately and include it in this commit (feature + docs = atomic unit)
+
+### 3. Group by Feature
+Each commit = one complete, testable change.
+
+### 4. Commit Each Group
+```bash
+git add <files>        # Stage related files
+bun run precommit      # Validate (already quiet)
+# If fails: fix issues, re-stage, re-validate
+git commit -m "..."    # Commit when valid
+```
+
+### 5. Handle Untracked Files
+- **Should commit**: Add to appropriate commit
+- **Should ignore**: Suggest `.gitignore` (ask first)
+- **Intentional**: Document why untracked
+
+Goal: Clean `git status` after commits
+
+## Safety
+
+- Never force push
+- Never amend commits from other sessions
+- Ask if unsure about grouping
+- Always report final `git status`

package/skills/knowledge-search/SKILL.md

@@ -0,0 +1,54 @@
+---
+name: knowledge-search
+description: Teaches how to use Bluera Knowledge for accessing library sources and reference material. Explains two approaches - vector search via MCP/slash commands for discovery, or direct Grep/Read access to cloned repos for precision. Shows when to use each method and example workflows for querying library internals.
+---
+
+# Using Bluera Knowledge (BK)
+
+BK provides access to **definitive library sources** for your project dependencies.
+
+## Purpose: Authoritative References, Not Project Search
+
+**CKB is for**: Reference material and external sources
+- **Library sources**: Clone Vue.js/Pydantic/Hono for authoritative API reference
+- **Specifications**: Add project requirements, API specs, RFCs
+- **Documentation**: Add design docs, architecture guides, research papers
+- **Reference material**: Best practices, coding standards, examples
+
+**CKB is NOT for**: Searching your current project code
+- Use Grep/Read directly on project files
+- BK stores are for external reference material
+
+## Two Ways to Access Library Sources
+
+### 1. Vector Search (MCP or slash command)
+Find concepts and patterns across library docs:
+```
+search("vue reactivity system")
+/bluera-knowledge:search "pydantic custom validators"
+```
+
+### 2. Direct File Access (Grep/Read)
+Precise lookups in cloned library source:
+```
+Grep: pattern="defineReactive" path=".bluera/bluera-knowledge/repos/vue/"
+Read: .bluera/bluera-knowledge/repos/pydantic/pydantic/validators.py
+```
+
+## Both Are Valid!
+
+You can use **either or both** approaches on the same cloned repo:
+- Vector search to discover relevant files
+- Grep/Read to find specific functions/classes
+- Or just Grep/Read if you know what you're looking for
+
+## Example Workflow
+
+User: "How does Vue's computed properties work internally?"
+
+Claude:
+1. Check stores: `list_stores` MCP tool → vue store exists
+2. Vector search: `search("vue computed properties")` → finds computed.ts
+3. Read file: `.bluera/bluera-knowledge/repos/vue/packages/reactivity/src/computed.ts`
+4. Grep for implementation: pattern="class ComputedRefImpl"
+5. Explain with authoritative source code examples