bluera-knowledge 0.31.0 → 0.33.0

package/scripts/doctor.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+# Bluera Knowledge Doctor - Comprehensive Diagnostics
+# Run with: /bluera-knowledge:doctor
+
+PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(dirname "$(dirname "$0")")}"
+
+echo "═══════════════════════════════════════════════════════════"
+echo "  Bluera Knowledge Doctor"
+echo "═══════════════════════════════════════════════════════════"
+echo ""
+
+ISSUES=0
+
+# Check 1: Build tools
+echo "Checking build tools..."
+if command -v make &>/dev/null; then
+    echo "   [OK] make found"
+else
+    echo "   [FAIL] make NOT found - REQUIRED for native modules"
+    echo ""
+    echo "   FIX: Install build tools:"
+    echo "     Debian/Ubuntu: sudo apt install build-essential"
+    echo "     Fedora/RHEL:   sudo dnf groupinstall 'Development Tools'"
+    echo "     macOS:         xcode-select --install"
+    echo ""
+    ISSUES=$((ISSUES + 1))
+fi
+
+# Check 2: Node.js
+echo "Checking Node.js..."
+if command -v node &>/dev/null; then
+    NODE_VERSION=$(node --version)
+    NODE_MAJOR=$(echo "$NODE_VERSION" | cut -d. -f1 | tr -d 'v')
+    if [ "$NODE_MAJOR" -ge 24 ]; then
+        echo "   [WARN] Node.js $NODE_VERSION - v24+ may have native module issues"
+        echo ""
+        echo "   Native modules (tree-sitter, lancedb) may not compile on Node.js v24+"
+        echo "   due to V8 API changes. Recommended: Use Node.js v20.x or v22.x (LTS)"
+        echo ""
+    else
+        echo "   [OK] Node.js $NODE_VERSION"
+    fi
+else
+    echo "   [FAIL] Node.js NOT found"
+    ISSUES=$((ISSUES + 1))
+fi
+
+# Check 3: node_modules
+echo "Checking plugin dependencies..."
+if [ -d "$PLUGIN_ROOT/node_modules" ]; then
+    echo "   [OK] node_modules installed"
+else
+    echo "   [FAIL] node_modules missing"
+    echo ""
+    echo "   FIX: Run setup manually:"
+    echo "     $PLUGIN_ROOT/scripts/setup.sh"
+    echo ""
+    ISSUES=$((ISSUES + 1))
+fi
+
+# Check 4: MCP wrapper
+WRAPPER_PATH="$HOME/.local/bin/bluera-knowledge-mcp"
+echo "Checking MCP wrapper..."
+if [ -f "$WRAPPER_PATH" ]; then
+    echo "   [OK] MCP wrapper installed at $WRAPPER_PATH"
+else
+    echo "   [FAIL] MCP wrapper NOT installed"
+    echo ""
+    echo "   FIX: Run setup manually:"
+    echo "     $PLUGIN_ROOT/scripts/setup.sh"
+    echo ""
+    ISSUES=$((ISSUES + 1))
+fi
+
+# Check 5: Python 3
+echo "Checking Python 3..."
+if command -v python3 &>/dev/null; then
+    PY_VERSION=$(python3 --version 2>&1)
+    echo "   [OK] $PY_VERSION"
+else
+    echo "   [WARN] Python 3 not found (optional, needed for embeddings)"
+fi
+
+# Check 6: Playwright
+PLAYWRIGHT_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$HOME/.cache/ms-playwright}"
+echo "Checking Playwright browser..."
+if ls "$PLAYWRIGHT_PATH"/chromium-* 1>/dev/null 2>&1; then
+    echo "   [OK] Playwright Chromium installed"
+else
+    echo "   [WARN] Playwright Chromium not found (optional, needed for web crawling)"
+    echo "   FIX: npx playwright install chromium"
+fi
+
+echo ""
+echo "═══════════════════════════════════════════════════════════"
+if [ $ISSUES -eq 0 ]; then
+    echo "  All required checks passed!"
+    echo ""
+    echo "  If MCP is still failing, restart Claude Code."
+else
+    echo "  Found $ISSUES issue(s) - see FIX instructions above"
+fi
+echo "═══════════════════════════════════════════════════════════"

package/scripts/download-models.ts

@@ -0,0 +1,188 @@
+#!/usr/bin/env bun
+/**
+ * Download Embedding Models
+ *
+ * Pre-downloads all registered embedding models to ~/.cache/huggingface-transformers/
+ * This ensures models are available offline and avoids download time during benchmarks.
+ *
+ * Usage:
+ *   bun run models:download           # Download all models
+ *   bun run models:download --small   # Download only small models
+ *   bun run models:download --list    # List available models
+ */
+
+import { pipeline, env } from '@huggingface/transformers';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { MODEL_REGISTRY, RERANKER_REGISTRY, listModels } from '../src/models/registry.js';
+import type { ModelConfig } from '../src/models/registry.js';
+
+// Set cache directory
+env.cacheDir = join(homedir(), '.cache', 'huggingface-transformers');
+
+interface DownloadResult {
+  model: string;
+  success: boolean;
+  error?: string;
+  timeMs: number;
+}
+
+async function downloadModel(config: ModelConfig): Promise<DownloadResult> {
+  const startTime = Date.now();
+  try {
+    console.log(`  ↓ Downloading ${config.name} (${config.id})...`);
+
+    // Load the model to trigger download
+    const pipe = await pipeline('feature-extraction', config.id, {
+      dtype: 'fp32',
+    });
+
+    // Dispose to free memory
+    await pipe.dispose();
+
+    const timeMs = Date.now() - startTime;
+    console.log(`  ✓ ${config.name} downloaded (${(timeMs / 1000).toFixed(1)}s)`);
+
+    return { model: config.id, success: true, timeMs };
+  } catch (error) {
+    const timeMs = Date.now() - startTime;
+    const message = error instanceof Error ? error.message : String(error);
+    console.log(`  ✗ ${config.name} failed: ${message}`);
+
+    return { model: config.id, success: false, error: message, timeMs };
+  }
+}
+
+async function downloadReranker(id: string, name: string): Promise<DownloadResult> {
+  const startTime = Date.now();
+  try {
+    console.log(`  ↓ Downloading reranker ${name} (${id})...`);
+
+    // Rerankers use text-classification pipeline
+    const { AutoModelForSequenceClassification, AutoTokenizer } = await import(
+      '@huggingface/transformers'
+    );
+
+    const [model, tokenizer] = await Promise.all([
+      AutoModelForSequenceClassification.from_pretrained(id, { dtype: 'fp32' }),
+      AutoTokenizer.from_pretrained(id),
+    ]);
+
+    // Dispose to free memory
+    await model.dispose();
+    // Tokenizer doesn't have dispose
+
+    const timeMs = Date.now() - startTime;
+    console.log(`  ✓ ${name} downloaded (${(timeMs / 1000).toFixed(1)}s)`);
+
+    return { model: id, success: true, timeMs };
+  } catch (error) {
+    const timeMs = Date.now() - startTime;
+    const message = error instanceof Error ? error.message : String(error);
+    console.log(`  ✗ ${name} failed: ${message}`);
+
+    return { model: id, success: false, error: message, timeMs };
+  }
+}
+
+function printModelList(): void {
+  console.log('\nAvailable Embedding Models:\n');
+
+  const categories = ['bge', 'e5', 'minilm', 'gte', 'nomic', 'other'] as const;
+
+  for (const category of categories) {
+    const models = listModels({ category });
+    if (models.length === 0) continue;
+
+    console.log(`  ${category.toUpperCase()} Models:`);
+    for (const model of models) {
+      const size = model.sizeCategory.padEnd(5);
+      const dims = String(model.dimensions).padStart(4);
+      console.log(`    [${size}] ${dims}d  ${model.name}`);
+      if (model.notes) {
+        console.log(`                    ${model.notes}`);
+      }
+    }
+    console.log('');
+  }
+
+  console.log('  Reranker Models:');
+  for (const [key, config] of Object.entries(RERANKER_REGISTRY)) {
+    console.log(`    ${config.name} (${key})`);
+    if (config.notes) {
+      console.log(`      ${config.notes}`);
+    }
+  }
+  console.log('');
+}
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2);
+
+  if (args.includes('--list') || args.includes('-l')) {
+    printModelList();
+    return;
+  }
+
+  const smallOnly = args.includes('--small') || args.includes('-s');
+  const skipRerankers = args.includes('--no-rerankers');
+  const specificModel = args.find((a) => !a.startsWith('-'));
+
+  console.log('\n📦 Model Download Script\n');
+  console.log(`Cache directory: ${env.cacheDir}\n`);
+
+  const results: DownloadResult[] = [];
+
+  // Download embedding models
+  let models: ModelConfig[];
+  if (specificModel !== undefined) {
+    const config = MODEL_REGISTRY[specificModel];
+    if (config === undefined) {
+      console.error(`Unknown model: ${specificModel}`);
+      console.error(`Run with --list to see available models`);
+      process.exit(1);
+    }
+    models = [config];
+  } else if (smallOnly) {
+    models = listModels({ sizeCategory: 'small' });
+    console.log(`Downloading ${models.length} small embedding models...\n`);
+  } else {
+    models = Object.values(MODEL_REGISTRY);
+    console.log(`Downloading ${models.length} embedding models...\n`);
+  }
+
+  for (const config of models) {
+    const result = await downloadModel(config);
+    results.push(result);
+  }
+
+  // Download rerankers
+  if (!skipRerankers && specificModel === undefined) {
+    console.log('\nDownloading reranker models...\n');
+    for (const [, config] of Object.entries(RERANKER_REGISTRY)) {
+      const result = await downloadReranker(config.id, config.name);
+      results.push(result);
+    }
+  }
+
+  // Summary
+  console.log('\n' + '─'.repeat(50));
+  const succeeded = results.filter((r) => r.success).length;
+  const failed = results.filter((r) => !r.success).length;
+  const totalTime = results.reduce((sum, r) => sum + r.timeMs, 0);
+
+  console.log(`\n✓ ${succeeded} models downloaded`);
+  if (failed > 0) {
+    console.log(`✗ ${failed} models failed`);
+  }
+  console.log(`Total time: ${(totalTime / 1000).toFixed(1)}s\n`);
+
+  if (failed > 0) {
+    process.exit(1);
+  }
+}
+
+main().catch((error) => {
+  console.error('Fatal error:', error);
+  process.exit(1);
+});

package/scripts/export-web-store.ts

@@ -0,0 +1,142 @@
+#!/usr/bin/env bun
+/**
+ * Export web store documents to markdown fixtures
+ *
+ * Usage: bun run scripts/export-web-store.ts <store-name> <output-dir>
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+interface StoreEntry {
+  type: string;
+  id: string;
+  name: string;
+  url?: string;
+  status: string;
+}
+
+interface StoresData {
+  stores: StoreEntry[];
+}
+
+interface DocumentRow {
+  id: string;
+  content: string;
+  metadata: string;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isStoresData(value: unknown): value is StoresData {
+  if (!isRecord(value)) return false;
+  if (!Array.isArray(value['stores'])) return false;
+  return true;
+}
+
+function isDocumentRow(value: unknown): value is DocumentRow {
+  if (!isRecord(value)) return false;
+  return (
+    typeof value['id'] === 'string' &&
+    typeof value['content'] === 'string' &&
+    typeof value['metadata'] === 'string'
+  );
+}
+
+async function main(): Promise<void> {
+  const [storeName, outputDir] = process.argv.slice(2);
+
+  if (!storeName || !outputDir) {
+    console.error('Usage: bun run scripts/export-web-store.ts <store-name> <output-dir>');
+    process.exit(1);
+  }
+
+  // Find store in stores.json
+  const storesPath = path.join(process.cwd(), '.bluera/bluera-knowledge/data/stores.json');
+  if (!fs.existsSync(storesPath)) {
+    console.error('stores.json not found');
+    process.exit(1);
+  }
+
+  const storesJson: unknown = JSON.parse(fs.readFileSync(storesPath, 'utf-8'));
+  if (!isStoresData(storesJson)) {
+    console.error('Invalid stores.json format');
+    process.exit(1);
+  }
+
+  const store = storesJson.stores.find((s) => s.name === storeName);
+  if (!store) {
+    console.error(`Store not found: ${storeName}`);
+    console.error('Available stores:', storesJson.stores.map((s) => s.name).join(', '));
+    process.exit(1);
+  }
+
+  if (store.type !== 'web') {
+    console.error(`Store ${storeName} is not a web store (type: ${store.type})`);
+    process.exit(1);
+  }
+
+  // Load documents from LanceDB
+  const lanceDbPath = path.join(process.cwd(), `.bluera/bluera-knowledge/data/documents_${store.id}.lance`);
+  if (!fs.existsSync(lanceDbPath)) {
+    console.error(`LanceDB not found: ${lanceDbPath}`);
+    process.exit(1);
+  }
+
+  // Dynamic import for lancedb
+  const lancedb = await import('@lancedb/lancedb');
+  const db = await lancedb.connect(path.join(process.cwd(), '.bluera/bluera-knowledge/data'));
+  const table = await db.openTable(`documents_${store.id}`);
+
+  // Query all documents
+  const results = await table.query().select(['id', 'content', 'metadata']).toArray();
+
+  // Group chunks by source URL
+  const pageChunks = new Map<string, { content: string; title: string; chunks: string[] }>();
+
+  for (const row of results) {
+    if (!isDocumentRow(row)) continue;
+    const metadata: unknown = JSON.parse(row.metadata);
+    if (!isRecord(metadata)) continue;
+
+    const url = typeof metadata['url'] === 'string' ? metadata['url'] : 'unknown';
+    const title = typeof metadata['title'] === 'string' ? metadata['title'] : url;
+
+    if (!pageChunks.has(url)) {
+      pageChunks.set(url, { content: '', title, chunks: [] });
+    }
+    const page = pageChunks.get(url);
+    if (page) {
+      page.chunks.push(row.content);
+    }
+  }
+
+  // Create output directory
+  const fullOutputDir = path.resolve(outputDir);
+  if (!fs.existsSync(fullOutputDir)) {
+    fs.mkdirSync(fullOutputDir, { recursive: true });
+  }
+
+  // Write each page as a markdown file
+  let fileCount = 0;
+  for (const [url, page] of pageChunks) {
+    // Generate filename from URL
+    const urlPath = new URL(url).pathname.replace(/^\//, '').replace(/\/$/, '') || 'index';
+    const filename = urlPath.replace(/\//g, '-').replace(/[^a-zA-Z0-9-]/g, '') + '.md';
+
+    // Combine chunks (they may overlap, just concatenate for now)
+    const content = `# ${page.title}\n\nSource: ${url}\n\n---\n\n${page.chunks.join('\n\n---\n\n')}`;
+
+    fs.writeFileSync(path.join(fullOutputDir, filename), content);
+    fileCount++;
+  }
+
+  console.log(`Exported ${fileCount} pages to ${fullOutputDir}`);
+}
+
+main().catch((err: unknown) => {
+  console.error(err instanceof Error ? err.message : String(err));
+  process.exit(1);
+});

package/scripts/lib/mock-server.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+#
+# mock-server.sh
+#
+# Start a simple HTTP server for testing crawl functionality.
+# Serves static files from tests/fixtures/mock-server/
+#
+# Usage:
+#   # Start server in background, get PID:
+#   MOCK_PID=$(bash scripts/lib/mock-server.sh start)
+#
+#   # Stop server:
+#   bash scripts/lib/mock-server.sh stop $MOCK_PID
+#
+#   # Wait for server to be ready:
+#   bash scripts/lib/mock-server.sh wait
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+MOCK_SERVER_DIR="$REPO_ROOT/tests/fixtures/mock-server"
+PORT="${MOCK_SERVER_PORT:-8765}"
+
+start_server() {
+    # Start Python HTTP server in background
+    cd "$MOCK_SERVER_DIR"
+    python3 -m http.server "$PORT" --bind 127.0.0.1 >/dev/null 2>&1 &
+    local pid=$!
+    echo "$pid"
+}
+
+stop_server() {
+    local pid="$1"
+    if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
+        kill "$pid" 2>/dev/null || true
+        wait "$pid" 2>/dev/null || true
+    fi
+}
+
+wait_for_server() {
+    local max_attempts=30
+    local attempt=0
+    while [ $attempt -lt $max_attempts ]; do
+        if curl -s "http://127.0.0.1:$PORT/" >/dev/null 2>&1; then
+            return 0
+        fi
+        attempt=$((attempt + 1))
+        sleep 0.1
+    done
+    echo "Error: Mock server failed to start on port $PORT" >&2
+    return 1
+}
+
+case "${1:-start}" in
+    start)
+        start_server
+        ;;
+    stop)
+        stop_server "${2:-}"
+        ;;
+    wait)
+        wait_for_server
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|wait}" >&2
+        exit 1
+        ;;
+esac

package/scripts/mcp-wrapper.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# =====================================================================
+# bluera-knowledge MCP Wrapper Script
+# =====================================================================
+#
+# WORKAROUND: ${CLAUDE_PLUGIN_ROOT} not expanding in plugin .mcp.json
+#
+# Claude Code's ${CLAUDE_PLUGIN_ROOT} environment variable is documented
+# to work in plugin .mcp.json files, but doesn't expand correctly in
+# some environments. This wrapper script locates the plugin installation
+# directory dynamically and runs bootstrap.js.
+#
+# References:
+# - Bug: https://github.com/anthropics/claude-code/issues/9427
+#   "env variable expansion not working in plugin .mcp.json"
+#   (Closed Jan 2026, but still occurs on some systems)
+#
+# - Docs: https://code.claude.com/docs/en/mcp
+#   "Plugin MCP features: Use ${CLAUDE_PLUGIN_ROOT} for plugin-relative paths"
+#
+# This script reads ~/.claude/plugins/installed_plugins.json to find the
+# correct plugin path, with fallbacks to cache directory scanning.
+# =====================================================================
+
+set -e
+
+# Find plugin root from installed_plugins.json
+# Claude Code stores plugin metadata here when plugins are installed
+INSTALLED_PLUGINS="$HOME/.claude/plugins/installed_plugins.json"
+
+if [ -f "$INSTALLED_PLUGINS" ]; then
+    # Use jq if available (more reliable JSON parsing)
+    if command -v jq &> /dev/null; then
+        PLUGIN_ROOT=$(jq -r '.["bluera-knowledge"].path // empty' "$INSTALLED_PLUGINS" 2>/dev/null)
+    else
+        # Fallback: extract path with grep/sed (less reliable but works without jq)
+        # This handles the JSON structure: {"bluera-knowledge": {"path": "/path/to/plugin"}}
+        PLUGIN_ROOT=$(grep -o '"bluera-knowledge"[^}]*"path"[[:space:]]*:[[:space:]]*"[^"]*"' "$INSTALLED_PLUGINS" 2>/dev/null | sed 's/.*"path"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/')
+    fi
+fi
+
+# Fallback: scan cache directories for valid plugin installations
+# Plugins are cached at ~/.claude/plugins/cache/<org>/<name>/<version>/
+#
+# CRITICAL: Sort by version number (descending) to use the LATEST version!
+# Bug fixed in v0.22.7: Alphabetical sort put 0.20.0 before 0.22.x, causing
+# old bootstrap.ts (without --legacy-peer-deps) to run and fail.
+# The -V flag does proper version sorting: 0.22.7 > 0.22.6 > 0.21.0 > 0.20.0
+#
+if [ -z "$PLUGIN_ROOT" ] || [ ! -d "$PLUGIN_ROOT" ]; then
+    CACHE_BASE="$HOME/.claude/plugins/cache/bluera/bluera-knowledge"
+    if [ -d "$CACHE_BASE" ]; then
+        # Sort versions numerically (descending) with -V flag and use the latest valid one
+        for cache_dir in $(ls -d "$CACHE_BASE"/*/ 2>/dev/null | sort -V -r); do
+            if [ -f "$cache_dir/dist/mcp/bootstrap.js" ]; then
+                PLUGIN_ROOT="$cache_dir"
+                break
+            fi
+        done
+    fi
+fi
+
+# Verify we found a valid plugin installation
+if [ -z "$PLUGIN_ROOT" ] || [ ! -f "$PLUGIN_ROOT/dist/mcp/bootstrap.js" ]; then
+    echo "ERROR: Could not locate bluera-knowledge plugin" >&2
+    echo "Expected bootstrap.js at: \$PLUGIN_ROOT/dist/mcp/bootstrap.js" >&2
+    echo "Checked:" >&2
+    echo "  - installed_plugins.json: $INSTALLED_PLUGINS" >&2
+    echo "  - Cache dir: ~/.claude/plugins/cache/bluera/bluera-knowledge/" >&2
+    # Exit 2 = blocking error, stderr shown to user (exit 1 is non-blocking, hidden)
+    # See: https://code.claude.com/docs/en/hooks
+    exit 2
+fi
+
+# Check if build tools are available for native module compilation
+# Required for native modules (LanceDB) - check BEFORE attempting to run bootstrap
+if ! command -v make &>/dev/null; then
+    echo "ERROR: Build tools (make) not found - required for native modules." >&2
+    echo "" >&2
+    echo "Install build tools, then restart Claude Code:" >&2
+    echo "  Debian/Ubuntu: sudo apt install build-essential" >&2
+    echo "  Fedora/RHEL:   sudo dnf groupinstall 'Development Tools'" >&2
+    echo "  macOS:         xcode-select --install" >&2
+    # Exit 2 = blocking error, stderr shown to user (exit 1 is non-blocking, hidden)
+    # See: https://code.claude.com/docs/en/hooks
+    exit 2
+fi
+
+# Run bootstrap.js with all environment variables passed through
+# The bootstrap script handles dependency installation and MCP server startup
+exec node "$PLUGIN_ROOT/dist/mcp/bootstrap.js" "$@"