chub-dev 0.1.0 → 0.1.2-beta.0

package/src/lib/registry.js

@@ -1,8 +1,10 @@
-import { loadSourceRegistry } from './cache.js';
+import { loadSourceRegistry, loadSearchIndex } from './cache.js';
 import { loadConfig } from './config.js';
 import { normalizeLanguage } from './normalize.js';
+import { search as bm25Search } from './bm25.js';
 
 let _merged = null;
+let _searchIndex = null;
 
 /**
  * Load and merge entries from all configured sources.
@@ -14,11 +16,16 @@ function getMerged() {
   const config = loadConfig();
   const allDocs = [];
   const allSkills = [];
+  const searchIndexes = [];
 
   for (const source of config.sources) {
     const registry = loadSourceRegistry(source);
     if (!registry) continue;
 
+    // Load BM25 search index if available
+    const idx = loadSearchIndex(source);
+    if (idx) searchIndexes.push(idx);
+
     // Support both new format (docs/skills) and old format (entries)
     if (registry.docs) {
       for (const doc of registry.docs) {
@@ -46,6 +53,53 @@ function getMerged() {
     }
   }
 
+  // Merge search indexes (combine documents and recompute IDF)
+  if (searchIndexes.length > 0) {
+    if (searchIndexes.length === 1) {
+      _searchIndex = searchIndexes[0];
+    } else {
+      // Merge multiple indexes: combine documents, recompute global IDF
+      const allDocuments = searchIndexes.flatMap((idx) => idx.documents);
+      const N = allDocuments.length;
+      const dfMap = {};
+      const fieldLengths = { name: [], description: [], tags: [] };
+
+      for (const doc of allDocuments) {
+        const allTerms = new Set([
+          ...(doc.tokens.name || []),
+          ...(doc.tokens.description || []),
+          ...(doc.tokens.tags || []),
+        ]);
+        for (const term of allTerms) {
+          dfMap[term] = (dfMap[term] || 0) + 1;
+        }
+        fieldLengths.name.push((doc.tokens.name || []).length);
+        fieldLengths.description.push((doc.tokens.description || []).length);
+        fieldLengths.tags.push((doc.tokens.tags || []).length);
+      }
+
+      const idf = {};
+      for (const [term, df] of Object.entries(dfMap)) {
+        idf[term] = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+      }
+
+      const avg = (arr) => arr.length === 0 ? 0 : arr.reduce((a, b) => a + b, 0) / arr.length;
+      _searchIndex = {
+        version: '1.0.0',
+        algorithm: 'bm25',
+        params: searchIndexes[0].params,
+        totalDocs: N,
+        avgFieldLengths: {
+          name: avg(fieldLengths.name),
+          description: avg(fieldLengths.description),
+          tags: avg(fieldLengths.tags),
+        },
+        idf,
+        documents: allDocuments,
+      };
+    }
+  }
+
   _merged = { docs: allDocs, skills: allSkills };
   return _merged;
 }
@@ -121,11 +175,10 @@ export function getDisplayId(entry) {
 
 /**
  * Search entries by query string. Searches both docs and skills.
+ * Uses BM25 when a search index is available, falls back to keyword matching.
  */
 export function searchEntries(query, filters = {}) {
   const entries = applySourceFilter(getAllEntries());
-  const q = query.toLowerCase();
-  const words = q.split(/\s+/);
 
   // Deduplicate: same id+source appearing as both doc and skill → show once
   const seen = new Set();
@@ -138,27 +191,50 @@ export function searchEntries(query, filters = {}) {
     }
   }
 
-  let results = deduped.map((entry) => {
-    let score = 0;
+  // Build entry lookup by id
+  const entryById = new Map();
+  for (const entry of deduped) {
+    entryById.set(entry.id, entry);
+  }
+
+  let results;
+
+  if (_searchIndex) {
+    // BM25 search
+    const bm25Results = bm25Search(query, _searchIndex);
+    results = bm25Results
+      .map((r) => {
+        const entry = entryById.get(r.id);
+        return entry ? { entry, score: r.score } : null;
+      })
+      .filter(Boolean);
+  } else {
+    // Fallback: keyword matching
+    const q = query.toLowerCase();
+    const words = q.split(/\s+/);
 
-    if (entry.id === q) score += 100;
-    else if (entry.id.includes(q)) score += 50;
+    results = deduped.map((entry) => {
+      let score = 0;
 
-    const nameLower = entry.name.toLowerCase();
-    if (nameLower === q) score += 80;
-    else if (nameLower.includes(q)) score += 40;
+      if (entry.id === q) score += 100;
+      else if (entry.id.includes(q)) score += 50;
 
-    for (const word of words) {
-      if (entry.id.includes(word)) score += 10;
-      if (nameLower.includes(word)) score += 10;
-      if (entry.description?.toLowerCase().includes(word)) score += 5;
-      if (entry.tags?.some((t) => t.toLowerCase().includes(word))) score += 15;
-    }
+      const nameLower = entry.name.toLowerCase();
+      if (nameLower === q) score += 80;
+      else if (nameLower.includes(q)) score += 40;
 
-    return { entry, score };
-  });
+      for (const word of words) {
+        if (entry.id.includes(word)) score += 10;
+        if (nameLower.includes(word)) score += 10;
+        if (entry.description?.toLowerCase().includes(word)) score += 5;
+        if (entry.tags?.some((t) => t.toLowerCase().includes(word))) score += 15;
+      }
+
+      return { entry, score };
+    });
 
-  results = results.filter((r) => r.score > 0);
+    results = results.filter((r) => r.score > 0);
+  }
 
   const filtered = applyFilters(results.map((r) => r.entry), filters);
   const filteredSet = new Set(filtered);
@@ -255,6 +331,13 @@ export function resolveDocPath(entry, language, version) {
   let verObj = null;
   if (version) {
     verObj = langObj.versions?.find((v) => v.version === version);
+    if (!verObj) {
+      return {
+        versionNotFound: true,
+        requested: version,
+        available: langObj.versions?.map((v) => v.version) || [],
+      };
+    }
   } else {
     const rec = langObj.recommendedVersion;
     verObj = langObj.versions?.find((v) => v.version === rec) || langObj.versions?.[0];
@@ -272,7 +355,7 @@ export function resolveDocPath(entry, language, version) {
  * Given a resolved path and a type ("doc" or "skill"), return the entry file path.
  */
 export function resolveEntryFile(resolved, type) {
-  if (!resolved || resolved.needsLanguage) return { error: 'unresolved' };
+  if (!resolved || resolved.needsLanguage || resolved.versionNotFound) return { error: 'unresolved' };
 
   const fileName = type === 'skill' ? 'SKILL.md' : 'DOC.md';
 

package/src/lib/telemetry.js

@@ -0,0 +1,86 @@
+import { loadConfig } from './config.js';
+
+const DEFAULT_TELEMETRY_URL = 'https://api.aichub.org/v1';
+
+export function isTelemetryEnabled() {
+  if (process.env.CHUB_TELEMETRY === '0' || process.env.CHUB_TELEMETRY === 'false') return false;
+  const config = loadConfig();
+  return config.telemetry !== false;
+}
+
+export function getTelemetryUrl() {
+  const url = process.env.CHUB_TELEMETRY_URL;
+  if (url) return url;
+  const config = loadConfig();
+  return config.telemetry_url || DEFAULT_TELEMETRY_URL;
+}
+
+/**
+ * Send feedback to the API.
+ *
+ * @param {string} entryId - e.g. "openai/chat"
+ * @param {string} entryType - "doc" or "skill"
+ * @param {string} rating - "up" or "down"
+ * @param {object} opts - Additional context
+ * @param {string} [opts.comment]
+ * @param {string} [opts.docLang] - Language variant fetched
+ * @param {string} [opts.docVersion] - Version fetched
+ * @param {string} [opts.targetFile] - Specific file within the entry
+ * @param {string[]} [opts.labels] - Structured feedback labels
+ * @param {string} [opts.agent] - Agent name override
+ * @param {string} [opts.model] - LLM model override
+ * @param {string} [opts.cliVersion]
+ * @param {string} [opts.source] - Registry source name
+ */
+export async function sendFeedback(entryId, entryType, rating, opts = {}) {
+  if (!isTelemetryEnabled()) return { status: 'skipped', reason: 'telemetry_disabled' };
+
+  const { getOrCreateClientId, detectAgent, detectAgentVersion } = await import('./identity.js');
+  const clientId = await getOrCreateClientId();
+  const telemetryUrl = getTelemetryUrl();
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 3000);
+
+  try {
+    const res = await fetch(`${telemetryUrl}/feedback`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-Client-ID': clientId,
+      },
+      body: JSON.stringify({
+        entry_id: entryId,
+        entry_type: entryType,
+        rating,
+        // Doc-specific dimensions
+        doc_lang: opts.docLang || undefined,
+        doc_version: opts.docVersion || undefined,
+        target_file: opts.targetFile || undefined,
+        // Structured feedback
+        labels: opts.labels || undefined,
+        comment: opts.comment || undefined,
+        // Agent info
+        agent: {
+          name: opts.agent || detectAgent(),
+          version: detectAgentVersion(),
+          model: opts.model || undefined,
+        },
+        // Context
+        cli_version: opts.cliVersion || undefined,
+        source: opts.source || undefined,
+      }),
+      signal: controller.signal,
+    });
+    clearTimeout(timeout);
+
+    if (res.ok) {
+      const data = await res.json();
+      return { status: 'sent', feedback_id: data.feedback_id || data.id };
+    }
+    return { status: 'error', code: res.status };
+  } catch (err) {
+    clearTimeout(timeout);
+    return { status: 'error', reason: 'network' };
+  }
+}

package/src/mcp/server.js

@@ -0,0 +1,177 @@
+/**
+ * Context Hub MCP Server.
+ *
+ * Exposes chub search, get, list, annotate, and feedback as MCP tools
+ * for use with Claude Code, Cursor, and other MCP-compatible agents.
+ */
+
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { z } from 'zod';
+import { ensureRegistry } from '../lib/cache.js';
+import { listEntries } from '../lib/registry.js';
+import { handleSearch, handleGet, handleList, handleAnnotate, handleFeedback } from './tools.js';
+
+// Prevent console.log from corrupting the stdio JSON-RPC protocol.
+// Any transitive dependency (e.g. posthog-node) that calls console.log
+// would break the MCP transport without this redirect.
+const _stderr = process.stderr;
+console.log = (...args) => _stderr.write(args.join(' ') + '\n');
+console.warn = (...args) => _stderr.write('[warn] ' + args.join(' ') + '\n');
+console.info = (...args) => _stderr.write('[info] ' + args.join(' ') + '\n');
+console.debug = (...args) => _stderr.write('[debug] ' + args.join(' ') + '\n');
+
+// Read package version
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const pkg = JSON.parse(readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf8'));
+
+// Create server
+const server = new McpServer({
+  name: 'chub',
+  version: pkg.version,
+});
+
+// --- Register Tools ---
+
+server.tool(
+  'chub_search',
+  'Search Context Hub for docs and skills by query, tags, or language',
+  {
+    query: z.string().optional().describe('Search query. Omit to list all entries.'),
+    tags: z.string().optional().describe('Comma-separated tag filter (e.g. "openai,chat")'),
+    lang: z.string().optional().describe('Filter by language (e.g. "python", "js")'),
+    limit: z.number().int().min(1).max(100).optional().describe('Max results (default 20)'),
+  },
+  async (args) => handleSearch(args),
+);
+
+server.tool(
+  'chub_get',
+  'Fetch the content of a doc or skill by ID from Context Hub',
+  {
+    id: z.string().describe('Entry ID (e.g. "openai/chat", "stripe/api"). Use source:id for disambiguation.'),
+    lang: z.string().optional().describe('Language variant (e.g. "python", "js"). Auto-selected if only one.'),
+    version: z.string().optional().describe('Specific version (e.g. "1.52.0"). Defaults to recommended.'),
+    full: z.boolean().optional().describe('Fetch all files, not just the entry point (default false)'),
+    file: z.string().optional().describe('Fetch a specific file by path (e.g. "references/streaming.md")'),
+  },
+  async (args) => handleGet(args),
+);
+
+server.tool(
+  'chub_list',
+  'List all available docs and skills in Context Hub',
+  {
+    tags: z.string().optional().describe('Comma-separated tag filter'),
+    lang: z.string().optional().describe('Filter by language'),
+    limit: z.number().int().min(1).max(500).optional().describe('Max entries (default 50)'),
+  },
+  async (args) => handleList(args),
+);
+
+server.tool(
+  'chub_annotate',
+  'Read, write, clear, or list agent annotations. Modes: (1) list=true to list all, (2) id+note to write, (3) id+clear=true to delete, (4) id alone to read. Annotations persist locally across sessions.',
+  {
+    id: z.string().optional().describe('Entry ID to annotate (e.g. "openai/chat"). Required unless using list mode.'),
+    note: z.string().optional().describe('Annotation text to save. Omit to read existing annotation.'),
+    clear: z.boolean().optional().describe('Remove the annotation for this entry (default false)'),
+    list: z.boolean().optional().describe('List all annotations (default false). When true, id is not needed.'),
+  },
+  async (args) => handleAnnotate(args),
+);
+
+server.tool(
+  'chub_feedback',
+  'Send quality feedback (thumbs up/down) for a doc or skill to help authors improve content',
+  {
+    id: z.string().describe('Entry ID to rate (e.g. "openai/chat")'),
+    rating: z.enum(['up', 'down']).describe('Thumbs up or down'),
+    comment: z.string().optional().describe('Optional comment explaining the rating'),
+    type: z.enum(['doc', 'skill']).optional().describe('Entry type. Auto-detected if omitted.'),
+    lang: z.string().optional().describe('Language variant rated'),
+    version: z.string().optional().describe('Version rated'),
+    file: z.string().optional().describe('Specific file rated'),
+    labels: z.array(z.enum([
+      'accurate', 'well-structured', 'helpful', 'good-examples',
+      'outdated', 'inaccurate', 'incomplete', 'wrong-examples',
+      'wrong-version', 'poorly-structured',
+    ])).optional().describe('Structured feedback labels'),
+  },
+  async (args) => handleFeedback(args),
+);
+
+// --- Register Resource ---
+
+server.resource(
+  'registry',
+  'chub://registry',
+  {
+    title: 'Context Hub Registry',
+    description: 'Browse the full Context Hub registry of docs and skills',
+    mimeType: 'application/json',
+  },
+  async (uri) => {
+    try {
+      const entries = listEntries({});
+      const simplified = entries.map((entry) => ({
+        id: entry.id,
+        name: entry.name,
+        type: entry._type || (entry.languages ? 'doc' : 'skill'),
+        description: entry.description,
+        tags: entry.tags || [],
+        ...(entry.languages
+          ? {
+            languages: entry.languages.map((l) => ({
+              language: l.language,
+              versions: l.versions?.map((v) => v.version) || [],
+              recommended: l.recommendedVersion,
+            })),
+          }
+          : {}),
+      }));
+      return {
+        contents: [{
+          uri: uri.href,
+          mimeType: 'application/json',
+          text: JSON.stringify({ entries: simplified, total: simplified.length }, null, 2),
+        }],
+      };
+    } catch (err) {
+      console.warn(`Registry resource error: ${err.message}`);
+      return {
+        contents: [{
+          uri: uri.href,
+          mimeType: 'application/json',
+          text: JSON.stringify({ error: 'Registry not loaded. Run "chub update" first.' }),
+        }],
+      };
+    }
+  },
+);
+
+// --- Process Safety ---
+
+// Prevent the server from crashing on unhandled errors (long-lived process)
+process.on('uncaughtException', (err) => {
+  _stderr.write(`[chub-mcp] Uncaught exception: ${err.message}\n`);
+});
+process.on('unhandledRejection', (reason) => {
+  _stderr.write(`[chub-mcp] Unhandled rejection: ${reason}\n`);
+});
+
+// --- Start Server ---
+
+// Best-effort registry load — server starts even if this fails
+try {
+  await ensureRegistry();
+} catch (err) {
+  _stderr.write(`[chub-mcp] Warning: Registry not loaded: ${err.message}\n`);
+}
+
+const transport = new StdioServerTransport();
+await server.connect(transport);
+_stderr.write(`[chub-mcp] Server started (v${pkg.version})\n`);

package/src/mcp/tools.js

@@ -0,0 +1,251 @@
+/**
+ * MCP tool handler implementations.
+ * Each handler wraps existing lib/ functions and returns MCP-compatible results.
+ */
+
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join, resolve, relative } from 'node:path';
+import { searchEntries, getEntry, listEntries, resolveDocPath, resolveEntryFile } from '../lib/registry.js';
+import { fetchDoc, fetchDocFull } from '../lib/cache.js';
+import { readAnnotation, writeAnnotation, clearAnnotation, listAnnotations } from '../lib/annotations.js';
+import { sendFeedback, isTelemetryEnabled } from '../lib/telemetry.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+let _cliVersion;
+function getCliVersion() {
+  if (_cliVersion) return _cliVersion;
+  try {
+    const pkg = JSON.parse(readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf8'));
+    _cliVersion = pkg.version;
+  } catch {
+    _cliVersion = 'unknown';
+  }
+  return _cliVersion;
+}
+
+function textResult(data) {
+  return {
+    content: [{ type: 'text', text: typeof data === 'string' ? data : JSON.stringify(data, null, 2) }],
+  };
+}
+
+function errorResult(message, details = {}) {
+  return {
+    content: [{ type: 'text', text: JSON.stringify({ error: message, ...details }, null, 2) }],
+    isError: true,
+  };
+}
+
+/**
+ * Simplify an entry for agent-friendly output (strip internal fields).
+ */
+function simplifyEntry(entry) {
+  const result = {
+    id: entry.id,
+    name: entry.name,
+    type: entry._type || (entry.languages ? 'doc' : 'skill'),
+    description: entry.description,
+    tags: entry.tags || [],
+  };
+  if (entry.languages) {
+    result.languages = entry.languages.map((l) => l.language);
+  }
+  return result;
+}
+
+// --- Tool Handlers ---
+
+export async function handleSearch({ query, tags, lang, limit = 20 }) {
+  try {
+    let entries;
+    if (query) {
+      entries = searchEntries(query, { tags, lang });
+    } else {
+      entries = listEntries({ tags, lang });
+    }
+    const sliced = entries.slice(0, limit);
+    return textResult({
+      results: sliced.map(simplifyEntry),
+      total: entries.length,
+      showing: sliced.length,
+    });
+  } catch (err) {
+    return errorResult(`Search failed: ${err.message}`);
+  }
+}
+
+export async function handleGet({ id, lang, version, full = false, file }) {
+  try {
+    // Validate file parameter early (before entry lookup) to reject path traversal
+    if (file) {
+      const normalizedFile = resolve('/', file).slice(1);
+      if (normalizedFile !== file || file.includes('..')) {
+        return errorResult(`Invalid file path: "${file}". Path traversal is not allowed.`);
+      }
+    }
+
+    const result = getEntry(id);
+
+    if (result.ambiguous) {
+      return errorResult(`Ambiguous entry ID "${id}". Be specific:`, {
+        alternatives: result.alternatives,
+      });
+    }
+
+    if (!result.entry) {
+      return errorResult(`Entry "${id}" not found.`, {
+        suggestion: 'Use chub_search to find available entries.',
+      });
+    }
+
+    const entry = result.entry;
+    const type = entry.languages ? 'doc' : 'skill';
+    const resolved = resolveDocPath(entry, lang, version);
+
+    if (!resolved) {
+      return errorResult(`Could not resolve path for "${id}".`);
+    }
+
+    if (resolved.versionNotFound) {
+      return errorResult(`Version "${resolved.requested}" not found for "${id}".`, {
+        available: resolved.available,
+      });
+    }
+
+    if (resolved.needsLanguage) {
+      return errorResult(`Multiple languages available for "${id}". Specify the lang parameter.`, {
+        available: resolved.available,
+      });
+    }
+
+    const entryFile = resolveEntryFile(resolved, type);
+    if (entryFile.error) {
+      return errorResult(`"${id}": ${entryFile.error}`);
+    }
+
+    let content;
+
+    if (file) {
+      // Fetch a specific file
+      if (!resolved.files.includes(file)) {
+        const entryFileName = type === 'skill' ? 'SKILL.md' : 'DOC.md';
+        const available = resolved.files.filter((f) => f !== entryFileName);
+        return errorResult(`File "${file}" not found in ${id}.`, {
+          available: available.length > 0 ? available : '(none)',
+        });
+      }
+      content = await fetchDoc(resolved.source, join(resolved.path, file));
+    } else if (full && resolved.files.length > 0) {
+      // Fetch all files
+      const allFiles = await fetchDocFull(resolved.source, resolved.path, resolved.files);
+      content = allFiles.map((f) => `# FILE: ${f.name}\n\n${f.content}`).join('\n\n---\n\n');
+    } else {
+      // Fetch entry point only
+      content = await fetchDoc(resolved.source, entryFile.filePath);
+    }
+
+    // Append annotation if present
+    const annotation = readAnnotation(entry.id);
+    if (annotation) {
+      content += `\n\n---\n[Agent note — ${annotation.updatedAt}]\n${annotation.note}\n`;
+    }
+
+    return textResult(content);
+  } catch (err) {
+    return errorResult(`Failed to fetch "${id}": ${err.message}`);
+  }
+}
+
+export async function handleList({ tags, lang, limit = 50 }) {
+  try {
+    const entries = listEntries({ tags, lang });
+    const sliced = entries.slice(0, limit);
+    return textResult({
+      entries: sliced.map(simplifyEntry),
+      total: entries.length,
+      showing: sliced.length,
+    });
+  } catch (err) {
+    return errorResult(`List failed: ${err.message}`);
+  }
+}
+
+export async function handleAnnotate({ id, note, clear = false, list = false }) {
+  try {
+    if (list) {
+      const annotations = listAnnotations();
+      return textResult({ annotations, total: annotations.length });
+    }
+
+    if (!id) {
+      return errorResult('Missing required parameter: id. Provide an entry ID or use list mode.');
+    }
+
+    // Validate entry ID to prevent path traversal or filesystem abuse
+    if (id.length > 200) {
+      return errorResult('Entry ID too long (max 200 characters).');
+    }
+    if (!/^[a-zA-Z0-9._\-\/]+$/.test(id)) {
+      return errorResult('Entry ID contains invalid characters. Use only alphanumeric, hyphens, underscores, dots, and slashes.');
+    }
+
+    if (clear) {
+      const removed = clearAnnotation(id);
+      return textResult({
+        status: removed ? 'cleared' : 'not_found',
+        id,
+      });
+    }
+
+    if (note) {
+      const saved = writeAnnotation(id, note);
+      return textResult({ status: 'saved', annotation: saved });
+    }
+
+    // Read mode
+    const annotation = readAnnotation(id);
+    if (annotation) {
+      return textResult({ annotation });
+    }
+    return textResult({ status: 'no_annotation', id });
+  } catch (err) {
+    return errorResult(`Annotation failed: ${err.message}`);
+  }
+}
+
+export async function handleFeedback({ id, rating, comment, type, lang, version, file, labels }) {
+  try {
+    if (!isTelemetryEnabled()) {
+      return textResult({ status: 'skipped', reason: 'telemetry_disabled' });
+    }
+
+    // Auto-detect entry type if not provided
+    let entryType = type;
+    if (!entryType) {
+      try {
+        const result = getEntry(id);
+        if (result.entry) {
+          entryType = result.entry.languages ? 'doc' : 'skill';
+        }
+      } catch {
+        // Fall through with undefined type
+      }
+      entryType = entryType || 'doc';
+    }
+
+    const result = await sendFeedback(id, entryType, rating, {
+      comment,
+      docLang: lang,
+      docVersion: version,
+      targetFile: file,
+      labels,
+      agent: 'mcp-server',
+      cliVersion: getCliVersion(),
+    });
+
+    return textResult(result);
+  } catch (err) {
+    return errorResult(`Feedback failed: ${err.message}`);
+  }
+}