chub-dev 0.2.0-beta.3 → 0.2.0-beta.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "chub-dev",
-  "version": "0.2.0-beta.3",
+  "version": "0.2.0-beta.4",
   "description": "CLI for Context Hub - search and retrieve LLM-optimized docs and skills",
   "type": "module",
   "bin": {
@@ -46,6 +46,6 @@
     "yaml": "^2.3.0"
   },
   "devDependencies": {
-    "vitest": "^3.0.0"
+    "vitest": "^4.0.18"
   }
 }

package/src/commands/annotate.js

@@ -0,0 +1,83 @@
+import chalk from 'chalk';
+import { readAnnotation, writeAnnotation, clearAnnotation, listAnnotations } from '../lib/annotations.js';
+import { output, error, info } from '../lib/output.js';
+
+export function registerAnnotateCommand(program) {
+  program
+    .command('annotate [id] [note]')
+    .description('Attach agent notes to a doc or skill')
+    .option('--clear', 'Remove annotation for this entry')
+    .option('--list', 'List all annotations')
+    .action((id, note, opts) => {
+      const globalOpts = program.optsWithGlobals();
+
+      if (opts.list) {
+        const annotations = listAnnotations();
+        output(
+          annotations,
+          (data) => {
+            if (data.length === 0) {
+              console.log('No annotations.');
+              return;
+            }
+            for (const a of data) {
+              console.log(`${chalk.bold(a.id)} ${chalk.dim(`(${a.updatedAt})`)}`);
+              console.log(`  ${a.note}`);
+              console.log();
+            }
+          },
+          globalOpts
+        );
+        return;
+      }
+
+      if (!id) {
+        error('Usage: chub annotate <id> <note> | chub annotate <id> --clear | chub annotate --list', globalOpts);
+      }
+
+      if (opts.clear) {
+        const removed = clearAnnotation(id);
+        output(
+          { id, cleared: removed },
+          (data) => {
+            if (data.cleared) {
+              console.log(`Annotation cleared for ${chalk.bold(id)}.`);
+            } else {
+              console.log(`No annotation found for ${chalk.bold(id)}.`);
+            }
+          },
+          globalOpts
+        );
+        return;
+      }
+
+      if (!note) {
+        // Show existing annotation
+        const existing = readAnnotation(id);
+        if (existing) {
+          output(
+            existing,
+            (data) => {
+              console.log(`${chalk.bold(data.id)} ${chalk.dim(`(${data.updatedAt})`)}`);
+              console.log(data.note);
+            },
+            globalOpts
+          );
+        } else {
+          output(
+            { id, note: null },
+            () => console.log(`No annotation for ${chalk.bold(id)}.`),
+            globalOpts
+          );
+        }
+        return;
+      }
+
+      const data = writeAnnotation(id, note);
+      output(
+        data,
+        (d) => console.log(`Annotation saved for ${chalk.bold(d.id)}.`),
+        globalOpts
+      );
+    });
+}

package/src/commands/build.js

@@ -4,6 +4,7 @@ import chalk from 'chalk';
 import { parseFrontmatter } from '../lib/frontmatter.js';
 import { info } from '../lib/output.js';
 import { trackEvent } from '../lib/analytics.js';
+import { buildIndex } from '../lib/bm25.js';
 
 /**
  * Recursively find all DOC.md and SKILL.md files under a directory.
@@ -301,6 +302,14 @@ export function registerBuildCommand(program) {
       mkdirSync(outputDir, { recursive: true });
       writeFileSync(join(outputDir, 'registry.json'), JSON.stringify(registry, null, 2));
 
+      // Build and write BM25 search index
+      const allEntries = [
+        ...allDocs.map((d) => ({ ...d, _type: 'doc' })),
+        ...allSkills.map((s) => ({ ...s, _type: 'skill' })),
+      ];
+      const searchIndex = buildIndex(allEntries);
+      writeFileSync(join(outputDir, 'search-index.json'), JSON.stringify(searchIndex));
+
       // Copy content tree
       for (const authorEntry of topLevel) {
         const src = join(contentDir, authorEntry.name);

package/src/commands/get.js

@@ -5,6 +5,7 @@ import { getEntry, resolveDocPath, resolveEntryFile } from '../lib/registry.js';
 import { fetchDoc, fetchDocFull } from '../lib/cache.js';
 import { output, error, info } from '../lib/output.js';
 import { trackEvent } from '../lib/analytics.js';
+import { readAnnotation } from '../lib/annotations.js';
 
 /**
  * Fetch one or more entries by ID. Auto-detects doc vs skill per entry.
@@ -35,6 +36,13 @@ async function fetchEntries(ids, opts, globalOpts) {
       error(`Could not resolve path for "${id}" ${opts.lang || ''} ${opts.version || ''}`.trim(), globalOpts);
     }
 
+    if (resolved.versionNotFound) {
+      error(
+        `Version "${resolved.requested}" not found for "${id}". Available versions: ${resolved.available.join(', ')}`,
+        globalOpts
+      );
+    }
+
     if (resolved.needsLanguage) {
       error(
         `Multiple languages available for "${id}": ${resolved.available.join(', ')}. Specify --lang.`,
@@ -47,13 +55,32 @@ async function fetchEntries(ids, opts, globalOpts) {
       error(`"${id}" ${entryFile.error}`, globalOpts);
     }
 
+    // Determine which reference files exist (beyond DOC.md/SKILL.md)
+    const entryFileName = type === 'skill' ? 'SKILL.md' : 'DOC.md';
+    const refFiles = resolved.files.filter((f) => f !== entryFileName);
+
     try {
-      if (opts.full && resolved.files.length > 0) {
+      if (opts.file) {
+        // --file mode: fetch specific file(s) by path
+        const requested = opts.file.split(',').map((f) => f.trim());
+        const invalid = requested.filter((f) => !resolved.files.includes(f));
+        if (invalid.length > 0) {
+          const available = refFiles.length > 0 ? refFiles.join(', ') : '(none)';
+          error(`File "${invalid[0]}" not found in ${id}. Available: ${available}`, globalOpts);
+        }
+        if (requested.length === 1) {
+          const content = await fetchDoc(resolved.source, join(resolved.path, requested[0]));
+          results.push({ id: entry.id, type, content, path: join(resolved.path, requested[0]) });
+        } else {
+          const allFiles = await fetchDocFull(resolved.source, resolved.path, requested);
+          results.push({ id: entry.id, type, files: allFiles, path: resolved.path });
+        }
+      } else if (opts.full && resolved.files.length > 0) {
         const allFiles = await fetchDocFull(resolved.source, resolved.path, resolved.files);
         results.push({ id: entry.id, type, files: allFiles, path: resolved.path });
       } else {
         const content = await fetchDoc(resolved.source, entryFile.filePath);
-        results.push({ id: entry.id, type, content, path: entryFile.filePath });
+        results.push({ id: entry.id, type, content, path: entryFile.filePath, additionalFiles: refFiles });
       }
     } catch (err) {
       error(err.message, globalOpts);
@@ -112,9 +139,25 @@ async function fetchEntries(ids, opts, globalOpts) {
     }
   } else {
     if (results.length === 1 && !results[0].files) {
+      const r = results[0];
+      const extraFiles = r.additionalFiles || [];
+      const annotation = readAnnotation(r.id);
+      const jsonData = { id: r.id, type: r.type, content: r.content, path: r.path };
+      if (extraFiles.length > 0) jsonData.additionalFiles = extraFiles;
+      if (annotation) jsonData.annotation = annotation;
       output(
-        { id: results[0].id, type: results[0].type, content: results[0].content, path: results[0].path },
-        (data) => process.stdout.write(data.content),
+        jsonData,
+        (data) => {
+          process.stdout.write(data.content);
+          if (annotation) {
+            process.stdout.write(`\n\n---\n[Agent note — ${annotation.updatedAt}]\n${annotation.note}\n`);
+          }
+          if (extraFiles.length > 0) {
+            const fileList = extraFiles.map((f) => `  ${f}`).join('\n');
+            const example = `chub get ${r.id} --file ${extraFiles[0]}`;
+            process.stdout.write(`\n\n---\nAdditional files available (use --file to fetch):\n${fileList}\nExample: ${example}\n`);
+          }
+        },
         globalOpts
       );
     } else {
@@ -142,6 +185,7 @@ export function registerGetCommand(program) {
     .option('--version <version>', 'Specific version (for docs)')
     .option('-o, --output <path>', 'Write to file or directory')
     .option('--full', 'Fetch all files (not just entry point)')
+    .option('--file <paths>', 'Fetch specific file(s) by path (comma-separated)')
     .action(async (ids, opts) => {
       const globalOpts = program.optsWithGlobals();
       await fetchEntries(ids, opts, globalOpts);

package/src/index.js

@@ -10,6 +10,7 @@ import { registerSearchCommand } from './commands/search.js';
 import { registerGetCommand } from './commands/get.js';
 import { registerBuildCommand } from './commands/build.js';
 import { registerFeedbackCommand } from './commands/feedback.js';
+import { registerAnnotateCommand } from './commands/annotate.js';
 import { trackEvent, shutdownAnalytics } from './lib/analytics.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -77,14 +78,14 @@ const program = new Command();
 program
   .name('chub')
   .description('Context Hub - search and retrieve LLM-optimized docs and skills')
-  .version(pkg.version)
+  .version(pkg.version, '-V, --cli-version')
   .option('--json', 'Output as JSON (machine-readable)')
   .action(() => {
     printUsage();
   });
 
 // Commands that don't need registry
-const SKIP_REGISTRY = ['update', 'cache', 'build', 'feedback', 'help'];
+const SKIP_REGISTRY = ['update', 'cache', 'build', 'feedback', 'annotate', 'help'];
 
 program.hook('preAction', async (thisCommand) => {
   const cmdName = thisCommand.args?.[0] || thisCommand.name();
@@ -111,6 +112,7 @@ registerSearchCommand(program);
 registerGetCommand(program);
 registerBuildCommand(program);
 registerFeedbackCommand(program);
+registerAnnotateCommand(program);
 
 program.parse();
 

package/src/lib/annotations.js

@@ -0,0 +1,57 @@
+import { readFileSync, writeFileSync, mkdirSync, unlinkSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { getChubDir } from './config.js';
+
+function getAnnotationsDir() {
+  return join(getChubDir(), 'annotations');
+}
+
+function annotationPath(entryId) {
+  const safe = entryId.replace(/\//g, '--');
+  return join(getAnnotationsDir(), `${safe}.json`);
+}
+
+export function readAnnotation(entryId) {
+  try {
+    return JSON.parse(readFileSync(annotationPath(entryId), 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+export function writeAnnotation(entryId, note) {
+  const dir = getAnnotationsDir();
+  mkdirSync(dir, { recursive: true });
+  const data = {
+    id: entryId,
+    note,
+    updatedAt: new Date().toISOString(),
+  };
+  writeFileSync(annotationPath(entryId), JSON.stringify(data, null, 2));
+  return data;
+}
+
+export function clearAnnotation(entryId) {
+  try {
+    unlinkSync(annotationPath(entryId));
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function listAnnotations() {
+  const dir = getAnnotationsDir();
+  try {
+    const files = readdirSync(dir).filter((f) => f.endsWith('.json'));
+    return files.map((f) => {
+      try {
+        return JSON.parse(readFileSync(join(dir, f), 'utf8'));
+      } catch {
+        return null;
+      }
+    }).filter(Boolean);
+  } catch {
+    return [];
+  }
+}

package/src/lib/bm25.js

@@ -0,0 +1,170 @@
+/**
+ * BM25 search implementation for Context Hub.
+ * Index is built at `chub build` time, scoring happens at search time.
+ * Tokenizer is shared between build and search to ensure consistency.
+ */
+
+const STOP_WORDS = new Set([
+  'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
+  'has', 'have', 'in', 'is', 'it', 'its', 'of', 'on', 'or', 'that',
+  'the', 'to', 'was', 'were', 'will', 'with', 'this', 'but', 'not',
+  'you', 'your', 'can', 'do', 'does', 'how', 'if', 'may', 'no',
+  'so', 'than', 'too', 'very', 'just', 'about', 'into', 'over',
+  'such', 'then', 'them', 'these', 'those', 'through', 'under',
+  'use', 'using', 'used',
+]);
+
+// BM25 default parameters
+const DEFAULT_K1 = 1.5;
+const DEFAULT_B = 0.75;
+
+// Field weights for multi-field scoring
+const FIELD_WEIGHTS = {
+  name: 3.0,
+  tags: 2.0,
+  description: 1.0,
+};
+
+/**
+ * Tokenize text into lowercase terms with stop word removal.
+ * Must be used identically at build time and search time.
+ */
+export function tokenize(text) {
+  if (!text) return [];
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, ' ')
+    .split(/[\s-]+/)
+    .filter((t) => t.length > 1 && !STOP_WORDS.has(t));
+}
+
+/**
+ * Build a BM25 search index from registry entries.
+ * Called during `chub build`.
+ *
+ * @param {Array} entries - Combined docs and skills from registry
+ * @returns {Object} The search index
+ */
+export function buildIndex(entries) {
+  const documents = [];
+  const dfMap = {}; // document frequency per term (across all fields)
+  const fieldLengths = { name: [], description: [], tags: [] };
+
+  for (const entry of entries) {
+    const nameTokens = tokenize(entry.name);
+    const descTokens = tokenize(entry.description || '');
+    const tagTokens = (entry.tags || []).flatMap((t) => tokenize(t));
+
+    documents.push({
+      id: entry.id,
+      tokens: {
+        name: nameTokens,
+        description: descTokens,
+        tags: tagTokens,
+      },
+    });
+
+    fieldLengths.name.push(nameTokens.length);
+    fieldLengths.description.push(descTokens.length);
+    fieldLengths.tags.push(tagTokens.length);
+
+    // Count document frequency — a term counts once per document (union of all fields)
+    const allTerms = new Set([...nameTokens, ...descTokens, ...tagTokens]);
+    for (const term of allTerms) {
+      dfMap[term] = (dfMap[term] || 0) + 1;
+    }
+  }
+
+  const N = documents.length;
+
+  // Compute IDF for each term
+  const idf = {};
+  for (const [term, df] of Object.entries(dfMap)) {
+    idf[term] = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+  }
+
+  // Compute average field lengths
+  const avg = (arr) => arr.length === 0 ? 0 : arr.reduce((a, b) => a + b, 0) / arr.length;
+  const avgFieldLengths = {
+    name: avg(fieldLengths.name),
+    description: avg(fieldLengths.description),
+    tags: avg(fieldLengths.tags),
+  };
+
+  return {
+    version: '1.0.0',
+    algorithm: 'bm25',
+    params: { k1: DEFAULT_K1, b: DEFAULT_B },
+    totalDocs: N,
+    avgFieldLengths,
+    idf,
+    documents,
+  };
+}
+
+/**
+ * Compute BM25 score for a single field.
+ */
+function scoreField(queryTerms, fieldTokens, idf, avgFieldLen, k1, b) {
+  if (fieldTokens.length === 0) return 0;
+
+  // Build term frequency map for this field
+  const tf = {};
+  for (const t of fieldTokens) {
+    tf[t] = (tf[t] || 0) + 1;
+  }
+
+  let score = 0;
+  const dl = fieldTokens.length;
+
+  for (const term of queryTerms) {
+    const termFreq = tf[term] || 0;
+    if (termFreq === 0) continue;
+
+    const termIdf = idf[term] || 0;
+    const numerator = termFreq * (k1 + 1);
+    const denominator = termFreq + k1 * (1 - b + b * (dl / (avgFieldLen || 1)));
+    score += termIdf * (numerator / denominator);
+  }
+
+  return score;
+}
+
+/**
+ * Search the BM25 index with a query string.
+ *
+ * @param {string} query - The search query
+ * @param {Object} index - The pre-built BM25 index
+ * @param {Object} opts - Options: { limit }
+ * @returns {Array} Sorted results: [{ id, score }]
+ */
+export function search(query, index, opts = {}) {
+  const queryTerms = tokenize(query);
+  if (queryTerms.length === 0) return [];
+
+  const { k1, b } = index.params;
+  const results = [];
+
+  for (const doc of index.documents) {
+    let totalScore = 0;
+
+    for (const [field, weight] of Object.entries(FIELD_WEIGHTS)) {
+      const fieldTokens = doc.tokens[field] || [];
+      const avgLen = index.avgFieldLengths[field] || 1;
+      const fieldScore = scoreField(queryTerms, fieldTokens, index.idf, avgLen, k1, b);
+      totalScore += fieldScore * weight;
+    }
+
+    if (totalScore > 0) {
+      results.push({ id: doc.id, score: totalScore });
+    }
+  }
+
+  results.sort((a, b) => b.score - a.score);
+
+  if (opts.limit) {
+    return results.slice(0, opts.limit);
+  }
+
+  return results;
+}

package/src/lib/cache.js

@@ -225,6 +225,20 @@ export function loadSourceRegistry(source) {
   return JSON.parse(readFileSync(regPath, 'utf8'));
 }
 
+/**
+ * Load BM25 search index for a single source (if available).
+ */
+export function loadSearchIndex(source) {
+  const basePath = source.path || getSourceDir(source.name);
+  const indexPath = join(basePath, 'search-index.json');
+  if (!existsSync(indexPath)) return null;
+  try {
+    return JSON.parse(readFileSync(indexPath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Get cache stats.
  */

package/src/lib/config.js

@@ -18,7 +18,7 @@ const DEFAULTS = {
 let _config = null;
 
 export function getChubDir() {
-  return join(homedir(), '.chub');
+  return process.env.CHUB_DIR || join(homedir(), '.chub');
 }
 
 export function loadConfig() {

package/src/lib/registry.js

@@ -1,8 +1,10 @@
-import { loadSourceRegistry } from './cache.js';
+import { loadSourceRegistry, loadSearchIndex } from './cache.js';
 import { loadConfig } from './config.js';
 import { normalizeLanguage } from './normalize.js';
+import { search as bm25Search } from './bm25.js';
 
 let _merged = null;
+let _searchIndex = null;
 
 /**
  * Load and merge entries from all configured sources.
@@ -14,11 +16,16 @@ function getMerged() {
   const config = loadConfig();
   const allDocs = [];
   const allSkills = [];
+  const searchIndexes = [];
 
   for (const source of config.sources) {
     const registry = loadSourceRegistry(source);
     if (!registry) continue;
 
+    // Load BM25 search index if available
+    const idx = loadSearchIndex(source);
+    if (idx) searchIndexes.push(idx);
+
     // Support both new format (docs/skills) and old format (entries)
     if (registry.docs) {
       for (const doc of registry.docs) {
@@ -46,6 +53,53 @@ function getMerged() {
     }
   }
 
+  // Merge search indexes (combine documents and recompute IDF)
+  if (searchIndexes.length > 0) {
+    if (searchIndexes.length === 1) {
+      _searchIndex = searchIndexes[0];
+    } else {
+      // Merge multiple indexes: combine documents, recompute global IDF
+      const allDocuments = searchIndexes.flatMap((idx) => idx.documents);
+      const N = allDocuments.length;
+      const dfMap = {};
+      const fieldLengths = { name: [], description: [], tags: [] };
+
+      for (const doc of allDocuments) {
+        const allTerms = new Set([
+          ...(doc.tokens.name || []),
+          ...(doc.tokens.description || []),
+          ...(doc.tokens.tags || []),
+        ]);
+        for (const term of allTerms) {
+          dfMap[term] = (dfMap[term] || 0) + 1;
+        }
+        fieldLengths.name.push((doc.tokens.name || []).length);
+        fieldLengths.description.push((doc.tokens.description || []).length);
+        fieldLengths.tags.push((doc.tokens.tags || []).length);
+      }
+
+      const idf = {};
+      for (const [term, df] of Object.entries(dfMap)) {
+        idf[term] = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+      }
+
+      const avg = (arr) => arr.length === 0 ? 0 : arr.reduce((a, b) => a + b, 0) / arr.length;
+      _searchIndex = {
+        version: '1.0.0',
+        algorithm: 'bm25',
+        params: searchIndexes[0].params,
+        totalDocs: N,
+        avgFieldLengths: {
+          name: avg(fieldLengths.name),
+          description: avg(fieldLengths.description),
+          tags: avg(fieldLengths.tags),
+        },
+        idf,
+        documents: allDocuments,
+      };
+    }
+  }
+
   _merged = { docs: allDocs, skills: allSkills };
   return _merged;
 }
@@ -121,11 +175,10 @@ export function getDisplayId(entry) {
 
 /**
  * Search entries by query string. Searches both docs and skills.
+ * Uses BM25 when a search index is available, falls back to keyword matching.
  */
 export function searchEntries(query, filters = {}) {
   const entries = applySourceFilter(getAllEntries());
-  const q = query.toLowerCase();
-  const words = q.split(/\s+/);
 
   // Deduplicate: same id+source appearing as both doc and skill → show once
   const seen = new Set();
@@ -138,27 +191,50 @@ export function searchEntries(query, filters = {}) {
     }
   }
 
-  let results = deduped.map((entry) => {
-    let score = 0;
+  // Build entry lookup by id
+  const entryById = new Map();
+  for (const entry of deduped) {
+    entryById.set(entry.id, entry);
+  }
+
+  let results;
+
+  if (_searchIndex) {
+    // BM25 search
+    const bm25Results = bm25Search(query, _searchIndex);
+    results = bm25Results
+      .map((r) => {
+        const entry = entryById.get(r.id);
+        return entry ? { entry, score: r.score } : null;
+      })
+      .filter(Boolean);
+  } else {
+    // Fallback: keyword matching
+    const q = query.toLowerCase();
+    const words = q.split(/\s+/);
 
-    if (entry.id === q) score += 100;
-    else if (entry.id.includes(q)) score += 50;
+    results = deduped.map((entry) => {
+      let score = 0;
 
-    const nameLower = entry.name.toLowerCase();
-    if (nameLower === q) score += 80;
-    else if (nameLower.includes(q)) score += 40;
+      if (entry.id === q) score += 100;
+      else if (entry.id.includes(q)) score += 50;
 
-    for (const word of words) {
-      if (entry.id.includes(word)) score += 10;
-      if (nameLower.includes(word)) score += 10;
-      if (entry.description?.toLowerCase().includes(word)) score += 5;
-      if (entry.tags?.some((t) => t.toLowerCase().includes(word))) score += 15;
-    }
+      const nameLower = entry.name.toLowerCase();
+      if (nameLower === q) score += 80;
+      else if (nameLower.includes(q)) score += 40;
 
-    return { entry, score };
-  });
+      for (const word of words) {
+        if (entry.id.includes(word)) score += 10;
+        if (nameLower.includes(word)) score += 10;
+        if (entry.description?.toLowerCase().includes(word)) score += 5;
+        if (entry.tags?.some((t) => t.toLowerCase().includes(word))) score += 15;
+      }
+
+      return { entry, score };
+    });
 
-  results = results.filter((r) => r.score > 0);
+    results = results.filter((r) => r.score > 0);
+  }
 
   const filtered = applyFilters(results.map((r) => r.entry), filters);
   const filteredSet = new Set(filtered);
@@ -255,6 +331,13 @@ export function resolveDocPath(entry, language, version) {
   let verObj = null;
   if (version) {
     verObj = langObj.versions?.find((v) => v.version === version);
+    if (!verObj) {
+      return {
+        versionNotFound: true,
+        requested: version,
+        available: langObj.versions?.map((v) => v.version) || [],
+      };
+    }
   } else {
     const rec = langObj.recommendedVersion;
     verObj = langObj.versions?.find((v) => v.version === rec) || langObj.versions?.[0];
@@ -272,7 +355,7 @@ export function resolveDocPath(entry, language, version) {
  * Given a resolved path and a type ("doc" or "skill"), return the entry file path.
  */
 export function resolveEntryFile(resolved, type) {
-  if (!resolved || resolved.needsLanguage) return { error: 'unresolved' };
+  if (!resolved || resolved.needsLanguage || resolved.versionNotFound) return { error: 'unresolved' };
 
   const fileName = type === 'skill' ? 'SKILL.md' : 'DOC.md';