chub-dev 0.2.0-beta.3 → 0.3.0

package/src/index.js

@@ -1,4 +1,3 @@
-import chalk from 'chalk';
 import { Command } from 'commander';
 import { readFileSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
@@ -10,87 +9,55 @@ import { registerSearchCommand } from './commands/search.js';
 import { registerGetCommand } from './commands/get.js';
 import { registerBuildCommand } from './commands/build.js';
 import { registerFeedbackCommand } from './commands/feedback.js';
-import { trackEvent, shutdownAnalytics } from './lib/analytics.js';
+import { registerAnnotateCommand } from './commands/annotate.js';
+import { registerHelpCommand } from './commands/help.js';
+import { trackEvent, shutdownAnalytics, setCliVersion } from './lib/analytics.js';
+import { error } from './lib/output.js';
+import { showWelcomeIfNeeded } from './lib/welcome.js';
+import { getLocalHelpText } from './lib/help.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf8'));
-
-function printUsage() {
-  console.log(`
-${chalk.bold('chub')} — Context Hub CLI v${pkg.version}
-Search and retrieve LLM-optimized docs and skills.
-
-${chalk.bold.underline('Getting Started')}
-
-  ${chalk.dim('$')} chub update                                ${chalk.dim('# download the registry')}
-  ${chalk.dim('$')} chub search                                ${chalk.dim('# list everything available')}
-  ${chalk.dim('$')} chub search "stripe"                       ${chalk.dim('# fuzzy search')}
-  ${chalk.dim('$')} chub search stripe/payments                ${chalk.dim('# exact id → full detail')}
-  ${chalk.dim('$')} chub get stripe/api                        ${chalk.dim('# print doc to terminal')}
-  ${chalk.dim('$')} chub get stripe/api -o doc.md              ${chalk.dim('# save to file')}
-  ${chalk.dim('$')} chub get openai/chat --lang py             ${chalk.dim('# specific language')}
-  ${chalk.dim('$')} chub get pw-community/login-flows          ${chalk.dim('# fetch a skill')}
-  ${chalk.dim('$')} chub get openai/chat stripe/api            ${chalk.dim('# fetch multiple')}
-
-${chalk.bold.underline('Commands')}
-
-  ${chalk.bold('search')} [query]              Search docs and skills (no query = list all)
-  ${chalk.bold('get')} <ids...>                 Fetch docs or skills by ID
-  ${chalk.bold('update')}                      Refresh the cached registry
-  ${chalk.bold('cache')} status|clear          Manage the local cache
-  ${chalk.bold('build')} <content-dir>        Build registry from content directory
-
-${chalk.bold.underline('Flags')}
-
-  --json                 Structured JSON output (for agents and piping)
-  --tags <csv>           Filter by tags (e.g. docs, skill, openai, browser)
-  --lang <language>      Language variant (js, py, ts)
-  --full                 Fetch all files, not just the entry point
-  -o, --output <path>    Write content to file or directory
-
-${chalk.bold.underline('Agent Piping Patterns')}
-
-  ${chalk.dim('# Get the top result id')}
-  ${chalk.dim('$')} chub search "stripe" --json | jq -r '.results[0].id'
-
-  ${chalk.dim('# Search → pick → fetch → save')}
-  ${chalk.dim('$')} ID=$(chub search "stripe" --json | jq -r '.results[0].id')
-  ${chalk.dim('$')} chub get "$ID" --lang js -o .context/stripe.md
-
-  ${chalk.dim('# Fetch multiple at once')}
-  ${chalk.dim('$')} chub get openai/chat stripe/api -o .context/
-
-${chalk.bold.underline('Multi-Source Config')} ${chalk.dim('(~/.chub/config.yaml)')}
-
-  ${chalk.dim('sources:')}
-  ${chalk.dim('  - name: community')}
-  ${chalk.dim('    url: https://cdn.aichub.org/v1')}
-  ${chalk.dim('  - name: internal')}
-  ${chalk.dim('    path: /path/to/local/docs')}
-
-  ${chalk.dim('# On id collision, use source: prefix: chub get internal:openai/chat')}
-`);
-}
+setCliVersion(pkg.version);
 
 const program = new Command();
 
 program
   .name('chub')
   .description('Context Hub - search and retrieve LLM-optimized docs and skills')
-  .version(pkg.version)
+  .version(pkg.version, '-V, --cli-version')
+  .addHelpCommand(false)
   .option('--json', 'Output as JSON (machine-readable)')
   .action(() => {
-    printUsage();
+    console.log(getLocalHelpText(pkg.version));
   });
 
 // Commands that don't need registry
-const SKIP_REGISTRY = ['update', 'cache', 'build', 'feedback', 'help'];
+const SKIP_REGISTRY = ['update', 'cache', 'build', 'feedback', 'annotate', 'help'];
 
 program.hook('preAction', async (thisCommand) => {
+  const globalOpts = thisCommand.optsWithGlobals?.() || {};
+  showWelcomeIfNeeded(globalOpts);
+
   const cmdName = thisCommand.args?.[0] || thisCommand.name();
-  // Track command usage (fire-and-forget, never blocks)
   if (cmdName !== 'chub') {
-    trackEvent('command_run', { command: cmdName }).catch(() => {});
+    // Only initialize identity and track if telemetry is enabled
+    // Respects CHUB_TELEMETRY=0 — no client_id file created, no events sent
+    try {
+      const { isTelemetryEnabled } = await import('./lib/telemetry.js');
+      if (isTelemetryEnabled()) {
+        const { getOrCreateClientId, isFirstRun } = await import('./lib/identity.js');
+        await getOrCreateClientId();
+
+        // Fire-and-forget — don't block command on PostHog network I/O
+        trackEvent('command_run', { command: cmdName }).catch(() => {});
+        if (isFirstRun()) {
+          trackEvent('first_run', { command: cmdName }).catch(() => {});
+        }
+      }
+    } catch {
+      // Identity/telemetry failure — silently skip, don't block the command
+    }
   }
   if (SKIP_REGISTRY.includes(cmdName)) return;
   if (thisCommand.parent?.name() === 'cache') return;
@@ -99,9 +66,8 @@ program.hook('preAction', async (thisCommand) => {
   try {
     await ensureRegistry();
   } catch (err) {
-    process.stderr.write(`Warning: Could not load registry: ${err.message}\n`);
-    process.stderr.write(`Run \`chub update\` to initialize.\n`);
-    process.exit(1);
+    await trackEvent('command_error', { command: cmdName, error_type: 'registry_unavailable' });
+    error(`Registry not available: ${err.message}. Run \`chub update\` to refresh remote registries, or check that local source paths in ~/.chub/config.yaml are correct.`, globalOpts);
   }
 });
 
@@ -111,6 +77,8 @@ registerSearchCommand(program);
 registerGetCommand(program);
 registerBuildCommand(program);
 registerFeedbackCommand(program);
+registerAnnotateCommand(program);
+registerHelpCommand(program, pkg.version);
 
 program.parse();
 

package/src/lib/analytics.js

@@ -4,13 +4,14 @@
  * Tracks: command usage, search patterns, doc/skill popularity, errors.
  * Does NOT track feedback ratings (those go to the custom API via telemetry.js).
  *
- * Respects the same telemetry opt-out: `telemetry: false` in config or CHUB_TELEMETRY=0.
+ * Respects telemetry opt-out: `telemetry: false` in config or CHUB_TELEMETRY=0.
+ * Feedback has a separate opt-out: `feedback: false` in config or CHUB_FEEDBACK=0.
  */
 
 import { isTelemetryEnabled } from './telemetry.js';
 
 // PostHog project API key (public — standard for client-side analytics)
-const POSTHOG_KEY = 'phc_cUPXY1tAUkIOU9perzGcFYEtFQeCgUhUO6ejT79YLIk';
+const POSTHOG_KEY = 'phc_tO9mXIgcCuBccfN2Ut0quf6UFsd06u3Y6g1kqMaYdQX';
 const POSTHOG_HOST = 'https://us.i.posthog.com';
 
 let _posthog = null;
@@ -65,6 +66,7 @@ export async function trackEvent(event, properties = {}) {
         ...properties,
         platform: process.platform,
         node_version: process.version,
+        cli_version: _cliVersion || undefined,
       },
     });
 
@@ -75,6 +77,15 @@ export async function trackEvent(event, properties = {}) {
   }
 }
 
+let _cliVersion;
+/**
+ * Set the CLI version for inclusion in all events.
+ * Called once from index.js at startup.
+ */
+export function setCliVersion(version) {
+  _cliVersion = version;
+}
+
 /**
  * Shut down the PostHog client gracefully.
  * Call this before process exit if possible.

package/src/lib/annotations.js

@@ -0,0 +1,57 @@
+import { readFileSync, writeFileSync, mkdirSync, unlinkSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { getChubDir } from './config.js';
+
+function getAnnotationsDir() {
+  return join(getChubDir(), 'annotations');
+}
+
+function annotationPath(entryId) {
+  const safe = entryId.replace(/\//g, '--');
+  return join(getAnnotationsDir(), `${safe}.json`);
+}
+
+export function readAnnotation(entryId) {
+  try {
+    return JSON.parse(readFileSync(annotationPath(entryId), 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+export function writeAnnotation(entryId, note) {
+  const dir = getAnnotationsDir();
+  mkdirSync(dir, { recursive: true });
+  const data = {
+    id: entryId,
+    note,
+    updatedAt: new Date().toISOString(),
+  };
+  writeFileSync(annotationPath(entryId), JSON.stringify(data, null, 2));
+  return data;
+}
+
+export function clearAnnotation(entryId) {
+  try {
+    unlinkSync(annotationPath(entryId));
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function listAnnotations() {
+  const dir = getAnnotationsDir();
+  try {
+    const files = readdirSync(dir).filter((f) => f.endsWith('.json'));
+    return files.map((f) => {
+      try {
+        return JSON.parse(readFileSync(join(dir, f), 'utf8'));
+      } catch {
+        return null;
+      }
+    }).filter(Boolean);
+  } catch {
+    return [];
+  }
+}

package/src/lib/bm25.js

@@ -0,0 +1,303 @@
+/**
+ * BM25 search implementation for Context Hub.
+ * Index is built at `chub build` time, scoring happens at search time.
+ * Tokenizer is shared between build and search to ensure consistency.
+ */
+
+const STOP_WORDS = new Set([
+  'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
+  'has', 'have', 'in', 'is', 'it', 'its', 'of', 'on', 'or', 'that',
+  'the', 'to', 'was', 'were', 'will', 'with', 'this', 'but', 'not',
+  'you', 'your', 'can', 'do', 'does', 'how', 'if', 'may', 'no',
+  'so', 'than', 'too', 'very', 'just', 'about', 'into', 'over',
+  'such', 'then', 'them', 'these', 'those', 'through', 'under',
+  'use', 'using', 'used',
+]);
+
+// BM25 default parameters
+const DEFAULT_K1 = 1.5;
+const DEFAULT_B = 0.75;
+
+// Field weights for multi-field scoring
+const FIELD_WEIGHTS = {
+  id: 4.0,
+  name: 3.0,
+  tags: 2.0,
+  description: 1.0,
+};
+
+function getDefaultParams() {
+  return { k1: DEFAULT_K1, b: DEFAULT_B };
+}
+
+function isSearchableToken(token) {
+  return (token.length > 1 || /^\d+$/.test(token)) && !STOP_WORDS.has(token);
+}
+
+export function compactIdentifier(text) {
+  return String(text || '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]/g, '');
+}
+
+function splitAlphaNumeric(text) {
+  return text
+    .replace(/([a-z])(\d)/g, '$1 $2')
+    .replace(/(\d)([a-z])/g, '$1 $2');
+}
+
+/**
+ * Tokenize text into lowercase terms with stop word removal.
+ * Must be used identically at build time and search time.
+ */
+export function tokenize(text) {
+  if (!text) return [];
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, ' ')
+    .split(/[\s-]+/)
+    .filter(isSearchableToken);
+}
+
+/**
+ * Tokenize identifiers more aggressively than free text so package ids
+ * still match joined/split variants like "nodefetch" and "auth 0".
+ */
+export function tokenizeIdentifier(text) {
+  if (!text) return [];
+
+  const tokens = new Set(tokenize(text));
+  const raw = String(text);
+  const compact = compactIdentifier(raw);
+  const segments = new Set([
+    ...raw.split('/').map((segment) => compactIdentifier(segment)),
+    ...raw.split(/[\/_.\s-]+/).map((segment) => compactIdentifier(segment)),
+  ]);
+
+  if (isSearchableToken(compact)) {
+    tokens.add(compact);
+  }
+
+  for (const token of tokenize(splitAlphaNumeric(compact))) {
+    tokens.add(token);
+  }
+
+  for (const segment of segments) {
+    if (!segment) continue;
+    if (isSearchableToken(segment)) {
+      tokens.add(segment);
+    }
+    for (const token of tokenize(splitAlphaNumeric(segment))) {
+      tokens.add(token);
+    }
+  }
+
+  return [...tokens];
+}
+
+function buildInvertedIndex(documents) {
+  const invertedIndex = Object.create(null);
+
+  for (const [docIndex, doc] of documents.entries()) {
+    const allTerms = new Set([
+      ...(doc.tokens.id || []),
+      ...(doc.tokens.name || []),
+      ...(doc.tokens.description || []),
+      ...(doc.tokens.tags || []),
+    ]);
+
+    for (const term of allTerms) {
+      if (!invertedIndex[term]) invertedIndex[term] = [];
+      invertedIndex[term].push(docIndex);
+    }
+  }
+
+  return invertedIndex;
+}
+
+export function buildIndexFromDocuments(documents, params = getDefaultParams()) {
+  const dfMap = Object.create(null); // document frequency per term (across all fields)
+  const fieldLengths = { id: [], name: [], description: [], tags: [] };
+
+  for (const doc of documents) {
+    const idTokens = doc.tokens.id || [];
+    const nameTokens = doc.tokens.name || [];
+    const descTokens = doc.tokens.description || [];
+    const tagTokens = doc.tokens.tags || [];
+
+    fieldLengths.id.push(idTokens.length);
+    fieldLengths.name.push(nameTokens.length);
+    fieldLengths.description.push(descTokens.length);
+    fieldLengths.tags.push(tagTokens.length);
+
+    const allTerms = new Set([...idTokens, ...nameTokens, ...descTokens, ...tagTokens]);
+    for (const term of allTerms) {
+      dfMap[term] = (dfMap[term] || 0) + 1;
+    }
+  }
+
+  const N = documents.length;
+  const idf = Object.create(null);
+  for (const [term, df] of Object.entries(dfMap)) {
+    idf[term] = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+  }
+
+  const avg = (arr) => arr.length === 0 ? 0 : arr.reduce((a, b) => a + b, 0) / arr.length;
+  return {
+    version: '1.0.0',
+    algorithm: 'bm25',
+    params,
+    totalDocs: N,
+    avgFieldLengths: {
+      id: avg(fieldLengths.id),
+      name: avg(fieldLengths.name),
+      description: avg(fieldLengths.description),
+      tags: avg(fieldLengths.tags),
+    },
+    idf,
+    documents,
+    invertedIndex: buildInvertedIndex(documents),
+  };
+}
+
+/**
+ * Build a BM25 search index from registry entries.
+ * Called during `chub build`.
+ *
+ * @param {Array} entries - Combined docs and skills from registry
+ * @returns {Object} The search index
+ */
+export function buildIndex(entries) {
+  const documents = [];
+
+  for (const entry of entries) {
+    const idTokens = tokenizeIdentifier(entry.id);
+    const nameTokens = tokenize(entry.name);
+    const descTokens = tokenize(entry.description || '');
+    const tagTokens = (entry.tags || []).flatMap((t) => tokenize(t));
+
+    documents.push({
+      id: entry.id,
+      tokens: {
+        id: idTokens,
+        name: nameTokens,
+        description: descTokens,
+        tags: tagTokens,
+      },
+    });
+  }
+  return buildIndexFromDocuments(documents);
+}
+
+/**
+ * Compute BM25 score for a single field.
+ */
+function scoreField(queryTerms, fieldTokens, idf, avgFieldLen, k1, b) {
+  if (fieldTokens.length === 0) return 0;
+
+  // Build term frequency map for this field
+  const tf = Object.create(null);
+  for (const t of fieldTokens) {
+    tf[t] = (tf[t] || 0) + 1;
+  }
+
+  let score = 0;
+  const dl = fieldTokens.length;
+
+  for (const term of queryTerms) {
+    const termFreq = tf[term] || 0;
+    if (termFreq === 0) continue;
+
+    const termIdf = idf[term] || 0;
+    const numerator = termFreq * (k1 + 1);
+    const denominator = termFreq + k1 * (1 - b + b * (dl / (avgFieldLen || 1)));
+    score += termIdf * (numerator / denominator);
+  }
+
+  return score;
+}
+
+function getCandidateDocIndexes(queryTerms, index) {
+  if (!index.invertedIndex) {
+    return index.documents.map((_, docIndex) => docIndex);
+  }
+
+  const candidateIndexes = new Set();
+  for (const term of new Set(queryTerms)) {
+    const postings = index.invertedIndex[term];
+    if (!postings) continue;
+    for (const docIndex of postings) {
+      candidateIndexes.add(docIndex);
+    }
+  }
+
+  return [...candidateIndexes];
+}
+
+function runSearch(query, index, opts = {}) {
+  const queryTerms = tokenize(query);
+  const totalDocs = index.documents.length;
+
+  if (queryTerms.length === 0) {
+    return {
+      results: [],
+      stats: {
+        totalDocs,
+        candidateDocCount: 0,
+        scoredDocCount: 0,
+        matchedDocCount: 0,
+        usedInvertedIndex: !!index.invertedIndex,
+      },
+    };
+  }
+
+  const { k1, b } = index.params;
+  const results = [];
+  const candidateDocIndexes = getCandidateDocIndexes(queryTerms, index);
+
+  for (const docIndex of candidateDocIndexes) {
+    const doc = index.documents[docIndex];
+    let totalScore = 0;
+
+    for (const [field, weight] of Object.entries(FIELD_WEIGHTS)) {
+      const fieldTokens = doc.tokens[field] || [];
+      const avgLen = index.avgFieldLengths[field] || 1;
+      const fieldScore = scoreField(queryTerms, fieldTokens, index.idf, avgLen, k1, b);
+      totalScore += fieldScore * weight;
+    }
+
+    if (totalScore > 0) {
+      results.push({ id: doc.id, score: totalScore });
+    }
+  }
+
+  results.sort((a, b) => b.score - a.score);
+  const limitedResults = opts.limit ? results.slice(0, opts.limit) : results;
+
+  return {
+    results: limitedResults,
+    stats: {
+      totalDocs,
+      candidateDocCount: candidateDocIndexes.length,
+      scoredDocCount: candidateDocIndexes.length,
+      matchedDocCount: results.length,
+      usedInvertedIndex: !!index.invertedIndex,
+    },
+  };
+}
+
+/**
+ * Search the BM25 index with a query string.
+ *
+ * @param {string} query - The search query
+ * @param {Object} index - The pre-built BM25 index
+ * @param {Object} opts - Options: { limit }
+ * @returns {Array} Sorted results: [{ id, score }]
+ */
+export function search(query, index, opts = {}) {
+  return runSearch(query, index, opts).results;
+}
+
+export function searchWithStats(query, index, opts = {}) {
+  return runSearch(query, index, opts);
+}