chub-dev 0.2.0-beta.3 → 0.3.0

package/src/lib/cache.js

@@ -31,6 +31,10 @@ function getSourceRegistryPath(sourceName) {
   return join(getSourceDir(sourceName), 'registry.json');
 }
 
+function getSourceSearchIndexPath(sourceName) {
+  return join(getSourceDir(sourceName), 'search-index.json');
+}
+
 function readMeta(sourceName) {
   try {
     return JSON.parse(readFileSync(getSourceMetaPath(sourceName), 'utf8'));
@@ -47,38 +51,99 @@ function writeMeta(sourceName, meta) {
 
 function isSourceCacheFresh(sourceName) {
   const meta = readMeta(sourceName);
-  if (!meta.lastUpdated) return false;
+  if (!meta.lastUpdated && meta.lastUpdated !== 0) return false;
   const config = loadConfig();
   const age = (Date.now() - meta.lastUpdated) / 1000;
   return age < config.refresh_interval;
 }
 
-/**
- * Fetch registry for a single remote source.
- */
-async function fetchRemoteRegistry(source, force = false) {
-  if (!force && isSourceCacheFresh(source.name) && existsSync(getSourceRegistryPath(source.name))) {
-    return;
+function isTimestampFresh(timestamp) {
+  if (timestamp === undefined || timestamp === null) return false;
+  const config = loadConfig();
+  const age = (Date.now() - timestamp) / 1000;
+  return age < config.refresh_interval;
+}
+
+function hasFreshSearchIndexState(sourceName) {
+  if (existsSync(getSourceSearchIndexPath(sourceName))) {
+    return true;
   }
 
-  const url = `${source.url}/registry.json`;
+  const meta = readMeta(sourceName);
+  return meta.searchIndexAvailable === false && isTimestampFresh(meta.searchIndexCheckedAt);
+}
+
+function shouldFetchRemoteRegistry(sourceName, force = false) {
+  if (force) return true;
+  return !(
+    isSourceCacheFresh(sourceName)
+    && existsSync(getSourceRegistryPath(sourceName))
+    && hasFreshSearchIndexState(sourceName)
+  );
+}
+
+async function fetchRemoteText(url) {
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), 30000);
-  let res;
   try {
-    res = await fetch(url, { signal: controller.signal });
+    const res = await fetch(url, { signal: controller.signal });
+    if (!res.ok) {
+      throw new Error(`${res.status} ${res.statusText}`);
+    }
+    return await res.text();
   } finally {
     clearTimeout(timeout);
   }
-  if (!res.ok) {
-    throw new Error(`Failed to fetch registry from ${source.name}: ${res.status} ${res.statusText}`);
+}
+
+/**
+ * Fetch registry for a single remote source.
+ */
+async function fetchRemoteRegistry(source, force = false) {
+  if (!shouldFetchRemoteRegistry(source.name, force)) {
+    return;
+  }
+
+  const registryUrl = `${source.url}/registry.json`;
+  let registryText;
+  try {
+    registryText = await fetchRemoteText(registryUrl);
+  } catch (err) {
+    throw new Error(`Failed to fetch registry from ${source.name}: ${err.message}`);
   }
 
-  const data = await res.text();
   const dir = getSourceDir(source.name);
   mkdirSync(dir, { recursive: true });
-  writeFileSync(getSourceRegistryPath(source.name), data);
-  writeMeta(source.name, { ...readMeta(source.name), lastUpdated: Date.now() });
+  writeFileSync(getSourceRegistryPath(source.name), registryText);
+
+  const searchIndexUrl = `${source.url}/search-index.json`;
+  const searchIndexCheckedAt = Date.now();
+  let searchIndexAvailable;
+  try {
+    const searchIndexText = await fetchRemoteText(searchIndexUrl);
+    writeFileSync(getSourceSearchIndexPath(source.name), searchIndexText);
+    searchIndexAvailable = true;
+  } catch (err) {
+    // Avoid serving a stale local search index after a registry refresh.
+    rmSync(getSourceSearchIndexPath(source.name), { force: true });
+    if (err.message?.startsWith('404 ')) {
+      searchIndexAvailable = false;
+    }
+  }
+
+  const nextMeta = {
+    ...readMeta(source.name),
+    lastUpdated: Date.now(),
+  };
+  delete nextMeta.searchIndexAvailable;
+  delete nextMeta.searchIndexCheckedAt;
+
+  if (searchIndexAvailable !== undefined) {
+    nextMeta.searchIndexAvailable = searchIndexAvailable;
+    nextMeta.searchIndexCheckedAt = searchIndexCheckedAt;
+  }
+
+  writeMeta(source.name, nextMeta);
 }
 
 /**
@@ -141,6 +206,14 @@ export async function fetchFullBundle(sourceName) {
     writeFileSync(getSourceRegistryPath(sourceName), regData);
   }
 
+  const extractedSearchIndex = join(dataDir, 'search-index.json');
+  if (existsSync(extractedSearchIndex)) {
+    const searchIndexData = readFileSync(extractedSearchIndex, 'utf8');
+    writeFileSync(getSourceSearchIndexPath(sourceName), searchIndexData);
+  } else {
+    rmSync(getSourceSearchIndexPath(sourceName), { force: true });
+  }
+
   writeMeta(sourceName, { ...readMeta(sourceName), lastUpdated: Date.now(), fullBundle: true });
   rmSync(tmpPath, { force: true });
 }
@@ -187,7 +260,7 @@ export async function fetchDoc(source, docPath) {
   const content = await res.text();
 
   // Cache locally
-  const dir = cachedPath.substring(0, cachedPath.lastIndexOf('/'));
+  const dir = dirname(cachedPath);
   mkdirSync(dir, { recursive: true });
   writeFileSync(cachedPath, content);
 
@@ -225,6 +298,20 @@ export function loadSourceRegistry(source) {
   return JSON.parse(readFileSync(regPath, 'utf8'));
 }
 
+/**
+ * Load BM25 search index for a single source (if available).
+ */
+export function loadSearchIndex(source) {
+  const basePath = source.path || getSourceDir(source.name);
+  const indexPath = join(basePath, 'search-index.json');
+  if (!existsSync(indexPath)) return null;
+  try {
+    return JSON.parse(readFileSync(indexPath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Get cache stats.
  */
@@ -313,7 +400,7 @@ export async function ensureRegistry() {
     // Auto-refresh stale remote registries (best-effort)
     for (const source of config.sources) {
       if (source.path) continue;
-      if (!isSourceCacheFresh(source.name)) {
+      if (shouldFetchRemoteRegistry(source.name)) {
         try { await fetchRemoteRegistry(source); } catch { /* use stale */ }
       }
     }
@@ -327,6 +414,10 @@ export async function ensureRegistry() {
     const defaultDir = getSourceDir('default');
     mkdirSync(defaultDir, { recursive: true });
     writeFileSync(getSourceRegistryPath('default'), readFileSync(bundledRegistry, 'utf8'));
+    const bundledSearchIndex = join(getBundledDir(), 'search-index.json');
+    if (existsSync(bundledSearchIndex)) {
+      writeFileSync(getSourceSearchIndexPath('default'), readFileSync(bundledSearchIndex, 'utf8'));
+    }
     writeMeta('default', { lastUpdated: 0, bundledSeed: true }); // lastUpdated=0 → stale, so chub update will refresh
     return;
   }

package/src/lib/config.js

@@ -5,20 +5,25 @@ import { parse as parseYaml } from 'yaml';
 
 const DEFAULT_CDN_URL = 'https://cdn.aichub.org/v1';
 const DEFAULT_TELEMETRY_URL = 'https://api.aichub.org/v1';
+const DEFAULT_HELP_URL = 'https://cdn.aichub.org/v1/help.json';
+const DEFAULT_HELP_TIMEOUT_MS = 2000;
 
 const DEFAULTS = {
   output_dir: '.context',
-  refresh_interval: 86400,
+  refresh_interval: 21600,
   output_format: 'human',
   source: 'official,maintainer,community',
   telemetry: true,
+  feedback: true,
   telemetry_url: DEFAULT_TELEMETRY_URL,
+  help_url: DEFAULT_HELP_URL,
+  help_timeout_ms: DEFAULT_HELP_TIMEOUT_MS,
 };
 
 let _config = null;
 
 export function getChubDir() {
-  return join(homedir(), '.chub');
+  return process.env.CHUB_DIR || join(homedir(), '.chub');
 }
 
 export function loadConfig() {
@@ -50,7 +55,15 @@ export function loadConfig() {
     output_format: fileConfig.output_format || DEFAULTS.output_format,
     source: fileConfig.source || DEFAULTS.source,
     telemetry: fileConfig.telemetry !== undefined ? fileConfig.telemetry : DEFAULTS.telemetry,
+    feedback: fileConfig.feedback !== undefined ? fileConfig.feedback : DEFAULTS.feedback,
     telemetry_url: fileConfig.telemetry_url || DEFAULTS.telemetry_url,
+    help_url: process.env.CHUB_HELP_URL
+      ?? (fileConfig.help_url !== undefined ? fileConfig.help_url : DEFAULTS.help_url),
+    help_timeout_ms: Number.parseInt(
+      process.env.CHUB_HELP_TIMEOUT_MS
+      ?? (fileConfig.help_timeout_ms ?? DEFAULTS.help_timeout_ms),
+      10
+    ) || DEFAULTS.help_timeout_ms,
   };
 
   return _config;

package/src/lib/help.js

@@ -0,0 +1,158 @@
+import { loadConfig } from './config.js';
+
+function normalizeHelpUrl(value) {
+  if (value === false || value === null) return null;
+  if (typeof value !== 'string') return value || null;
+
+  const trimmed = value.trim();
+  if (!trimmed) return null;
+
+  const normalized = trimmed.toLowerCase();
+  if (['0', 'false', 'off', 'disabled', 'none', 'local'].includes(normalized)) {
+    return null;
+  }
+
+  return trimmed;
+}
+
+function buildRemoteHelpUrl(helpUrl, cliVersion) {
+  try {
+    const url = new URL(helpUrl);
+    if (!['http:', 'https:'].includes(url.protocol)) {
+      return helpUrl;
+    }
+    url.searchParams.set('cli_version', cliVersion);
+    return url.toString();
+  } catch {
+    return helpUrl;
+  }
+}
+
+function parseRemoteHelpPayload(raw, contentType = '') {
+  const trimmed = raw.trim();
+  const looksLikeJson = contentType.includes('json') || (trimmed.startsWith('{') && trimmed.endsWith('}'));
+
+  if (!looksLikeJson) {
+    return { content: raw };
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return { content: raw };
+  }
+
+  if (typeof parsed === 'string') {
+    return { content: parsed };
+  }
+
+  if (!parsed || typeof parsed.content !== 'string') {
+    throw new Error('Invalid remote help payload');
+  }
+
+  const result = { content: parsed.content };
+  if (typeof parsed.updatedAt === 'string') result.updatedAt = parsed.updatedAt;
+  if (typeof parsed.version === 'string') result.version = parsed.version;
+  if (typeof parsed.minimumCliVersion === 'string') {
+    result.minimumCliVersion = parsed.minimumCliVersion;
+  }
+
+  return result;
+}
+
+export function getLocalHelpText(cliVersion) {
+  return [
+    `chub — Context Hub CLI v${cliVersion}`,
+    'Search and retrieve LLM-optimized docs and skills.',
+    '',
+    'Bootstrap workflow for coding agents',
+    '',
+    '  1. Start with chub before writing code against an external API, SDK, or library.',
+    '  2. Find the best entry with `chub search`.',
+    '  3. Fetch the exact doc or skill with `chub get`.',
+    '  4. Use that content instead of guessing from training data.',
+    '  5. If you discover a gap, save it with `chub annotate`.',
+    '  6. Always leave feedback after using a doc or skill.',
+    '',
+    'Quick start',
+    '',
+    '  $ chub search "openai" --json',
+    '  $ chub get openai/chat --lang py',
+    '  $ chub feedback openai/chat up --label accurate "Current and clear"',
+    '',
+    'Distribution surfaces',
+    '',
+    '  $ npx skills add chub-dev',
+    '  Copy `skills/get-api-docs/SKILL.md` into your agent\'s skill or rule directory when manual install is needed.',
+    '  Use `chub-mcp` for MCP-compatible agents that prefer tools over local skill files.',
+    '',
+    'If chub is not installed yet',
+    '',
+    '  $ npm install -g chub-dev',
+    '',
+    'Core commands',
+    '',
+    '  search [query]              Search docs and skills (no query = list all)',
+    '  get <ids...>                Fetch docs or skills by ID',
+    '  annotate [id] [note]        Save a note that appears on future fetches',
+    '  feedback <id> <up|down>     Rate a doc or skill after using it',
+    '  update                      Refresh the cached registry',
+    '  cache status|clear          Manage the local cache',
+    '',
+    'Feedback loop',
+    '',
+    '  After using a doc or skill, always run one of these:',
+    '  $ chub feedback <id> up --label accurate "What was useful"',
+    '  $ chub feedback <id> down --label outdated "What needs to be fixed"',
+    '',
+    '  Valid labels: accurate, well-structured, helpful, good-examples, outdated, inaccurate, incomplete, wrong-examples, wrong-version, poorly-structured',
+  ].join('\n');
+}
+
+export async function loadHelpContent(cliVersion, { fetchImpl = globalThis.fetch } = {}) {
+  const config = loadConfig();
+  const helpUrl = normalizeHelpUrl(config.help_url);
+
+  if (!helpUrl || typeof fetchImpl !== 'function') {
+    return {
+      source: 'local',
+      content: getLocalHelpText(cliVersion),
+      url: helpUrl || null,
+    };
+  }
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), config.help_timeout_ms || 2000);
+
+  try {
+    const response = await fetchImpl(buildRemoteHelpUrl(helpUrl, cliVersion), {
+      signal: controller.signal,
+      headers: {
+        accept: 'application/json, text/plain;q=0.9, text/markdown;q=0.8',
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`${response.status} ${response.statusText}`);
+    }
+
+    const contentType = response.headers?.get?.('content-type') || '';
+    const payload = parseRemoteHelpPayload(await response.text(), contentType);
+
+    return {
+      source: 'remote',
+      url: helpUrl,
+      ...payload,
+    };
+  } catch (err) {
+    return {
+      source: 'local',
+      content: getLocalHelpText(cliVersion),
+      url: helpUrl,
+      fallbackReason: err?.name === 'AbortError' ? 'timeout' : (err?.message || 'remote_help_unavailable'),
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}

package/src/lib/identity.js

@@ -63,7 +63,7 @@ export async function getOrCreateClientId() {
     // File doesn't exist or is unreadable
   }
 
-  // Generate from machine UUID
+  // Generate from machine UUID — this is a first-time user
   const uuid = getMachineUUID();
   const hash = createHash('sha256').update(uuid).digest('hex');
 
@@ -74,9 +74,20 @@ export async function getOrCreateClientId() {
 
   writeFileSync(idPath, hash, 'utf8');
   _cachedClientId = hash;
+  _isFirstRun = true;
   return hash;
 }
 
+let _isFirstRun = false;
+
+/**
+ * Returns true if this is the first time the CLI has run on this machine.
+ * Only valid after getOrCreateClientId() has been called.
+ */
+export function isFirstRun() {
+  return _isFirstRun;
+}
+
 /**
  * Auto-detect the AI coding tool from environment variables.
  */