chub-dev 0.1.0

package/src/commands/update.js

@@ -0,0 +1,56 @@
+import chalk from 'chalk';
+import { fetchAllRegistries, fetchFullBundle } from '../lib/cache.js';
+import { loadConfig } from '../lib/config.js';
+import { output, info } from '../lib/output.js';
+
+export function registerUpdateCommand(program) {
+  program
+    .command('update')
+    .description('Refresh the cached registry index')
+    .option('--force', 'Force re-download even if cache is fresh')
+    .option('--full', 'Download the full bundle for offline use')
+    .action(async (opts) => {
+      const globalOpts = program.optsWithGlobals();
+      const config = loadConfig();
+
+      try {
+        if (opts.full) {
+          // Download full bundle for each remote source
+          for (const source of config.sources) {
+            if (source.path) {
+              info(`Skipping local source: ${source.name}`);
+              continue;
+            }
+            info(`Downloading full bundle for ${source.name}...`);
+            await fetchFullBundle(source.name);
+          }
+          output(
+            { status: 'ok', mode: 'full' },
+            () => console.log(chalk.green('Full bundle(s) downloaded and extracted.')),
+            globalOpts
+          );
+        } else {
+          info('Updating registries...');
+          const errors = await fetchAllRegistries(opts.force || true);
+          if (errors.length > 0) {
+            for (const e of errors) {
+              process.stderr.write(chalk.yellow(`Warning: ${e.source}: ${e.error}\n`));
+            }
+          }
+          const updated = config.sources.filter((s) => !s.path).length - errors.length;
+          output(
+            { status: 'ok', mode: 'registry', updated, errors },
+            () => console.log(chalk.green(`Registry updated (${updated} remote source(s)).`)),
+            globalOpts
+          );
+        }
+      } catch (err) {
+        output(
+          { error: err.message },
+          () => console.error(chalk.red(`Update failed: ${err.message}`)),
+          globalOpts
+        );
+        process.exit(1);
+      }
+    });
+}

package/src/index.js

@@ -0,0 +1,109 @@
+import chalk from 'chalk';
+import { Command } from 'commander';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+import { ensureRegistry } from './lib/cache.js';
+import { registerUpdateCommand } from './commands/update.js';
+import { registerCacheCommand } from './commands/cache.js';
+import { registerSearchCommand } from './commands/search.js';
+import { registerGetCommand } from './commands/get.js';
+import { registerBuildCommand } from './commands/build.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf8'));
+
+function printUsage() {
+  console.log(`
+${chalk.bold('chub')} — Context Hub CLI v${pkg.version}
+Search and retrieve LLM-optimized docs and skills.
+
+${chalk.bold.underline('Getting Started')}
+
+  ${chalk.dim('$')} chub update                                ${chalk.dim('# download the registry')}
+  ${chalk.dim('$')} chub search                                ${chalk.dim('# list everything available')}
+  ${chalk.dim('$')} chub search "stripe"                       ${chalk.dim('# fuzzy search')}
+  ${chalk.dim('$')} chub search stripe/payments                ${chalk.dim('# exact id → full detail')}
+  ${chalk.dim('$')} chub get docs stripe/payments              ${chalk.dim('# print doc to terminal')}
+  ${chalk.dim('$')} chub get docs stripe/payments -o doc.md    ${chalk.dim('# save to file')}
+  ${chalk.dim('$')} chub get docs stripe/payments --lang py    ${chalk.dim('# specific language')}
+  ${chalk.dim('$')} chub get skills pw/login-flows             ${chalk.dim('# fetch a skill')}
+  ${chalk.dim('$')} chub get docs openai/chat stripe/payments  ${chalk.dim('# fetch multiple')}
+
+${chalk.bold.underline('Commands')}
+
+  ${chalk.bold('search')} [query]              Search docs and skills (no query = list all)
+  ${chalk.bold('get docs')} <ids...>           Fetch documentation content
+  ${chalk.bold('get skills')} <ids...>         Fetch skill content
+  ${chalk.bold('update')}                      Refresh the cached registry
+  ${chalk.bold('cache')} status|clear          Manage the local cache
+  ${chalk.bold('build')} <content-dir>        Build registry from content directory
+
+${chalk.bold.underline('Flags')}
+
+  --json                 Structured JSON output (for agents and piping)
+  --tags <csv>           Filter by tags (e.g. docs, skill, openai, browser)
+  --lang <language>      Language variant (js, py, ts)
+  --full                 Fetch all files, not just the entry point
+  -o, --output <path>    Write content to file or directory
+
+${chalk.bold.underline('Agent Piping Patterns')}
+
+  ${chalk.dim('# Get the top result id')}
+  ${chalk.dim('$')} chub search "stripe" --json | jq -r '.results[0].id'
+
+  ${chalk.dim('# Search → pick → fetch → save')}
+  ${chalk.dim('$')} ID=$(chub search "stripe" --json | jq -r '.results[0].id')
+  ${chalk.dim('$')} chub get docs "$ID" --lang js -o .context/stripe.md
+
+  ${chalk.dim('# Fetch multiple docs at once')}
+  ${chalk.dim('$')} chub get docs openai/chat stripe/payments -o .context/
+
+${chalk.bold.underline('Multi-Source Config')} ${chalk.dim('(~/.chub/config.yaml)')}
+
+  ${chalk.dim('sources:')}
+  ${chalk.dim('  - name: community')}
+  ${chalk.dim('    url: https://cdn.contexthub.dev/v1')}
+  ${chalk.dim('  - name: internal')}
+  ${chalk.dim('    path: /path/to/local/docs')}
+
+  ${chalk.dim('# On id collision, use source: prefix: chub get docs internal:openai/chat')}
+`);
+}
+
+const program = new Command();
+
+program
+  .name('chub')
+  .description('Context Hub - search and retrieve LLM-optimized docs and skills')
+  .version(pkg.version)
+  .option('--json', 'Output as JSON (machine-readable)')
+  .action(() => {
+    printUsage();
+  });
+
+// Commands that don't need registry
+const SKIP_REGISTRY = ['update', 'cache', 'build', 'help'];
+
+program.hook('preAction', async (thisCommand) => {
+  const cmdName = thisCommand.args?.[0] || thisCommand.name();
+  if (SKIP_REGISTRY.includes(cmdName)) return;
+  if (thisCommand.parent?.name() === 'cache') return;
+  // Don't fetch registry for default action (no command)
+  if (cmdName === 'chub') return;
+  try {
+    await ensureRegistry();
+  } catch (err) {
+    process.stderr.write(`Warning: Could not load registry: ${err.message}\n`);
+    process.stderr.write(`Run \`chub update\` to initialize.\n`);
+    process.exit(1);
+  }
+});
+
+registerUpdateCommand(program);
+registerCacheCommand(program);
+registerSearchCommand(program);
+registerGetCommand(program);
+registerBuildCommand(program);
+
+program.parse();

package/src/lib/cache.js

@@ -0,0 +1,287 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import { createWriteStream } from 'node:fs';
+import { getChubDir, loadConfig } from './config.js';
+
+function getSourceDir(sourceName) {
+  return join(getChubDir(), 'sources', sourceName);
+}
+
+function getSourceDataDir(sourceName) {
+  return join(getSourceDir(sourceName), 'data');
+}
+
+function getSourceMetaPath(sourceName) {
+  return join(getSourceDir(sourceName), 'meta.json');
+}
+
+function getSourceRegistryPath(sourceName) {
+  return join(getSourceDir(sourceName), 'registry.json');
+}
+
+function readMeta(sourceName) {
+  try {
+    return JSON.parse(readFileSync(getSourceMetaPath(sourceName), 'utf8'));
+  } catch {
+    return {};
+  }
+}
+
+function writeMeta(sourceName, meta) {
+  const dir = getSourceDir(sourceName);
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(getSourceMetaPath(sourceName), JSON.stringify(meta, null, 2));
+}
+
+function isSourceCacheFresh(sourceName) {
+  const meta = readMeta(sourceName);
+  if (!meta.lastUpdated) return false;
+  const config = loadConfig();
+  const age = (Date.now() - meta.lastUpdated) / 1000;
+  return age < config.refresh_interval;
+}
+
+/**
+ * Fetch registry for a single remote source.
+ */
+async function fetchRemoteRegistry(source, force = false) {
+  if (!force && isSourceCacheFresh(source.name) && existsSync(getSourceRegistryPath(source.name))) {
+    return;
+  }
+
+  const url = `${source.url}/registry.json`;
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(`Failed to fetch registry from ${source.name}: ${res.status} ${res.statusText}`);
+  }
+
+  const data = await res.text();
+  const dir = getSourceDir(source.name);
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(getSourceRegistryPath(source.name), data);
+  writeMeta(source.name, { ...readMeta(source.name), lastUpdated: Date.now() });
+}
+
+/**
+ * Fetch registries for all configured sources.
+ */
+export async function fetchAllRegistries(force = false) {
+  const config = loadConfig();
+  const errors = [];
+
+  for (const source of config.sources) {
+    if (source.path) continue; // Local sources don't need fetching
+    try {
+      await fetchRemoteRegistry(source, force);
+    } catch (err) {
+      errors.push({ source: source.name, error: err.message });
+    }
+  }
+
+  return errors;
+}
+
+/**
+ * Download full bundle for a remote source.
+ */
+export async function fetchFullBundle(sourceName) {
+  const config = loadConfig();
+  const source = config.sources.find((s) => s.name === sourceName);
+  if (!source || source.path) {
+    throw new Error(`Source "${sourceName}" is not a remote source.`);
+  }
+
+  const url = `${source.url}/bundle.tar.gz`;
+  const tmpPath = join(getSourceDir(sourceName), 'bundle.tar.gz');
+
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(`Failed to fetch bundle from ${sourceName}: ${res.status} ${res.statusText}`);
+  }
+
+  const dir = getSourceDir(sourceName);
+  mkdirSync(dir, { recursive: true });
+  await pipeline(res.body, createWriteStream(tmpPath));
+
+  const { extract } = await import('tar');
+  const dataDir = getSourceDataDir(sourceName);
+  mkdirSync(dataDir, { recursive: true });
+  await extract({ file: tmpPath, cwd: dataDir });
+
+  // Copy registry.json from extracted bundle if present
+  const extractedRegistry = join(dataDir, 'registry.json');
+  if (existsSync(extractedRegistry)) {
+    const regData = readFileSync(extractedRegistry, 'utf8');
+    writeFileSync(getSourceRegistryPath(sourceName), regData);
+  }
+
+  writeMeta(sourceName, { ...readMeta(sourceName), lastUpdated: Date.now(), fullBundle: true });
+  rmSync(tmpPath, { force: true });
+}
+
+/**
+ * Fetch a single doc. Source object must have name + (url or path).
+ */
+export async function fetchDoc(source, docPath) {
+  // Local source: read directly
+  if (source.path) {
+    const localPath = join(source.path, docPath);
+    if (!existsSync(localPath)) {
+      throw new Error(`File not found: ${localPath}`);
+    }
+    return readFileSync(localPath, 'utf8');
+  }
+
+  // Remote source: check cache first
+  const cachedPath = join(getSourceDataDir(source.name), docPath);
+  if (existsSync(cachedPath)) {
+    return readFileSync(cachedPath, 'utf8');
+  }
+
+  // Fetch from CDN
+  const url = `${source.url}/${docPath}`;
+  const res = await fetch(url);
+  if (!res.ok) {
+    throw new Error(`Failed to fetch ${docPath} from ${source.name}: ${res.status} ${res.statusText}`);
+  }
+
+  const content = await res.text();
+
+  // Cache locally
+  const dir = cachedPath.substring(0, cachedPath.lastIndexOf('/'));
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(cachedPath, content);
+
+  return content;
+}
+
+/**
+ * Fetch all files in an entry directory.
+ * Returns array of { name, content }.
+ */
+export async function fetchDocFull(source, basePath, files) {
+  const results = [];
+  for (const file of files) {
+    const filePath = `${basePath}/${file}`;
+    const content = await fetchDoc(source, filePath);
+    results.push({ name: file, content });
+  }
+  return results;
+}
+
+/**
+ * Load cached/local registry for a single source.
+ */
+export function loadSourceRegistry(source) {
+  if (source.path) {
+    // Local source: read registry.json from the folder
+    const regPath = join(source.path, 'registry.json');
+    if (!existsSync(regPath)) return null;
+    return JSON.parse(readFileSync(regPath, 'utf8'));
+  }
+
+  // Remote source: read from cache
+  const regPath = getSourceRegistryPath(source.name);
+  if (!existsSync(regPath)) return null;
+  return JSON.parse(readFileSync(regPath, 'utf8'));
+}
+
+/**
+ * Get cache stats.
+ */
+export function getCacheStats() {
+  const chubDir = getChubDir();
+  if (!existsSync(chubDir)) {
+    return { exists: false, sources: [] };
+  }
+
+  const config = loadConfig();
+  const sourceStats = [];
+
+  for (const source of config.sources) {
+    if (source.path) {
+      sourceStats.push({ name: source.name, type: 'local', path: source.path });
+      continue;
+    }
+
+    const meta = readMeta(source.name);
+    const dataDir = getSourceDataDir(source.name);
+    let dataSize = 0;
+    let fileCount = 0;
+
+    if (existsSync(dataDir)) {
+      const walk = (dir) => {
+        for (const entry of readdirSync(dir, { withFileTypes: true })) {
+          const full = join(dir, entry.name);
+          if (entry.isDirectory()) walk(full);
+          else { dataSize += statSync(full).size; fileCount++; }
+        }
+      };
+      walk(dataDir);
+    }
+
+    sourceStats.push({
+      name: source.name,
+      type: 'remote',
+      hasRegistry: existsSync(getSourceRegistryPath(source.name)),
+      lastUpdated: meta.lastUpdated ? new Date(meta.lastUpdated).toISOString() : null,
+      fullBundle: meta.fullBundle || false,
+      fileCount,
+      dataSize,
+    });
+  }
+
+  return { exists: true, sources: sourceStats };
+}
+
+/**
+ * Clear the cache (preserves config.yaml).
+ */
+export function clearCache() {
+  const chubDir = getChubDir();
+  const configPath = join(chubDir, 'config.yaml');
+  let configContent = null;
+  if (existsSync(configPath)) {
+    configContent = readFileSync(configPath, 'utf8');
+  }
+
+  rmSync(chubDir, { recursive: true, force: true });
+
+  if (configContent) {
+    mkdirSync(chubDir, { recursive: true });
+    writeFileSync(configPath, configContent);
+  }
+}
+
+/**
+ * Ensure at least one registry is available.
+ */
+export async function ensureRegistry() {
+  const config = loadConfig();
+
+  // Check if any source has a registry available
+  let hasAny = false;
+  for (const source of config.sources) {
+    if (source.path) {
+      const regPath = join(source.path, 'registry.json');
+      if (existsSync(regPath)) { hasAny = true; break; }
+    } else {
+      if (existsSync(getSourceRegistryPath(source.name))) { hasAny = true; break; }
+    }
+  }
+
+  if (hasAny) {
+    // Auto-refresh stale remote registries (best-effort)
+    for (const source of config.sources) {
+      if (source.path) continue;
+      if (!isSourceCacheFresh(source.name)) {
+        try { await fetchRemoteRegistry(source); } catch { /* use stale */ }
+      }
+    }
+    return;
+  }
+
+  // No registries at all — must download remote ones
+  await fetchAllRegistries(true);
+}

package/src/lib/config.js

@@ -0,0 +1,52 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import { parse as parseYaml } from 'yaml';
+
+const DEFAULT_CDN_URL = 'https://github.com/context-hub/context-hub/releases/latest/download';
+
+const DEFAULTS = {
+  output_dir: '.context',
+  refresh_interval: 86400,
+  output_format: 'human',
+  source: 'official,maintainer,community',
+};
+
+let _config = null;
+
+export function getChubDir() {
+  return join(homedir(), '.chub');
+}
+
+export function loadConfig() {
+  if (_config) return _config;
+
+  let fileConfig = {};
+  const configPath = join(getChubDir(), 'config.yaml');
+  try {
+    const raw = readFileSync(configPath, 'utf8');
+    fileConfig = parseYaml(raw) || {};
+  } catch {
+    // No config file, use defaults
+  }
+
+  // Build sources list
+  let sources;
+  if (fileConfig.sources && Array.isArray(fileConfig.sources)) {
+    sources = fileConfig.sources;
+  } else {
+    // Backward compat: single cdn_url becomes a single source
+    const url = process.env.CHUB_BUNDLE_URL || fileConfig.cdn_url || DEFAULT_CDN_URL;
+    sources = [{ name: 'default', url }];
+  }
+
+  _config = {
+    sources,
+    output_dir: fileConfig.output_dir || DEFAULTS.output_dir,
+    refresh_interval: fileConfig.refresh_interval ?? DEFAULTS.refresh_interval,
+    output_format: fileConfig.output_format || DEFAULTS.output_format,
+    source: fileConfig.source || DEFAULTS.source,
+  };
+
+  return _config;
+}

package/src/lib/frontmatter.js

@@ -0,0 +1,14 @@
+import { parse as parseYaml } from 'yaml';
+
+/**
+ * Parse YAML frontmatter from markdown content.
+ * Returns { attributes, body } where attributes is the parsed YAML object.
+ */
+export function parseFrontmatter(content) {
+  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
+  if (!match) return { attributes: {}, body: content };
+  return {
+    attributes: parseYaml(match[1]) || {},
+    body: match[2],
+  };
+}

package/src/lib/normalize.js

@@ -0,0 +1,25 @@
+const ALIASES = {
+  js: 'javascript',
+  ts: 'typescript',
+  py: 'python',
+  rb: 'ruby',
+  cs: 'csharp',
+};
+
+const DISPLAY = {
+  javascript: 'js',
+  typescript: 'ts',
+  python: 'py',
+  ruby: 'rb',
+  csharp: 'cs',
+};
+
+export function normalizeLanguage(lang) {
+  if (!lang) return null;
+  const lower = lang.toLowerCase();
+  return ALIASES[lower] || lower;
+}
+
+export function displayLanguage(lang) {
+  return DISPLAY[lang] || lang;
+}

package/src/lib/output.js

@@ -0,0 +1,33 @@
+/**
+ * Dual-mode output: human-friendly (default) or JSON (--json flag).
+ *
+ * Every command calls `output(data, humanFormatter, opts)`.
+ * - In JSON mode: prints JSON to stdout, nothing else.
+ * - In human mode: calls humanFormatter(data) which prints with chalk.
+ */
+export function output(data, humanFormatter, opts) {
+  if (opts?.json) {
+    console.log(JSON.stringify(data, null, 2));
+  } else {
+    humanFormatter(data);
+  }
+}
+
+/**
+ * Print a message to stderr (for confirmations when -o is used).
+ */
+export function info(msg) {
+  process.stderr.write(msg + '\n');
+}
+
+/**
+ * Print an error and exit.
+ */
+export function error(msg, opts) {
+  if (opts?.json) {
+    console.log(JSON.stringify({ error: msg }));
+  } else {
+    process.stderr.write(`Error: ${msg}\n`);
+  }
+  process.exit(1);
+}