cryptoserve 0.2.1 → 0.3.2

package/bin/cryptoserve.mjs

@@ -20,6 +20,7 @@
  *   cryptoserve vault init|set|get|list|delete|run|import|export
  *   cryptoserve login [--server URL]
  *   cryptoserve status
+ *   cryptoserve census [--format json|html] [--output file] [--no-cache] [--verbose]
  */
 
 import { readFileSync, writeFileSync } from 'node:fs';
@@ -40,7 +41,7 @@ const OPTIONS_WITH_VALUES = new Set([
 
 const KNOWN_FLAGS = new Set([
   '--insecure-storage', '--verbose', '--binary', '--fail-on-weak',
-  '--help', '--version',
+  '--help', '--version', '--no-cache',
 ]);
 
 function getFlag(args, name) {
@@ -91,6 +92,9 @@ async function cmdHelp() {
   console.log(`    ${info('cbom [path] [--format F] [--output O]')} Generate Crypto Bill of Materials`);
   console.log(`    ${info('gate [path] [--max-risk R]')}            CI/CD gate (exit 0=pass, 1=fail)`);
   console.log();
+  console.log(`  ${bold('Research')}`);
+  console.log(`    ${info('census [--format json|html]')}            Global crypto census (11 ecosystems + NVD)`);
+  console.log();
   console.log(`  ${bold('Encryption')}`);
   console.log(`    ${info('encrypt "text" [--context C]')}          Encrypt with context-aware algorithm selection`);
   console.log(`    ${info('encrypt "text" [--password P]')}         Encrypt text (interactive password if omitted)`);
@@ -988,6 +992,59 @@ function timeSince(date) {
   return `${days}d ago`;
 }
 
+// ---------------------------------------------------------------------------
+// Census — global crypto adoption survey
+// ---------------------------------------------------------------------------
+
+async function cmdCensus(args) {
+  const {
+    compactHeader, section, labelValue, tableHeader, tableRow,
+    warning, info, dim, bold, divider, progressBar,
+  } = await import('../lib/cli-style.mjs');
+
+  const format = getOption(args, '--format', 'text');
+  const output = getOption(args, '--output', null);
+  const verbose = getFlag(args, '--verbose');
+  const noCache = getFlag(args, '--no-cache');
+
+  const { runCensus } = await import('../lib/census/index.mjs');
+
+  if (format === 'text') {
+    console.log(compactHeader('census'));
+    console.log(dim('  Collecting data from 11 ecosystems + NVD + GitHub...'));
+    console.log(dim('  This may take 90-120 seconds on first run.\n'));
+  }
+
+  const data = await runCensus({ verbose, noCache });
+
+  if (format === 'json') {
+    const json = JSON.stringify(data, null, 2);
+    if (output) {
+      writeFileSync(resolve(output), json);
+      console.log(`Census data written to ${output}`);
+    } else {
+      console.log(json);
+    }
+    return;
+  }
+
+  if (format === 'html') {
+    const { generateHtml } = await import('../lib/census/report-html.mjs');
+    const html = generateHtml(data);
+    const outFile = output || 'crypto-census.html';
+    writeFileSync(resolve(outFile), html);
+    console.log(`HTML report written to ${outFile}`);
+    return;
+  }
+
+  // Default: terminal report
+  const { renderTerminal } = await import('../lib/census/report-terminal.mjs');
+  renderTerminal(data, {
+    compactHeader, section, labelValue, tableHeader, tableRow,
+    warning, info, dim, bold, divider, progressBar,
+  });
+}
+
 // ---------------------------------------------------------------------------
 // Main router
 // ---------------------------------------------------------------------------
@@ -1050,6 +1107,9 @@ try {
     case 'status':
       await cmdStatus();
       break;
+    case 'census':
+      await cmdCensus(commandArgs);
+      break;
     default:
       console.error(`Unknown command: ${command}\nRun "cryptoserve help" for usage.`);
       process.exit(1);

package/lib/census/aggregator.mjs

@@ -0,0 +1,226 @@
+/**
+ * Aggregate raw census data into headline metrics.
+ *
+ * Supports all 11 ecosystems: npm, PyPI, Go, Maven, crates.io, Packagist, NuGet,
+ * RubyGems, Hex (Elixir), pub.dev (Dart), and CocoaPods (Swift/ObjC).
+ * Includes project-level transparency stats when available.
+ */
+
+import { TIERS, getCatalogSize } from './package-catalog.mjs';
+
+// NIST Post-Quantum Cryptography deadlines
+const NIST_2030 = new Date('2030-01-01T00:00:00Z');
+const NIST_2035 = new Date('2035-01-01T00:00:00Z');
+
+const ECOSYSTEM_IDS = ['npm', 'pypi', 'go', 'maven', 'crates', 'packagist', 'nuget', 'rubygems', 'hex', 'pub', 'cocoapods'];
+
+/**
+ * Sum downloads for a given tier from a packages array.
+ */
+function sumByTier(packages, tier) {
+  return packages
+    .filter(p => p.tier === tier)
+    .reduce((sum, p) => sum + p.downloads, 0);
+}
+
+/**
+ * Get top N packages sorted by downloads descending.
+ */
+function topPackages(packages, n = 10) {
+  return [...packages]
+    .sort((a, b) => b.downloads - a.downloads)
+    .slice(0, n);
+}
+
+/**
+ * Calculate days remaining until a target date.
+ */
+function daysUntil(target) {
+  const now = new Date();
+  const diff = target.getTime() - now.getTime();
+  return Math.max(0, Math.ceil(diff / (1000 * 60 * 60 * 24)));
+}
+
+/**
+ * Format days as "X yrs, Y days".
+ */
+function formatDaysRemaining(totalDays) {
+  const years = Math.floor(totalDays / 365);
+  const days = totalDays % 365;
+  if (years === 0) return `${days} days`;
+  return `${years} yr${years !== 1 ? 's' : ''}, ${days} days`;
+}
+
+/**
+ * Format a large number with suffix (M, K).
+ */
+export function formatNumber(n) {
+  if (n >= 1_000_000_000) return (n / 1_000_000_000).toFixed(1) + 'B';
+  if (n >= 1_000_000) return (n / 1_000_000).toFixed(1) + 'M';
+  if (n >= 1_000) return (n / 1_000).toFixed(1) + 'K';
+  return String(n);
+}
+
+/**
+ * Build a per-ecosystem breakdown from a packages array.
+ */
+function buildEcosystemBreakdown(pkgs, period) {
+  const weak = sumByTier(pkgs, TIERS.WEAK);
+  const modern = sumByTier(pkgs, TIERS.MODERN);
+  const pqc = sumByTier(pkgs, TIERS.PQC);
+  return {
+    weak,
+    modern,
+    pqc,
+    total: weak + modern + pqc,
+    topPackages: topPackages(pkgs, 15),
+    period,
+  };
+}
+
+/**
+ * Aggregate all census data into headline metrics.
+ *
+ * @param {Object} data
+ * @param {Object} data.npm - Result from collectNpmDownloads
+ * @param {Object} data.pypi - Result from collectPypiDownloads
+ * @param {Object} [data.go] - Result from collectGoDownloads
+ * @param {Object} [data.maven] - Result from collectMavenDownloads
+ * @param {Object} [data.crates] - Result from collectCratesDownloads
+ * @param {Object} [data.packagist] - Result from collectPackagistDownloads
+ * @param {Object} [data.nuget] - Result from collectNugetDownloads
+ * @param {Object} [data.rubygems] - Result from collectRubygemsDownloads
+ * @param {Object} [data.hex] - Result from collectHexDownloads
+ * @param {Object} [data.pub] - Result from collectPubDownloads
+ * @param {Object} [data.cocoapods] - Result from collectCocoapodsDownloads
+ * @param {Object} [data.nvd] - Result from collectNvdCves
+ * @param {Object} [data.github] - Result from collectGithubAdvisories
+ * @param {Object} [data.projectDeps] - Result from collectProjectDeps
+ * @returns {Object} Aggregated metrics matching CensusData type
+ */
+export function aggregate(data) {
+  // Gather all package arrays
+  const npmPkgs = data.npm?.packages || [];
+  const pypiPkgs = data.pypi?.packages || [];
+  const goPkgs = data.go?.packages || [];
+  const mavenPkgs = data.maven?.packages || [];
+  const cratesPkgs = data.crates?.packages || [];
+  const packagistPkgs = data.packagist?.packages || [];
+  const nugetPkgs = data.nuget?.packages || [];
+  const rubygemsPkgs = data.rubygems?.packages || [];
+  const hexPkgs = data.hex?.packages || [];
+  const pubPkgs = data.pub?.packages || [];
+  const cocoapodsPkgs = data.cocoapods?.packages || [];
+  const allPkgs = [...npmPkgs, ...pypiPkgs, ...goPkgs, ...mavenPkgs, ...cratesPkgs, ...packagistPkgs, ...nugetPkgs, ...rubygemsPkgs, ...hexPkgs, ...pubPkgs, ...cocoapodsPkgs];
+
+  // Download totals by tier
+  const totalWeakDownloads = sumByTier(allPkgs, TIERS.WEAK);
+  const totalModernDownloads = sumByTier(allPkgs, TIERS.MODERN);
+  const totalPqcDownloads = sumByTier(allPkgs, TIERS.PQC);
+  const totalDownloads = totalWeakDownloads + totalModernDownloads + totalPqcDownloads;
+
+  // Percentages
+  const weakPercentage = totalDownloads > 0
+    ? Math.round((totalWeakDownloads / totalDownloads) * 1000) / 10
+    : 0;
+  const modernPercentage = totalDownloads > 0
+    ? Math.round((totalModernDownloads / totalDownloads) * 1000) / 10
+    : 0;
+  const pqcPercentage = totalDownloads > 0
+    ? Math.round((totalPqcDownloads / totalDownloads) * 1000) / 10
+    : 0;
+
+  // The headline ratio
+  const weakToPqcRatio = totalPqcDownloads > 0
+    ? Math.round(totalWeakDownloads / totalPqcDownloads)
+    : null;
+
+  // Per-ecosystem breakdowns
+  const npm = buildEcosystemBreakdown(npmPkgs, data.npm?.period);
+  const pypi = buildEcosystemBreakdown(pypiPkgs, data.pypi?.period);
+  const go = buildEcosystemBreakdown(goPkgs, data.go?.period);
+  const maven = buildEcosystemBreakdown(mavenPkgs, data.maven?.period);
+  const crates = buildEcosystemBreakdown(cratesPkgs, data.crates?.period);
+  const packagist = buildEcosystemBreakdown(packagistPkgs, data.packagist?.period);
+  const nuget = buildEcosystemBreakdown(nugetPkgs, data.nuget?.period);
+  const rubygems = buildEcosystemBreakdown(rubygemsPkgs, data.rubygems?.period);
+  const hex = buildEcosystemBreakdown(hexPkgs, data.hex?.period);
+  const pub = buildEcosystemBreakdown(pubPkgs, data.pub?.period);
+  const cocoapods = buildEcosystemBreakdown(cocoapodsPkgs, data.cocoapods?.period);
+
+  // CVE totals
+  const nvdCves = data.nvd?.cves || [];
+  const totalCryptoCves = nvdCves.reduce((sum, c) => sum + c.totalCount, 0);
+
+  // GitHub advisories
+  const ghAdvisories = data.github?.advisories || [];
+  const totalAdvisories = ghAdvisories.reduce((sum, a) => sum + a.count, 0);
+
+  // Merge severity counts across CWEs
+  const advisorySeverity = { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 };
+  for (const adv of ghAdvisories) {
+    for (const [sev, count] of Object.entries(adv.bySeverity || {})) {
+      advisorySeverity[sev] = (advisorySeverity[sev] || 0) + count;
+    }
+  }
+
+  // NIST deadlines
+  const nistDeadline2030Days = daysUntil(NIST_2030);
+  const nistDeadline2035Days = daysUntil(NIST_2035);
+
+  // Project-level stats (if collected)
+  const projectDeps = data.projectDeps;
+  const projectStats = projectDeps?.stats || null;
+
+  // Count active ecosystems
+  const activeEcosystems = ECOSYSTEM_IDS.filter(eco => {
+    const d = data[eco];
+    return d?.packages?.length > 0;
+  });
+
+  return {
+    // Headline numbers
+    totalDownloads,
+    totalWeakDownloads,
+    totalModernDownloads,
+    totalPqcDownloads,
+    weakPercentage,
+    modernPercentage,
+    pqcPercentage,
+    weakToPqcRatio,
+
+    // Per-ecosystem
+    npm,
+    pypi,
+    go,
+    maven,
+    crates,
+    packagist,
+    nuget,
+    rubygems,
+    hex,
+    pub,
+    cocoapods,
+
+    // Project-level transparency
+    ...(projectStats ? { projectStats } : {}),
+
+    // Vulnerabilities
+    totalCryptoCves,
+    cveBreakdown: nvdCves,
+    totalAdvisories,
+    advisorySeverity,
+    advisoryBreakdown: ghAdvisories,
+
+    // Deadlines
+    nistDeadline2030: formatDaysRemaining(nistDeadline2030Days),
+    nistDeadline2030Days,
+    nistDeadline2035: formatDaysRemaining(nistDeadline2035Days),
+    nistDeadline2035Days,
+
+    // Metadata
+    collectedAt: data.npm?.collectedAt || data.pypi?.collectedAt || new Date().toISOString(),
+    catalogSize: getCatalogSize(),
+    ecosystemCount: activeEcosystems.length || ECOSYSTEM_IDS.length,
+  };
+}

package/lib/census/collectors/cocoapods-downloads.mjs

@@ -0,0 +1,72 @@
+/**
+ * Collect download counts from the CocoaPods trunk API.
+ *
+ * Endpoint: GET https://trunk.cocoapods.org/api/v1/pods/{name}
+ * - CocoaPods has no public download stats API
+ * - Use Libraries.io API as fallback for estimated downloads
+ * - Estimate based on GitHub stars/dependents if available
+ */
+
+const TRUNK_API = 'https://trunk.cocoapods.org/api/v1/pods';
+const LIBRARIES_API = 'https://libraries.io/api/cocoapods';
+const REQUEST_DELAY_MS = 500;
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Fetch CocoaPods pod metadata. Since CocoaPods has no download stats,
+ * we fetch from trunk API for verification and use conservative estimates
+ * based on pod popularity metrics (stars, dependents, rank).
+ *
+ * @param {import('../package-catalog.mjs').CatalogEntry[]} packages
+ * @param {Object} [options]
+ * @param {Function} [options.fetchFn]
+ * @param {boolean} [options.verbose]
+ * @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
+ */
+export async function collectCocoapodsDownloads(packages, options = {}) {
+  const fetchFn = options.fetchFn || globalThis.fetch;
+  const verbose = options.verbose || false;
+
+  const results = [];
+
+  for (let i = 0; i < packages.length; i++) {
+    const pkg = packages[i];
+
+    if (verbose) {
+      process.stderr.write(`  cocoapods ${i + 1}/${packages.length}: ${pkg.name}\n`);
+    }
+
+    try {
+      const res = await fetchFn(`${TRUNK_API}/${pkg.name}`, {
+        headers: { 'Accept': 'application/json' },
+      });
+
+      if (!res.ok) {
+        if (verbose) process.stderr.write(`  cocoapods ${pkg.name}: ${res.status}\n`);
+        results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+      } else {
+        // Trunk API confirms pod exists but has no download stats.
+        // Use conservative estimate: CocoaPods ecosystem is smaller,
+        // most crypto pods get 1K-50K installs/month based on GitHub activity.
+        // We set 0 and rely on scanner data if available.
+        results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+      }
+    } catch (err) {
+      if (verbose) process.stderr.write(`  cocoapods ${pkg.name} error: ${err.message}\n`);
+      results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+    }
+
+    if (i < packages.length - 1) {
+      await sleep(REQUEST_DELAY_MS);
+    }
+  }
+
+  return {
+    packages: results.sort((a, b) => b.downloads - a.downloads),
+    period: 'estimated_monthly',
+    collectedAt: new Date().toISOString(),
+  };
+}

package/lib/census/collectors/crates-downloads.mjs

@@ -0,0 +1,71 @@
+/**
+ * Collect download counts from the crates.io API.
+ *
+ * Endpoint: GET https://crates.io/api/v1/crates/{name}
+ * - Returns total downloads and recent_downloads (last 90 days)
+ * - Requires User-Agent header
+ * - No authentication required
+ * - Rate limit: 1 request per second recommended
+ */
+
+const CRATES_API = 'https://crates.io/api/v1/crates';
+const REQUEST_DELAY_MS = 300;
+const USER_AGENT = 'crypto-census/1.0 (https://census.cryptoserve.dev)';
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Fetch crates.io download counts for a list of packages.
+ *
+ * @param {import('../package-catalog.mjs').CatalogEntry[]} packages
+ * @param {Object} [options]
+ * @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
+ * @param {boolean} [options.verbose] - Log progress
+ * @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
+ */
+export async function collectCratesDownloads(packages, options = {}) {
+  const fetchFn = options.fetchFn || globalThis.fetch;
+  const verbose = options.verbose || false;
+
+  const results = [];
+
+  for (let i = 0; i < packages.length; i++) {
+    const pkg = packages[i];
+
+    if (verbose) {
+      process.stderr.write(`  crates ${i + 1}/${packages.length}: ${pkg.name}\n`);
+    }
+
+    try {
+      const res = await fetchFn(`${CRATES_API}/${pkg.name}`, {
+        headers: { 'User-Agent': USER_AGENT },
+      });
+
+      if (!res.ok) {
+        if (verbose) process.stderr.write(`  crates ${pkg.name}: ${res.status}\n`);
+        results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+      } else {
+        const data = await res.json();
+        // recent_downloads = last 90 days, divide by 3 for monthly estimate
+        const recentDownloads = data?.crate?.recent_downloads || 0;
+        const monthlyEstimate = Math.round(recentDownloads / 3);
+        results.push({ name: pkg.name, downloads: monthlyEstimate, tier: pkg.tier });
+      }
+    } catch (err) {
+      if (verbose) process.stderr.write(`  crates ${pkg.name} error: ${err.message}\n`);
+      results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+    }
+
+    if (i < packages.length - 1) {
+      await sleep(REQUEST_DELAY_MS);
+    }
+  }
+
+  return {
+    packages: results.sort((a, b) => b.downloads - a.downloads),
+    period: 'last_month_estimated',
+    collectedAt: new Date().toISOString(),
+  };
+}

package/lib/census/collectors/github-advisories.mjs

@@ -0,0 +1,132 @@
+/**
+ * Collect crypto-related security advisories from the GitHub Advisory Database.
+ *
+ * Endpoint: GET https://api.github.com/advisories?per_page=100
+ * - The REST API does NOT support CWE filtering -- we fetch and filter client-side
+ * - Free, no authentication required (60 req/hr unauthenticated)
+ * - We fetch up to MAX_PAGES pages and filter for crypto-related CWEs
+ */
+
+const GITHUB_API = 'https://api.github.com/advisories';
+const REQUEST_DELAY_MS = 2000;
+const MAX_PAGES = 5; // 500 advisories max to stay under rate limits
+
+const CRYPTO_CWE_IDS = new Set(['CWE-327', 'CWE-326', 'CWE-328']);
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Check if an advisory is crypto-related based on its CWEs.
+ */
+function isCryptoRelated(advisory) {
+  const cwes = advisory.cwes || [];
+  return cwes.some(c => CRYPTO_CWE_IDS.has(c.cwe_id));
+}
+
+/**
+ * Get the crypto CWE IDs from an advisory.
+ */
+function getCryptoCweIds(advisory) {
+  return (advisory.cwes || [])
+    .filter(c => CRYPTO_CWE_IDS.has(c.cwe_id))
+    .map(c => c.cwe_id);
+}
+
+/**
+ * Fetch crypto-related advisory counts from GitHub Advisory Database.
+ *
+ * @param {Object} [options]
+ * @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
+ * @param {boolean} [options.verbose] - Log progress
+ * @returns {Promise<{advisories: Array<{cweId: string, count: number, bySeverity: Object, byEcosystem: Object}>, collectedAt: string}>}
+ */
+export async function collectGithubAdvisories(options = {}) {
+  const fetchFn = options.fetchFn || globalThis.fetch;
+  const verbose = options.verbose || false;
+
+  // Accumulate crypto advisories across all pages
+  const byCwe = {};
+  for (const cweId of CRYPTO_CWE_IDS) {
+    byCwe[cweId] = {
+      count: 0,
+      bySeverity: { critical: 0, high: 0, medium: 0, low: 0, unknown: 0 },
+      byEcosystem: {},
+    };
+  }
+
+  let url = `${GITHUB_API}?per_page=100&type=reviewed`;
+  let page = 0;
+  let totalScanned = 0;
+
+  while (url && page < MAX_PAGES) {
+    page++;
+    if (verbose) process.stderr.write(`  github page ${page}/${MAX_PAGES}\n`);
+
+    try {
+      const res = await fetchFn(url, {
+        headers: { 'Accept': 'application/vnd.github+json' },
+      });
+
+      if (!res.ok) {
+        if (verbose) process.stderr.write(`  github page ${page}: HTTP ${res.status}\n`);
+        break;
+      }
+
+      const data = await res.json();
+      if (!Array.isArray(data) || data.length === 0) break;
+
+      totalScanned += data.length;
+
+      for (const adv of data) {
+        if (!isCryptoRelated(adv)) continue;
+
+        const cweIds = getCryptoCweIds(adv);
+        const sev = (adv.severity || 'unknown').toLowerCase();
+
+        for (const cweId of cweIds) {
+          const entry = byCwe[cweId];
+          entry.count++;
+          if (sev in entry.bySeverity) {
+            entry.bySeverity[sev]++;
+          } else {
+            entry.bySeverity.unknown++;
+          }
+
+          const vulnerabilities = adv.vulnerabilities || [];
+          for (const vuln of vulnerabilities) {
+            const eco = vuln?.package?.ecosystem || 'other';
+            entry.byEcosystem[eco] = (entry.byEcosystem[eco] || 0) + 1;
+          }
+        }
+      }
+
+      // Check for next page
+      const linkHeader = res.headers?.get?.('link') || '';
+      const nextMatch = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
+      url = nextMatch ? nextMatch[1] : null;
+
+      if (url) await sleep(REQUEST_DELAY_MS);
+    } catch (err) {
+      if (verbose) process.stderr.write(`  github page ${page} error: ${err.message}\n`);
+      break;
+    }
+  }
+
+  if (verbose) {
+    process.stderr.write(`  github scanned ${totalScanned} advisories across ${page} pages\n`);
+  }
+
+  const results = [...CRYPTO_CWE_IDS].map(cweId => ({
+    cweId,
+    count: byCwe[cweId].count,
+    bySeverity: byCwe[cweId].bySeverity,
+    byEcosystem: byCwe[cweId].byEcosystem,
+  }));
+
+  return {
+    advisories: results,
+    collectedAt: new Date().toISOString(),
+  };
+}