cryptoserve 0.3.0 → 0.3.2

package/lib/census/collectors/nuget-downloads.mjs

@@ -0,0 +1,72 @@
+/**
+ * Collect download counts from the NuGet API.
+ *
+ * Endpoint: GET https://api.nuget.org/v3/registration5-semver1/{id}/index.json
+ * - Returns per-version download counts
+ * - No authentication required
+ * - Rate limit: be polite, 300ms between requests
+ *
+ * Alternative: NuGet search API for total downloads
+ * GET https://azuresearch-usnc.nuget.org/query?q=packageid:{name}&take=1
+ */
+
+const NUGET_SEARCH = 'https://azuresearch-usnc.nuget.org/query';
+const REQUEST_DELAY_MS = 300;
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Fetch NuGet download counts for a list of packages.
+ *
+ * @param {import('../package-catalog.mjs').CatalogEntry[]} packages
+ * @param {Object} [options]
+ * @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
+ * @param {boolean} [options.verbose] - Log progress
+ * @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
+ */
+export async function collectNugetDownloads(packages, options = {}) {
+  const fetchFn = options.fetchFn || globalThis.fetch;
+  const verbose = options.verbose || false;
+
+  const results = [];
+
+  for (let i = 0; i < packages.length; i++) {
+    const pkg = packages[i];
+
+    if (verbose) {
+      process.stderr.write(`  nuget ${i + 1}/${packages.length}: ${pkg.name}\n`);
+    }
+
+    try {
+      const url = `${NUGET_SEARCH}?q=packageid:${encodeURIComponent(pkg.name)}&take=1`;
+      const res = await fetchFn(url);
+
+      if (!res.ok) {
+        if (verbose) process.stderr.write(`  nuget ${pkg.name}: ${res.status}\n`);
+        results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+      } else {
+        const data = await res.json();
+        const entry = data?.data?.[0];
+        // NuGet returns total downloads, estimate monthly as total / 36 (3 years average)
+        const totalDownloads = entry?.totalDownloads || 0;
+        const monthlyEstimate = Math.round(totalDownloads / 36);
+        results.push({ name: pkg.name, downloads: monthlyEstimate, tier: pkg.tier });
+      }
+    } catch (err) {
+      if (verbose) process.stderr.write(`  nuget ${pkg.name} error: ${err.message}\n`);
+      results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+    }
+
+    if (i < packages.length - 1) {
+      await sleep(REQUEST_DELAY_MS);
+    }
+  }
+
+  return {
+    packages: results.sort((a, b) => b.downloads - a.downloads),
+    period: 'last_month_estimated',
+    collectedAt: new Date().toISOString(),
+  };
+}

package/lib/census/collectors/packagist-downloads.mjs

@@ -0,0 +1,65 @@
+/**
+ * Collect download counts from the Packagist API.
+ *
+ * Endpoint: GET https://packagist.org/packages/{name}.json
+ * - Returns total downloads and monthly downloads
+ * - No authentication required
+ * - Rate limit: be polite, 300ms between requests
+ */
+
+const PACKAGIST_API = 'https://packagist.org/packages';
+const REQUEST_DELAY_MS = 300;
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Fetch Packagist download counts for a list of packages.
+ *
+ * @param {import('../package-catalog.mjs').CatalogEntry[]} packages
+ * @param {Object} [options]
+ * @param {Function} [options.fetchFn] - Fetch implementation (defaults to globalThis.fetch)
+ * @param {boolean} [options.verbose] - Log progress
+ * @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
+ */
+export async function collectPackagistDownloads(packages, options = {}) {
+  const fetchFn = options.fetchFn || globalThis.fetch;
+  const verbose = options.verbose || false;
+
+  const results = [];
+
+  for (let i = 0; i < packages.length; i++) {
+    const pkg = packages[i];
+
+    if (verbose) {
+      process.stderr.write(`  packagist ${i + 1}/${packages.length}: ${pkg.name}\n`);
+    }
+
+    try {
+      const res = await fetchFn(`${PACKAGIST_API}/${pkg.name}.json`);
+
+      if (!res.ok) {
+        if (verbose) process.stderr.write(`  packagist ${pkg.name}: ${res.status}\n`);
+        results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+      } else {
+        const data = await res.json();
+        const monthlyDownloads = data?.package?.downloads?.monthly || 0;
+        results.push({ name: pkg.name, downloads: monthlyDownloads, tier: pkg.tier });
+      }
+    } catch (err) {
+      if (verbose) process.stderr.write(`  packagist ${pkg.name} error: ${err.message}\n`);
+      results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+    }
+
+    if (i < packages.length - 1) {
+      await sleep(REQUEST_DELAY_MS);
+    }
+  }
+
+  return {
+    packages: results.sort((a, b) => b.downloads - a.downloads),
+    period: 'last_month',
+    collectedAt: new Date().toISOString(),
+  };
+}

package/lib/census/collectors/pub-downloads.mjs

@@ -0,0 +1,65 @@
+/**
+ * Collect download counts from the pub.dev API.
+ *
+ * Endpoint: GET https://pub.dev/api/packages/{name}/score
+ * - Returns downloadCount30Days (or estimate from likes/popularity)
+ * - No authentication required
+ */
+
+const PUB_API = 'https://pub.dev/api/packages';
+const REQUEST_DELAY_MS = 300;
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Fetch pub.dev download counts for a list of packages.
+ *
+ * @param {import('../package-catalog.mjs').CatalogEntry[]} packages
+ * @param {Object} [options]
+ * @param {Function} [options.fetchFn]
+ * @param {boolean} [options.verbose]
+ * @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
+ */
+export async function collectPubDownloads(packages, options = {}) {
+  const fetchFn = options.fetchFn || globalThis.fetch;
+  const verbose = options.verbose || false;
+
+  const results = [];
+
+  for (let i = 0; i < packages.length; i++) {
+    const pkg = packages[i];
+
+    if (verbose) {
+      process.stderr.write(`  pub ${i + 1}/${packages.length}: ${pkg.name}\n`);
+    }
+
+    try {
+      const res = await fetchFn(`${PUB_API}/${pkg.name}/score`);
+
+      if (!res.ok) {
+        if (verbose) process.stderr.write(`  pub ${pkg.name}: ${res.status}\n`);
+        results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+      } else {
+        const data = await res.json();
+        // pub.dev score endpoint has downloadCount30Days
+        const downloads = data?.downloadCount30Days || 0;
+        results.push({ name: pkg.name, downloads: downloads, tier: pkg.tier });
+      }
+    } catch (err) {
+      if (verbose) process.stderr.write(`  pub ${pkg.name} error: ${err.message}\n`);
+      results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+    }
+
+    if (i < packages.length - 1) {
+      await sleep(REQUEST_DELAY_MS);
+    }
+  }
+
+  return {
+    packages: results.sort((a, b) => b.downloads - a.downloads),
+    period: 'last_month',
+    collectedAt: new Date().toISOString(),
+  };
+}

package/lib/census/collectors/rubygems-downloads.mjs

@@ -0,0 +1,67 @@
+/**
+ * Collect download counts from the RubyGems API.
+ *
+ * Endpoint: GET https://rubygems.org/api/v1/gems/{name}.json
+ * - Returns total downloads (no monthly breakdown)
+ * - Estimate monthly = total / 120 (approx 10 years of data)
+ * - No authentication required
+ */
+
+const RUBYGEMS_API = 'https://rubygems.org/api/v1/gems';
+const REQUEST_DELAY_MS = 300;
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Fetch RubyGems download counts for a list of packages.
+ *
+ * @param {import('../package-catalog.mjs').CatalogEntry[]} packages
+ * @param {Object} [options]
+ * @param {Function} [options.fetchFn]
+ * @param {boolean} [options.verbose]
+ * @returns {Promise<{packages: Array<{name: string, downloads: number, tier: string}>, period: string, collectedAt: string}>}
+ */
+export async function collectRubygemsDownloads(packages, options = {}) {
+  const fetchFn = options.fetchFn || globalThis.fetch;
+  const verbose = options.verbose || false;
+
+  const results = [];
+
+  for (let i = 0; i < packages.length; i++) {
+    const pkg = packages[i];
+
+    if (verbose) {
+      process.stderr.write(`  rubygems ${i + 1}/${packages.length}: ${pkg.name}\n`);
+    }
+
+    try {
+      const res = await fetchFn(`${RUBYGEMS_API}/${pkg.name}.json`);
+
+      if (!res.ok) {
+        if (verbose) process.stderr.write(`  rubygems ${pkg.name}: ${res.status}\n`);
+        results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+      } else {
+        const data = await res.json();
+        // RubyGems only provides total downloads; estimate monthly
+        const totalDownloads = data?.downloads || 0;
+        const monthlyEstimate = Math.round(totalDownloads / 120);
+        results.push({ name: pkg.name, downloads: monthlyEstimate, tier: pkg.tier });
+      }
+    } catch (err) {
+      if (verbose) process.stderr.write(`  rubygems ${pkg.name} error: ${err.message}\n`);
+      results.push({ name: pkg.name, downloads: 0, tier: pkg.tier });
+    }
+
+    if (i < packages.length - 1) {
+      await sleep(REQUEST_DELAY_MS);
+    }
+  }
+
+  return {
+    packages: results.sort((a, b) => b.downloads - a.downloads),
+    period: 'estimated_monthly',
+    collectedAt: new Date().toISOString(),
+  };
+}

package/lib/census/index.mjs

@@ -1,14 +1,30 @@
 /**
  * Census orchestrator: run collectors, aggregate, cache results.
+ *
+ * Supports 11 ecosystems: npm, PyPI, Go, Maven, crates.io, Packagist, NuGet,
+ * RubyGems, Hex (Elixir), pub.dev (Dart), and CocoaPods (Swift/ObjC).
  */
 
 import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { homedir } from 'node:os';
 
-import { NPM_PACKAGES, PYPI_PACKAGES } from './package-catalog.mjs';
+import {
+  NPM_PACKAGES, PYPI_PACKAGES, GO_PACKAGES,
+  MAVEN_PACKAGES, CRATES_PACKAGES, PACKAGIST_PACKAGES, NUGET_PACKAGES,
+  RUBYGEMS_PACKAGES, HEX_PACKAGES, PUB_PACKAGES, COCOAPODS_PACKAGES,
+} from './package-catalog.mjs';
 import { collectNpmDownloads } from './collectors/npm-downloads.mjs';
 import { collectPypiDownloads } from './collectors/pypi-downloads.mjs';
+import { collectGoDownloads } from './collectors/go-downloads.mjs';
+import { collectMavenDownloads } from './collectors/maven-downloads.mjs';
+import { collectCratesDownloads } from './collectors/crates-downloads.mjs';
+import { collectPackagistDownloads } from './collectors/packagist-downloads.mjs';
+import { collectNugetDownloads } from './collectors/nuget-downloads.mjs';
+import { collectRubygemsDownloads } from './collectors/rubygems-downloads.mjs';
+import { collectHexDownloads } from './collectors/hex-downloads.mjs';
+import { collectPubDownloads } from './collectors/pub-downloads.mjs';
+import { collectCocoapodsDownloads } from './collectors/cocoapods-downloads.mjs';
 import { collectNvdCves } from './collectors/nvd-cves.mjs';
 import { collectGithubAdvisories } from './collectors/github-advisories.mjs';
 import { aggregate } from './aggregator.mjs';
@@ -68,42 +84,66 @@ export async function runCensus(options = {}) {
     }
   }
 
-  const enabledSources = sources || ['npm', 'pypi', 'nvd', 'github'];
+  const enabledSources = sources || [
+    'npm', 'pypi', 'go', 'maven', 'crates', 'packagist', 'nuget',
+    'rubygems', 'hex', 'pub', 'cocoapods',
+    'nvd', 'github',
+  ];
   const collectorOpts = { verbose, fetchFn };
-
-  if (verbose) process.stderr.write('Collecting census data...\n');
-
-  // Phase 1: Package downloads (npm + PyPI in parallel)
-  const downloadPromises = [];
-  if (enabledSources.includes('npm')) {
-    if (verbose) process.stderr.write('\nFetching npm download counts...\n');
-    downloadPromises.push(collectNpmDownloads(NPM_PACKAGES, collectorOpts));
-  } else {
-    downloadPromises.push(Promise.resolve({ packages: [], period: {}, collectedAt: new Date().toISOString() }));
-  }
-  if (enabledSources.includes('pypi')) {
-    if (verbose) process.stderr.write('\nFetching PyPI download counts...\n');
-    downloadPromises.push(collectPypiDownloads(PYPI_PACKAGES, collectorOpts));
-  } else {
-    downloadPromises.push(Promise.resolve({ packages: [], period: 'last_month', collectedAt: new Date().toISOString() }));
-  }
-
-  const [npmData, pypiData] = await Promise.all(downloadPromises);
+  const empty = { packages: [], period: 'last_month', collectedAt: new Date().toISOString() };
+
+  if (verbose) process.stderr.write('Collecting census data across 11 ecosystems...\n');
+
+  // Phase 1: Package downloads (all 11 ecosystems in parallel)
+  const downloadPromises = [
+    enabledSources.includes('npm')
+      ? (verbose && process.stderr.write('\nFetching npm download counts...\n'), collectNpmDownloads(NPM_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('pypi')
+      ? (verbose && process.stderr.write('\nFetching PyPI download counts...\n'), collectPypiDownloads(PYPI_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('go')
+      ? (verbose && process.stderr.write('\nFetching Go module stats...\n'), collectGoDownloads(GO_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('maven')
+      ? (verbose && process.stderr.write('\nFetching Maven Central stats...\n'), collectMavenDownloads(MAVEN_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('crates')
+      ? (verbose && process.stderr.write('\nFetching crates.io download counts...\n'), collectCratesDownloads(CRATES_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('packagist')
+      ? (verbose && process.stderr.write('\nFetching Packagist download counts...\n'), collectPackagistDownloads(PACKAGIST_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('nuget')
+      ? (verbose && process.stderr.write('\nFetching NuGet download counts...\n'), collectNugetDownloads(NUGET_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('rubygems')
+      ? (verbose && process.stderr.write('\nFetching RubyGems download counts...\n'), collectRubygemsDownloads(RUBYGEMS_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('hex')
+      ? (verbose && process.stderr.write('\nFetching Hex.pm download counts...\n'), collectHexDownloads(HEX_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('pub')
+      ? (verbose && process.stderr.write('\nFetching pub.dev download counts...\n'), collectPubDownloads(PUB_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+    enabledSources.includes('cocoapods')
+      ? (verbose && process.stderr.write('\nFetching CocoaPods pod counts...\n'), collectCocoapodsDownloads(COCOAPODS_PACKAGES, collectorOpts))
+      : Promise.resolve(empty),
+  ];
+
+  const [npmData, pypiData, goData, mavenData, cratesData, packagistData, nugetData,
+         rubygemsData, hexData, pubData, cocoapodsData] =
+    await Promise.all(downloadPromises);
 
   // Phase 2: Vulnerability data (NVD + GitHub in parallel)
-  const vulnPromises = [];
-  if (enabledSources.includes('nvd')) {
-    if (verbose) process.stderr.write('\nFetching NVD CVE data...\n');
-    vulnPromises.push(collectNvdCves(collectorOpts));
-  } else {
-    vulnPromises.push(Promise.resolve({ cves: [], collectedAt: new Date().toISOString() }));
-  }
-  if (enabledSources.includes('github')) {
-    if (verbose) process.stderr.write('\nFetching GitHub advisories...\n');
-    vulnPromises.push(collectGithubAdvisories(collectorOpts));
-  } else {
-    vulnPromises.push(Promise.resolve({ advisories: [], collectedAt: new Date().toISOString() }));
-  }
+  const vulnPromises = [
+    enabledSources.includes('nvd')
+      ? (verbose && process.stderr.write('\nFetching NVD CVE data...\n'), collectNvdCves(collectorOpts))
+      : Promise.resolve({ cves: [], collectedAt: new Date().toISOString() }),
+    enabledSources.includes('github')
+      ? (verbose && process.stderr.write('\nFetching GitHub advisories...\n'), collectGithubAdvisories(collectorOpts))
+      : Promise.resolve({ advisories: [], collectedAt: new Date().toISOString() }),
+  ];
 
   const [nvdData, githubData] = await Promise.all(vulnPromises);
 
@@ -111,6 +151,15 @@ export async function runCensus(options = {}) {
   const result = aggregate({
     npm: npmData,
     pypi: pypiData,
+    go: goData,
+    maven: mavenData,
+    crates: cratesData,
+    packagist: packagistData,
+    nuget: nugetData,
+    rubygems: rubygemsData,
+    hex: hexData,
+    pub: pubData,
+    cocoapods: cocoapodsData,
     nvd: nvdData,
     github: githubData,
   });