dep-oracle 1.2.0 → 1.3.0

package/dist/{chunk-TXSNFX3N.js → chunk-DLWG22RC.js}

@@ -940,6 +940,87 @@ function createLogger(label) {
   };
 }
 
+// src/utils/rate-limiter.ts
+var RateLimiter = class {
+  tokens;
+  maxTokens;
+  refillIntervalMs;
+  lastRefill;
+  waitQueue = [];
+  /**
+   * @param maxRequests  Maximum number of requests allowed in the window
+   * @param windowMs     Window duration in milliseconds
+   */
+  constructor(maxRequests, windowMs) {
+    this.maxTokens = maxRequests;
+    this.tokens = maxRequests;
+    this.refillIntervalMs = windowMs;
+    this.lastRefill = Date.now();
+  }
+  /**
+   * Acquire a token. Resolves immediately when tokens are available,
+   * otherwise waits until the bucket is refilled.
+   */
+  async acquire() {
+    this.refill();
+    if (this.tokens > 0) {
+      this.tokens--;
+      return;
+    }
+    return new Promise((resolve) => {
+      this.waitQueue.push(resolve);
+      this.scheduleRefill();
+    });
+  }
+  /**
+   * Return the number of tokens currently available (without waiting).
+   */
+  get remaining() {
+    this.refill();
+    return this.tokens;
+  }
+  /**
+   * Return the number of milliseconds until the next refill.
+   */
+  get msUntilRefill() {
+    const elapsed = Date.now() - this.lastRefill;
+    return Math.max(0, this.refillIntervalMs - elapsed);
+  }
+  // -----------------------------------------------------------------------
+  // Internal
+  // -----------------------------------------------------------------------
+  refill() {
+    const now = Date.now();
+    const elapsed = now - this.lastRefill;
+    if (elapsed >= this.refillIntervalMs) {
+      const periods = Math.floor(elapsed / this.refillIntervalMs);
+      this.tokens = Math.min(this.maxTokens, this.tokens + periods * this.maxTokens);
+      this.lastRefill = now - elapsed % this.refillIntervalMs;
+      this.drainWaitQueue();
+    }
+  }
+  scheduleRefill() {
+    const delay = this.msUntilRefill;
+    if (delay <= 0) {
+      this.refill();
+      return;
+    }
+    setTimeout(() => {
+      this.refill();
+    }, delay);
+  }
+  drainWaitQueue() {
+    while (this.waitQueue.length > 0 && this.tokens > 0) {
+      this.tokens--;
+      const resolve = this.waitQueue.shift();
+      resolve?.();
+    }
+  }
+};
+var githubRateLimiter = new RateLimiter(5e3, 36e5);
+var npmRateLimiter = new RateLimiter(300, 6e4);
+var pypiRateLimiter = new RateLimiter(100, 6e4);
+
 // src/collectors/orchestrator.ts
 import pLimit from "p-limit";
 
@@ -1055,6 +1136,7 @@ var RegistryCollector = class extends BaseCollector {
   async fetchPackument(packageName) {
     const url = `https://registry.npmjs.org/${encodeURIComponent(packageName)}`;
     logger.debug(`Fetching packument: ${url}`);
+    await npmRateLimiter.acquire();
     const res = await fetch(url, {
       headers: { Accept: "application/json" }
     });
@@ -1067,6 +1149,7 @@ var RegistryCollector = class extends BaseCollector {
     const url = `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent(packageName)}`;
     logger.debug(`Fetching weekly downloads: ${url}`);
     try {
+      await npmRateLimiter.acquire();
       const res = await fetch(url, {
         headers: { Accept: "application/json" }
       });
@@ -1094,6 +1177,166 @@ var RegistryCollector = class extends BaseCollector {
   }
 };
 
+// src/collectors/pypi-registry.ts
+var PyPIRegistryCollector = class extends BaseCollector {
+  name = "pypi-registry";
+  constructor(cache) {
+    super(cache);
+  }
+  async collect(packageName, version) {
+    const cached = await this.getCached(packageName, version);
+    if (cached) return cached;
+    try {
+      const [metadataResult, downloadsResult] = await Promise.allSettled([
+        this.fetchMetadata(packageName),
+        this.fetchWeeklyDownloads(packageName)
+      ]);
+      const metadata = metadataResult.status === "fulfilled" ? metadataResult.value : null;
+      const downloads = downloadsResult.status === "fulfilled" ? downloadsResult.value : 0;
+      if (!metadata) {
+        throw new Error(`PyPI registry returned no data for ${packageName}`);
+      }
+      const versionCount = metadata.releases ? Object.keys(metadata.releases).length : 0;
+      const lastPublishDate = this.findLastPublishDate(metadata.releases);
+      const deprecated = this.checkYanked(metadata.releases, version);
+      const data = {
+        packageName,
+        version: version === "latest" ? metadata.info.version : version,
+        description: metadata.info.summary ?? null,
+        lastPublishDate,
+        versionCount,
+        deprecated,
+        weeklyDownloads: downloads,
+        license: metadata.info.license ?? null,
+        repositoryUrl: this.extractRepoUrl(metadata.info)
+      };
+      await this.setCache(packageName, version, data);
+      return {
+        status: "success",
+        data,
+        collectedAt: (/* @__PURE__ */ new Date()).toISOString()
+      };
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      logger.error(
+        `PyPIRegistryCollector failed for ${packageName}@${version}: ${message}`
+      );
+      return {
+        status: "error",
+        data: null,
+        error: message,
+        collectedAt: (/* @__PURE__ */ new Date()).toISOString()
+      };
+    }
+  }
+  // ---------------------------------------------------------------------------
+  // Private helpers
+  // ---------------------------------------------------------------------------
+  async fetchMetadata(packageName) {
+    await pypiRateLimiter.acquire();
+    const url = `https://pypi.org/pypi/${encodeURIComponent(packageName)}/json`;
+    logger.debug(`Fetching PyPI metadata: ${url}`);
+    const res = await fetch(url, {
+      headers: { Accept: "application/json" }
+    });
+    if (!res.ok) {
+      if (res.status === 404) return null;
+      throw new Error(`PyPI registry returned ${res.status} for ${packageName}`);
+    }
+    return await res.json();
+  }
+  async fetchWeeklyDownloads(packageName) {
+    await pypiRateLimiter.acquire();
+    const url = `https://pypistats.org/api/packages/${encodeURIComponent(packageName)}/recent`;
+    logger.debug(`Fetching PyPI weekly downloads: ${url}`);
+    try {
+      const res = await fetch(url, {
+        headers: { Accept: "application/json" }
+      });
+      if (!res.ok) {
+        logger.warn(
+          `PyPI stats API returned ${res.status} for ${packageName}`
+        );
+        return 0;
+      }
+      const body = await res.json();
+      return body.data?.last_week ?? 0;
+    } catch {
+      logger.warn(`Could not fetch PyPI download stats for ${packageName}`);
+      return 0;
+    }
+  }
+  /**
+   * Find the most recent upload date across all releases.
+   */
+  findLastPublishDate(releases) {
+    if (!releases) return null;
+    const dates = [];
+    for (const files of Object.values(releases)) {
+      for (const file of files) {
+        const dateStr = file.upload_time_iso_8601 ?? file.upload_time;
+        if (dateStr) {
+          const ts = new Date(dateStr).getTime();
+          if (!isNaN(ts)) dates.push(ts);
+        }
+      }
+    }
+    if (dates.length === 0) return null;
+    dates.sort((a, b) => b - a);
+    return new Date(dates[0]).toISOString();
+  }
+  /**
+   * Check if the requested version is yanked (PyPI's deprecation mechanism).
+   * Returns the yank reason string, or null if not yanked.
+   */
+  checkYanked(releases, version) {
+    if (!releases || version === "latest") return null;
+    const files = releases[version];
+    if (!files || files.length === 0) return null;
+    const yankedFile = files.find((f) => f.yanked);
+    if (yankedFile) {
+      return yankedFile.yanked_reason || "This version has been yanked";
+    }
+    return null;
+  }
+  /**
+   * Extract a normalised repository URL from PyPI project_urls or home_page.
+   */
+  extractRepoUrl(info) {
+    const projectUrls = info.project_urls ?? {};
+    const repoKeys = [
+      "Source",
+      "Source Code",
+      "Repository",
+      "GitHub",
+      "Code",
+      "Homepage",
+      "source",
+      "source_code",
+      "repository",
+      "github",
+      "code",
+      "homepage"
+    ];
+    for (const key of repoKeys) {
+      const url = projectUrls[key];
+      if (url && (url.includes("github.com") || url.includes("gitlab.com") || url.includes("bitbucket.org"))) {
+        return url.replace(/\.git$/, "");
+      }
+    }
+    for (const url of Object.values(projectUrls)) {
+      if (url && (url.includes("github.com") || url.includes("gitlab.com") || url.includes("bitbucket.org"))) {
+        return url.replace(/\.git$/, "");
+      }
+    }
+    const homePage = info.home_page;
+    if (homePage && (homePage.includes("github.com") || homePage.includes("gitlab.com"))) {
+      return homePage.replace(/\.git$/, "");
+    }
+    return null;
+  }
+};
+
 // src/collectors/github.ts
 var GitHubCollector = class extends BaseCollector {
   name = "github";
@@ -1165,6 +1408,7 @@ var GitHubCollector = class extends BaseCollector {
   async resolveRepoSlug(packageName) {
     try {
       const url = `https://registry.npmjs.org/${encodeURIComponent(packageName)}`;
+      await npmRateLimiter.acquire();
       const res = await fetch(url, {
         headers: { Accept: "application/json" }
       });
@@ -1207,6 +1451,7 @@ var GitHubCollector = class extends BaseCollector {
   async fetchRepoInfo(owner, repo) {
     const url = `https://api.github.com/repos/${owner}/${repo}`;
     logger.debug(`GitHub: fetching repo info ${url}`);
+    await githubRateLimiter.acquire();
     const res = await fetch(url, { headers: this.headers() });
     if (!res.ok) {
       throw new Error(`GitHub API ${res.status} for ${url}`);
@@ -1221,6 +1466,7 @@ var GitHubCollector = class extends BaseCollector {
     const url = `https://api.github.com/repos/${owner}/${repo}/contributors?per_page=1&anon=true`;
     logger.debug(`GitHub: fetching contributor count ${url}`);
     try {
+      await githubRateLimiter.acquire();
       const res = await fetch(url, { headers: this.headers() });
       if (!res.ok) return 0;
       const count = this.extractLastPage(res.headers.get("link"));
@@ -1241,6 +1487,7 @@ var GitHubCollector = class extends BaseCollector {
     const url = `https://api.github.com/repos/${owner}/${repo}/commits?since=${since}&per_page=1`;
     logger.debug(`GitHub: fetching recent commit count ${url}`);
     try {
+      await githubRateLimiter.acquire();
       const res = await fetch(url, { headers: this.headers() });
       if (!res.ok) return 0;
       const count = this.extractLastPage(res.headers.get("link"));
@@ -1256,6 +1503,7 @@ var GitHubCollector = class extends BaseCollector {
     const url = `https://api.github.com/repos/${owner}/${repo}/commits?per_page=1`;
     logger.debug(`GitHub: fetching latest commit ${url}`);
     try {
+      await githubRateLimiter.acquire();
       const res = await fetch(url, { headers: this.headers() });
       if (!res.ok) return null;
       const body = await res.json();
@@ -1272,6 +1520,7 @@ var GitHubCollector = class extends BaseCollector {
     const url = `https://api.github.com/repos/${owner}/${repo}/contents/.github/FUNDING.yml`;
     logger.debug(`GitHub: checking FUNDING.yml ${url}`);
     try {
+      await githubRateLimiter.acquire();
       const res = await fetch(url, { headers: this.headers() });
       return res.ok;
     } catch {
@@ -1303,11 +1552,11 @@ var SecurityCollector = class extends BaseCollector {
   constructor(cache) {
     super(cache);
   }
-  async collect(packageName, version) {
+  async collect(packageName, version, ecosystem) {
     const cached = await this.getCached(packageName, version);
     if (cached) return cached;
     try {
-      const vulns = await this.queryOsv(packageName);
+      const vulns = await this.queryOsv(packageName, ecosystem);
       const totalVulnerabilities = vulns.length;
       const severityCounts = {
         critical: 0,
@@ -1357,16 +1606,16 @@ var SecurityCollector = class extends BaseCollector {
   // ---------------------------------------------------------------------------
   // OSV API
   // ---------------------------------------------------------------------------
-  async queryOsv(packageName) {
+  async queryOsv(packageName, ecosystem = "npm") {
     const url = "https://api.osv.dev/v1/query";
-    logger.debug(`OSV: querying vulnerabilities for ${packageName}`);
+    logger.debug(`OSV: querying vulnerabilities for ${packageName} (ecosystem=${ecosystem})`);
     const res = await fetch(url, {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
         package: {
           name: packageName,
-          ecosystem: "npm"
+          ecosystem
         }
       })
     });
@@ -1530,6 +1779,7 @@ var FundingCollector = class extends BaseCollector {
   async fetchNpmFunding(packageName) {
     try {
       const url = `https://registry.npmjs.org/${encodeURIComponent(packageName)}`;
+      await npmRateLimiter.acquire();
       const res = await fetch(url, {
         headers: { Accept: "application/json" }
       });
@@ -1564,6 +1814,7 @@ var FundingCollector = class extends BaseCollector {
       if (this.githubToken) {
         headers.Authorization = `Bearer ${this.githubToken}`;
       }
+      await githubRateLimiter.acquire();
       const res = await fetch(url, { headers });
       if (!res.ok) return null;
       return await res.text();
@@ -1594,6 +1845,7 @@ var FundingCollector = class extends BaseCollector {
   async resolveRepoSlug(packageName) {
     try {
       const url = `https://registry.npmjs.org/${encodeURIComponent(packageName)}`;
+      await npmRateLimiter.acquire();
       const res = await fetch(url, {
         headers: { Accept: "application/json" }
       });
@@ -1712,6 +1964,7 @@ var PopularityCollector = class extends BaseCollector {
     const url = `https://api.npmjs.org/downloads/point/${period}/${encodeURIComponent(packageName)}`;
     logger.debug(`Popularity: fetching ${period} downloads: ${url}`);
     try {
+      await npmRateLimiter.acquire();
       const res = await fetch(url, {
         headers: { Accept: "application/json" }
       });
@@ -1739,6 +1992,7 @@ var PopularityCollector = class extends BaseCollector {
     const url = `https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(packageName)}&size=1`;
     logger.debug(`Popularity: fetching dependent count: ${url}`);
     try {
+      await npmRateLimiter.acquire();
       const res = await fetch(url, {
         headers: { Accept: "application/json" }
       });
@@ -1764,6 +2018,7 @@ var PopularityCollector = class extends BaseCollector {
   async fetchDependentCountFromRegistry(packageName) {
     try {
       const url = `https://www.npmjs.com/package/${encodeURIComponent(packageName)}`;
+      await npmRateLimiter.acquire();
       const res = await fetch(url, {
         headers: {
           Accept: "text/html",
@@ -1931,6 +2186,7 @@ var LicenseCollector = class extends BaseCollector {
   async fetchLicense(packageName, version) {
     const url = `https://registry.npmjs.org/${encodeURIComponent(packageName)}`;
     logger.debug(`License: fetching packument ${url}`);
+    await npmRateLimiter.acquire();
     const res = await fetch(url, {
       headers: { Accept: "application/json" }
     });
@@ -1990,6 +2246,7 @@ var CollectorOrchestrator = class {
   cache;
   options;
   registryCollector;
+  pypiRegistryCollector;
   githubCollector;
   securityCollector;
   fundingCollector;
@@ -2003,6 +2260,7 @@ var CollectorOrchestrator = class {
       concurrency: options.concurrency ?? 10
     };
     this.registryCollector = new RegistryCollector(this.cache);
+    this.pypiRegistryCollector = new PyPIRegistryCollector(this.cache);
     this.githubCollector = new GitHubCollector(this.cache, this.options.githubToken || void 0);
     this.securityCollector = new SecurityCollector(this.cache);
     this.fundingCollector = new FundingCollector(this.cache, this.options.githubToken || void 0);
@@ -2016,13 +2274,15 @@ var CollectorOrchestrator = class {
    * only the cache is consulted; if there is no cached entry the result gets
    * `status: 'offline'` with `data: null`.
    */
-  async collectAll(packageName, version) {
+  async collectAll(packageName, version, ecosystem = "npm") {
     logger.info(
-      `Collecting data for ${packageName}@${version} (offline=${String(this.options.offline)})`
+      `Collecting data for ${packageName}@${version} (ecosystem=${ecosystem}, offline=${String(this.options.offline)})`
     );
     const limit = pLimit(this.options.concurrency);
+    const activeRegistryCollector = ecosystem === "pypi" ? this.pypiRegistryCollector : this.registryCollector;
+    const osvEcosystem = ecosystem === "pypi" ? "PyPI" : "npm";
     const entries = [
-      { key: "registry", collector: this.registryCollector },
+      { key: "registry", collector: activeRegistryCollector },
       { key: "github", collector: this.githubCollector },
       { key: "security", collector: this.securityCollector },
       { key: "funding", collector: this.fundingCollector },
@@ -2038,12 +2298,21 @@ var CollectorOrchestrator = class {
           result = await this.offlineCollect(collector, packageName, version);
         } else {
           try {
-            result = await Promise.race([
-              this.onlineCollect(collector, packageName, version),
-              new Promise(
-                (_, reject) => setTimeout(() => reject(new Error("Collector timeout")), COLLECTOR_TIMEOUT)
-              )
-            ]);
+            if (key === "security") {
+              result = await Promise.race([
+                this.securityCollector.collect(packageName, version, osvEcosystem),
+                new Promise(
+                  (_, reject) => setTimeout(() => reject(new Error("Collector timeout")), COLLECTOR_TIMEOUT)
+                )
+              ]);
+            } else {
+              result = await Promise.race([
+                this.onlineCollect(collector, packageName, version),
+                new Promise(
+                  (_, reject) => setTimeout(() => reject(new Error("Collector timeout")), COLLECTOR_TIMEOUT)
+                )
+              ]);
+            }
           } catch {
             logger.warn(`[${collector.name}] ${packageName}@${version} => timeout (${COLLECTOR_TIMEOUT}ms)`);
             result = {
@@ -2168,7 +2437,7 @@ var TrustScoreEngine = class {
     return {
       trustScore: Math.round(trustScore),
       metrics,
-      insufficientData: unavailableMetrics.length >= 3,
+      insufficientData: unavailableMetrics.length >= 2,
       unavailableMetrics
     };
   }
@@ -2310,6 +2579,11 @@ var TrustScoreEngine = class {
       const effectiveWeight = m.weight / totalAvailableWeight;
       score += m.score * effectiveWeight;
     }
+    const missingWeightFraction = 1 - totalAvailableWeight;
+    if (missingWeightFraction > 0) {
+      const penalty = (score - 50) * missingWeightFraction;
+      score = score - penalty;
+    }
     return clamp(Math.round(score));
   }
 };
@@ -4764,7 +5038,115 @@ var POPULAR_PACKAGES = [
   "request",
   "tslint",
   "node-pre-gyp",
-  "npm-lifecycle"
+  "npm-lifecycle",
+  // ---------------------------------------------------------------------------
+  // Popular Python packages (PyPI)
+  // ---------------------------------------------------------------------------
+  "requests",
+  "flask",
+  "django",
+  "fastapi",
+  "numpy",
+  "pandas",
+  "scipy",
+  "matplotlib",
+  "tensorflow",
+  "torch",
+  "scikit-learn",
+  "keras",
+  "pytorch-lightning",
+  "xgboost",
+  "lightgbm",
+  "pytest",
+  "black",
+  "mypy",
+  "ruff",
+  "pylint",
+  "flake8",
+  "isort",
+  "celery",
+  "redis",
+  "sqlalchemy",
+  "alembic",
+  "pydantic",
+  "httpx",
+  "aiohttp",
+  "uvicorn",
+  "gunicorn",
+  "starlette",
+  "boto3",
+  "botocore",
+  "awscli",
+  "google-cloud-storage",
+  "azure-storage-blob",
+  "pillow",
+  "opencv-python",
+  "beautifulsoup4",
+  "lxml",
+  "scrapy",
+  "cryptography",
+  "paramiko",
+  "fabric",
+  "ansible",
+  "click",
+  "typer",
+  "rich",
+  "tqdm",
+  "colorama",
+  "tabulate",
+  "setuptools",
+  "wheel",
+  "pip",
+  "twine",
+  "poetry",
+  "pdm",
+  "hatch",
+  "jinja2",
+  "mako",
+  "markupsafe",
+  "werkzeug",
+  "psycopg2",
+  "pymongo",
+  "motor",
+  "peewee",
+  "tortoise-orm",
+  "marshmallow",
+  "attrs",
+  "dataclasses-json",
+  "sentry-sdk",
+  "prometheus-client",
+  "opentelemetry-api",
+  "transformers",
+  "huggingface-hub",
+  "tokenizers",
+  "datasets",
+  "langchain",
+  "openai",
+  "anthropic",
+  "tiktoken",
+  "pytest-cov",
+  "pytest-asyncio",
+  "pytest-mock",
+  "coverage",
+  "pyyaml",
+  "toml",
+  "python-dotenv",
+  "decouple",
+  "arrow",
+  "pendulum",
+  "python-dateutil",
+  "stripe",
+  "twilio",
+  "sendgrid",
+  "selenium",
+  "playwright",
+  "httptools",
+  "orjson",
+  "ujson",
+  "msgpack",
+  "networkx",
+  "sympy",
+  "statsmodels"
 ];
 var TyposquatDetector = class _TyposquatDetector {
   popularPackages;
@@ -6576,6 +6958,10 @@ export {
   isDebug,
   logger,
   createLogger,
+  RateLimiter,
+  githubRateLimiter,
+  npmRateLimiter,
+  pypiRateLimiter,
   CollectorOrchestrator,
   TrustScoreEngine,
   ZombieDetector,
@@ -6586,4 +6972,4 @@ export {
   getBlastRadius,
   getImportingFiles
 };
-//# sourceMappingURL=chunk-TXSNFX3N.js.map
+//# sourceMappingURL=chunk-DLWG22RC.js.map