@cdxgen/cdxgen 0.0.1 → 13.0.0

package/lib/inventory/fetchRate.js

@@ -0,0 +1,255 @@
+/**
+ * Per-host rate policy for batched registry fetches.
+ *
+ * This is the single source of truth for the JS transport's concurrency and
+ * minimum-interval decisions. The numbers are shared with the Rust transport
+ * (`cdxgen-plugins-bin/thirdparty/cdxrs/src/fetch/rate.rs`); a change to one
+ * side must be reflected in the other, and `fetchRate.poku.js` pins them so a
+ * drift is visible.
+ *
+ * Why a per-host table rather than a single global interval: a single-registry
+ * project — which is to say, almost every npm project — has exactly one host,
+ * so a conservative per-host cap *is* the global cap. Applying crates.io's
+ * 250 ms to every host caps an npm run at 4 requests per second, which is
+ * slower than the serial JS path once connection reuse is taken into account.
+ * Hosts that publish a limit get an interval; hosts that do not are bounded by
+ * the per-host concurrency cap alone, and by whatever `Retry-After` they send.
+ */
+
+import { readEnvironmentVariable } from "../core/activity.js";
+
+/**
+ * Whether a credential will be sent to the host.
+ *
+ * GitHub is the reason this exists: 60 requests an hour anonymously against
+ * 5000 with a token is not a difference of degree. Applying the anonymous
+ * budget to a token-bearing client throws away the point of having set
+ * `GITHUB_TOKEN`.
+ *
+ * @typedef {("anonymous"|"authenticated")} Credentials
+ */
+
+/**
+ * The policy for one host: the minimum gap between two requests and the
+ * maximum number in flight.
+ *
+ * @typedef {Object} HostPolicy
+ * @property {number} minInterval Minimum gap between requests, in milliseconds.
+ * @property {number} maxConcurrency Maximum requests in flight to this host.
+ */
+
+/**
+ * Hosts that publish a rate limit, and what they publish.
+ *
+ * Columns: host, anonymous interval (ms), anonymous concurrency, authenticated
+ * interval (ms), authenticated concurrency. A host whose limit does not change
+ * with a credential repeats the same pair.
+ *
+ * Suffix match is used so `index.crates.io` inherits `crates.io`.
+ */
+const HOST_POLICIES = [
+  // crates.io's published budget is per IP and does not improve with a token.
+  {
+    host: "crates.io",
+    anonInterval: 250,
+    anonConcurrency: 4,
+    authInterval: 250,
+    authConcurrency: 4,
+  },
+  // GitHub: 60 req/h anonymous, 5000 req/h authenticated. The limit is an
+  // hourly budget rather than a rate, so a batch of a few hundred lookups is
+  // well inside it once a token is present.
+  {
+    host: "api.github.com",
+    anonInterval: 250,
+    anonConcurrency: 4,
+    authInterval: 0,
+    authConcurrency: 8,
+  },
+  {
+    host: "github.com",
+    anonInterval: 250,
+    anonConcurrency: 4,
+    authInterval: 0,
+    authConcurrency: 8,
+  },
+  // GitLab: 2000 req/min authenticated against 500 unauthenticated.
+  {
+    host: "gitlab.com",
+    anonInterval: 250,
+    anonConcurrency: 4,
+    authInterval: 0,
+    authConcurrency: 8,
+  },
+  {
+    host: "pkg.go.dev",
+    anonInterval: 250,
+    anonConcurrency: 4,
+    authInterval: 250,
+    authConcurrency: 4,
+  },
+];
+
+/**
+ * Concurrency allowed for a host with no published limit. Registries that are
+ * CDN-fronted document store fronts (npm, PyPI, RubyGems, NuGet, pub.dev) sit
+ * here; the global cap is what bounds them.
+ */
+export const DEFAULT_HOST_CONCURRENCY = 16;
+
+/**
+ * Global cap on in-flight requests across every host. Per-host caps cannot
+ * sum past this; a 16-way global batch saturates a large npm run without
+ * putting any single host above its published allowance.
+ */
+export const DEFAULT_GLOBAL_CONCURRENCY = 16;
+
+/**
+ * Resolve the credential class for a host based on the environment.
+ *
+ * Only token presence is considered, never its value. A host we hold no token
+ * for keeps the anonymous budget even if some other host has one. Today only
+ * GitHub tokens are honoured, matching the Rust client.
+ *
+ * @param {string} host The hostname to classify.
+ * @returns {Credentials}
+ */
+export function credentialsFor(host) {
+  const lower = host.toLowerCase();
+  const isGitHub =
+    lower === "api.github.com" ||
+    lower === "github.com" ||
+    lower.endsWith(".github.com");
+  if (isGitHub && readEnvironmentVariable("GITHUB_TOKEN")) {
+    return "authenticated";
+  }
+  return "anonymous";
+}
+
+/**
+ * Resolve the published policy for a host, or the CDN default.
+ *
+ * @param {string} host The hostname.
+ * @param {Credentials} [credentials] Credential class; defaults to what the
+ *   environment would actually send to this host.
+ * @returns {HostPolicy}
+ */
+export function policyFor(host, credentials) {
+  const lower = (host || "").toLowerCase();
+  const cred = credentials || credentialsFor(host);
+  for (const candidate of HOST_POLICIES) {
+    const matches =
+      lower === candidate.host || lower.endsWith(`.${candidate.host}`);
+    if (!matches) {
+      continue;
+    }
+    if (cred === "authenticated") {
+      return {
+        minInterval: candidate.authInterval,
+        maxConcurrency: candidate.authConcurrency,
+      };
+    }
+    return {
+      minInterval: candidate.anonInterval,
+      maxConcurrency: candidate.anonConcurrency,
+    };
+  }
+  return { minInterval: 0, maxConcurrency: DEFAULT_HOST_CONCURRENCY };
+}
+
+/**
+ * The table of published policies, exposed for the parity test. The shape is
+ * stable: changing an entry is a policy change that must be mirrored in
+ * `rate.rs`.
+ */
+export const PUBLISHED_HOST_POLICIES = HOST_POLICIES.map((entry) => ({
+  ...entry,
+}));
+
+/**
+ * Create a minimum-interval gate for one host.
+ *
+ * `wait()` queues the caller so that successive calls are at least
+ * `minInterval` apart. `externalDelay()` pushes the gate out by a server-side
+ * back-off (`Retry-After`, `X-RateLimit-Reset`) and is counted.
+ */
+export class RateLimiter {
+  /**
+   * @param {number} minInterval Minimum gap between requests, in milliseconds.
+   */
+  constructor(minInterval) {
+    this._minInterval = minInterval;
+    this._nextAvailable = 0;
+    this._externalDelays = 0;
+  }
+
+  /**
+   * Wait until this host's gate opens, then reserve the next slot. Callers
+   * that arrive while the gate is closed queue up rather than all being
+   * released together.
+   *
+   * @returns {Promise<void>}
+   */
+  async wait() {
+    const now = Date.now();
+    // The slot is reserved synchronously, before any await, so concurrent
+    // callers each take a distinct slot instead of all reading the same gate.
+    const slot = Math.max(this._nextAvailable, now);
+    this._nextAvailable = slot + this._minInterval;
+    const delay = slot - now;
+    if (delay > 0) {
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+
+  /**
+   * Record a server-imposed delay and push the gate out to at least now plus
+   * the delay. The caller is still responsible for actually waiting before it
+   * retries; recording alone does not sleep.
+   *
+   * @param {number} delayMs Delay in milliseconds.
+   */
+  externalDelay(delayMs) {
+    this._externalDelays += 1;
+    const target = Date.now() + delayMs;
+    if (target > this._nextAvailable) {
+      this._nextAvailable = target;
+    }
+  }
+
+  /** @returns {number} How many times this host asked us to slow down. */
+  get externalDelayCount() {
+    return this._externalDelays;
+  }
+}
+
+/**
+ * Create a counting semaphore with a FIFO wait queue. Used both for the global
+ * concurrency cap and the per-host cap.
+ *
+ * @param {number} limit Maximum simultaneous holders.
+ * @returns {{acquire: () => Promise<() => void>}} An `acquire` function that
+ *   resolves once a slot is free, with a `release` function as its value.
+ */
+export function makeSemaphore(limit) {
+  let available = Math.max(1, limit);
+  const waiters = [];
+  return {
+    async acquire() {
+      if (available > 0) {
+        available -= 1;
+        return release;
+      }
+      await new Promise((resolve) => waiters.push(resolve));
+      return release;
+    },
+  };
+  function release() {
+    const next = waiters.shift();
+    if (next) {
+      next();
+    } else {
+      available += 1;
+    }
+  }
+}

package/lib/inventory/forgeEnricher.js

@@ -0,0 +1,300 @@
+import { cdxgenAgent, readEnvironmentVariable } from "../core/activity.js";
+import { getOriginUrl } from "./envcontext.js";
+import { prefetchJson } from "./fetchBatch.js";
+
+/**
+ * Fetch the commit-level lookups for a whole commit list in one round.
+ *
+ * Both forges answer "which pull/merge request contains this commit?" from a
+ * URL built entirely from the commit hash, so the whole list is knowable before
+ * the loop. The review and approval lookups that follow are not: their URL
+ * carries a number that only arrives with the response, so they stay serial.
+ *
+ * The requests carry a caller-supplied credential, which the cdxrs protocol has
+ * no field for, so `prefetchJson` routes them to the JS pool.
+ *
+ * @param {Array<string>} urls Commit-level URLs.
+ * @param {Object} headers Headers to send with each, including the credential.
+ * @returns {Promise<Map<string, {ok?: boolean, body?: *}>>} Map keyed by URL.
+ */
+async function prefetchCommitLookups(urls, headers) {
+  return await prefetchJson(
+    urls.map((url) => ({ headers, responseType: "json", url })),
+  );
+}
+
+/**
+ * Read a prefetched commit lookup as the response shape the loop expects.
+ *
+ * The loop tests `statusCode === 200`, and a batch entry records only whether
+ * the request succeeded, so a hit becomes a 200 and anything else is left for
+ * the caller to fetch itself.
+ *
+ * @param {Map<string, {ok?: boolean, body?: *}>} prefetched
+ * @param {string} url
+ * @returns {{statusCode: number, body: *}|undefined}
+ */
+function prefetchedCommitLookup(prefetched, url) {
+  const entry = prefetched.get(url);
+  return entry?.ok ? { body: entry.body, statusCode: 200 } : undefined;
+}
+
+// Only accept single path segments made of git-safe characters, and never the
+// `.`/`..` traversal segments, so a malicious remote.origin.url cannot steer the
+// authenticated request to a different GitHub/GitLab API path.
+const SAFE_SEGMENT = /^[A-Za-z0-9._-]+$/;
+function isSafeSegment(seg) {
+  return Boolean(seg) && seg !== "." && seg !== ".." && SAFE_SEGMENT.test(seg);
+}
+
+/**
+ * Parse a GitHub repository URL into its owner and repository name.
+ *
+ * Only accepts URLs where github.com is the actual host (rejecting spoofed
+ * hosts such as `https://evil.com/github.com/o/r`) and only git-safe path
+ * segments, so an untrusted remote URL cannot steer authenticated API requests.
+ *
+ * @param {string} url The git origin url
+ * @returns {{owner: string, repo: string}|null} Object containing owner and repo, or null
+ */
+export function parseGitHubUrl(url) {
+  if (!url) return null;
+  // Require github.com to be the actual host (preceded by start, `//`, or `@`),
+  // so URLs like `https://evil.com/github.com/o/r` do not match.
+  const match = url.match(
+    /(?:^|@|\/\/)github\.com[:/]([^/]+)\/([^/]+?)(?:\.git)?\/?$/,
+  );
+  if (match && isSafeSegment(match[1]) && isSafeSegment(match[2])) {
+    return { owner: match[1], repo: match[2] };
+  }
+  return null;
+}
+
+/**
+ * Parses GitLab repository origin URL to extract project path/ID.
+ *
+ * @param {string} url The git origin url
+ * @returns {string|null} The project path (owner/repo) or null
+ */
+export function parseGitLabUrl(url) {
+  if (!url) return null;
+  const match = url.match(
+    /(?:^|@|\/\/)gitlab\.com[:/]([^/]+)\/([^/]+?)(?:\.git)?\/?$/,
+  );
+  if (match && isSafeSegment(match[1]) && isSafeSegment(match[2])) {
+    return `${match[1]}/${match[2]}`;
+  }
+  return null;
+}
+
+/**
+ * Enriches AI commits with details from GitHub or GitLab API if tokens are present.
+ *
+ * @param {string} dir Root directory of the repository
+ * @param {Array<Object>} aiCommits List of AI commit objects
+ * @param {Object} options Options containing forgeToken or env context
+ * @returns {Promise<Object>} Object containing reviews list and authoritative flags
+ */
+export async function enrichFromForge(dir, aiCommits, options = {}) {
+  const result = {
+    dataSources: [],
+    prReviews: [], // Array of review states { commitHash, hasIndependentApproval, selfApproved, reviewLatencySeconds }
+  };
+
+  const originUrl = getOriginUrl(dir);
+  if (!originUrl) {
+    return result;
+  }
+
+  const githubInfo = parseGitHubUrl(originUrl);
+  const gitlabProject = parseGitLabUrl(originUrl);
+
+  const githubToken =
+    options.forgeToken || readEnvironmentVariable("GITHUB_TOKEN");
+  const gitlabToken =
+    options.forgeToken || readEnvironmentVariable("GITLAB_TOKEN");
+
+  if (githubInfo && githubToken) {
+    result.dataSources.push("github-api");
+    const { owner, repo } = githubInfo;
+
+    // To avoid hitting API rate limits or spamming, limit to the most recent 5 AI commits
+    const commitsToQuery = aiCommits.slice(0, 5);
+    const githubHeaders = {
+      Accept: "application/vnd.github+json",
+      Authorization: `Bearer ${githubToken}`,
+      "User-Agent": "cdxgen-oversight",
+    };
+    const pullsUrlFor = (commit) =>
+      `https://api.github.com/repos/${owner}/${repo}/commits/${commit.hash}/pulls`;
+    const prefetchedPulls = await prefetchCommitLookups(
+      commitsToQuery.map(pullsUrlFor),
+      githubHeaders,
+    );
+
+    for (const commit of commitsToQuery) {
+      try {
+        const pullsUrl = pullsUrlFor(commit);
+        const pullsRes =
+          prefetchedCommitLookup(prefetchedPulls, pullsUrl) ||
+          (await cdxgenAgent.get(pullsUrl, {
+            headers: githubHeaders,
+            responseType: "json",
+            throwHttpErrors: false,
+          }));
+
+        if (
+          pullsRes?.statusCode === 200 &&
+          Array.isArray(pullsRes.body) &&
+          pullsRes.body.length > 0
+        ) {
+          const pr = pullsRes.body[0];
+          const prNumber = pr.number;
+          const prAuthor = pr.user?.login;
+
+          // Fetch reviews
+          const reviewsRes = await cdxgenAgent.get(
+            `https://api.github.com/repos/${owner}/${repo}/pulls/${prNumber}/reviews`,
+            {
+              headers: githubHeaders,
+              responseType: "json",
+              throwHttpErrors: false,
+            },
+          );
+
+          if (
+            reviewsRes?.statusCode === 200 &&
+            Array.isArray(reviewsRes.body)
+          ) {
+            let hasIndependentApproval = false;
+            let selfApproved = false;
+            let approvedTime = null;
+
+            for (const review of reviewsRes.body) {
+              if (review.state === "APPROVED") {
+                const reviewer = review.user?.login;
+                const isBot =
+                  review.user?.type === "Bot" || /\[bot\]/i.test(reviewer);
+
+                if (reviewer === prAuthor) {
+                  selfApproved = true;
+                } else if (!isBot) {
+                  hasIndependentApproval = true;
+                  approvedTime = new Date(review.submitted_at);
+                }
+              }
+            }
+
+            let reviewLatencySeconds = null;
+            if (approvedTime && pr.created_at) {
+              const createdTime = new Date(pr.created_at);
+              reviewLatencySeconds = Math.max(
+                0,
+                Math.floor((approvedTime - createdTime) / 1000),
+              );
+            }
+
+            result.prReviews.push({
+              commitHash: commit.hash,
+              hasIndependentApproval,
+              selfApproved,
+              reviewLatencySeconds,
+              prNumber,
+              prAuthor,
+            });
+          }
+        }
+      } catch (_err) {
+        // Degrade gracefully
+      }
+    }
+  } else if (gitlabProject && gitlabToken) {
+    result.dataSources.push("gitlab-api");
+    const projectEncoded = encodeURIComponent(gitlabProject);
+    const commitsToQuery = aiCommits.slice(0, 5);
+    const gitlabHeaders = {
+      "PRIVATE-TOKEN": gitlabToken,
+      "User-Agent": "cdxgen-oversight",
+    };
+    const mrUrlFor = (commit) =>
+      `https://gitlab.com/api/v4/projects/${projectEncoded}/repository/commits/${commit.hash}/merge_requests`;
+    const prefetchedMrs = await prefetchCommitLookups(
+      commitsToQuery.map(mrUrlFor),
+      gitlabHeaders,
+    );
+
+    for (const commit of commitsToQuery) {
+      try {
+        const mrUrl = mrUrlFor(commit);
+        const mrRes =
+          prefetchedCommitLookup(prefetchedMrs, mrUrl) ||
+          (await cdxgenAgent.get(mrUrl, {
+            headers: gitlabHeaders,
+            responseType: "json",
+            throwHttpErrors: false,
+          }));
+
+        if (
+          mrRes?.statusCode === 200 &&
+          Array.isArray(mrRes.body) &&
+          mrRes.body.length > 0
+        ) {
+          const mr = mrRes.body[0];
+          const mrIid = mr.iid;
+          const mrAuthor = mr.author?.username;
+
+          // Fetch approvals/reviews
+          const approvalsRes = await cdxgenAgent.get(
+            `https://gitlab.com/api/v4/projects/${projectEncoded}/merge_requests/${mrIid}/approvals`,
+            {
+              headers: gitlabHeaders,
+              responseType: "json",
+              throwHttpErrors: false,
+            },
+          );
+
+          if (approvalsRes?.statusCode === 200 && approvalsRes.body) {
+            const approvals = approvalsRes.body;
+            let hasIndependentApproval = false;
+            let selfApproved = false;
+
+            const approvedBy = approvals.approved_by || [];
+            for (const approval of approvedBy) {
+              const approver = approval.user?.username;
+              const isBot = /\[bot\]/i.test(approver);
+
+              if (approver === mrAuthor) {
+                selfApproved = true;
+              } else if (!isBot) {
+                hasIndependentApproval = true;
+              }
+            }
+
+            let reviewLatencySeconds = null;
+            if (hasIndependentApproval && mr.created_at && mr.updated_at) {
+              const createdTime = new Date(mr.created_at);
+              const approvedTime = new Date(mr.updated_at);
+              reviewLatencySeconds = Math.max(
+                0,
+                Math.floor((approvedTime - createdTime) / 1000),
+              );
+            }
+
+            result.prReviews.push({
+              commitHash: commit.hash,
+              hasIndependentApproval,
+              selfApproved,
+              reviewLatencySeconds,
+              prNumber: mrIid,
+              prAuthor: mrAuthor,
+            });
+          }
+        }
+      } catch (_err) {
+        // Degrade gracefully
+      }
+    }
+  }
+
+  return result;
+}