@cdxgen/cdxgen 0.0.1 → 13.0.0

package/lib/core/logger.js

@@ -0,0 +1,142 @@
+import { Console } from "node:console";
+import fs from "node:fs";
+import process from "node:process";
+
+// Environment variables here are read with raw `process.env` rather than
+// `readEnvironmentVariable`: that helper lives in activity.js, which imports
+// this module, so routing these through it would form an import cycle.
+
+// Stream contract: human-readable diagnostics default to stderr so stdout
+// carries only the BOM payload. CDXGEN_LOG_STREAM=stdout restores the
+// pre-v13 behaviour for callers that scraped cdxgen's stdout.
+const logStream =
+  process.env.CDXGEN_LOG_STREAM === "stdout" ? process.stdout : process.stderr;
+
+// Custom lightweight colors formatting helper replacing yoctocolors
+const colors = {
+  cyanBright: (text) => `\x1b[96m${text}\x1b[39m`,
+  dim: (text) => `\x1b[2m${text}\x1b[22m`,
+};
+
+/** True when thought/reasoning logging is enabled (CDXGEN_THOUGHT_LOG, CDXGEN_THINK_MODE, or verbose debug mode). */
+export const THINK_MODE =
+  process.env.CDXGEN_THOUGHT_LOG ||
+  ["true", "1"].includes(process.env.CDXGEN_THINK_MODE) ||
+  process.env.CDXGEN_DEBUG_MODE === "verbose";
+
+const thinkOutput = process.env.CDXGEN_THOUGHT_LOG
+  ? fs.createWriteStream(process.env.CDXGEN_THOUGHT_LOG)
+  : logStream;
+const thinkErrorOutput = process.env.CDXGEN_THOUGHT_LOG
+  ? fs.createWriteStream(process.env.CDXGEN_THOUGHT_LOG)
+  : process.stderr;
+const thinkLogger = new Console({
+  stdout: thinkOutput,
+  stderr: thinkErrorOutput,
+  colorMode: process.env.CDXGEN_THOUGHT_LOG ? false : "auto",
+});
+
+/** True when structured command/HTTP trace logging is enabled (CDXGEN_TRACE_LOG, CDXGEN_TRACE_ID, CDXGEN_TRACE_MODE, or verbose debug mode). */
+export const TRACE_MODE =
+  process.env.CDXGEN_TRACE_LOG ||
+  process.env.CDXGEN_TRACE_ID ||
+  ["true", "1"].includes(process.env.CDXGEN_TRACE_MODE) ||
+  process.env.CDXGEN_DEBUG_MODE === "verbose";
+
+const traceOutput = process.env.CDXGEN_TRACE_LOG
+  ? fs.createWriteStream(process.env.CDXGEN_TRACE_LOG)
+  : logStream;
+const traceErrorOutput = process.env.CDXGEN_TRACE_LOG
+  ? fs.createWriteStream(process.env.CDXGEN_TRACE_LOG)
+  : process.stderr;
+const traceLogger = new Console({
+  stdout: traceOutput,
+  stderr: traceErrorOutput,
+  colorMode: process.env.CDXGEN_TRACE_LOG ? false : "auto",
+});
+
+if (THINK_MODE) {
+  thinkLogger.group(colorizeText("<think>"));
+}
+/**
+ * Logs a thought message to the think logger if THINK_MODE is enabled.
+ * Automatically appends a period to the message if it lacks terminal punctuation.
+ *
+ * @param {string} s The thought message to log
+ * @param {Object} [args] Optional additional arguments to log alongside the message
+ * @returns {void}
+ */
+export function thoughtLog(s, args) {
+  if (!THINK_MODE) {
+    return;
+  }
+  if (!s?.endsWith(".") && !s?.endsWith("?") && !s?.endsWith("!")) {
+    s = `${s}.`;
+  }
+  s = s.replaceAll("'.'", "'<project dir>'");
+  if (args) {
+    thinkLogger.log(colorizeText(`${s}`), args);
+  } else {
+    thinkLogger.log(colorizeText(`${s}`));
+  }
+}
+/**
+ * Closes the think log group by emitting the closing `</think>` marker.
+ * Has no effect if THINK_MODE is not enabled.
+ *
+ * @returns {void}
+ */
+export function thoughtEnd() {
+  if (THINK_MODE) {
+    thinkLogger.groupEnd();
+    thinkLogger.log(colorizeText("</think>"));
+  }
+}
+
+function colorizeText(s) {
+  // Only colorize for an interactive terminal stream; piped/CI/file output
+  // stays free of escape bytes so the diagnostic stream is grep-safe.
+  if (process.env.CDXGEN_THOUGHT_LOG || !logStream.isTTY) {
+    return s;
+  }
+  s = s.replace(/(\d+)/g, colors.cyanBright("$1"));
+  return colors.dim(s);
+}
+
+/**
+ * Log trace messages
+ *
+ * @param {String} traceType Trace type
+ * @param {Object} args Additional arguments
+ */
+export function traceLog(traceType, args) {
+  if (!TRACE_MODE || !traceType || !args) {
+    return;
+  }
+  const traceId = process.env.CDXGEN_TRACE_ID;
+  const message = {};
+  if (traceId) {
+    message.traceId = traceId;
+  }
+  message["timestamp"] = new Date().toISOString();
+  if (traceType) {
+    message.type = traceType;
+  }
+  if (args) {
+    for (const k of [
+      "command",
+      "cwd",
+      "protocol",
+      "host",
+      "path",
+      "pathname",
+    ]) {
+      if (args[k]) {
+        message[k] = args[k];
+      }
+    }
+  }
+  if (Object.keys(message).length) {
+    traceLogger.log(JSON.stringify(message, null, null));
+  }
+}

package/lib/core/parallel.js

@@ -0,0 +1,323 @@
+/**
+ * Bounded concurrency utilities for CPU-bound and I/O-bound parallel work.
+ *
+ * Two primitives live here:
+ *
+ * - {@link mapWithConcurrency} — a promise pool for async I/O work (file
+ *   hashing, registry fetches). Tasks are async; results preserve input order.
+ *
+ * - {@link mapInWorkers} — a worker-thread pool for synchronous CPU work (AST
+ *   parsing). Below a configurable threshold tasks run inline; above it they
+ *   are distributed across `worker_threads` workers. Any failure of the thread
+ *   machinery itself — an unavailable `worker_threads`, a worker entry that
+ *   will not load, a worker that dies mid-task — falls back to running the
+ *   remaining tasks inline, so the result is the same whether or not threads
+ *   were usable.
+ */
+
+import { availableParallelism } from "node:os";
+
+import { readEnvironmentVariable } from "./activity.js";
+
+const _isDeno = globalThis.Deno?.version?.deno !== undefined;
+const _isBun = globalThis.Bun?.version !== undefined;
+
+const DEFAULT_HASH_CONCURRENCY = Math.min(
+  Math.max(availableParallelism(), 4),
+  16,
+);
+
+const WORKER_THRESHOLD = Number.parseInt(
+  readEnvironmentVariable("CDXGEN_WORKER_THRESHOLD") || "24",
+  10,
+);
+
+const MAX_WORKERS = Number.parseInt(
+  readEnvironmentVariable("CDXGEN_MAX_WORKERS") ||
+    String(Math.min(availableParallelism() || 4, 8)),
+  10,
+);
+
+/**
+ * Run an async mapper over an array with bounded concurrency.
+ *
+ * Results are placed at their original index so order is preserved regardless
+ * of completion order. Below the concurrency limit tasks start immediately;
+ * above it they queue in FIFO order.
+ *
+ * The bound is per call, so two pools running at once can together hold twice
+ * the limit open. Every caller today awaits its pool before the next phase
+ * begins, so the effective bound is the limit; a caller that starts a pool
+ * without awaiting it should pass a smaller `concurrency`.
+ *
+ * @template T, R
+ * @param {Array<T>} items
+ * @param {(item: T, index: number) => Promise<R>|R} mapper
+ * @param {number} [concurrency]
+ * @returns {Promise<Array<R>>}
+ */
+export async function mapWithConcurrency(items, mapper, concurrency) {
+  const limit = Math.max(1, concurrency || DEFAULT_HASH_CONCURRENCY);
+  const results = new Array(items.length);
+  let nextIndex = 0;
+
+  async function runner() {
+    while (true) {
+      const i = nextIndex;
+      nextIndex++;
+      if (i >= items.length) return;
+      results[i] = await mapper(items[i], i);
+    }
+  }
+
+  const runners = [];
+  for (let i = 0; i < Math.min(limit, items.length); i++) {
+    runners.push(runner());
+  }
+  await Promise.all(runners);
+  return results;
+}
+
+/**
+ * Maximum number of worker threads the pool will create for a single call.
+ *
+ * @returns {number}
+ */
+export function getMaxWorkers() {
+  return Math.max(1, MAX_WORKERS);
+}
+
+/**
+ * The task count at or above which {@link mapInWorkers} dispatches to threads
+ * instead of running inline.
+ *
+ * @returns {number}
+ */
+export function getWorkerThreshold() {
+  return WORKER_THRESHOLD;
+}
+
+let _workerSupported = null;
+let lastWorkerCount = 0;
+
+/**
+ * Workers created by the most recent {@link mapInWorkers} call. Scoped to that
+ * one call rather than to the process, so a batch that ran inline reports 0
+ * rather than the widest earlier batch.
+ *
+ * @returns {number}
+ */
+export function lastWorkerPoolSize() {
+  return lastWorkerCount;
+}
+
+async function checkWorkerSupport() {
+  if (_workerSupported !== null) return _workerSupported;
+  if (_isDeno || _isBun) {
+    _workerSupported = false;
+    return false;
+  }
+  try {
+    await import("node:worker_threads");
+    _workerSupported = true;
+  } catch {
+    _workerSupported = false;
+  }
+  return _workerSupported;
+}
+
+/**
+ * A task failed inside a worker, as opposed to the worker itself failing. The
+ * two are handled differently: the first belongs to one task, the second means
+ * threads cannot be used at all.
+ */
+class TaskError extends Error {}
+
+/**
+ * Apply `inlineFn` to the tasks at the given indices on the calling thread.
+ *
+ * @template T, R
+ * @param {Array<T>} tasks
+ * @param {(task: T) => R} inlineFn
+ * @param {Array<R|null>} results Filled in place at each index.
+ * @param {Iterable<number>} indices
+ * @param {boolean} swallowErrors
+ * @param {R|null} errorResult
+ */
+function runInline(
+  tasks,
+  inlineFn,
+  results,
+  indices,
+  swallowErrors,
+  errorResult,
+) {
+  for (const i of indices) {
+    try {
+      results[i] = inlineFn(tasks[i]);
+    } catch (err) {
+      if (!swallowErrors) {
+        throw err;
+      }
+      results[i] = errorResult;
+    }
+  }
+}
+
+/**
+ * Run synchronous CPU-bound tasks across worker threads.
+ *
+ * Below the threshold (default 24), tasks run inline on the calling thread,
+ * because starting a worker costs more than a small batch saves. Above it,
+ * tasks are distributed to a pool of ephemeral workers that are terminated
+ * before the call returns, so a completed scan does not hold the event loop
+ * open.
+ *
+ * Results are always returned in the original task order.
+ *
+ * Two kinds of failure are distinguished. A task that fails inside the worker
+ * is reported by the worker as `{error}` and is treated as that one task
+ * failing: it yields `options.errorResult` when `options.swallowErrors` is set,
+ * and rejects otherwise. A failure of the thread machinery — `worker_threads`
+ * missing, the entry module refusing to load, a worker exiting mid-task — is
+ * not a property of any task, so the tasks that had not completed are run
+ * inline instead. That keeps the result independent of whether threads were
+ * usable, which matters because callers of this function treat an empty result
+ * as "nothing found" rather than as an error.
+ *
+ * @template T, R
+ * @param {URL|string} workerUrl - Resolved URL of the worker entry module
+ * @param {Array<T>} tasks - Structured-cloneable task descriptors
+ * @param {(task: T) => R} inlineFn - Called per task when running inline
+ * @param {Object} [options]
+ * @param {number} [options.threshold] - Minimum task count to use threads
+ * @param {number} [options.maxWorkers] - Cap on thread count
+ * @param {boolean} [options.swallowErrors] - Return errorResult instead of rejecting
+ * @param {R} [options.errorResult] - Value used when an error is swallowed
+ * @returns {Promise<Array<R|null>>}
+ */
+export async function mapInWorkers(workerUrl, tasks, inlineFn, options = {}) {
+  const threshold = options.threshold ?? WORKER_THRESHOLD;
+  const swallowErrors = Boolean(options.swallowErrors);
+  const errorResult = options.errorResult ?? null;
+  const results = new Array(tasks.length).fill(null);
+  const useWorkers = tasks.length >= threshold && (await checkWorkerSupport());
+
+  lastWorkerCount = 0;
+  if (!useWorkers) {
+    runInline(
+      tasks,
+      inlineFn,
+      results,
+      tasks.keys(),
+      swallowErrors,
+      errorResult,
+    );
+    return results;
+  }
+
+  const maxWorkers = Math.min(options.maxWorkers ?? MAX_WORKERS, tasks.length);
+  const done = new Set();
+  let nextTask = 0;
+  let workersUsable = true;
+
+  const { Worker } = await import("node:worker_threads");
+  const workers = [];
+
+  function runOnWorker(worker, task) {
+    return new Promise((resolve, reject) => {
+      const cleanup = () => {
+        worker.removeListener("message", onMessage);
+        worker.removeListener("error", onError);
+        worker.removeListener("exit", onExit);
+      };
+      const onMessage = (msg) => {
+        cleanup();
+        if (msg?.error) {
+          reject(new TaskError(msg.error));
+        } else {
+          resolve(msg?.result);
+        }
+      };
+      const onError = (err) => {
+        cleanup();
+        reject(err);
+      };
+      // A worker can end without ever emitting `error` — it can be terminated,
+      // call `process.exit`, or be killed for running out of memory. Without
+      // this the pending request would never settle and the scan would hang.
+      const onExit = (code) => {
+        cleanup();
+        reject(new Error(`Worker exited with code ${code} before replying`));
+      };
+      worker.on("message", onMessage);
+      worker.on("error", onError);
+      worker.on("exit", onExit);
+      worker.postMessage(task);
+    });
+  }
+
+  async function workerLoop() {
+    let worker;
+    try {
+      worker = new Worker(workerUrl);
+    } catch {
+      workersUsable = false;
+      return;
+    }
+    workers.push(worker);
+    lastWorkerCount = workers.length;
+    try {
+      while (workersUsable) {
+        const i = nextTask;
+        nextTask++;
+        if (i >= tasks.length) {
+          break;
+        }
+        try {
+          results[i] = await runOnWorker(worker, tasks[i]);
+          done.add(i);
+        } catch (err) {
+          if (err instanceof TaskError) {
+            if (!swallowErrors) {
+              throw err;
+            }
+            results[i] = errorResult;
+            done.add(i);
+            continue;
+          }
+          // The worker, not the task, failed. Stop the pool and let the
+          // caller finish the outstanding tasks inline.
+          workersUsable = false;
+          return;
+        }
+      }
+    } finally {
+      await worker.terminate().catch(() => {
+        /* already exited */
+      });
+    }
+  }
+
+  try {
+    const loops = [];
+    for (let i = 0; i < maxWorkers; i++) {
+      loops.push(workerLoop());
+    }
+    await Promise.all(loops);
+  } finally {
+    await Promise.allSettled(workers.map((w) => w.terminate()));
+  }
+
+  if (!workersUsable) {
+    const pending = [];
+    for (let i = 0; i < tasks.length; i++) {
+      if (!done.has(i)) {
+        pending.push(i);
+      }
+    }
+    runInline(tasks, inlineFn, results, pending, swallowErrors, errorResult);
+  }
+
+  return results;
+}

package/lib/core/paths.js

@@ -0,0 +1,70 @@
+/**
+ * Repo-root resolution, shared by every helper that loads a file from `data/`.
+ *
+ * This lives in its own leaf module — importing nothing from `utils.js` — because
+ * the value is consumed at *module-evaluation* time (`JSON.parse(readFileSync(
+ * join(dirNameStr, "data", …)))`). A module in a cycle with `utils.js` that reads
+ * `dirNameStr` while `utils.js` is still evaluating gets the binding in its
+ * temporal dead zone and throws `ReferenceError: Cannot access 'dirNameStr'
+ * before initialization`. Keeping the definition in a leaf guarantees it is
+ * initialized before any consumer runs.
+ *
+ * The `file://` normalization below is load-bearing, not defensive noise:
+ * `jsr.json` publishes `lib/cli/index.js` to JSR, so `import.meta.url` can be an
+ * `https://` URL, and `fileURLToPath` throws `ERR_INVALID_URL_SCHEME` on those.
+ */
+
+import { platform } from "node:os";
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+let url = import.meta?.url;
+if (url && !url.startsWith("file://")) {
+  url = new URL(`file://${import.meta.url}`).toString();
+}
+// TODO: verify if this is a good method (Prabhu)
+// this is due to dirNameStr being "cdxgen/lib/helpers" which causes errors
+/** Absolute path to the cdxgen repository root, resolved from the module URL. */
+export const dirNameStr = url
+  ? dirname(dirname(dirname(fileURLToPath(url))))
+  : __dirname;
+
+/** True when running on Windows. */
+export const isWin = platform() === "win32";
+/** True when running on macOS. */
+export const isMac = platform() === "darwin";
+
+/**
+ * Validate that a string is a Windows drive root such as `C:\`.
+ *
+ * @param {string} root Candidate drive root string.
+ * @returns {boolean} True when the string is a valid Windows drive root.
+ */
+export function isValidDriveRoot(root) {
+  if (root.length > 3) {
+    return false;
+  }
+  const driveLetter = root.charAt(0);
+  const colon = root.charAt(1);
+  const backslash = root.charAt(2);
+  const charCode = driveLetter.charCodeAt(0);
+  const isAsciiLetter =
+    (charCode >= 65 && charCode <= 90) || (charCode >= 97 && charCode <= 122);
+  if (!isAsciiLetter) {
+    return false;
+  }
+  if (colon.charCodeAt(0) !== 0x3a) {
+    return false;
+  }
+  return !(backslash && backslash.charCodeAt(0) !== 0x5c);
+}
+
+/**
+ * Convert a kebab-case string to camelCase.
+ *
+ * @param {string} str Kebab-case input string.
+ * @returns {string} camelCase representation of the input.
+ */
+export function toCamel(str) {
+  return str.replace(/-([a-z])/g, (_, g) => g.toUpperCase());
+}

package/lib/core/projectConfig.js

@@ -0,0 +1,122 @@
+import path from "node:path";
+
+import { toCamel } from "./paths.js";
+
+/**
+ * Config file names cdxgen reads from the directory it is invoked in.
+ */
+export const PROJECT_CONFIG_FILENAMES = [
+  ".cdxgenrc",
+  ".cdxgen.json",
+  ".cdxgen.yml",
+  ".cdxgen.yaml",
+];
+
+/**
+ * Options whose values name a file or directory cdxgen writes to or reads
+ * from. A config file may set these, but only to a location inside its own
+ * directory.
+ */
+const PATH_OPTIONS = new Set([
+  "output",
+  "evinse-output",
+  "proto-bin-file",
+  "deps-slices-file",
+  "usages-slices-file",
+  "data-flow-slices-file",
+  "reachables-slices-file",
+  "semantics-slices-file",
+  "openapi-spec-file",
+  "license-policy",
+  "bom-audit-rules-dir",
+]);
+
+/**
+ * Options that change where cdxgen sends data or how much of the host it
+ * touches. These stay usable, but never silently.
+ */
+const ANNOUNCED_OPTIONS = new Set([
+  "server-url",
+  "api-key",
+  "include-formulation",
+  "install-deps",
+  "deep",
+  "tea-publish",
+  "tea-fetch",
+  "tea-artifact-url",
+  "tea-token",
+]);
+
+/**
+ * Both spellings a config file may use for an option.
+ *
+ * @param {string} option kebab-case option name
+ * @returns {string[]} kebab-case and camelCase spellings
+ */
+function optionSpellings(option) {
+  const camel = toCamel(option);
+  return camel === option ? [option] : [option, camel];
+}
+
+/**
+ * Determine whether a config-supplied path stays inside the config's own
+ * directory.
+ *
+ * @param {string} configDir directory the config file was read from
+ * @param {unknown} value config-supplied path value
+ * @returns {boolean} true when the path resolves inside configDir
+ */
+function isContainedPath(configDir, value) {
+  if (typeof value !== "string" || !value.length) {
+    return false;
+  }
+  const relativePath = path.relative(
+    path.resolve(configDir),
+    path.resolve(configDir, value),
+  );
+  return (
+    relativePath === "" ||
+    (!relativePath.startsWith("..") && !path.isAbsolute(relativePath))
+  );
+}
+
+/**
+ * Apply the trust boundary between cdxgen and the directory it is scanning.
+ *
+ * A config file lives in the tree under analysis, so it is attacker-controlled
+ * whenever that tree is. Path options are therefore confined to the config's
+ * own directory - a scanned project may choose where in itself the BOM lands,
+ * but not write outside it - and options that redirect data off the host or
+ * widen what cdxgen executes are announced on the console.
+ *
+ * @param {Object} config parsed config file contents
+ * @param {string} configDir directory the config file was read from
+ * @returns {{config: Object, rejected: string[], announced: string[]}} sanitized config and what it did
+ */
+export function sanitizeProjectConfig(config, configDir) {
+  if (!config || typeof config !== "object") {
+    return { announced: [], config: {}, rejected: [] };
+  }
+  const sanitized = { ...config };
+  const rejected = [];
+  const announced = [];
+  for (const option of PATH_OPTIONS) {
+    for (const key of optionSpellings(option)) {
+      if (sanitized[key] === undefined) {
+        continue;
+      }
+      if (!isContainedPath(configDir, sanitized[key])) {
+        rejected.push(`${key}=${sanitized[key]}`);
+        delete sanitized[key];
+      }
+    }
+  }
+  for (const option of ANNOUNCED_OPTIONS) {
+    for (const key of optionSpellings(option)) {
+      if (sanitized[key] !== undefined) {
+        announced.push(key);
+      }
+    }
+  }
+  return { announced, config: sanitized, rejected };
+}