@cdxgen/cdxgen 0.0.1 → 13.0.1

package/lib/core/logger.js

@@ -0,0 +1,280 @@
+import { Console } from "node:console";
+import fs from "node:fs";
+import process from "node:process";
+import { Writable } from "node:stream";
+import { isMainThread } from "node:worker_threads";
+
+// Environment variables here are read with raw `process.env` rather than
+// `readEnvironmentVariable`: that helper lives in activity.js, which imports
+// this module, so routing these through it would form an import cycle.
+
+// Stream contract: human-readable diagnostics default to stderr so stdout
+// carries only the BOM payload. CDXGEN_LOG_STREAM=stdout restores the
+// pre-v13 behaviour for callers that scraped cdxgen's stdout.
+const logStream =
+  process.env.CDXGEN_LOG_STREAM === "stdout" ? process.stdout : process.stderr;
+
+/**
+ * The stream every human-readable diagnostic is written to.
+ *
+ * Exported so that the live region and the table renderer resolve the same
+ * destination as the loggers do. A caller that sets `CDXGEN_LOG_STREAM=stdout`
+ * gets the pre-v13 behaviour for all of it, not just for the thought and trace
+ * logs.
+ *
+ * @type {NodeJS.WritableStream}
+ */
+export const diagnosticStream = logStream;
+
+// Custom lightweight colors formatting helper replacing yoctocolors
+const colors = {
+  cyanBright: (text) => `\x1b[96m${text}\x1b[39m`,
+  dim: (text) => `\x1b[2m${text}\x1b[22m`,
+};
+
+/** True when thought/reasoning logging is enabled (CDXGEN_THOUGHT_LOG, CDXGEN_THINK_MODE, or verbose debug mode). */
+export const THINK_MODE =
+  process.env.CDXGEN_THOUGHT_LOG ||
+  ["true", "1"].includes(process.env.CDXGEN_THINK_MODE) ||
+  process.env.CDXGEN_DEBUG_MODE === "verbose";
+
+// Log files are written synchronously rather than through
+// `fs.createWriteStream`. A write stream buffers, so anything still queued when
+// the process exits is lost, leaving readers that tail these files a truncated
+// log with no closing `</think>` marker. Writing through one shared descriptor
+// per path also keeps the two Console channels from interleaving at
+// independent file offsets.
+const syncFileWriters = new Map();
+
+/**
+ * Open a synchronous writer for a log file path.
+ *
+ * `Console` requires a real stream, so this is a `Writable` whose `_write`
+ * completes synchronously: bytes reach the descriptor as each record is
+ * logged, and nothing is left queued for an event loop turn that a process
+ * exit may never take.
+ *
+ * @param {string} filePath Destination path
+ * @returns {Writable} Writer, or the diagnostic stream when the path is unwritable
+ */
+function openSyncFileWriter(filePath) {
+  const existing = syncFileWriters.get(filePath);
+  if (existing) {
+    return existing;
+  }
+  let fd;
+  try {
+    // Worker threads log to the same path through their own instance of this
+    // module, so the descriptor is opened in append mode: O_APPEND lands every
+    // write at the end of the file no matter how many writers share it, where
+    // a truncating open gives each writer its own offset and they punch holes
+    // through each other's records. The main thread truncates once so a run
+    // still starts from an empty log.
+    if (isMainThread) {
+      try {
+        fs.truncateSync(filePath, 0);
+      } catch {
+        // No file yet: the append open below creates it.
+      }
+    }
+    fd = fs.openSync(filePath, "a");
+  } catch {
+    // An unwritable path must not take the whole run down: fall back to the
+    // diagnostic stream so the records stay visible somewhere.
+    return logStream;
+  }
+  const writer = new Writable({
+    write(chunk, _encoding, callback) {
+      if (fd !== undefined) {
+        try {
+          fs.writeSync(fd, chunk);
+        } catch {
+          // A full disk or a revoked descriptor must not abort BOM generation.
+        }
+      }
+      callback();
+    },
+  });
+  writer.closeSync = () => {
+    if (fd === undefined) {
+      return;
+    }
+    try {
+      fs.closeSync(fd);
+    } finally {
+      fd = undefined;
+      syncFileWriters.delete(filePath);
+    }
+  };
+  syncFileWriters.set(filePath, writer);
+  return writer;
+}
+
+const thinkOutput = process.env.CDXGEN_THOUGHT_LOG
+  ? openSyncFileWriter(process.env.CDXGEN_THOUGHT_LOG)
+  : logStream;
+const thinkErrorOutput = process.env.CDXGEN_THOUGHT_LOG
+  ? thinkOutput
+  : process.stderr;
+const thinkLogger = new Console({
+  stdout: thinkOutput,
+  stderr: thinkErrorOutput,
+  colorMode: process.env.CDXGEN_THOUGHT_LOG ? false : "auto",
+});
+
+/** True when structured command/HTTP trace logging is enabled (CDXGEN_TRACE_LOG, CDXGEN_TRACE_ID, CDXGEN_TRACE_MODE, or verbose debug mode). */
+export const TRACE_MODE =
+  process.env.CDXGEN_TRACE_LOG ||
+  process.env.CDXGEN_TRACE_ID ||
+  ["true", "1"].includes(process.env.CDXGEN_TRACE_MODE) ||
+  process.env.CDXGEN_DEBUG_MODE === "verbose";
+
+const traceOutput = process.env.CDXGEN_TRACE_LOG
+  ? openSyncFileWriter(process.env.CDXGEN_TRACE_LOG)
+  : logStream;
+const traceErrorOutput = process.env.CDXGEN_TRACE_LOG
+  ? traceOutput
+  : process.stderr;
+const traceLogger = new Console({
+  stdout: traceOutput,
+  stderr: traceErrorOutput,
+  colorMode: process.env.CDXGEN_TRACE_LOG ? false : "auto",
+});
+
+if (THINK_MODE) {
+  thinkLogger.group(colorizeText("<think>"));
+}
+/**
+ * Logs a thought message to the think logger if THINK_MODE is enabled.
+ * Automatically appends a period to the message if it lacks terminal punctuation.
+ *
+ * @param {string} s The thought message to log
+ * @param {Object} [args] Optional additional arguments to log alongside the message
+ * @returns {void}
+ */
+export function thoughtLog(s, args) {
+  if (!THINK_MODE) {
+    return;
+  }
+  if (!s?.endsWith(".") && !s?.endsWith("?") && !s?.endsWith("!")) {
+    s = `${s}.`;
+  }
+  s = s.replaceAll("'.'", "'<project dir>'");
+  if (args) {
+    thinkLogger.log(colorizeText(`${s}`), args);
+  } else {
+    thinkLogger.log(colorizeText(`${s}`));
+  }
+}
+let thoughtBlockClosed = false;
+
+/**
+ * Closes the think log group by emitting the closing `</think>` marker.
+ * Has no effect if THINK_MODE is not enabled, and emits the marker at most
+ * once so an explicit call followed by the exit handler cannot produce two.
+ *
+ * @returns {void}
+ */
+export function thoughtEnd() {
+  if (THINK_MODE && !thoughtBlockClosed) {
+    thoughtBlockClosed = true;
+    thinkLogger.groupEnd();
+    thinkLogger.log(colorizeText("</think>"));
+  }
+}
+
+function colorizeText(s) {
+  // Only colorize for an interactive terminal stream; piped/CI/file output
+  // stays free of escape bytes so the diagnostic stream is grep-safe.
+  if (process.env.CDXGEN_THOUGHT_LOG || !logStream.isTTY) {
+    return s;
+  }
+  s = s.replace(/(\d+)/g, colors.cyanBright("$1"));
+  return colors.dim(s);
+}
+
+// Fields serialized for each trace type. The allow-list is per-type because a
+// single shared list either drops the fields a type exists to carry or leaks
+// unbounded caller objects into the stream: `activity` and `phase` records
+// carry no field in common with `spawn` and `http`.
+const TRACE_FIELDS = {
+  activity: [
+    "identifier",
+    "kind",
+    "status",
+    "target",
+    "reason",
+    "networkIntent",
+    "command",
+    "cwd",
+  ],
+  phase: ["phase", "state", "detail", "note", "done", "total", "elapsedMs"],
+};
+
+const DEFAULT_TRACE_FIELDS = [
+  "command",
+  "cwd",
+  "protocol",
+  "host",
+  "path",
+  "pathname",
+];
+
+/**
+ * Log trace messages
+ *
+ * @param {String} traceType Trace type
+ * @param {Object} args Additional arguments
+ */
+export function traceLog(traceType, args) {
+  if (!TRACE_MODE || !traceType || !args) {
+    return;
+  }
+  const traceId = process.env.CDXGEN_TRACE_ID;
+  const message = {};
+  if (traceId) {
+    message.traceId = traceId;
+  }
+  message["timestamp"] = new Date().toISOString();
+  if (traceType) {
+    message.type = traceType;
+  }
+  if (args) {
+    for (const k of TRACE_FIELDS[traceType] || DEFAULT_TRACE_FIELDS) {
+      // Numeric progress counters are meaningful at zero, so presence is
+      // tested rather than truthiness.
+      if (args[k] !== undefined && args[k] !== null && args[k] !== "") {
+        message[k] = args[k];
+      }
+    }
+  }
+  if (Object.keys(message).length) {
+    traceLogger.log(JSON.stringify(message, null, null));
+  }
+}
+
+let logStreamsClosed = false;
+
+/**
+ * Terminate the think block and close the log file descriptors.
+ *
+ * Readers tail these files as a live feed, so a run that ends without the
+ * closing `</think>` marker leaves the consumer holding an unterminated block.
+ * Registered on process exit and safe to call more than once.
+ *
+ * @returns {void}
+ */
+export function closeLogStreams() {
+  if (logStreamsClosed) {
+    return;
+  }
+  logStreamsClosed = true;
+  thoughtEnd();
+  for (const writer of [...syncFileWriters.values()]) {
+    writer.closeSync();
+  }
+}
+
+// The writers are synchronous, so this handler only has to emit the terminator
+// and release the descriptors — work that is legal during `exit`.
+process.on("exit", closeLogStreams);

package/lib/core/parallel.js

@@ -0,0 +1,323 @@
+/**
+ * Bounded concurrency utilities for CPU-bound and I/O-bound parallel work.
+ *
+ * Two primitives live here:
+ *
+ * - {@link mapWithConcurrency} — a promise pool for async I/O work (file
+ *   hashing, registry fetches). Tasks are async; results preserve input order.
+ *
+ * - {@link mapInWorkers} — a worker-thread pool for synchronous CPU work (AST
+ *   parsing). Below a configurable threshold tasks run inline; above it they
+ *   are distributed across `worker_threads` workers. Any failure of the thread
+ *   machinery itself — an unavailable `worker_threads`, a worker entry that
+ *   will not load, a worker that dies mid-task — falls back to running the
+ *   remaining tasks inline, so the result is the same whether or not threads
+ *   were usable.
+ */
+
+import { availableParallelism } from "node:os";
+
+import { readEnvironmentVariable } from "./activity.js";
+
+const _isDeno = globalThis.Deno?.version?.deno !== undefined;
+const _isBun = globalThis.Bun?.version !== undefined;
+
+const DEFAULT_HASH_CONCURRENCY = Math.min(
+  Math.max(availableParallelism(), 4),
+  16,
+);
+
+const WORKER_THRESHOLD = Number.parseInt(
+  readEnvironmentVariable("CDXGEN_WORKER_THRESHOLD") || "24",
+  10,
+);
+
+const MAX_WORKERS = Number.parseInt(
+  readEnvironmentVariable("CDXGEN_MAX_WORKERS") ||
+    String(Math.min(availableParallelism() || 4, 8)),
+  10,
+);
+
+/**
+ * Run an async mapper over an array with bounded concurrency.
+ *
+ * Results are placed at their original index so order is preserved regardless
+ * of completion order. Below the concurrency limit tasks start immediately;
+ * above it they queue in FIFO order.
+ *
+ * The bound is per call, so two pools running at once can together hold twice
+ * the limit open. Every caller today awaits its pool before the next phase
+ * begins, so the effective bound is the limit; a caller that starts a pool
+ * without awaiting it should pass a smaller `concurrency`.
+ *
+ * @template T, R
+ * @param {Array<T>} items
+ * @param {(item: T, index: number) => Promise<R>|R} mapper
+ * @param {number} [concurrency]
+ * @returns {Promise<Array<R>>}
+ */
+export async function mapWithConcurrency(items, mapper, concurrency) {
+  const limit = Math.max(1, concurrency || DEFAULT_HASH_CONCURRENCY);
+  const results = new Array(items.length);
+  let nextIndex = 0;
+
+  async function runner() {
+    while (true) {
+      const i = nextIndex;
+      nextIndex++;
+      if (i >= items.length) return;
+      results[i] = await mapper(items[i], i);
+    }
+  }
+
+  const runners = [];
+  for (let i = 0; i < Math.min(limit, items.length); i++) {
+    runners.push(runner());
+  }
+  await Promise.all(runners);
+  return results;
+}
+
+/**
+ * Maximum number of worker threads the pool will create for a single call.
+ *
+ * @returns {number}
+ */
+export function getMaxWorkers() {
+  return Math.max(1, MAX_WORKERS);
+}
+
+/**
+ * The task count at or above which {@link mapInWorkers} dispatches to threads
+ * instead of running inline.
+ *
+ * @returns {number}
+ */
+export function getWorkerThreshold() {
+  return WORKER_THRESHOLD;
+}
+
+let _workerSupported = null;
+let lastWorkerCount = 0;
+
+/**
+ * Workers created by the most recent {@link mapInWorkers} call. Scoped to that
+ * one call rather than to the process, so a batch that ran inline reports 0
+ * rather than the widest earlier batch.
+ *
+ * @returns {number}
+ */
+export function lastWorkerPoolSize() {
+  return lastWorkerCount;
+}
+
+async function checkWorkerSupport() {
+  if (_workerSupported !== null) return _workerSupported;
+  if (_isDeno || _isBun) {
+    _workerSupported = false;
+    return false;
+  }
+  try {
+    await import("node:worker_threads");
+    _workerSupported = true;
+  } catch {
+    _workerSupported = false;
+  }
+  return _workerSupported;
+}
+
+/**
+ * A task failed inside a worker, as opposed to the worker itself failing. The
+ * two are handled differently: the first belongs to one task, the second means
+ * threads cannot be used at all.
+ */
+class TaskError extends Error {}
+
+/**
+ * Apply `inlineFn` to the tasks at the given indices on the calling thread.
+ *
+ * @template T, R
+ * @param {Array<T>} tasks
+ * @param {(task: T) => R} inlineFn
+ * @param {Array<R|null>} results Filled in place at each index.
+ * @param {Iterable<number>} indices
+ * @param {boolean} swallowErrors
+ * @param {R|null} errorResult
+ */
+function runInline(
+  tasks,
+  inlineFn,
+  results,
+  indices,
+  swallowErrors,
+  errorResult,
+) {
+  for (const i of indices) {
+    try {
+      results[i] = inlineFn(tasks[i]);
+    } catch (err) {
+      if (!swallowErrors) {
+        throw err;
+      }
+      results[i] = errorResult;
+    }
+  }
+}
+
+/**
+ * Run synchronous CPU-bound tasks across worker threads.
+ *
+ * Below the threshold (default 24), tasks run inline on the calling thread,
+ * because starting a worker costs more than a small batch saves. Above it,
+ * tasks are distributed to a pool of ephemeral workers that are terminated
+ * before the call returns, so a completed scan does not hold the event loop
+ * open.
+ *
+ * Results are always returned in the original task order.
+ *
+ * Two kinds of failure are distinguished. A task that fails inside the worker
+ * is reported by the worker as `{error}` and is treated as that one task
+ * failing: it yields `options.errorResult` when `options.swallowErrors` is set,
+ * and rejects otherwise. A failure of the thread machinery — `worker_threads`
+ * missing, the entry module refusing to load, a worker exiting mid-task — is
+ * not a property of any task, so the tasks that had not completed are run
+ * inline instead. That keeps the result independent of whether threads were
+ * usable, which matters because callers of this function treat an empty result
+ * as "nothing found" rather than as an error.
+ *
+ * @template T, R
+ * @param {URL|string} workerUrl - Resolved URL of the worker entry module
+ * @param {Array<T>} tasks - Structured-cloneable task descriptors
+ * @param {(task: T) => R} inlineFn - Called per task when running inline
+ * @param {Object} [options]
+ * @param {number} [options.threshold] - Minimum task count to use threads
+ * @param {number} [options.maxWorkers] - Cap on thread count
+ * @param {boolean} [options.swallowErrors] - Return errorResult instead of rejecting
+ * @param {R} [options.errorResult] - Value used when an error is swallowed
+ * @returns {Promise<Array<R|null>>}
+ */
+export async function mapInWorkers(workerUrl, tasks, inlineFn, options = {}) {
+  const threshold = options.threshold ?? WORKER_THRESHOLD;
+  const swallowErrors = Boolean(options.swallowErrors);
+  const errorResult = options.errorResult ?? null;
+  const results = new Array(tasks.length).fill(null);
+  const useWorkers = tasks.length >= threshold && (await checkWorkerSupport());
+
+  lastWorkerCount = 0;
+  if (!useWorkers) {
+    runInline(
+      tasks,
+      inlineFn,
+      results,
+      tasks.keys(),
+      swallowErrors,
+      errorResult,
+    );
+    return results;
+  }
+
+  const maxWorkers = Math.min(options.maxWorkers ?? MAX_WORKERS, tasks.length);
+  const done = new Set();
+  let nextTask = 0;
+  let workersUsable = true;
+
+  const { Worker } = await import("node:worker_threads");
+  const workers = [];
+
+  function runOnWorker(worker, task) {
+    return new Promise((resolve, reject) => {
+      const cleanup = () => {
+        worker.removeListener("message", onMessage);
+        worker.removeListener("error", onError);
+        worker.removeListener("exit", onExit);
+      };
+      const onMessage = (msg) => {
+        cleanup();
+        if (msg?.error) {
+          reject(new TaskError(msg.error));
+        } else {
+          resolve(msg?.result);
+        }
+      };
+      const onError = (err) => {
+        cleanup();
+        reject(err);
+      };
+      // A worker can end without ever emitting `error` — it can be terminated,
+      // call `process.exit`, or be killed for running out of memory. Without
+      // this the pending request would never settle and the scan would hang.
+      const onExit = (code) => {
+        cleanup();
+        reject(new Error(`Worker exited with code ${code} before replying`));
+      };
+      worker.on("message", onMessage);
+      worker.on("error", onError);
+      worker.on("exit", onExit);
+      worker.postMessage(task);
+    });
+  }
+
+  async function workerLoop() {
+    let worker;
+    try {
+      worker = new Worker(workerUrl);
+    } catch {
+      workersUsable = false;
+      return;
+    }
+    workers.push(worker);
+    lastWorkerCount = workers.length;
+    try {
+      while (workersUsable) {
+        const i = nextTask;
+        nextTask++;
+        if (i >= tasks.length) {
+          break;
+        }
+        try {
+          results[i] = await runOnWorker(worker, tasks[i]);
+          done.add(i);
+        } catch (err) {
+          if (err instanceof TaskError) {
+            if (!swallowErrors) {
+              throw err;
+            }
+            results[i] = errorResult;
+            done.add(i);
+            continue;
+          }
+          // The worker, not the task, failed. Stop the pool and let the
+          // caller finish the outstanding tasks inline.
+          workersUsable = false;
+          return;
+        }
+      }
+    } finally {
+      await worker.terminate().catch(() => {
+        /* already exited */
+      });
+    }
+  }
+
+  try {
+    const loops = [];
+    for (let i = 0; i < maxWorkers; i++) {
+      loops.push(workerLoop());
+    }
+    await Promise.all(loops);
+  } finally {
+    await Promise.allSettled(workers.map((w) => w.terminate()));
+  }
+
+  if (!workersUsable) {
+    const pending = [];
+    for (let i = 0; i < tasks.length; i++) {
+      if (!done.has(i)) {
+        pending.push(i);
+      }
+    }
+    runInline(tasks, inlineFn, results, pending, swallowErrors, errorResult);
+  }
+
+  return results;
+}

package/lib/core/paths.js

@@ -0,0 +1,70 @@
+/**
+ * Repo-root resolution, shared by every helper that loads a file from `data/`.
+ *
+ * This lives in its own leaf module — importing nothing from `utils.js` — because
+ * the value is consumed at *module-evaluation* time (`JSON.parse(readFileSync(
+ * join(dirNameStr, "data", …)))`). A module in a cycle with `utils.js` that reads
+ * `dirNameStr` while `utils.js` is still evaluating gets the binding in its
+ * temporal dead zone and throws `ReferenceError: Cannot access 'dirNameStr'
+ * before initialization`. Keeping the definition in a leaf guarantees it is
+ * initialized before any consumer runs.
+ *
+ * The `file://` normalization below is load-bearing, not defensive noise:
+ * `jsr.json` publishes `lib/cli/index.js` to JSR, so `import.meta.url` can be an
+ * `https://` URL, and `fileURLToPath` throws `ERR_INVALID_URL_SCHEME` on those.
+ */
+
+import { platform } from "node:os";
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+let url = import.meta?.url;
+if (url && !url.startsWith("file://")) {
+  url = new URL(`file://${import.meta.url}`).toString();
+}
+// TODO: verify if this is a good method (Prabhu)
+// this is due to dirNameStr being "cdxgen/lib/helpers" which causes errors
+/** Absolute path to the cdxgen repository root, resolved from the module URL. */
+export const dirNameStr = url
+  ? dirname(dirname(dirname(fileURLToPath(url))))
+  : __dirname;
+
+/** True when running on Windows. */
+export const isWin = platform() === "win32";
+/** True when running on macOS. */
+export const isMac = platform() === "darwin";
+
+/**
+ * Validate that a string is a Windows drive root such as `C:\`.
+ *
+ * @param {string} root Candidate drive root string.
+ * @returns {boolean} True when the string is a valid Windows drive root.
+ */
+export function isValidDriveRoot(root) {
+  if (root.length > 3) {
+    return false;
+  }
+  const driveLetter = root.charAt(0);
+  const colon = root.charAt(1);
+  const backslash = root.charAt(2);
+  const charCode = driveLetter.charCodeAt(0);
+  const isAsciiLetter =
+    (charCode >= 65 && charCode <= 90) || (charCode >= 97 && charCode <= 122);
+  if (!isAsciiLetter) {
+    return false;
+  }
+  if (colon.charCodeAt(0) !== 0x3a) {
+    return false;
+  }
+  return !(backslash && backslash.charCodeAt(0) !== 0x5c);
+}
+
+/**
+ * Convert a kebab-case string to camelCase.
+ *
+ * @param {string} str Kebab-case input string.
+ * @returns {string} camelCase representation of the input.
+ */
+export function toCamel(str) {
+  return str.replace(/-([a-z])/g, (_, g) => g.toUpperCase());
+}