npxconfuse 1.0.0

package/src/sources/local.js

@@ -0,0 +1,117 @@
+import { readFile, stat } from "node:fs/promises";
+import { resolve, basename, extname } from "node:path";
+import { glob } from "glob";
+import logger from "../utils/logger.js";
+
+/**
+ * Scan a local filesystem path for scannable files.
+ *
+ * @param {string} targetPath - File or directory to scan
+ * @param {object} options
+ * @param {boolean} options.deep - Also scan JS bundles (slower)
+ * @returns {Promise<Array<{filepath: string, content: string, type: string}>>}
+ */
+export async function scanLocal(targetPath, options = {}) {
+  const absPath = resolve(targetPath);
+  const results = [];
+
+  // Check if it's a file or directory
+  let stats;
+  try {
+    stats = await stat(absPath);
+  } catch (err) {
+    throw new Error(`Cannot access "${absPath}": ${err.message}`);
+  }
+
+  if (stats.isFile()) {
+    const file = await readSingleFile(absPath);
+    if (file) results.push(file);
+    return results;
+  }
+
+  if (!stats.isDirectory()) {
+    throw new Error(`"${absPath}" is not a file or directory`);
+  }
+
+  logger.info(`Scanning directory: ${absPath}`);
+
+  // Define glob patterns
+  const ignorePatterns = [
+    "**/node_modules/**",
+    "**/.git/**",
+    "**/dist/**",
+    "**/build/**",
+    "**/.next/**",
+    "**/coverage/**",
+    "**/.nyc_output/**",
+  ];
+
+  const patterns = ["**/package.json"];
+
+  if (options.deep) {
+    patterns.push("**/*.js", "**/*.mjs", "**/*.cjs");
+    logger.info(
+      "Deep scan enabled — also scanning JS bundles (this may take a while)",
+    );
+  }
+
+  // Run glob
+  const files = await glob(patterns, {
+    cwd: absPath,
+    ignore: ignorePatterns,
+    nodir: true,
+    absolute: true,
+  });
+
+  logger.info(`Found ${files.length} files to analyze`);
+
+  // Read each file
+  for (const filepath of files) {
+    try {
+      const content = await readFile(filepath, "utf-8");
+      const type = classifyFile(filepath);
+      if (type) {
+        results.push({ filepath, content, type });
+      }
+    } catch (err) {
+      logger.warn(`Skipping unreadable file: ${filepath} (${err.message})`);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Read and classify a single file.
+ */
+async function readSingleFile(filepath) {
+  try {
+    const content = await readFile(filepath, "utf-8");
+    const type = classifyFile(filepath);
+    if (!type) {
+      logger.warn(`Unrecognized file type: ${filepath}`);
+      return null;
+    }
+    return { filepath, content, type };
+  } catch (err) {
+    logger.error(`Cannot read file: ${filepath} (${err.message})`);
+    return null;
+  }
+}
+
+/**
+ * Classify a file by its name/extension.
+ */
+function classifyFile(filepath) {
+  const name = basename(filepath).toLowerCase();
+  const ext = extname(filepath).toLowerCase();
+
+  if (name === "package.json") return "package-json";
+  if (name === "package-lock.json") return "package-json"; // may contain useful data
+
+  if ([".js", ".mjs", ".cjs"].includes(ext)) return "js-bundle";
+
+  return null;
+}
+
+export default scanLocal;

package/src/sources/web.js

@@ -0,0 +1,182 @@
+import { readFile } from 'node:fs/promises';
+import pLimit from 'p-limit';
+import { httpGet, httpGetText } from '../utils/http.js';
+import { WEB_PROBE_PATHS, SKIP_EXTENSIONS } from '../utils/constants.js';
+import logger from '../utils/logger.js';
+
+/**
+ * Scan web domains for exposed package manifests and JS bundles.
+ *
+ * @param {string} domainsFile - Path to file with one domain/URL per line
+ * @param {object} options
+ * @param {number} options.concurrency - Parallel domain scans (default 10)
+ * @param {number} options.timeout - HTTP timeout in ms
+ * @returns {Promise<Array<{filepath: string, content: string, type: string}>>}
+ */
+export async function scanWeb(domainsFile, options = {}) {
+  const content = await readFile(domainsFile, 'utf-8');
+  const domains = content
+    .split('\n')
+    .map(line => line.trim())
+    .filter(line => line && !line.startsWith('#'));
+
+  if (domains.length === 0) {
+    throw new Error(`No domains found in ${domainsFile}`);
+  }
+
+  logger.info(`Loaded ${domains.length} domains from ${domainsFile}`);
+
+  const concurrency = options.concurrency || 10;
+  const limit = pLimit(concurrency);
+  const results = [];
+  let processed = 0;
+
+  const tasks = domains.map(domain =>
+    limit(async () => {
+      try {
+        const domainResults = await scanDomain(normalizeDomain(domain), options);
+        results.push(...domainResults);
+      } catch (err) {
+        logger.debug(`Error scanning ${domain}: ${err.message}`);
+      }
+
+      processed++;
+      if (processed % 20 === 0 || processed === domains.length) {
+        logger.info(`Progress: ${processed}/${domains.length} domains scanned (${results.length} files found)`);
+      }
+    })
+  );
+
+  await Promise.all(tasks);
+  logger.success(`Web scan complete: ${results.length} files from ${domains.length} domains`);
+
+  return results;
+}
+
+/**
+ * Normalize a domain string to a full URL.
+ */
+function normalizeDomain(domain) {
+  if (domain.startsWith('http://') || domain.startsWith('https://')) {
+    return domain.replace(/\/+$/, ''); // strip trailing slashes
+  }
+  return `https://${domain}`;
+}
+
+/**
+ * Scan a single domain for exposed package files and JS bundles.
+ */
+async function scanDomain(baseUrl, options = {}) {
+  const results = [];
+
+  // ── 1. Probe known paths ──
+  for (const probePath of WEB_PROBE_PATHS) {
+    const url = `${baseUrl}${probePath}`;
+    try {
+      const response = await httpGet(url, {
+        timeout: options.timeout,
+        retries: 1,
+      });
+
+      if (response.status === 200 && response.data) {
+        const content = typeof response.data === 'string'
+          ? response.data
+          : JSON.stringify(response.data);
+
+        // Validate it looks like a real package.json (not an HTML error page)
+        if (probePath.includes('package.json') && isLikelyJson(content)) {
+          results.push({
+            filepath: url,
+            content,
+            type: 'package-json',
+          });
+          logger.debug(`Found exposed ${probePath} at ${baseUrl}`);
+        }
+      }
+    } catch (err) {
+      // Expected for most domains
+    }
+  }
+
+  // ── 2. Fetch root page and extract JS bundle URLs ──
+  try {
+    const rootResponse = await httpGetText(`${baseUrl}/`, {
+      timeout: options.timeout,
+      retries: 1,
+    });
+
+    if (rootResponse.status === 200 && typeof rootResponse.data === 'string') {
+      const scriptUrls = extractScriptUrls(rootResponse.data, baseUrl);
+      const maxBundles = 10;
+
+      for (const scriptUrl of scriptUrls.slice(0, maxBundles)) {
+        // Skip non-JS assets
+        const ext = scriptUrl.split('?')[0].split('#')[0].split('.').pop()?.toLowerCase();
+        if (SKIP_EXTENSIONS.has(`.${ext}`)) continue;
+
+        try {
+          const bundleResponse = await httpGetText(scriptUrl, {
+            timeout: options.timeout,
+            retries: 0,
+          });
+
+          if (bundleResponse.status === 200 && typeof bundleResponse.data === 'string') {
+            // Only process files that look like JS (not HTML error pages)
+            if (bundleResponse.data.length > 100 && !bundleResponse.data.startsWith('<!')) {
+              results.push({
+                filepath: scriptUrl,
+                content: bundleResponse.data,
+                type: 'js-bundle',
+              });
+            }
+          }
+        } catch {
+          // Skip failed bundle fetches
+        }
+      }
+    }
+  } catch {
+    // Root page fetch failed — that's fine
+  }
+
+  return results;
+}
+
+/**
+ * Extract <script src="..."> URLs from HTML.
+ */
+function extractScriptUrls(html, baseUrl) {
+  const urls = [];
+  const pattern = /<script[^>]+src=["']([^"']+)["']/gi;
+  let match;
+
+  while ((match = pattern.exec(html)) !== null) {
+    let src = match[1];
+
+    // Skip inline data URIs and blobs
+    if (src.startsWith('data:') || src.startsWith('blob:')) continue;
+
+    // Resolve relative URLs
+    if (src.startsWith('//')) {
+      src = `https:${src}`;
+    } else if (src.startsWith('/')) {
+      src = `${baseUrl}${src}`;
+    } else if (!src.startsWith('http')) {
+      src = `${baseUrl}/${src}`;
+    }
+
+    urls.push(src);
+  }
+
+  return urls;
+}
+
+/**
+ * Check if a string looks like JSON (not an HTML page).
+ */
+function isLikelyJson(content) {
+  const trimmed = content.trim();
+  return trimmed.startsWith('{') || trimmed.startsWith('[');
+}
+
+export default scanWeb;

package/src/utils/constants.js

@@ -0,0 +1,181 @@
+/**
+ * Constants used throughout npxconfuse
+ */
+
+// Node.js built-in modules to filter out during extraction
+export const NODE_BUILTINS = new Set([
+  "_http_agent",
+  "_http_client",
+  "_http_common",
+  "_http_incoming",
+  "_http_outgoing",
+  "_http_server",
+  "_stream_duplex",
+  "_stream_passthrough",
+  "_stream_readable",
+  "_stream_transform",
+  "_stream_wrap",
+  "_stream_writable",
+  "_tls_common",
+  "_tls_wrap",
+  "assert",
+  "assert/strict",
+  "async_hooks",
+  "buffer",
+  "child_process",
+  "cluster",
+  "console",
+  "constants",
+  "crypto",
+  "dgram",
+  "diagnostics_channel",
+  "dns",
+  "dns/promises",
+  "domain",
+  "events",
+  "fs",
+  "fs/promises",
+  "http",
+  "http2",
+  "https",
+  "inspector",
+  "inspector/promises",
+  "module",
+  "net",
+  "os",
+  "path",
+  "path/posix",
+  "path/win32",
+  "perf_hooks",
+  "process",
+  "punycode",
+  "querystring",
+  "readline",
+  "readline/promises",
+  "repl",
+  "stream",
+  "stream/consumers",
+  "stream/promises",
+  "stream/web",
+  "string_decoder",
+  "sys",
+  "timers",
+  "timers/promises",
+  "tls",
+  "trace_events",
+  "tty",
+  "url",
+  "util",
+  "util/types",
+  "v8",
+  "vm",
+  "wasi",
+  "worker_threads",
+  "zlib",
+  // Prefixed forms
+  "node:assert",
+  "node:buffer",
+  "node:child_process",
+  "node:cluster",
+  "node:console",
+  "node:constants",
+  "node:crypto",
+  "node:dgram",
+  "node:diagnostics_channel",
+  "node:dns",
+  "node:domain",
+  "node:events",
+  "node:fs",
+  "node:http",
+  "node:http2",
+  "node:https",
+  "node:inspector",
+  "node:module",
+  "node:net",
+  "node:os",
+  "node:path",
+  "node:perf_hooks",
+  "node:process",
+  "node:punycode",
+  "node:querystring",
+  "node:readline",
+  "node:repl",
+  "node:stream",
+  "node:string_decoder",
+  "node:sys",
+  "node:timers",
+  "node:tls",
+  "node:trace_events",
+  "node:tty",
+  "node:url",
+  "node:util",
+  "node:v8",
+  "node:vm",
+  "node:wasi",
+  "node:worker_threads",
+  "node:zlib",
+]);
+
+// Registry endpoints
+export const REGISTRIES = {
+  npm: {
+    registry: "https://registry.npmjs.org",
+    downloads: "https://api.npmjs.org/downloads/point/last-month",
+  },
+};
+
+// Default concurrency for parallel HTTP requests
+export const DEFAULT_CONCURRENCY = 20;
+
+// Default HTTP timeout in ms
+export const DEFAULT_TIMEOUT = 10000;
+
+// Severity levels
+export const SEVERITY = {
+  CRITICAL: "CRITICAL",
+  HIGH: "HIGH",
+  MEDIUM: "MEDIUM",
+  LOW: "LOW",
+  INFO: "INFO",
+};
+
+// Finding types
+export const FINDING_TYPE = {
+  NPX_CONFUSION: "npx-confusion",
+  DEPENDENCY_CONFUSION: "dependency-confusion",
+  BIN_MISMATCH: "bin-mismatch",
+  NAME_CLASH: "name-clash",
+};
+
+// Common patterns to skip in web scraping
+export const SKIP_EXTENSIONS = new Set([
+  ".png",
+  ".jpg",
+  ".jpeg",
+  ".gif",
+  ".svg",
+  ".ico",
+  ".woff",
+  ".woff2",
+  ".ttf",
+  ".eot",
+  ".mp4",
+  ".webm",
+  ".mp3",
+  ".pdf",
+  ".zip",
+  ".tar",
+]);
+
+// Paths to probe on web targets
+export const WEB_PROBE_PATHS = [
+  "/package.json",
+  "/package-lock.json",
+  "/npm-shrinkwrap.json",
+];
+
+// GitHub API defaults
+export const GITHUB_DEFAULTS = {
+  perPage: 100,
+  maxRepos: 1000,
+};

package/src/utils/http.js

@@ -0,0 +1,179 @@
+import { DEFAULT_TIMEOUT } from "./constants.js";
+import logger from "./logger.js";
+
+/**
+ * HTTP client wrapper with retry, backoff, and rate limiting.
+ * Uses Node.js built-in fetch (available in Node 18+).
+ */
+
+// Simple token-bucket rate limiter
+class RateLimiter {
+  constructor(maxTokens, refillRate) {
+    this.maxTokens = maxTokens;
+    this.tokens = maxTokens;
+    this.refillRate = refillRate; // tokens per second
+    this.lastRefill = Date.now();
+  }
+
+  async acquire() {
+    this._refill();
+    if (this.tokens > 0) {
+      this.tokens--;
+      return;
+    }
+    // Wait for a token
+    const waitMs = (1 / this.refillRate) * 1000;
+    await new Promise((resolve) => setTimeout(resolve, waitMs));
+    this._refill();
+    this.tokens--;
+  }
+
+  _refill() {
+    const now = Date.now();
+    const elapsed = (now - this.lastRefill) / 1000;
+    this.tokens = Math.min(
+      this.maxTokens,
+      this.tokens + elapsed * this.refillRate,
+    );
+    this.lastRefill = now;
+  }
+}
+
+// Per-host rate limiters
+const limiters = new Map();
+
+function getLimiter(hostname, rate = 30) {
+  if (!limiters.has(hostname)) {
+    limiters.set(hostname, new RateLimiter(rate, rate));
+  }
+  return limiters.get(hostname);
+}
+
+/**
+ * Fetch with retry and exponential backoff.
+ * @param {string} url
+ * @param {object} options
+ * @param {number} options.timeout - ms
+ * @param {number} options.retries - max retry count
+ * @param {number} options.rateLimit - requests per second per host
+ * @param {object} options.headers - additional headers
+ * @returns {Promise<{status: number, data: any, headers: Headers}>}
+ */
+export async function httpGet(url, options = {}) {
+  const {
+    timeout = DEFAULT_TIMEOUT,
+    retries = 3,
+    rateLimit = 30,
+    headers = {},
+  } = options;
+
+  const parsedUrl = new URL(url);
+  const limiter = getLimiter(parsedUrl.hostname, rateLimit);
+
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    await limiter.acquire();
+
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+
+    try {
+      logger.debug(`HTTP GET ${url} (attempt ${attempt + 1})`);
+
+      const response = await fetch(url, {
+        signal: controller.signal,
+        headers: {
+          "User-Agent": "npxconfuse/1.0 (security-scanner)",
+          Accept: "application/json",
+          ...headers,
+        },
+      });
+
+      clearTimeout(timer);
+
+      // Don't retry 4xx (except 429)
+      if (
+        response.status >= 400 &&
+        response.status < 500 &&
+        response.status !== 429
+      ) {
+        return {
+          status: response.status,
+          data: null,
+          headers: response.headers,
+        };
+      }
+
+      // Retry on 429 or 5xx
+      if (response.status === 429 || response.status >= 500) {
+        const retryAfter = response.headers.get("retry-after");
+        const waitMs = retryAfter
+          ? parseInt(retryAfter, 10) * 1000
+          : Math.min(1000 * Math.pow(2, attempt), 30000);
+
+        logger.debug(
+          `Rate limited or server error (${response.status}), waiting ${waitMs}ms...`,
+        );
+
+        if (attempt < retries) {
+          await new Promise((resolve) => setTimeout(resolve, waitMs));
+          continue;
+        }
+
+        return {
+          status: response.status,
+          data: null,
+          headers: response.headers,
+        };
+      }
+
+      // Parse JSON or text
+      const contentType = response.headers.get("content-type") || "";
+      let data;
+      if (contentType.includes("application/json")) {
+        data = await response.json();
+      } else {
+        data = await response.text();
+      }
+
+      return {
+        status: response.status,
+        data,
+        headers: response.headers,
+      };
+    } catch (err) {
+      clearTimeout(timer);
+
+      if (err.name === "AbortError") {
+        logger.debug(`Request timed out: ${url}`);
+      } else {
+        logger.debug(`Request failed: ${url} — ${err.message}`);
+      }
+
+      if (attempt < retries) {
+        const waitMs = Math.min(1000 * Math.pow(2, attempt), 15000);
+        await new Promise((resolve) => setTimeout(resolve, waitMs));
+        continue;
+      }
+
+      return {
+        status: 0,
+        data: null,
+        headers: null,
+        error: err.message,
+      };
+    }
+  }
+}
+
+/**
+ * Fetch with HTML/text response (for web scraping).
+ */
+export async function httpGetText(url, options = {}) {
+  return httpGet(url, {
+    ...options,
+    headers: {
+      Accept: "text/html, application/json, text/plain, */*",
+      ...options.headers,
+    },
+  });
+}