@westbayberry/dg 1.0.52 → 1.0.56

package/dist/src/npm/registry.js

@@ -0,0 +1,299 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchPackageMetadata = fetchPackageMetadata;
+exports.fetchAllPackageMetadata = fetchAllPackageMetadata;
+exports.clearDownloadCache = clearDownloadCache;
+exports.setDownloadCachePersistence = setDownloadCachePersistence;
+exports.fetchWeeklyDownloads = fetchWeeklyDownloads;
+exports.prefetchDownloadCounts = prefetchDownloadCounts;
+exports.prefetchScopedDownloadCounts = prefetchScopedDownloadCounts;
+exports.getTarballUrl = getTarballUrl;
+const logger_1 = require("../logger");
+const h2pool_1 = require("./h2pool");
+const FETCH_TIMEOUT_MS = 30000; // 30s timeout for npm registry calls
+const NPM_TOKEN = process.env.NPM_TOKEN;
+function npmDownloadHeaders() {
+    const headers = { "User-Agent": "dependency-guardian" };
+    if (NPM_TOKEN)
+        headers["Authorization"] = `Bearer ${NPM_TOKEN}`;
+    return headers;
+}
+function npmRegistryUrl(name) {
+    if (name.startsWith("@")) {
+        return `https://registry.npmjs.org/${name.replace("/", "%2F")}`;
+    }
+    return `https://registry.npmjs.org/${name}`;
+}
+async function fetchPackageMetadata(packageName) {
+    try {
+        const url = npmRegistryUrl(packageName);
+        const response = await fetch(url, {
+            headers: { "User-Agent": "dependency-guardian" },
+            signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+        });
+        if (!response.ok)
+            return null;
+        return (await response.json());
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        logger_1.logger.warning(`Failed to fetch metadata for ${packageName}: ${msg}`);
+        return null;
+    }
+}
+/**
+ * Fetch metadata for all packages in bulk over a single HTTP/2 connection.
+ * 152 requests multiplex over 1 TCP connection instead of 6 concurrent HTTP/1.1 sockets.
+ */
+async function fetchAllPackageMetadata(packageNames) {
+    const result = new Map();
+    if (packageNames.length === 0)
+        return result;
+    const urls = packageNames.map(npmRegistryUrl);
+    const responses = await h2pool_1.h2pool.requestAll(urls, {
+        headers: { "user-agent": "dependency-guardian" },
+        timeout: FETCH_TIMEOUT_MS,
+    });
+    for (let i = 0; i < packageNames.length; i++) {
+        try {
+            if (responses[i].status === 200) {
+                result.set(packageNames[i], JSON.parse(responses[i].body));
+            }
+            else {
+                result.set(packageNames[i], null);
+            }
+        }
+        catch {
+            result.set(packageNames[i], null);
+        }
+    }
+    return result;
+}
+const downloadCache = new Map();
+const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours (fresh)
+const STALE_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 7 days (stale fallback)
+const MAX_CACHE_SIZE = 10000;
+function clearDownloadCache() {
+    downloadCache.clear();
+}
+let persistence = null;
+function setDownloadCachePersistence(p) {
+    persistence = p;
+}
+function cacheSet(key, entry) {
+    if (downloadCache.size >= MAX_CACHE_SIZE) {
+        // Evict entries beyond stale TTL first
+        const now = Date.now();
+        for (const [k, v] of downloadCache) {
+            if (now - v.fetchedAt >= STALE_TTL_MS) {
+                downloadCache.delete(k);
+            }
+        }
+        // If still at capacity, evict oldest (Map preserves insertion order)
+        if (downloadCache.size >= MAX_CACHE_SIZE) {
+            const firstKey = downloadCache.keys().next().value;
+            if (firstKey !== undefined)
+                downloadCache.delete(firstKey);
+        }
+    }
+    downloadCache.set(key, entry);
+}
+async function fetchWeeklyDownloads(packageName) {
+    const cached = downloadCache.get(packageName);
+    const now = Date.now();
+    // Fresh cache hit — return immediately without API call
+    if (cached !== undefined && now - cached.fetchedAt < CACHE_TTL_MS) {
+        return cached.count;
+    }
+    // Stale entry may exist — will be used as fallback on API failure
+    const staleValue = cached !== undefined && now - cached.fetchedAt < STALE_TTL_MS
+        ? cached.count
+        : null;
+    const encoded = packageName.startsWith("@")
+        ? packageName.replace("/", "%2F")
+        : packageName;
+    const url = `https://api.npmjs.org/downloads/point/last-week/${encoded}`;
+    for (let attempt = 0; attempt < 2; attempt++) {
+        try {
+            const response = await fetch(url, {
+                headers: npmDownloadHeaders(),
+                signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+            });
+            if (response.status === 429) {
+                // Rate limited — short backoff then retry once
+                await new Promise((r) => setTimeout(r, 2000));
+                continue;
+            }
+            if (!response.ok)
+                return staleValue;
+            const data = (await response.json());
+            const count = data.downloads ?? null;
+            if (count !== null) {
+                cacheSet(packageName, { count, fetchedAt: now });
+                // Write-through to persistent layer (fire-and-forget)
+                persistence?.setMany({ [packageName]: count }).catch(() => { });
+            }
+            return count;
+        }
+        catch {
+            if (attempt < 1) {
+                await new Promise((r) => setTimeout(r, 1000));
+                continue;
+            }
+            return staleValue;
+        }
+    }
+    return staleValue;
+}
+/**
+ * Bulk pre-fetch download counts for many packages at once.
+ * Uses npm's bulk API for unscoped packages (up to 128 per request)
+ * and falls back to individual fetches for scoped packages.
+ * Results are stored in the in-process cache so subsequent
+ * fetchWeeklyDownloads() calls get instant hits.
+ *
+ * On cold start (empty in-process cache), primes from the persistent
+ * layer (Redis) first — this avoids hammering the npm API after every
+ * process restart.
+ */
+async function prefetchDownloadCounts(packageNames) {
+    const now = Date.now();
+    // Step 1: Prime from persistent layer (Redis) on cold start.
+    // Any package not in the in-process cache but present in Redis gets
+    // loaded as a "stale" entry — good enough to prevent null-download
+    // false positives while the npm API catches up.
+    if (persistence) {
+        const coldMisses = packageNames.filter((name) => !downloadCache.has(name));
+        if (coldMisses.length > 0) {
+            try {
+                const persisted = await persistence.getMany(coldMisses);
+                for (const [name, count] of Object.entries(persisted)) {
+                    // Load as fetchedAt=0 so the fresh TTL is expired (triggers
+                    // npm API refresh) but the stale TTL still applies as fallback.
+                    if (!downloadCache.has(name)) {
+                        cacheSet(name, { count, fetchedAt: 0 });
+                    }
+                }
+            }
+            catch {
+                // Non-fatal — proceed without persistent cache
+            }
+        }
+    }
+    // Step 2: Identify packages needing a fresh npm API fetch
+    const needed = packageNames.filter((name) => {
+        const cached = downloadCache.get(name);
+        return !(cached && now - cached.fetchedAt < CACHE_TTL_MS);
+    });
+    if (needed.length === 0)
+        return;
+    // Split into scoped vs unscoped
+    const scoped = needed.filter((n) => n.startsWith("@"));
+    const unscoped = needed.filter((n) => !n.startsWith("@"));
+    // Bulk fetch unscoped in batches of 128
+    const BULK_SIZE = 128;
+    const bulkBatches = [];
+    for (let i = 0; i < unscoped.length; i += BULK_SIZE) {
+        bulkBatches.push(unscoped.slice(i, i + BULK_SIZE));
+    }
+    const writeThrough = {};
+    await Promise.all(bulkBatches.map(async (batch) => {
+        try {
+            const names = batch.join(",");
+            const url = `https://api.npmjs.org/downloads/point/last-week/${names}`;
+            const response = await fetch(url, {
+                headers: npmDownloadHeaders(),
+                signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+            });
+            if (response.ok) {
+                const data = (await response.json());
+                const fetchedAt = Date.now();
+                for (const [name, info] of Object.entries(data)) {
+                    if (info && typeof info.downloads === "number") {
+                        cacheSet(name, { count: info.downloads, fetchedAt });
+                        writeThrough[name] = info.downloads;
+                    }
+                }
+            }
+        }
+        catch {
+            // Non-fatal — individual fetchWeeklyDownloads will retry
+        }
+    }));
+    // Scoped packages are handled separately via prefetchScopedDownloadCounts()
+    // because they use the search API on registry.npmjs.org (same H2 connection
+    // as metadata) and must run AFTER metadata to avoid stream contention.
+    // Write-through to persistent layer for all newly-fetched counts.
+    // Scoped packages were already written individually via fetchWeeklyDownloads.
+    if (persistence && Object.keys(writeThrough).length > 0) {
+        persistence.setMany(writeThrough).catch(() => { });
+    }
+}
+/**
+ * Fetch download counts for scoped packages via npm search API.
+ * Uses registry.npmjs.org/-/v1/search (same H2 connection as metadata).
+ * Must be called AFTER metadata fetch completes to avoid H2 stream contention.
+ * Retries failed packages once after a short delay.
+ */
+async function prefetchScopedDownloadCounts(packageNames) {
+    const now = Date.now();
+    const scoped = packageNames
+        .filter((n) => n.startsWith("@"))
+        .filter((name) => {
+        const cached = downloadCache.get(name);
+        return !(cached && now - cached.fetchedAt < CACHE_TTL_MS);
+    });
+    if (scoped.length === 0)
+        return;
+    const writeThrough = {};
+    const fetchBatch = async (names) => {
+        const urls = names.map((name) => `https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(name)}&size=1`);
+        const responses = await h2pool_1.h2pool.requestAll(urls, {
+            headers: { "user-agent": "dependency-guardian" },
+            timeout: FETCH_TIMEOUT_MS,
+        });
+        const fetchedAt = Date.now();
+        const failed = [];
+        for (let i = 0; i < names.length; i++) {
+            try {
+                if (responses[i].status === 200 && responses[i].body) {
+                    const data = JSON.parse(responses[i].body);
+                    const match = data.objects?.find((o) => o.package.name === names[i]);
+                    if (match && typeof match.downloads?.weekly === "number") {
+                        cacheSet(names[i], { count: match.downloads.weekly, fetchedAt });
+                        writeThrough[names[i]] = match.downloads.weekly;
+                        continue;
+                    }
+                }
+                failed.push(names[i]);
+            }
+            catch {
+                failed.push(names[i]);
+            }
+        }
+        return failed;
+    };
+    // Fetch in batches of 30 via H2. Retry failures up to 3 times with delays —
+    // Cloudflare may throttle the search endpoint under concurrent load.
+    const SEARCH_BATCH = 30;
+    let remaining = scoped;
+    for (let attempt = 0; attempt < 3 && remaining.length > 0; attempt++) {
+        if (attempt > 0)
+            await new Promise((r) => setTimeout(r, 1000 * attempt));
+        const failed = [];
+        for (let i = 0; i < remaining.length; i += SEARCH_BATCH) {
+            if (i > 0)
+                await new Promise((r) => setTimeout(r, 100));
+            const batch = remaining.slice(i, i + SEARCH_BATCH);
+            const batchFailed = await fetchBatch(batch);
+            failed.push(...batchFailed);
+        }
+        remaining = failed;
+    }
+    if (persistence && Object.keys(writeThrough).length > 0) {
+        persistence.setMany(writeThrough).catch(() => { });
+    }
+}
+function getTarballUrl(meta, version) {
+    return meta.versions?.[version]?.dist?.tarball ?? null;
+}

package/dist/src/npm/tarball.js

@@ -0,0 +1,274 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.downloadAndExtract = downloadAndExtract;
+exports.cleanup = cleanup;
+const logger_1 = require("../logger");
+const crypto = __importStar(require("crypto"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
+const https = __importStar(require("https"));
+const tar = __importStar(require("tar"));
+// Size caps. Env-tunable so ops can raise/lower without a rebuild. Defaults
+// chosen to cover 99.5% of published packages (native-binary and ML-model
+// packages can blow past these — see MAX_TARBALL_MB=5000 to scan those too).
+// The streaming download and tar extraction both pipe entries through without
+// buffering the whole archive, so raising these does NOT linearly raise
+// memory — per-file caps bound RSS instead.
+const MAX_TARBALL_BYTES = (() => {
+    const mb = parseInt(process.env.MAX_TARBALL_MB || "2000", 10);
+    return (Number.isFinite(mb) ? mb : 2000) * 1024 * 1024;
+})(); // default 2 GB — catches kilocode/cli-* and friends, skips only truly pathological archives
+const MAX_EXTRACTED_BYTES = (() => {
+    const mb = parseInt(process.env.MAX_EXTRACTED_MB || "4000", 10);
+    return (Number.isFinite(mb) ? mb : 4000) * 1024 * 1024;
+})(); // 4 GB — zip-bomb ceiling, 2× the download cap
+const MAX_EXTRACTED_FILES = parseInt(process.env.MAX_EXTRACTED_FILES || "25000", 10);
+// npm packages should never contain these entry types — npm pack strips symlinks,
+// and device/FIFO entries have no legitimate use. Block them to prevent symlink-based
+// path escape attacks (defense-in-depth on top of tar@7's post-CVE mitigations).
+const BLOCKED_ENTRY_TYPES = new Set([
+    "SymbolicLink", "Link", "CharacterDevice", "BlockDevice", "FIFO",
+]);
+const MAX_REDIRECTS = 5;
+const ALLOWED_TARBALL_HOSTS = new Set([
+    "registry.npmjs.org",
+    "registry.yarnpkg.com",
+    "registry.npmmirror.com",
+]);
+// Shared agent reuses TCP+TLS connections across downloads (saves ~150ms per non-first request)
+const downloadAgent = new https.Agent({
+    keepAlive: true,
+    maxSockets: 100,
+    maxFreeSockets: 20,
+});
+// Persistent tarball cache — set TARBALL_CACHE_PATH to enable (e.g. /storage/dg-tarball-cache)
+const TARBALL_CACHE_DIR = process.env.TARBALL_CACHE_PATH || "";
+function tarballCacheKey(packageName, version) {
+    // safe filename: replace @ and / with double underscore
+    return `${packageName.replace(/[@/]/g, "__")}-${version}.tgz`;
+}
+function getCachedTarball(packageName, version) {
+    if (!TARBALL_CACHE_DIR)
+        return null;
+    const cached = path.join(TARBALL_CACHE_DIR, tarballCacheKey(packageName, version));
+    try {
+        if (fs.existsSync(cached))
+            return cached;
+    }
+    catch { /* ignore */ }
+    return null;
+}
+function saveTarballToCache(tarPath, packageName, version) {
+    if (!TARBALL_CACHE_DIR)
+        return;
+    try {
+        fs.mkdirSync(TARBALL_CACHE_DIR, { recursive: true });
+        const dest = path.join(TARBALL_CACHE_DIR, tarballCacheKey(packageName, version));
+        fs.copyFileSync(tarPath, dest);
+    }
+    catch { /* best effort */ }
+}
+async function downloadAndExtract(tarballUrl, packageName, version, expectedShasum) {
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "dg-"));
+    const tarPath = path.join(tmpDir, "pkg.tgz");
+    const extractDir = path.join(tmpDir, "extracted");
+    fs.mkdirSync(extractDir, { recursive: true });
+    try {
+        // Check persistent tarball cache first
+        const cached = getCachedTarball(packageName, version);
+        if (cached) {
+            fs.copyFileSync(cached, tarPath);
+        }
+        else {
+            await downloadToFile(tarballUrl, tarPath);
+            saveTarballToCache(tarPath, packageName, version);
+        }
+        if (expectedShasum) {
+            const hash = crypto.createHash("sha1");
+            const stream = fs.createReadStream(tarPath);
+            for await (const chunk of stream)
+                hash.update(chunk);
+            const actual = hash.digest("hex");
+            if (actual !== expectedShasum) {
+                logger_1.logger.warning(`Hash mismatch for ${packageName}@${version}: expected ${expectedShasum}, got ${actual}`);
+                cleanup(tmpDir);
+                return null;
+            }
+        }
+        const tarballSizeBytes = fs.statSync(tarPath).size;
+        let extractedBytes = 0;
+        let extractedFiles = 0;
+        let extractionAborted = false;
+        await tar.extract({
+            file: tarPath,
+            cwd: extractDir,
+            filter: (entryPath, entry) => {
+                if (extractionAborted)
+                    return false;
+                // Block symlink/hardlink/device/FIFO entries — these have no place in
+                // npm packages and can be used for path escape attacks.
+                if ("type" in entry && BLOCKED_ENTRY_TYPES.has(entry.type)) {
+                    const re = entry;
+                    logger_1.logger.warning(`Blocked ${re.type} entry in ${packageName}@${version}: ${entryPath}${re.linkpath ? ` -> ${re.linkpath}` : ""}`);
+                    return false;
+                }
+                if (entryPath.startsWith("/") || entryPath.includes("..")) {
+                    logger_1.logger.warning(`Blocked path traversal in ${packageName}@${version}: ${entryPath}`);
+                    return false;
+                }
+                extractedBytes += entry.size;
+                if (extractedBytes > MAX_EXTRACTED_BYTES) {
+                    extractionAborted = true;
+                    logger_1.logger.warning(`Extracted content for ${packageName}@${version} exceeds 200MB — aborting (zip bomb protection)`);
+                    return false;
+                }
+                extractedFiles++;
+                if (extractedFiles > MAX_EXTRACTED_FILES) {
+                    extractionAborted = true;
+                    logger_1.logger.warning(`File count for ${packageName}@${version} exceeds ${MAX_EXTRACTED_FILES} — aborting (many-file DoS protection)`);
+                    return false;
+                }
+                return true;
+            },
+        });
+        if (extractionAborted) {
+            cleanup(tmpDir);
+            return null;
+        }
+        const packageDir = path.join(extractDir, "package");
+        if (fs.existsSync(packageDir)) {
+            return { packageDir, rootTmpDir: tmpDir, tarballSizeBytes, tarballPath: tarPath };
+        }
+        return { packageDir: extractDir, rootTmpDir: tmpDir, tarballSizeBytes, tarballPath: tarPath };
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        logger_1.logger.warning(`Failed to download/extract ${packageName}@${version}: ${msg}`);
+        cleanup(tmpDir);
+        return null;
+    }
+}
+function validateTarballUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        throw new Error(`Invalid tarball URL: ${url}`);
+    }
+    if (parsed.protocol !== "https:") {
+        throw new Error(`Tarball URL must use HTTPS: ${url}`);
+    }
+    if (!ALLOWED_TARBALL_HOSTS.has(parsed.hostname)) {
+        throw new Error(`Tarball host not allowed: ${parsed.hostname}`);
+    }
+}
+function downloadToFile(url, destPath, redirectsRemaining = MAX_REDIRECTS) {
+    return new Promise((resolve, reject) => {
+        try {
+            validateTarballUrl(url);
+        }
+        catch (err) {
+            reject(err);
+            return;
+        }
+        https
+            .get(url, { agent: downloadAgent }, (response) => {
+            if ((response.statusCode === 301 || response.statusCode === 302) &&
+                response.headers.location) {
+                if (redirectsRemaining <= 0) {
+                    reject(new Error(`Too many redirects for tarball download`));
+                    return;
+                }
+                response.resume(); // consume redirect body to free socket
+                downloadToFile(response.headers.location, destPath, redirectsRemaining - 1)
+                    .then(resolve)
+                    .catch(reject);
+                return;
+            }
+            if (response.statusCode !== 200) {
+                reject(new Error(`HTTP ${response.statusCode} for ${url}`));
+                return;
+            }
+            // Defense-in-depth: enforce size limit during streaming, not just after download
+            let bytesReceived = 0;
+            let aborted = false;
+            const file = fs.createWriteStream(destPath);
+            response.on("data", (chunk) => {
+                bytesReceived += chunk.length;
+                if (!aborted && bytesReceived > MAX_TARBALL_BYTES) {
+                    aborted = true;
+                    response.destroy();
+                    file.destroy();
+                    try {
+                        fs.unlinkSync(destPath);
+                    }
+                    catch { }
+                    reject(new Error(`Tarball download exceeds ${MAX_TARBALL_BYTES / 1024 / 1024}MB limit during streaming — aborting`));
+                }
+            });
+            response.pipe(file);
+            file.on("finish", () => {
+                if (!aborted) {
+                    file.close();
+                    resolve();
+                }
+            });
+            file.on("error", (err) => {
+                if (!aborted) {
+                    try {
+                        fs.unlinkSync(destPath);
+                    }
+                    catch { }
+                    reject(err);
+                }
+            });
+        })
+            .on("error", reject);
+    });
+}
+function cleanup(dir) {
+    // Synchronous delete — blocks briefly but guarantees the temp dir is removed.
+    // The previous async fire-and-forget caused a leak: 4,022 orphaned /tmp/dg-*
+    // dirs (44GB) because processes restarted before fs.rm completed.
+    try {
+        fs.rmSync(dir, { recursive: true, force: true });
+    }
+    catch {
+        // Non-fatal — prefer moving forward over blocking on cleanup errors
+    }
+}