muaddib-scanner 2.11.64 → 2.11.66

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.64",
+  "version": "2.11.66",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.64.json → self-scan-v2.11.66.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-06T11:45:08.682Z",
+  "timestamp": "2026-06-06T20:23:04.305Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/ioc/updater.js

@@ -123,6 +123,11 @@ async function updateIOCs() {
   console.log('[4/4] Saved to cache: ' + CACHE_IOC_FILE);
   console.log('\n[OK] IOCs updated: ' + totalNpm + ' npm + ' + totalPyPI + ' PyPI packages');
 
+  // Fresh IOC files written — drop the in-process singleton so the next
+  // loadCachedIOCs() rebuilds from them (cross-process monitors pick the change
+  // up via the mtime/size source signature within SOURCE_CHECK_INTERVAL).
+  invalidateCache();
+
   return { total: totalNpm, totalPyPI: totalPyPI };
 }
 
@@ -202,16 +207,41 @@ function mergeIOCs(target, source) {
   return added;
 }
 
-// Cache to avoid reloading IOCs on each call
+// IOC store cache. The optimized store is large (~240K entries → hundreds of MB),
+// so it MUST be a stable singleton: rebuilding it duplicates that memory, and any
+// in-flight async scan (sandbox/deferred/network) that captured a prior copy pins
+// it — a periodic rebuild therefore accumulates copies. This was the monitor's
+// old_space → OOM leak: a heap snapshot showed 7+ live copies of the 421K-entry
+// Map retained via loadCachedIOCs closures + suspended Generators/Promises.
+// Fix: rebuild ONLY when a source file actually changes (mtime/size signature) or
+// on invalidateCache(); otherwise return the same object. The signature is
+// re-checked at most every SOURCE_CHECK_INTERVAL so the hot path (called per
+// scan/poll) does zero disk I/O.
+const IOCS_DIR = path.join(__dirname, '..', '..', 'iocs');
+const IOC_SOURCE_FILES = [
+  CACHE_IOC_FILE, LOCAL_IOC_FILE, LOCAL_COMPACT_FILE,
+  path.join(IOCS_DIR, 'packages.yaml'), path.join(IOCS_DIR, 'builtin.yaml'),
+  path.join(IOCS_DIR, 'hashes.yaml'), path.join(IOCS_DIR, 'string-iocs.yaml')
+];
+function iocSourcesSignature() {
+  let sig = '';
+  for (const f of IOC_SOURCE_FILES) { try { const s = fs.statSync(f); sig += s.mtimeMs + ':' + s.size + ';'; } catch { sig += '0;'; } }
+  return sig;
+}
+
 let cachedIOCsResult = null;
-let cachedIOCsTime = 0;
-const CACHE_TTL = 10000; // 10 seconds
+let cachedIOCsSig = null;
+let lastSourceCheck = 0;
+const SOURCE_CHECK_INTERVAL = 10000; // re-stat source files at most every 10s
 
 function loadCachedIOCs() {
-  // Return cache if still valid
   const now = Date.now();
-  if (cachedIOCsResult && (now - cachedIOCsTime) < CACHE_TTL) {
-    return cachedIOCsResult;
+  if (cachedIOCsResult) {
+    // Hot path: within the check window, return the singleton with no disk I/O.
+    if (now - lastSourceCheck < SOURCE_CHECK_INTERVAL) return cachedIOCsResult;
+    lastSourceCheck = now;
+    // Throttled freshness check: keep the singleton unless a source file changed.
+    if (iocSourcesSignature() === cachedIOCsSig) return cachedIOCsResult;
   }
 
   // Priority 1: YAML IOCs
@@ -279,9 +309,11 @@ function loadCachedIOCs() {
   // Create optimized structures for O(1) lookup
   const optimized = createOptimizedIOCs(merged);
 
-  // Store in cache
+  // Store as the shared singleton; record the source signature so we only rebuild
+  // when the IOC files actually change (see loadCachedIOCs header).
   cachedIOCsResult = optimized;
-  cachedIOCsTime = now;
+  cachedIOCsSig = iocSourcesSignature();
+  lastSourceCheck = now;
 
   return optimized;
 }
@@ -560,7 +592,8 @@ function expandCompactIOCs(compact) {
 
 function invalidateCache() {
   cachedIOCsResult = null;
-  cachedIOCsTime = 0;
+  cachedIOCsSig = null;
+  lastSourceCheck = 0;
 }
 
 /**

package/src/scanner/temporal-analysis.js

@@ -13,6 +13,54 @@ const _inflightRequests = new Map(); // packageName → Promise
 const METADATA_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
 const NEGATIVE_CACHE_TTL = 60 * 1000; // 60 seconds for failed fetches
 const METADATA_CACHE_MAX = 200;
+// Heap-leak fix: how many of the newest versions keep their FULL body in the cached
+// packument. Consumers (lifecycle/ast diff, maintainer-change) only diff the latest 2.
+const META_KEEP_VERSIONS = Math.max(2, parseInt(process.env.MUADDIB_META_KEEP_VERSIONS, 10) || 3);
+
+/**
+ * Shrink a full npm packument to what _metadataCache consumers actually need, so the
+ * cache never retains tens-of-MB packuments (packages with thousands of versions) —
+ * the root cause of the monitor's old_space leak → OOM restarts.
+ *
+ * Kept: every root field (small), the FULL `time` map (publish timeline — required by
+ * getLatestVersions + publish-anomaly), root `dist-tags`/`maintainers`, and the FULL
+ * bodies of the newest META_KEEP_VERSIONS versions (+ dist-tags.latest). Older versions
+ * are replaced by a truthy placeholder (1) so existence checks
+ * (`if (!versions[v]) continue`) and totalVersions counts stay correct without the bulk.
+ * The big optional blobs (`readme`, `_attachments`) are dropped.
+ * @param {object} parsed - full registry packument
+ * @returns {object} slimmed packument (safe drop-in for all current consumers)
+ */
+function projectPackument(parsed) {
+  if (!parsed || typeof parsed !== 'object' || !parsed.versions || typeof parsed.versions !== 'object') {
+    return parsed;
+  }
+  const versions = parsed.versions;
+  const time = (parsed.time && typeof parsed.time === 'object') ? parsed.time : {};
+
+  // Newest META_KEEP_VERSIONS versions by publish date (same ordering as getLatestVersions).
+  const dated = [];
+  for (const [v, t] of Object.entries(time)) {
+    if (v === 'created' || v === 'modified') continue;
+    if (!versions[v]) continue;
+    dated.push([v, t]);
+  }
+  dated.sort((a, b) => new Date(b[1]) - new Date(a[1]));
+  const keep = new Set(dated.slice(0, META_KEEP_VERSIONS).map(e => e[0]));
+  const distTags = parsed['dist-tags'];
+  if (distTags && distTags.latest && versions[distTags.latest]) keep.add(distTags.latest);
+
+  const slimVersions = {};
+  for (const v of Object.keys(versions)) {
+    slimVersions[v] = keep.has(v) ? versions[v] : 1; // truthy placeholder for old versions
+  }
+
+  const slim = { ...parsed, versions: slimVersions };
+  delete slim.readme;
+  delete slim.readmeFilename;
+  delete slim._attachments;
+  return slim;
+}
 
 const LIFECYCLE_SCRIPTS = [
   'preinstall',
@@ -99,14 +147,19 @@ function _fetchPackageMetadataHttp(packageName) {
         if (destroyed) return;
         try {
           const parsed = JSON.parse(data);
+          // Heap-leak fix: project to essentials BEFORE caching. A full packument can be
+          // tens of MB (packages with thousands of versions); retaining it whole bloated
+          // old_space → OOM restarts. Resolve the slim copy too so the full `parsed` is
+          // freed immediately (consumers only need time + the latest few version bodies).
+          const slim = projectPackument(parsed);
           // Store in cache on successful fetch
           if (_metadataCache.size >= METADATA_CACHE_MAX) {
             // Evict oldest entry
             const oldestKey = _metadataCache.keys().next().value;
             _metadataCache.delete(oldestKey);
           }
-          _metadataCache.set(packageName, { data: parsed, fetchedAt: Date.now() });
-          resolve(parsed);
+          _metadataCache.set(packageName, { data: slim, fetchedAt: Date.now() });
+          resolve(slim);
         } catch (e) {
           reject(new Error(`Invalid JSON from registry for ${packageName}: ${e.message}`));
         }
@@ -333,6 +386,7 @@ async function detectSuddenLifecycleChange(packageName) {
 module.exports = {
   fetchPackageMetadata,
   clearMetadataCache,
+  projectPackument,
   getLifecycleScripts,
   compareLifecycleScripts,
   getLatestVersions,