muaddib-scanner 2.10.66 → 2.10.67

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.66",
+  "version": "2.10.67",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/monitor/auto-labeler.js

@@ -18,7 +18,6 @@ const fs = require('fs');
 const path = require('path');
 const https = require('https');
 const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
-const { atomicWriteFileSync } = require('./state.js');
 
 const DEFAULT_INPUT = path.join(__dirname, '..', '..', 'data', 'ml-training.jsonl');
 const DEFAULT_OUTPUT = path.join(__dirname, '..', '..', 'data', 'ml-training-relabeled.jsonl');
@@ -191,52 +190,42 @@ async function relabelDataset(options = {}) {
   const dryRun = options.dryRun || false;
   const delayMs = options.delayMs != null ? options.delayMs : DEFAULT_DELAY_MS;
 
-  // 1. Read records
+  // 1. Build package map from input (records freed after block scope)
   if (!fs.existsSync(inputPath)) {
     throw new Error(`Input file not found: ${inputPath}`);
   }
-  const content = fs.readFileSync(inputPath, 'utf8');
-  const lines = content.split('\n');
-  const records = [];
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i].trim();
-    if (!line) continue;
-    try {
-      records.push({ idx: i, data: JSON.parse(line), raw: lines[i] });
-    } catch {
-      records.push({ idx: i, data: null, raw: lines[i] });
-    }
-  }
-
-  // 2. Extract unique packages eligible for relabeling
-  const packageMap = new Map(); // key → { name, ecosystem, score, timestamp, indices[] }
-  for (const rec of records) {
-    if (!rec.data) continue;
-    if (!RELABELABLE.has(rec.data.label)) continue;
-    const key = `${rec.data.ecosystem || 'npm'}/${rec.data.name}`;
-    if (!packageMap.has(key)) {
-      packageMap.set(key, {
-        name: rec.data.name,
-        ecosystem: rec.data.ecosystem || 'npm',
-        score: rec.data.score || 0,
-        timestamp: rec.data.timestamp,
-        indices: []
-      });
-    }
-    packageMap.get(key).indices.push(rec.idx);
-    // Use highest score seen for this package
-    if ((rec.data.score || 0) > packageMap.get(key).score) {
-      packageMap.get(key).score = rec.data.score;
-    }
-    // Use earliest timestamp
-    if (rec.data.timestamp && (!packageMap.get(key).timestamp || rec.data.timestamp < packageMap.get(key).timestamp)) {
-      packageMap.get(key).timestamp = rec.data.timestamp;
+  let recordCount = 0;
+  const packageMap = new Map(); // key → { name, ecosystem, score, timestamp }
+  {
+    const content = fs.readFileSync(inputPath, 'utf8');
+    const lines = content.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i].trim();
+      if (!line) continue;
+      recordCount++;
+      let data;
+      try { data = JSON.parse(line); } catch { continue; }
+      if (!RELABELABLE.has(data.label)) continue;
+      const key = `${data.ecosystem || 'npm'}/${data.name}`;
+      if (!packageMap.has(key)) {
+        packageMap.set(key, {
+          name: data.name,
+          ecosystem: data.ecosystem || 'npm',
+          score: data.score || 0,
+          timestamp: data.timestamp
+        });
+      }
+      const pkg = packageMap.get(key);
+      if ((data.score || 0) > pkg.score) pkg.score = data.score;
+      if (data.timestamp && (!pkg.timestamp || data.timestamp < pkg.timestamp)) {
+        pkg.timestamp = data.timestamp;
+      }
     }
-  }
+  } // content, lines — eligible for GC before registry checks
 
-  console.log(`[RELABEL] ${records.length} records, ${packageMap.size} unique packages to check`);
+  console.log(`[RELABEL] ${recordCount} records, ${packageMap.size} unique packages to check`);
 
-  // 3. Check each package against registry
+  // 2. Check each package against registry (crash-resilient)
   const summary = {
     checked: 0,
     relabeled_malicious: 0,
@@ -247,94 +236,115 @@ async function relabelDataset(options = {}) {
     errors: 0,
     records_updated: 0
   };
-
   const labelChanges = new Map(); // packageKey → { label, source }
+  let registryError = null;
 
-  const total = packageMap.size;
-  for (const [key, pkg] of packageMap) {
-    const t0 = Date.now();
-    let registryStatus;
-    try {
-      if (pkg.ecosystem === 'npm') {
-        registryStatus = await checkNpmStatus(pkg.name, { skipSemaphore: true });
-      } else if (pkg.ecosystem === 'pypi') {
-        registryStatus = await checkPyPIStatus(pkg.name);
-      } else {
-        summary.unchanged++;
+  try {
+    const total = packageMap.size;
+    for (const [key, pkg] of packageMap) {
+      const t0 = Date.now();
+      let registryStatus;
+      try {
+        if (pkg.ecosystem === 'npm') {
+          registryStatus = await checkNpmStatus(pkg.name, { skipSemaphore: true });
+        } else if (pkg.ecosystem === 'pypi') {
+          registryStatus = await checkPyPIStatus(pkg.name);
+        } else {
+          summary.unchanged++;
+          summary.checked++;
+          continue;
+        }
+      } catch (err) {
+        summary.errors++;
         summary.checked++;
+        console.log(`[RELABEL] ${key} → error (${Date.now() - t0}ms): ${err.message}`);
         continue;
       }
-    } catch (err) {
-      summary.errors++;
-      summary.checked++;
-      console.log(`[RELABEL] ${key} → error (${Date.now() - t0}ms): ${err.message}`);
-      continue;
-    }
 
-    if (registryStatus.status === 'error') {
-      summary.errors++;
-      summary.checked++;
-      console.log(`[RELABEL] ${key} → error (${Date.now() - t0}ms): ${registryStatus.detail}`);
-      if (delayMs > 0) await sleep(delayMs);
-      continue;
-    }
+      if (registryStatus.status === 'error') {
+        summary.errors++;
+        summary.checked++;
+        console.log(`[RELABEL] ${key} → error (${Date.now() - t0}ms): ${registryStatus.detail}`);
+        if (delayMs > 0) await sleep(delayMs);
+        continue;
+      }
 
-    const newLabel = computeNewLabel(pkg, registryStatus);
-    summary.checked++;
-    console.log(`[RELABEL] ${key} → ${newLabel ? newLabel.label : 'unchanged'} (${registryStatus.status}, ${Date.now() - t0}ms)`);
+      const newLabel = computeNewLabel(pkg, registryStatus);
+      summary.checked++;
+      console.log(`[RELABEL] ${key} → ${newLabel ? newLabel.label : 'unchanged'} (${registryStatus.status}, ${Date.now() - t0}ms)`);
 
-    if (summary.checked % 100 === 0) {
-      console.log(`[RELABEL] Progress: ${summary.checked}/${total} checked`);
-    }
+      if (summary.checked % 100 === 0) {
+        console.log(`[RELABEL] Progress: ${summary.checked}/${total} checked`);
+      }
 
-    if (newLabel) {
-      labelChanges.set(key, newLabel);
-      if (newLabel.label === 'confirmed_malicious') summary.relabeled_malicious++;
-      else if (newLabel.label === 'confirmed_benign') summary.relabeled_benign++;
-      else if (newLabel.label === 'likely_benign') summary.relabeled_likely_benign++;
-      else if (newLabel.label === 'removed_unlabeled') summary.removed_unlabeled++;
+      if (newLabel) {
+        labelChanges.set(key, newLabel);
+        if (newLabel.label === 'confirmed_malicious') summary.relabeled_malicious++;
+        else if (newLabel.label === 'confirmed_benign') summary.relabeled_benign++;
+        else if (newLabel.label === 'likely_benign') summary.relabeled_likely_benign++;
+        else if (newLabel.label === 'removed_unlabeled') summary.removed_unlabeled++;
 
-      if (dryRun) {
-        console.log(`[RELABEL] DRY-RUN: ${key} → ${newLabel.label} (${newLabel.source}, score=${pkg.score}, status=${registryStatus.status})`);
+        if (dryRun) {
+          console.log(`[RELABEL] DRY-RUN: ${key} → ${newLabel.label} (${newLabel.source}, score=${pkg.score}, status=${registryStatus.status})`);
+        }
+      } else {
+        summary.unchanged++;
       }
-    } else {
-      summary.unchanged++;
-    }
 
-    if (delayMs > 0) await sleep(delayMs);
-  }
-
-  // 4. Apply label changes to records
-  const outputLines = [];
-  for (const rec of records) {
-    if (!rec.data) {
-      outputLines.push(rec.raw);
-      continue;
-    }
-    const key = `${rec.data.ecosystem || 'npm'}/${rec.data.name}`;
-    const change = labelChanges.get(key);
-    if (change && RELABELABLE.has(rec.data.label)) {
-      rec.data.label = change.label;
-      rec.data.relabel_source = change.source;
-      rec.data.relabel_timestamp = new Date().toISOString();
-      outputLines.push(JSON.stringify(rec.data));
-      summary.records_updated++;
-    } else {
-      outputLines.push(rec.raw);
+      if (delayMs > 0) await sleep(delayMs);
     }
+  } catch (err) {
+    registryError = err;
+    console.error(`[RELABEL] Registry check interrupted at ${summary.checked}/${packageMap.size}: ${err.message}`);
   }
 
-  // 5. Write output
+  // 3. Stream output: re-read input, apply collected labelChanges, write line by line
   if (!dryRun) {
     const dir = path.dirname(outputPath);
     if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-    atomicWriteFileSync(outputPath, outputLines.join('\n'));
-    console.log(`[RELABEL] Written ${outputLines.length} records to ${path.basename(outputPath)} (${summary.records_updated} updated)`);
+    const tmpPath = outputPath + '.tmp';
+    const ws = fs.createWriteStream(tmpPath);
+    const relabelTs = new Date().toISOString();
+    const inputContent = fs.readFileSync(inputPath, 'utf8');
+    const inputLines = inputContent.split('\n');
+    let linesWritten = 0;
+    for (let i = 0; i < inputLines.length; i++) {
+      const raw = inputLines[i];
+      const trimmed = raw.trim();
+      let outputLine = raw;
+      if (trimmed) {
+        try {
+          const data = JSON.parse(trimmed);
+          const key = `${data.ecosystem || 'npm'}/${data.name}`;
+          const change = labelChanges.get(key);
+          if (change && RELABELABLE.has(data.label)) {
+            data.label = change.label;
+            data.relabel_source = change.source;
+            data.relabel_timestamp = relabelTs;
+            outputLine = JSON.stringify(data);
+            summary.records_updated++;
+          }
+        } catch { /* unparseable line — keep as-is */ }
+        linesWritten++;
+      }
+      ws.write(outputLine);
+      if (i < inputLines.length - 1) ws.write('\n');
+    }
+    await new Promise((resolve, reject) => {
+      ws.on('finish', resolve);
+      ws.on('error', reject);
+      ws.end();
+    });
+    fs.renameSync(tmpPath, outputPath);
+    console.log(`[RELABEL] Written ${linesWritten} records to ${path.basename(outputPath)} (${summary.records_updated} updated${registryError ? ', PARTIAL' : ''})`);
   } else {
-    console.log(`[RELABEL] DRY-RUN complete: ${summary.records_updated} records would be updated`);
+    console.log(`[RELABEL] DRY-RUN complete: ${summary.records_updated} records would be updated${registryError ? ' (PARTIAL)' : ''}`);
   }
 
   console.log(`[RELABEL] Summary: ${summary.relabeled_malicious} malicious, ${summary.relabeled_benign} benign, ${summary.relabeled_likely_benign} likely_benign, ${summary.removed_unlabeled} removed_unlabeled, ${summary.unchanged} unchanged, ${summary.errors} errors`);
+  if (registryError) {
+    console.error(`[RELABEL] WARNING: Partial results — registry check failed after ${summary.checked}/${packageMap.size} packages`);
+  }
 
   return summary;
 }