muaddib-scanner 2.10.35 → 2.10.36

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.35",
+  "version": "2.10.36",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/scripts/archive-cleanup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Supprime les archives de plus de 30 jours
+ARCHIVE_DIR="/opt/muaddib/archive"
+find "$ARCHIVE_DIR" -type d -name "20*" -mtime +30 -exec rm -rf {} + 2>/dev/null
+# Log
+TOTAL=$(du -sh "$ARCHIVE_DIR" 2>/dev/null | cut -f1)
+echo "[Archive Cleanup] $(date -Iseconds) — Total size: $TOTAL"

package/scripts/audit-archive.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Usage: ./audit-archive.sh [YYYY-MM-DD] [priority]
+# Exemples:
+#   ./audit-archive.sh                    → résumé de toutes les dates
+#   ./audit-archive.sh 2026-03-29         → liste les packages archivés ce jour
+#   ./audit-archive.sh 2026-03-29 P1      → filtre par priorité
+
+ARCHIVE_DIR="/opt/muaddib/archive"
+DATE=$1
+PRIORITY=$2
+
+if [ -z "$DATE" ]; then
+  echo "=== Archive Summary ==="
+  for dir in "$ARCHIVE_DIR"/20*; do
+    [ -d "$dir" ] || continue
+    day=$(basename "$dir")
+    count=$(ls "$dir"/*.tgz 2>/dev/null | wc -l)
+    size=$(du -sh "$dir" 2>/dev/null | cut -f1)
+    echo "$day : $count packages ($size)"
+  done
+  echo "---"
+  echo "Total: $(du -sh "$ARCHIVE_DIR" 2>/dev/null | cut -f1)"
+  exit 0
+fi
+
+DIR="$ARCHIVE_DIR/$DATE"
+if [ ! -d "$DIR" ]; then
+  echo "No archive for $DATE"
+  exit 1
+fi
+
+for json in "$DIR"/*.json; do
+  [ -f "$json" ] || continue
+  pkg=$(jq -r '.package' "$json")
+  ver=$(jq -r '.version' "$json")
+  prio=$(jq -r '.priority' "$json")
+  score=$(jq -r '.score' "$json")
+  llm=$(jq -r '.llm_verdict // "none"' "$json")
+
+  if [ -n "$PRIORITY" ] && [ "$prio" != "$PRIORITY" ]; then
+    continue
+  fi
+
+  printf "%-40s %-8s score=%-4s llm=%s\n" "$pkg@$ver" "$prio" "$score" "$llm"
+done

package/src/ml/llm-detective.js

@@ -240,6 +240,20 @@ Step 5 — COHERENCE: Does the complexity match the purpose?
 - Obfuscated code in a 10-line utility? → SUSPICIOUS
 - Minified dist/ in a large framework? → NORMAL
 
+Step 6 — FALSE POSITIVE CHECKS: Before declaring "malicious", verify the finding isn't one of these known benign patterns:
+
+6a. PHANTOM LIFECYCLE SCRIPTS: package.json declares preinstall/postinstall pointing to a script file, BUT the "files" field in package.json EXCLUDES that directory from the published tarball. The script does NOT exist in the package. npm silently ignores missing lifecycle scripts. → BENIGN (packaging oversight, not a threat).
+
+6b. ANONYMOUS INSTALL TELEMETRY: postinstall sends a single HTTP POST with ONLY: package version, process.platform, process.arch, process.version (Node), timestamp. NO process.env access, NO os.hostname(), NO os.userInfo(), NO file system reads, NO PII. This is standard practice (Next.js, Turborepo, Astro do the same). → BENIGN.
+
+6c. BINARY WRAPPER FROM GITHUB RELEASES: postinstall downloads a platform-specific binary from GitHub Releases (github.com/OWNER/REPO/releases/download/TAG/ASSET), verifies SHA256 checksum, and installs it. This is standard for native tool distribution (esbuild, turbo, biome, swc). → BENIGN if: (a) URL is GitHub Releases, (b) checksum is verified, (c) no fallback to a suspicious third-party domain.
+
+6d. APPLICATION CODE WITH HTTP CLIENTS: Source code contains fetch(), HttpClient, http.get() etc. but in application context (Angular services, React components, Express handlers). URLs are configurable templates (e.g. \`\${this.apiEndpoint}/...\`, \`\${process.env.NEXT_PUBLIC_API_URL}/...\`), not hardcoded suspicious domains. No lifecycle scripts performing network calls. → BENIGN — application code, not exfiltration.
+
+6e. CLI TOOLS USING EXEC/SPAWN FOR LOCAL OPERATIONS: Code uses child_process.execSync() or spawn() for legitimate local operations: git commands, npm/npx invocations, platform tools (screencapture, xdotool), CLI binary discovery. No encoded payloads, no eval(), no Buffer.from(base64) for code execution. → BENIGN — verify WHAT is being executed, not just the presence of execSync.
+
+6f. NPM PLACEHOLDER DEPENDENCIES: Package depends on "http@0.0.1-security" or similar npm placeholder packages. These are official npm placeholder modules (empty packages reserving names to prevent typosquatting). They are NOT suspicious HTTP modules. → BENIGN — ignore these dependencies entirely.
+
 ## GOLDEN RULE
 
 If sensitive data (env vars, credentials, keys) is only READ for self-configuration and never SENT to an external third-party, the package is BENIGN regardless of what the scanner says.
@@ -264,6 +278,22 @@ EXAMPLE 4 — FALSE POSITIVE:
 A web framework reads process.env.DATABASE_URL, process.env.API_KEY for configuration. It uses fetch() to call its own documented API endpoint. It uses dynamic require() to load user-configured plugins. Scanner flags env_access, dynamic_require, network_require — but all these are standard framework patterns. No data leaves the application boundary.
 → Verdict: BENIGN (confidence 0.95)
 
+EXAMPLE 5 — FALSE POSITIVE (phantom lifecycle script):
+Package "instructify@1.0.0" declares "postinstall": "node ./scripts/postinstall.js". But its "files" field is ["dist", ".cursor", "docs/README.md", "README.md", "LICENSE", "CHANGELOG.md", "CONTRIBUTING.md"]. The scripts/ directory does NOT exist in the published tarball because the "files" field excludes it. The postinstall script cannot execute — it is a packaging oversight. The GitHub repository shows the script only prints a welcome message.
+→ Verdict: BENIGN (confidence 0.95)
+
+EXAMPLE 6 — FALSE POSITIVE (anonymous telemetry):
+Package "delimit-cli@3.14.46" has a postinstall that prints CLI setup instructions, then sends anonymous telemetry: POST to delimit.ai/api/telemetry with body {event:'install', version, node:process.version, platform:process.platform, arch:process.arch, ts:ISO}. Silent fail on error, 3s timeout. No PII, no process.env access beyond process.version/platform/arch, no os.hostname(), no file reads. This is standard anonymous install telemetry identical to what Next.js, Turborepo, and Astro do.
+→ Verdict: BENIGN (confidence 0.92)
+
+EXAMPLE 7 — FALSE POSITIVE (binary wrapper with checksum):
+Package "plugin-kit-ai@1.0.1" has a postinstall that downloads a platform-specific binary from GitHub Releases (github.com/777genius/plugin-kit-ai/releases/download/vX.Y.Z/ASSET), verifies SHA256 checksum from checksums.txt, and extracts the binary to vendor/. No data exfiltration, no env access beyond optional GITHUB_TOKEN for rate limits. This is the standard binary distribution pattern used by esbuild, turbo, and biome.
+→ Verdict: BENIGN (confidence 0.95)
+
+EXAMPLE 8 — FALSE POSITIVE (application code with HTTP clients):
+Package "@craft-ng/core@0.1.2" is an Angular state management library. No lifecycle scripts (no postinstall/preinstall). Source contains fetch() and http references but ONLY in JSDoc examples ("const response = await fetch(\`/api/users/\${params}\`)") and Angular service patterns (this.httpClient.get(url)). These are application code patterns, not active network calls during install. No child_process, no eval, no Buffer manipulation.
+→ Verdict: BENIGN (confidence 0.95)
+
 ## KEY QUESTIONS TO ANSWER
 
 1. "Do sensitive data (env vars, credentials) LEAVE the package to a third party?"

package/src/monitor/queue.js

@@ -99,6 +99,9 @@ const {
 // From ./ingestion.js (will be created — currently in monitor.js)
 const { getNpmLatestTarball, getPyPITarballUrl, getWeeklyDownloads } = require('./ingestion.js');
 
+// From ./tarball-archive.js
+const { archiveSuspectTarball } = require('./tarball-archive.js');
+
 // --- Constants ---
 
 const SCAN_CONCURRENCY = Math.max(1, parseInt(process.env.MUADDIB_SCAN_CONCURRENCY, 10) || 5);
@@ -541,6 +544,16 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
 
         stats.suspect++;
 
+        // Fire-and-forget tarball archiving — never blocks the pipeline
+        archiveSuspectTarball(name, version, tarballUrl, {
+          score: riskScore,
+          priority: tierLabel,
+          rulesTriggered: (result.threats || []).map(t => t.ruleId || t.type).filter(Boolean),
+          llmVerdict: null // LLM runs after this point; updated by webhook if needed
+        }).catch(err => {
+          console.warn(`[Archive] Failed for ${name}@${version}: ${err.message}`);
+        });
+
         // Sandbox decision based on tier
         // T1a: mandatory sandbox (HC malice types, TIER1_TYPES non-LOW, lifecycle + intent compound)
         // T1b: conditional sandbox (HIGH/CRITICAL without HC type — bundler FP zone)

package/src/monitor/tarball-archive.js

@@ -0,0 +1,120 @@
+'use strict';
+
+/**
+ * Tarball archiving for suspect packages.
+ *
+ * Downloads and stores tarballs + metadata JSON for packages flagged as suspect,
+ * enabling retrospective audit when npm/PyPI unpublish the package.
+ *
+ * Fire-and-forget: never blocks the scan pipeline.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
+const { downloadToFile } = require('../shared/download.js');
+
+// Archive root — configurable via env for testing
+const ARCHIVE_DIR = process.env.MUADDIB_ARCHIVE_DIR || '/opt/muaddib/archive';
+const ARCHIVE_TIMEOUT_MS = 10_000;
+
+/**
+ * Get the date string in YYYY-MM-DD format (Paris timezone, consistent with monitor).
+ * Falls back to UTC if Intl is unavailable.
+ */
+function getArchiveDateString() {
+  try {
+    const now = new Date();
+    const parts = new Intl.DateTimeFormat('fr-CA', { timeZone: 'Europe/Paris', year: 'numeric', month: '2-digit', day: '2-digit' }).formatToParts(now);
+    const y = parts.find(p => p.type === 'year').value;
+    const m = parts.find(p => p.type === 'month').value;
+    const d = parts.find(p => p.type === 'day').value;
+    return `${y}-${m}-${d}`;
+  } catch {
+    return new Date().toISOString().slice(0, 10);
+  }
+}
+
+/**
+ * Sanitize package name for use in filenames.
+ * Replaces / (scoped packages) with __ and removes unsafe characters.
+ */
+function sanitizeForFilename(name) {
+  return name.replace(/^@/, '').replace(/\//g, '__').replace(/[^a-zA-Z0-9._-]/g, '_');
+}
+
+/**
+ * Compute SHA-256 hash of a file.
+ */
+function sha256File(filePath) {
+  const hash = crypto.createHash('sha256');
+  const data = fs.readFileSync(filePath);
+  hash.update(data);
+  return hash.digest('hex');
+}
+
+/**
+ * Archive a suspect package tarball and its scan metadata.
+ *
+ * @param {string} packageName - Package name (e.g. "evil-pkg" or "@scope/evil-pkg")
+ * @param {string} version - Package version
+ * @param {string} tarballUrl - Registry URL to download the tarball from
+ * @param {object} scanResult - Scan result object from the pipeline
+ * @param {number} scanResult.score - Risk score
+ * @param {string} scanResult.priority - Priority tier (e.g. "P1", "P2")
+ * @param {Array} [scanResult.rulesTriggered] - Array of triggered rule IDs
+ * @param {string} [scanResult.llmVerdict] - LLM detective verdict if available
+ * @returns {Promise<boolean>} true if archived, false if skipped/failed
+ */
+async function archiveSuspectTarball(packageName, version, tarballUrl, scanResult) {
+  if (!tarballUrl || !packageName || !version) return false;
+
+  const dateStr = getArchiveDateString();
+  const dayDir = path.join(ARCHIVE_DIR, dateStr);
+  const safeName = sanitizeForFilename(packageName);
+  const basename = `${safeName}-${version}`;
+  const tgzPath = path.join(dayDir, `${basename}.tgz`);
+  const jsonPath = path.join(dayDir, `${basename}.json`);
+
+  // Dedup: skip if already archived
+  if (fs.existsSync(tgzPath)) {
+    return false;
+  }
+
+  // Ensure day directory exists
+  fs.mkdirSync(dayDir, { recursive: true });
+
+  // Download with semaphore (shares concurrency with rest of pipeline)
+  await acquireRegistrySlot();
+  try {
+    await downloadToFile(tarballUrl, tgzPath, ARCHIVE_TIMEOUT_MS);
+  } finally {
+    releaseRegistrySlot();
+  }
+
+  // Compute hash and write metadata
+  const tarballSha256 = sha256File(tgzPath);
+  const metadata = {
+    package: packageName,
+    version,
+    timestamp: new Date().toISOString(),
+    score: scanResult.score || 0,
+    priority: scanResult.priority || null,
+    rules_triggered: scanResult.rulesTriggered || [],
+    llm_verdict: scanResult.llmVerdict || null,
+    tarball_sha256: tarballSha256
+  };
+
+  fs.writeFileSync(jsonPath, JSON.stringify(metadata, null, 2));
+  return true;
+}
+
+module.exports = {
+  archiveSuspectTarball,
+  ARCHIVE_DIR,
+  // Exported for testing
+  sanitizeForFilename,
+  sha256File,
+  getArchiveDateString
+};