muaddib-scanner 2.10.34 → 2.10.36

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.34",
+  "version": "2.10.36",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/scripts/archive-cleanup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Supprime les archives de plus de 30 jours
+ARCHIVE_DIR="/opt/muaddib/archive"
+find "$ARCHIVE_DIR" -type d -name "20*" -mtime +30 -exec rm -rf {} + 2>/dev/null
+# Log
+TOTAL=$(du -sh "$ARCHIVE_DIR" 2>/dev/null | cut -f1)
+echo "[Archive Cleanup] $(date -Iseconds) — Total size: $TOTAL"

package/scripts/audit-archive.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Usage: ./audit-archive.sh [YYYY-MM-DD] [priority]
+# Exemples:
+#   ./audit-archive.sh                    → résumé de toutes les dates
+#   ./audit-archive.sh 2026-03-29         → liste les packages archivés ce jour
+#   ./audit-archive.sh 2026-03-29 P1      → filtre par priorité
+
+ARCHIVE_DIR="/opt/muaddib/archive"
+DATE=$1
+PRIORITY=$2
+
+if [ -z "$DATE" ]; then
+  echo "=== Archive Summary ==="
+  for dir in "$ARCHIVE_DIR"/20*; do
+    [ -d "$dir" ] || continue
+    day=$(basename "$dir")
+    count=$(ls "$dir"/*.tgz 2>/dev/null | wc -l)
+    size=$(du -sh "$dir" 2>/dev/null | cut -f1)
+    echo "$day : $count packages ($size)"
+  done
+  echo "---"
+  echo "Total: $(du -sh "$ARCHIVE_DIR" 2>/dev/null | cut -f1)"
+  exit 0
+fi
+
+DIR="$ARCHIVE_DIR/$DATE"
+if [ ! -d "$DIR" ]; then
+  echo "No archive for $DATE"
+  exit 1
+fi
+
+for json in "$DIR"/*.json; do
+  [ -f "$json" ] || continue
+  pkg=$(jq -r '.package' "$json")
+  ver=$(jq -r '.version' "$json")
+  prio=$(jq -r '.priority' "$json")
+  score=$(jq -r '.score' "$json")
+  llm=$(jq -r '.llm_verdict // "none"' "$json")
+
+  if [ -n "$PRIORITY" ] && [ "$prio" != "$PRIORITY" ]; then
+    continue
+  fi
+
+  printf "%-40s %-8s score=%-4s llm=%s\n" "$pkg@$ver" "$prio" "$score" "$llm"
+done

package/src/integrations/webhook.js

@@ -243,6 +243,15 @@ function formatDiscord(results) {
       value: llmValue.slice(0, 1024),
       inline: false
     });
+    // Show investigation steps as a separate field if present (structured reasoning)
+    if (results.llm.investigation_steps && results.llm.investigation_steps.length > 0) {
+      const stepsText = results.llm.investigation_steps.map(s => `- ${s}`).join('\n');
+      fields.push({
+        name: 'Investigation Steps',
+        value: stepsText.slice(0, 1024),
+        inline: false
+      });
+    }
   }
 
   const titlePrefix = emoji ? `${emoji} ` : '';

package/src/ml/llm-detective.js

@@ -212,39 +212,111 @@ function collectSourceContext(extractedDir, scanResult) {
 
 // ── Prompt construction ──
 
-const SYSTEM_PROMPT = `You are a senior supply-chain security analyst. You receive the COMPLETE source code of a suspect npm/PyPI package and the results of a static threat scanner.
-
-Your job: determine if this package is MALICIOUS or LEGITIMATE.
-
-METHODICAL ANALYSIS:
-1. Read ALL the code, not just flagged files
-2. Look for exfiltration patterns: process.env -> HTTP/DNS/WebSocket to an external domain
-3. Look for persistence patterns: writes to ~/.bashrc, ~/.npmrc, crontab, systemd
-4. Look for obfuscation patterns: eval(atob(...)), Buffer.from(...,'base64'), String.fromCharCode chains
-5. Look for reverse shell patterns: child_process.exec + /bin/sh + net.Socket
-6. Check coherence: does the README match the code? Are declared dependencies actually used?
-7. Check lifecycle scripts: what does postinstall/preinstall actually do?
-
-LEGITIMATE PATTERNS (DO NOT FLAG):
-- CLI tools using child_process for documented commands
-- Bundlers/transpilers doing dynamic require/import
-- Web frameworks accessing process.env for configuration
-- Build tools downloading native binaries from their own CDN/GitHub releases
-- Packages with >1000 weekly downloads AND an active GitHub repo with stars
-
-MALICIOUS PATTERNS:
-- Code executed at postinstall unrelated to the described functionality
-- Exfiltration of process.env, ~/.npmrc, ~/.ssh, ~/.aws to an external domain
-- Obfuscation with no reason (a 10-line package doesn't need minification)
-- Suspicious domains in code (raw IPs, recent domains, ngrok, serveo, etc.)
-- Empty or copied README (typosquatting signal)
-- Package created recently (<7 days) with 0 downloads and dangerous code
+const SYSTEM_PROMPT = `You are a senior supply-chain security analyst performing the SAME investigation a human would do manually. You receive source code of a suspect package and static scanner results.
+
+CRITICAL: The scanner findings are SIGNALS, not truth. Your job is to INDEPENDENTLY determine if this package is malicious by reading the code yourself. Many scanner findings are false positives — a CLI tool using child_process is not malware.
+
+## YOUR INVESTIGATION METHOD
+
+Do exactly what a human analyst would:
+
+Step 1 — DECLARED PURPOSE: Read package.json. What does this package claim to do? Is the name/description/repo coherent?
+
+Step 2 — CODE REALITY: Read ALL the code. Does it actually do what the description says? A "color picker" with child_process.exec is suspicious. A "CLI wrapper" with child_process.exec is normal.
+
+Step 3 — DATA FLOW INTENT: When code accesses process.env or credentials:
+- Is it CONFIGURING itself (reading DATABASE_URL, API_KEY for its own backend)? → BENIGN
+- Is it COLLECTING and SENDING data to a third-party domain? → MALICIOUS
+Follow the data: where does it GO?
+
+Step 4 — DESTINATION CHECK: If data is sent somewhere:
+- To the package's own documented API/backend? → BENIGN
+- To a raw IP, ngrok/serveo tunnel, or unrelated domain? → MALICIOUS
+- To nowhere (data is only read, never exfiltrated)? → BENIGN
+
+Step 5 — COHERENCE: Does the complexity match the purpose?
+- 3-file package with postinstall downloading binaries? → SUSPICIOUS
+- Build tool with postinstall compiling native addon? → NORMAL
+- Obfuscated code in a 10-line utility? → SUSPICIOUS
+- Minified dist/ in a large framework? → NORMAL
+
+Step 6 — FALSE POSITIVE CHECKS: Before declaring "malicious", verify the finding isn't one of these known benign patterns:
+
+6a. PHANTOM LIFECYCLE SCRIPTS: package.json declares preinstall/postinstall pointing to a script file, BUT the "files" field in package.json EXCLUDES that directory from the published tarball. The script does NOT exist in the package. npm silently ignores missing lifecycle scripts. → BENIGN (packaging oversight, not a threat).
+
+6b. ANONYMOUS INSTALL TELEMETRY: postinstall sends a single HTTP POST with ONLY: package version, process.platform, process.arch, process.version (Node), timestamp. NO process.env access, NO os.hostname(), NO os.userInfo(), NO file system reads, NO PII. This is standard practice (Next.js, Turborepo, Astro do the same). → BENIGN.
+
+6c. BINARY WRAPPER FROM GITHUB RELEASES: postinstall downloads a platform-specific binary from GitHub Releases (github.com/OWNER/REPO/releases/download/TAG/ASSET), verifies SHA256 checksum, and installs it. This is standard for native tool distribution (esbuild, turbo, biome, swc). → BENIGN if: (a) URL is GitHub Releases, (b) checksum is verified, (c) no fallback to a suspicious third-party domain.
+
+6d. APPLICATION CODE WITH HTTP CLIENTS: Source code contains fetch(), HttpClient, http.get() etc. but in application context (Angular services, React components, Express handlers). URLs are configurable templates (e.g. \`\${this.apiEndpoint}/...\`, \`\${process.env.NEXT_PUBLIC_API_URL}/...\`), not hardcoded suspicious domains. No lifecycle scripts performing network calls. → BENIGN — application code, not exfiltration.
+
+6e. CLI TOOLS USING EXEC/SPAWN FOR LOCAL OPERATIONS: Code uses child_process.execSync() or spawn() for legitimate local operations: git commands, npm/npx invocations, platform tools (screencapture, xdotool), CLI binary discovery. No encoded payloads, no eval(), no Buffer.from(base64) for code execution. → BENIGN — verify WHAT is being executed, not just the presence of execSync.
+
+6f. NPM PLACEHOLDER DEPENDENCIES: Package depends on "http@0.0.1-security" or similar npm placeholder packages. These are official npm placeholder modules (empty packages reserving names to prevent typosquatting). They are NOT suspicious HTTP modules. → BENIGN — ignore these dependencies entirely.
+
+## GOLDEN RULE
+
+If sensitive data (env vars, credentials, keys) is only READ for self-configuration and never SENT to an external third-party, the package is BENIGN regardless of what the scanner says.
+
+If sensitive data is COLLECTED and EXFILTRATED to a domain unrelated to the package's stated purpose, it is MALICIOUS.
+
+## REFERENCE EXAMPLES
+
+EXAMPLE 1 — TRUE MALWARE:
+Package "slopex-cli" claims to be a "continuity patcher for OpenAI Codex". Postinstall downloads a binary from a personal GitHub repo and REPLACES the real Codex binary. The binary is not part of the described functionality — it's a trojan replacing a trusted tool.
+→ Verdict: MALICIOUS (backdoor, confidence 0.97)
+
+EXAMPLE 2 — FALSE POSITIVE:
+Package "@yeaft/webchat-agent" is a "remote agent for WebChat connecting worker machines". Code uses execSync to locate the Claude CLI binary, process.env to read PATH configuration. Scanner flags "detached_credential_exfil" (CRITICAL) — but the code is just spawning a documented CLI tool and reading PATH. No data is sent to any external domain. The functionality matches the description.
+→ Verdict: BENIGN (confidence 0.92)
+
+EXAMPLE 3 — TRUE MALWARE:
+Package "event-stream" (compromised via flatmap-stream dependency). Obfuscated code hidden in a nested dependency decrypts a payload targeting Bitcoin wallet data from Copay. The obfuscation has no legitimate reason — the parent package is a simple stream utility. The decrypted code specifically targets cryptocurrency credentials.
+→ Verdict: MALICIOUS (credential_exfil, confidence 0.98)
+
+EXAMPLE 4 — FALSE POSITIVE:
+A web framework reads process.env.DATABASE_URL, process.env.API_KEY for configuration. It uses fetch() to call its own documented API endpoint. It uses dynamic require() to load user-configured plugins. Scanner flags env_access, dynamic_require, network_require — but all these are standard framework patterns. No data leaves the application boundary.
+→ Verdict: BENIGN (confidence 0.95)
+
+EXAMPLE 5 — FALSE POSITIVE (phantom lifecycle script):
+Package "instructify@1.0.0" declares "postinstall": "node ./scripts/postinstall.js". But its "files" field is ["dist", ".cursor", "docs/README.md", "README.md", "LICENSE", "CHANGELOG.md", "CONTRIBUTING.md"]. The scripts/ directory does NOT exist in the published tarball because the "files" field excludes it. The postinstall script cannot execute — it is a packaging oversight. The GitHub repository shows the script only prints a welcome message.
+→ Verdict: BENIGN (confidence 0.95)
+
+EXAMPLE 6 — FALSE POSITIVE (anonymous telemetry):
+Package "delimit-cli@3.14.46" has a postinstall that prints CLI setup instructions, then sends anonymous telemetry: POST to delimit.ai/api/telemetry with body {event:'install', version, node:process.version, platform:process.platform, arch:process.arch, ts:ISO}. Silent fail on error, 3s timeout. No PII, no process.env access beyond process.version/platform/arch, no os.hostname(), no file reads. This is standard anonymous install telemetry identical to what Next.js, Turborepo, and Astro do.
+→ Verdict: BENIGN (confidence 0.92)
+
+EXAMPLE 7 — FALSE POSITIVE (binary wrapper with checksum):
+Package "plugin-kit-ai@1.0.1" has a postinstall that downloads a platform-specific binary from GitHub Releases (github.com/777genius/plugin-kit-ai/releases/download/vX.Y.Z/ASSET), verifies SHA256 checksum from checksums.txt, and extracts the binary to vendor/. No data exfiltration, no env access beyond optional GITHUB_TOKEN for rate limits. This is the standard binary distribution pattern used by esbuild, turbo, and biome.
+→ Verdict: BENIGN (confidence 0.95)
+
+EXAMPLE 8 — FALSE POSITIVE (application code with HTTP clients):
+Package "@craft-ng/core@0.1.2" is an Angular state management library. No lifecycle scripts (no postinstall/preinstall). Source contains fetch() and http references but ONLY in JSDoc examples ("const response = await fetch(\`/api/users/\${params}\`)") and Angular service patterns (this.httpClient.get(url)). These are application code patterns, not active network calls during install. No child_process, no eval, no Buffer manipulation.
+→ Verdict: BENIGN (confidence 0.95)
+
+## KEY QUESTIONS TO ANSWER
+
+1. "Do sensitive data (env vars, credentials) LEAVE the package to a third party?"
+2. "Does the code do something HIDDEN that the description doesn't mention?"
+3. "Is obfuscation justified (build tool output) or suspicious (tiny package, no build step)?"
+4. "Does the postinstall relate to the declared functionality?"
+5. "Could a reasonable developer have written this code for the stated purpose?"
+
+## COMMON FALSE POSITIVE PATTERNS (do NOT flag these)
+
+- CLI tools/wrappers using exec/spawn to run other CLI tools (their stated purpose)
+- SDK packages reading API keys from env vars (standard configuration)
+- Build tools with postinstall that compile native addons (node-gyp, prebuild)
+- Packages reading process.env for feature flags, logging config, or database URLs
+- Monorepo tooling with dynamic require for loading workspace packages
+- Test frameworks that use eval() or vm.runInContext for sandboxed test execution
 
 RESPOND IN STRICT JSON ONLY (nothing else):
 {
   "verdict": "malicious" | "benign" | "uncertain",
   "confidence": 0.0-1.0,
-  "reasoning": "Detailed explanation of your analysis",
+  "investigation_steps": ["Step 1: ...", "Step 2: ...", "Step 3: ..."],
+  "reasoning": "Final summary of your analysis",
   "iocs_found": ["domain.com", "1.2.3.4"],
   "attack_type": "credential_exfil" | "reverse_shell" | "crypto_miner" | "backdoor" | "typosquat" | "protestware" | null,
   "recommendation": "block" | "monitor" | "safe"
@@ -275,15 +347,15 @@ function buildPrompt(name, version, ecosystem, sourceContext, threats, npmRegist
     userContent += '\n';
   }
 
-  // Static scanner findings
+  // Static scanner findings — framed as signals to challenge
   if (threats && threats.length > 0) {
-    userContent += `## Static Scanner Findings (${threats.length} total)\n`;
+    userContent += `## Static Scanner Signals (${threats.length} total — these are SIGNALS to investigate, not confirmed threats)\n`;
     for (const t of threats.slice(0, 30)) {
       const loc = t.file ? ` in ${t.file}${t.line ? ':' + t.line : ''}` : '';
       userContent += `- [${t.severity}] ${t.type}${loc}: ${t.message || ''}\n`;
     }
     if (threats.length > 30) {
-      userContent += `... and ${threats.length - 30} more findings\n`;
+      userContent += `... and ${threats.length - 30} more signals\n`;
     }
     userContent += '\n';
   }
@@ -315,7 +387,7 @@ async function callAnthropicAPI(system, messages) {
 
   const body = JSON.stringify({
     model: MODEL_ID,
-    max_tokens: 1024,
+    max_tokens: 2048,
     system,
     messages
   });
@@ -384,6 +456,7 @@ function parseResponse(text) {
   const fallback = {
     verdict: 'uncertain',
     confidence: 0,
+    investigation_steps: [],
     reasoning: 'Failed to parse LLM response',
     iocs_found: [],
     attack_type: null,
@@ -434,6 +507,7 @@ function parseResponse(text) {
   return {
     verdict,
     confidence: Math.round(confidence * 1000) / 1000,
+    investigation_steps: Array.isArray(parsed.investigation_steps) ? parsed.investigation_steps.filter(x => typeof x === 'string').slice(0, 10) : [],
     reasoning: typeof parsed.reasoning === 'string' ? parsed.reasoning : '',
     iocs_found: Array.isArray(parsed.iocs_found) ? parsed.iocs_found.filter(x => typeof x === 'string').slice(0, 20) : [],
     attack_type: typeof parsed.attack_type === 'string' ? parsed.attack_type : null,

package/src/monitor/queue.js

@@ -99,6 +99,9 @@ const {
 // From ./ingestion.js (will be created — currently in monitor.js)
 const { getNpmLatestTarball, getPyPITarballUrl, getWeeklyDownloads } = require('./ingestion.js');
 
+// From ./tarball-archive.js
+const { archiveSuspectTarball } = require('./tarball-archive.js');
+
 // --- Constants ---
 
 const SCAN_CONCURRENCY = Math.max(1, parseInt(process.env.MUADDIB_SCAN_CONCURRENCY, 10) || 5);
@@ -541,6 +544,16 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
 
         stats.suspect++;
 
+        // Fire-and-forget tarball archiving — never blocks the pipeline
+        archiveSuspectTarball(name, version, tarballUrl, {
+          score: riskScore,
+          priority: tierLabel,
+          rulesTriggered: (result.threats || []).map(t => t.ruleId || t.type).filter(Boolean),
+          llmVerdict: null // LLM runs after this point; updated by webhook if needed
+        }).catch(err => {
+          console.warn(`[Archive] Failed for ${name}@${version}: ${err.message}`);
+        });
+
         // Sandbox decision based on tier
         // T1a: mandatory sandbox (HC malice types, TIER1_TYPES non-LOW, lifecycle + intent compound)
         // T1b: conditional sandbox (HIGH/CRITICAL without HC type — bundler FP zone)

package/src/monitor/tarball-archive.js

@@ -0,0 +1,120 @@
+'use strict';
+
+/**
+ * Tarball archiving for suspect packages.
+ *
+ * Downloads and stores tarballs + metadata JSON for packages flagged as suspect,
+ * enabling retrospective audit when npm/PyPI unpublish the package.
+ *
+ * Fire-and-forget: never blocks the scan pipeline.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
+const { downloadToFile } = require('../shared/download.js');
+
+// Archive root — configurable via env for testing
+const ARCHIVE_DIR = process.env.MUADDIB_ARCHIVE_DIR || '/opt/muaddib/archive';
+const ARCHIVE_TIMEOUT_MS = 10_000;
+
+/**
+ * Get the date string in YYYY-MM-DD format (Paris timezone, consistent with monitor).
+ * Falls back to UTC if Intl is unavailable.
+ */
+function getArchiveDateString() {
+  try {
+    const now = new Date();
+    const parts = new Intl.DateTimeFormat('fr-CA', { timeZone: 'Europe/Paris', year: 'numeric', month: '2-digit', day: '2-digit' }).formatToParts(now);
+    const y = parts.find(p => p.type === 'year').value;
+    const m = parts.find(p => p.type === 'month').value;
+    const d = parts.find(p => p.type === 'day').value;
+    return `${y}-${m}-${d}`;
+  } catch {
+    return new Date().toISOString().slice(0, 10);
+  }
+}
+
+/**
+ * Sanitize package name for use in filenames.
+ * Replaces / (scoped packages) with __ and removes unsafe characters.
+ */
+function sanitizeForFilename(name) {
+  return name.replace(/^@/, '').replace(/\//g, '__').replace(/[^a-zA-Z0-9._-]/g, '_');
+}
+
+/**
+ * Compute SHA-256 hash of a file.
+ */
+function sha256File(filePath) {
+  const hash = crypto.createHash('sha256');
+  const data = fs.readFileSync(filePath);
+  hash.update(data);
+  return hash.digest('hex');
+}
+
+/**
+ * Archive a suspect package tarball and its scan metadata.
+ *
+ * @param {string} packageName - Package name (e.g. "evil-pkg" or "@scope/evil-pkg")
+ * @param {string} version - Package version
+ * @param {string} tarballUrl - Registry URL to download the tarball from
+ * @param {object} scanResult - Scan result object from the pipeline
+ * @param {number} scanResult.score - Risk score
+ * @param {string} scanResult.priority - Priority tier (e.g. "P1", "P2")
+ * @param {Array} [scanResult.rulesTriggered] - Array of triggered rule IDs
+ * @param {string} [scanResult.llmVerdict] - LLM detective verdict if available
+ * @returns {Promise<boolean>} true if archived, false if skipped/failed
+ */
+async function archiveSuspectTarball(packageName, version, tarballUrl, scanResult) {
+  if (!tarballUrl || !packageName || !version) return false;
+
+  const dateStr = getArchiveDateString();
+  const dayDir = path.join(ARCHIVE_DIR, dateStr);
+  const safeName = sanitizeForFilename(packageName);
+  const basename = `${safeName}-${version}`;
+  const tgzPath = path.join(dayDir, `${basename}.tgz`);
+  const jsonPath = path.join(dayDir, `${basename}.json`);
+
+  // Dedup: skip if already archived
+  if (fs.existsSync(tgzPath)) {
+    return false;
+  }
+
+  // Ensure day directory exists
+  fs.mkdirSync(dayDir, { recursive: true });
+
+  // Download with semaphore (shares concurrency with rest of pipeline)
+  await acquireRegistrySlot();
+  try {
+    await downloadToFile(tarballUrl, tgzPath, ARCHIVE_TIMEOUT_MS);
+  } finally {
+    releaseRegistrySlot();
+  }
+
+  // Compute hash and write metadata
+  const tarballSha256 = sha256File(tgzPath);
+  const metadata = {
+    package: packageName,
+    version,
+    timestamp: new Date().toISOString(),
+    score: scanResult.score || 0,
+    priority: scanResult.priority || null,
+    rules_triggered: scanResult.rulesTriggered || [],
+    llm_verdict: scanResult.llmVerdict || null,
+    tarball_sha256: tarballSha256
+  };
+
+  fs.writeFileSync(jsonPath, JSON.stringify(metadata, null, 2));
+  return true;
+}
+
+module.exports = {
+  archiveSuspectTarball,
+  ARCHIVE_DIR,
+  // Exported for testing
+  sanitizeForFilename,
+  sha256File,
+  getArchiveDateString
+};

package/src/monitor/webhook.js

@@ -379,6 +379,7 @@ function buildAlertData(name, version, ecosystem, result, sandboxResult, llmResu
     webhookData.llm = {
       verdict: llmResult.verdict,
       confidence: llmResult.confidence,
+      investigation_steps: (llmResult.investigation_steps || []).slice(0, 5),
       reasoning: (llmResult.reasoning || '').slice(0, 200),
       attack_type: llmResult.attack_type || null,
       iocs_found: (llmResult.iocs_found || []).slice(0, 5),