muaddib-scanner 2.11.66 → 2.11.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "muaddib-scanner",
3
- "version": "2.11.66",
3
+ "version": "2.11.68",
4
4
  "description": "Supply-chain threat detection & response for npm & PyPI/Python",
5
5
  "main": "src/index.js",
6
6
  "bin": {
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "target": "node_modules",
3
- "timestamp": "2026-06-06T20:23:04.305Z",
3
+ "timestamp": "2026-06-07T13:41:08.649Z",
4
4
  "threats": [
5
5
  {
6
6
  "type": "string_mutation_obfuscation",
@@ -73,7 +73,9 @@ const HIGH_CONFIDENCE_MALICE_TYPES = new Set([
73
73
  // cap since the attack uses optionalDependencies + prepare hook (no direct lifecycle).
74
74
  'env_charcode_reconstruction', // fromCharCode + process.env[computed] (TeamPCP credential stealer)
75
75
  'ide_hook_autoexec', // .claude/settings.json SessionStart hook, .vscode/tasks.json folderOpen (Shai-Hulud)
76
- 'workflow_secrets_dump' // toJSON(secrets) in GitHub Actions workflow (Shai-Hulud)
76
+ 'workflow_secrets_dump', // toJSON(secrets) in GitHub Actions workflow (Shai-Hulud)
77
+ // Phantom Gyp 2026-06: binding.gyp command-substitution = install-time RCE, quasi-never legit in benign packages
78
+ 'gyp_command_exec'
77
79
  ]);
78
80
 
79
81
  // Lifecycle compound types that indicate real malicious intent beyond a simple postinstall
@@ -26,6 +26,7 @@ const {
26
26
  cacheTarball,
27
27
  updateScanStats,
28
28
  appendDetection,
29
+ appendScanLedger,
29
30
  maybePersistDailyStats,
30
31
  appendTemporalDetection,
31
32
  tarballCacheKey,
@@ -221,6 +222,20 @@ function recordTrainingSample(result, params) {
221
222
  sandboxResult: params.sandboxResult || null
222
223
  });
223
224
  appendTrainingRecord(record);
225
+ // Phase 0a: per-scan coverage ledger — record this terminal outcome (best-effort;
226
+ // appendScanLedger swallows its own write errors and never throws).
227
+ appendScanLedger({
228
+ name: params.name,
229
+ version: params.version,
230
+ ecosystem: params.ecosystem,
231
+ outcome: params.label || 'clean',
232
+ score: (result.summary && typeof result.summary.riskScore === 'number') ? result.summary.riskScore : null,
233
+ tier: params.tier,
234
+ maxSeverity: result.summary ? result.summary.riskLevel : null,
235
+ types: [...new Set((result.threats || []).map(t => t.type))],
236
+ sandbox: params.sandboxResult ? 'run' : 'none',
237
+ source: 'scan'
238
+ });
224
239
  } catch (err) {
225
240
  // Non-fatal: ML export must never crash the monitor
226
241
  console.error(`[ML] Failed to record training sample for ${params.name}: ${err.message}`);
@@ -521,6 +536,7 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
521
536
  stats.totalTimeMs += Date.now() - startTime;
522
537
  stats.clean++;
523
538
  updateScanStats('clean');
539
+ appendScanLedger({ name, version, ecosystem, outcome: 'size_skip', score: 0, source: 'size_skip_quick_clean' });
524
540
  return;
525
541
  }
526
542
  } catch {
@@ -541,6 +557,7 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
541
557
  stats.totalTimeMs += Date.now() - startTime;
542
558
  stats.clean++;
543
559
  updateScanStats('clean');
560
+ appendScanLedger({ name, version, ecosystem, outcome: 'size_skip', score: 0, source: 'size_skip_extract_failed' });
544
561
  return;
545
562
  }
546
563
  }
@@ -32,9 +32,20 @@ let _lastHardDropLog = 0;
32
32
  function enqueueScan(scanQueue, item, stats, max = MAX_SCAN_QUEUE) {
33
33
  let dropped = false;
34
34
  if (scanQueue.length >= max) {
35
- scanQueue.shift(); // drop oldest
35
+ const evicted = scanQueue.shift(); // drop oldest
36
36
  dropped = true;
37
37
  if (stats) stats.queueHardDrops = (stats.queueHardDrops || 0) + 1;
38
+ // Phase 0a: record the dropped item so a coverage loss keeps an identity — answers
39
+ // "which versions were never scanned" (e.g. the Miasma 72s/96-version burst). Lazy
40
+ // require avoids any top-level coupling with state.js; best-effort, never throws.
41
+ try {
42
+ if (evicted && evicted.name) {
43
+ require('./state.js').appendScanLedger({
44
+ name: evicted.name, version: evicted.version, ecosystem: evicted.ecosystem,
45
+ outcome: 'dropped', source: 'queue_cap'
46
+ });
47
+ }
48
+ } catch { /* ledger is best-effort */ }
38
49
  const now = Date.now();
39
50
  if (now - _lastHardDropLog > HARD_DROP_LOG_INTERVAL_MS) {
40
51
  _lastHardDropLog = now;
@@ -951,6 +951,224 @@ function _compactDetectionsJsonl() {
951
951
  }
952
952
  }
953
953
 
954
+ // --- Per-scan ledger (Phase 0a: operational coverage observability) ---
955
+ // Append-only record of EVERY package the monitor dequeues + its terminal outcome,
956
+ // so we can distinguish never-scanned vs scanned-clean vs suspect vs dropped and
957
+ // measure TRUE operational coverage (not just rule-TPR on the static corpus).
958
+ // Mirrors the detections JSONL machinery (chunked iterate + periodic compaction).
959
+ // Differences vs detections: (1) NO dedup — every scan event is a distinct record;
960
+ // (2) higher cap + compaction interval since this logs every scan, not just findings.
961
+ const SCAN_LEDGER_FILE = process.env.MUADDIB_SCAN_LEDGER_FILE || path.join(__dirname, '..', '..', 'data', 'scan-ledger.jsonl');
962
+ const MAX_SCAN_LEDGER = (() => {
963
+ const raw = process.env.MUADDIB_SCAN_LEDGER_MAX;
964
+ const n = raw ? parseInt(raw, 10) : NaN;
965
+ return (Number.isFinite(n) && n >= 10 && n <= 5_000_000) ? n : 500_000;
966
+ })();
967
+ const SCAN_LEDGER_COMPACT_INTERVAL = 2000;
968
+ let _scanLedgerAppendedSinceCompact = 0;
969
+
970
+ // Terminal outcomes a dequeued package can reach. Unknown values normalize to 'clean'
971
+ // so a typo at a call site can never crash the pipeline.
972
+ const SCAN_LEDGER_OUTCOMES = new Set([
973
+ 'clean', 'clean_low_signal', 'clean_tooling', 'suspect', 'ml_clean', 'llm_benign',
974
+ 'sandbox_inconclusive', 'sandbox_unconfirmed', 'confirmed',
975
+ 'static_timeout', 'size_skip', 'dropped'
976
+ ]);
977
+
978
+ /**
979
+ * Append one per-scan ledger entry recording the terminal outcome of a dequeued
980
+ * package. Best-effort: NEVER throws (a ledger failure must not break scanning).
981
+ * No dedup — repeated scans of the same package are intentionally all recorded.
982
+ *
983
+ * @param {object} e
984
+ * @param {string} e.name package name (required)
985
+ * @param {string} [e.version]
986
+ * @param {string} [e.ecosystem] 'npm' | 'pypi' | ...
987
+ * @param {string} [e.outcome] one of SCAN_LEDGER_OUTCOMES (default 'clean')
988
+ * @param {number} [e.score] riskScore at the terminal decision
989
+ * @param {string} [e.tier] suspect tier ('1a'|'1b'|2|3) if applicable
990
+ * @param {string} [e.maxSeverity]
991
+ * @param {string[]} [e.types] threat types (capped to 12)
992
+ * @param {string} [e.sandbox] 'none' | 'run' | 'deferred' | 'skip'
993
+ * @param {boolean} [e.firstPublish]
994
+ * @param {string} [e.source] where the record originated ('scan','queue_cap',...)
995
+ */
996
+ function appendScanLedger(e) {
997
+ try {
998
+ if (!e || !e.name) return;
999
+ const dir = path.dirname(SCAN_LEDGER_FILE);
1000
+ if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
1001
+ const entry = {
1002
+ ts: new Date().toISOString(),
1003
+ name: e.name,
1004
+ version: e.version || null,
1005
+ ecosystem: e.ecosystem || null,
1006
+ outcome: SCAN_LEDGER_OUTCOMES.has(e.outcome) ? e.outcome : 'clean',
1007
+ score: (typeof e.score === 'number') ? e.score : null,
1008
+ tier: (e.tier !== undefined && e.tier !== null) ? String(e.tier) : null,
1009
+ maxSeverity: e.maxSeverity || null,
1010
+ types: Array.isArray(e.types) ? e.types.slice(0, 12) : [],
1011
+ sandbox: e.sandbox || 'none',
1012
+ firstPublish: !!e.firstPublish,
1013
+ source: e.source || 'scan'
1014
+ };
1015
+ fs.appendFileSync(SCAN_LEDGER_FILE, JSON.stringify(entry) + '\n', 'utf8');
1016
+ _scanLedgerAppendedSinceCompact++;
1017
+ if (_scanLedgerAppendedSinceCompact >= SCAN_LEDGER_COMPACT_INTERVAL) {
1018
+ _scanLedgerAppendedSinceCompact = 0;
1019
+ _compactScanLedgerJsonl();
1020
+ }
1021
+ } catch (err) {
1022
+ if (err.code === 'EROFS' || err.code === 'EACCES' || err.code === 'EPERM') return;
1023
+ if (err.code === 'ENOSPC') {
1024
+ console.warn('[MONITOR] WARNING: disk full (ENOSPC) — cannot persist scan-ledger.');
1025
+ return;
1026
+ }
1027
+ console.error(`[MONITOR] Failed to write scan-ledger: ${err.message}`);
1028
+ }
1029
+ }
1030
+
1031
+ /**
1032
+ * Compact the scan-ledger JSONL: keep only the most recent MAX_SCAN_LEDGER entries.
1033
+ * No-op when already under cap. Streams (never loads the whole file at once).
1034
+ */
1035
+ function _compactScanLedgerJsonl() {
1036
+ try {
1037
+ const total = _countJsonlLines(SCAN_LEDGER_FILE);
1038
+ if (total <= MAX_SCAN_LEDGER) return;
1039
+ const toDrop = total - MAX_SCAN_LEDGER;
1040
+ let skipped = 0;
1041
+ const kept = [];
1042
+ _iterateJsonlSync(SCAN_LEDGER_FILE, (entry) => {
1043
+ if (skipped < toDrop) { skipped++; return; }
1044
+ kept.push(JSON.stringify(entry));
1045
+ });
1046
+ const tmpFile = SCAN_LEDGER_FILE + '.tmp';
1047
+ fs.writeFileSync(tmpFile, kept.length ? kept.join('\n') + '\n' : '', 'utf8');
1048
+ fs.renameSync(tmpFile, SCAN_LEDGER_FILE);
1049
+ console.log(`[MONITOR] COMPACT scan-ledger: ${total} -> ${kept.length} entries`);
1050
+ } catch (err) {
1051
+ console.error(`[MONITOR] Scan-ledger compaction failed: ${err.message}`);
1052
+ }
1053
+ }
1054
+
1055
+ /** Stream the scan-ledger into an array (tests + Phase 0b rollup). */
1056
+ function loadScanLedger() {
1057
+ const entries = [];
1058
+ try { _iterateJsonlSync(SCAN_LEDGER_FILE, (e) => { entries.push(e); }); } catch { /* ignore */ }
1059
+ return entries;
1060
+ }
1061
+
1062
+ // Bounded distinct-key tracking for the `vanished` cross-reference (CLAUDE.md §2).
1063
+ // Sits above the MAX_SCAN_LEDGER file ceiling so it is a pure safety valve: in normal
1064
+ // operation the in-window key sets are far smaller than the file, so `exactVanished`
1065
+ // stays true. Only an operator setting MUADDIB_SCAN_LEDGER_MAX above this would trip it.
1066
+ const MAX_ROLLUP_KEYS = 1_200_000;
1067
+
1068
+ /**
1069
+ * Phase 0b: roll up the per-scan ledger into operational-coverage metrics.
1070
+ *
1071
+ * Single streaming pass (never loads the whole file at once — same machinery as
1072
+ * getDetectionStats). It distinguishes:
1073
+ * - scanned : entries that reached a real verdict (outcome !== 'dropped')
1074
+ * - dropped : queue-cap evictions (outcome === 'dropped') — never scanned
1075
+ * - vanished: DISTINCT name@version that were dropped AND never (re)scanned in-window
1076
+ * = a permanent coverage hole (the "which Miasma versions never ran" case)
1077
+ *
1078
+ * HONEST METRIC NOTE — `alertRate` is (suspect+confirmed) / scanned, i.e. "of what we
1079
+ * scanned, the fraction we flagged". It is NOT a true-positive rate: the ledger carries
1080
+ * no ground truth. The GHSA-denominated operational TPR (the 105/429 audit number) needs
1081
+ * the ledger cross-referenced against the GHSA malware feed — that is the Phase 5
1082
+ * coverage-audit, not this rollup. Do not relabel `alertRate` as TPR (CLAUDE.md: pas
1083
+ * d'embellissement des métriques).
1084
+ *
1085
+ * @param {number|string|null} [sinceTs] window start — ms epoch, ISO string, or null for
1086
+ * "whole ledger". Entries with ts < sinceTs (or unparseable ts) are skipped.
1087
+ * @param {object} [opts]
1088
+ * @param {string} [opts.file] ledger path override (tests). Defaults to SCAN_LEDGER_FILE.
1089
+ * @returns {{
1090
+ * generatedAt:string, since:string|null, windowStart:string|null, windowEnd:string|null,
1091
+ * total:number, scanned:number, dropped:number, vanished:number, exactVanished:boolean,
1092
+ * alerted:number, alertRate:number|null,
1093
+ * byOutcome:Object.<string,number>,
1094
+ * byEcosystem:Object.<string,{total:number,scanned:number,dropped:number,alerted:number}>
1095
+ * }}
1096
+ */
1097
+ function computeLedgerRollup(sinceTs, opts = {}) {
1098
+ const file = opts.file || SCAN_LEDGER_FILE;
1099
+
1100
+ let sinceMs = null;
1101
+ if (typeof sinceTs === 'number' && Number.isFinite(sinceTs)) {
1102
+ sinceMs = sinceTs;
1103
+ } else if (typeof sinceTs === 'string') {
1104
+ const p = Date.parse(sinceTs);
1105
+ if (!Number.isNaN(p)) sinceMs = p;
1106
+ }
1107
+
1108
+ const byOutcome = Object.create(null);
1109
+ const byEcosystem = Object.create(null);
1110
+ let total = 0, scanned = 0, dropped = 0, alerted = 0;
1111
+ let earliest = null, latest = null;
1112
+ // Two sets so `vanished` is correct regardless of drop/scan ordering in the file.
1113
+ // droppedKeys is small (drops only happen under queue-cap pressure); scannedKeys is
1114
+ // bounded by the in-window line count (≤ MAX_SCAN_LEDGER), and further by MAX_ROLLUP_KEYS.
1115
+ const scannedKeys = new Set();
1116
+ const droppedKeys = new Set();
1117
+ let exactVanished = true;
1118
+
1119
+ _iterateJsonlSync(file, (e) => {
1120
+ if (!e || !e.name) return;
1121
+ let t = null;
1122
+ if (e.ts) { const p = Date.parse(e.ts); if (!Number.isNaN(p)) t = p; }
1123
+ if (sinceMs !== null && (t === null || t < sinceMs)) return;
1124
+
1125
+ total++;
1126
+ if (t !== null) {
1127
+ if (earliest === null || t < earliest) earliest = t;
1128
+ if (latest === null || t > latest) latest = t;
1129
+ }
1130
+
1131
+ const outcome = (typeof e.outcome === 'string' && e.outcome) ? e.outcome : 'clean';
1132
+ byOutcome[outcome] = (byOutcome[outcome] || 0) + 1;
1133
+
1134
+ const eco = e.ecosystem || 'unknown';
1135
+ let ecoNode = byEcosystem[eco];
1136
+ if (!ecoNode) ecoNode = byEcosystem[eco] = { total: 0, scanned: 0, dropped: 0, alerted: 0 };
1137
+ ecoNode.total++;
1138
+
1139
+ const key = `${e.name}@${e.version || ''}`;
1140
+ const underCap = exactVanished && (scannedKeys.size + droppedKeys.size) < MAX_ROLLUP_KEYS;
1141
+ if (outcome === 'dropped') {
1142
+ dropped++; ecoNode.dropped++;
1143
+ if (underCap) droppedKeys.add(key); else exactVanished = false;
1144
+ } else {
1145
+ scanned++; ecoNode.scanned++;
1146
+ if (outcome === 'suspect' || outcome === 'confirmed') { alerted++; ecoNode.alerted++; }
1147
+ if (underCap) scannedKeys.add(key); else exactVanished = false;
1148
+ }
1149
+ });
1150
+
1151
+ let vanished = 0;
1152
+ for (const k of droppedKeys) { if (!scannedKeys.has(k)) vanished++; }
1153
+
1154
+ return {
1155
+ generatedAt: new Date().toISOString(),
1156
+ since: sinceMs !== null ? new Date(sinceMs).toISOString() : null,
1157
+ windowStart: earliest !== null ? new Date(earliest).toISOString() : null,
1158
+ windowEnd: latest !== null ? new Date(latest).toISOString() : null,
1159
+ total,
1160
+ scanned,
1161
+ dropped,
1162
+ vanished,
1163
+ exactVanished,
1164
+ alerted,
1165
+ // NOT a TPR — see the HONEST METRIC NOTE above. null when nothing was scanned.
1166
+ alertRate: scanned > 0 ? alerted / scanned : null,
1167
+ byOutcome,
1168
+ byEcosystem
1169
+ };
1170
+ }
1171
+
954
1172
  // --- Scan stats (FP rate tracking) ---
955
1173
 
956
1174
  function loadScanStats() {
@@ -1420,6 +1638,8 @@ module.exports = {
1420
1638
  MAX_TEMPORAL_DETECTIONS,
1421
1639
  MAX_DAILY_ALERTS,
1422
1640
  DETECTION_COMPACT_INTERVAL,
1641
+ SCAN_LEDGER_FILE,
1642
+ MAX_SCAN_LEDGER,
1423
1643
 
1424
1644
  // Mutable state getters/setters
1425
1645
  getScanMemoryCache,
@@ -1456,6 +1676,10 @@ module.exports = {
1456
1676
  appendAlert,
1457
1677
  loadDetections,
1458
1678
  appendDetection,
1679
+ appendScanLedger,
1680
+ loadScanLedger,
1681
+ computeLedgerRollup,
1682
+ _compactScanLedgerJsonl,
1459
1683
  getDetectionStats,
1460
1684
  runStateMigrations,
1461
1685
  // Internal — exported for tests and for the daemon hourly housekeeping.
@@ -45,6 +45,18 @@ function getMinFreeBytes() {
45
45
  return gb * 1024 * 1024 * 1024;
46
46
  }
47
47
 
48
+ // Tarball download is gated on this score so the heavy .tgz is kept ONLY for
49
+ // alert-threshold packages; the cheap JSON metadata is still written for every
50
+ // suspect. Aligns with the webhook alert floor (20). Bounded to [0, 100], default 20.
51
+ const DEFAULT_TGZ_MIN_SCORE = 20;
52
+ function getArchiveTgzMinScore() {
53
+ const raw = process.env.MUADDIB_ARCHIVE_TGZ_MIN_SCORE;
54
+ if (raw === undefined || raw === '') return DEFAULT_TGZ_MIN_SCORE;
55
+ const n = parseInt(raw, 10);
56
+ if (!Number.isFinite(n) || n < 0 || n > 100) return DEFAULT_TGZ_MIN_SCORE;
57
+ return n;
58
+ }
59
+
48
60
  function hasEnoughSpace(targetDir) {
49
61
  try {
50
62
  if (typeof fs.statfsSync !== 'function') return true; // Node <18.15 — fail-open
@@ -109,14 +121,20 @@ async function archiveSuspectTarball(packageName, version, tarballUrl, scanResul
109
121
 
110
122
  // Defense-in-depth: never archive packages that are statically clean.
111
123
  // Callers in the pipeline already gate on tier 1a/1b/2 classification, but a
112
- // numeric score of 0 with no triggered rules is unambiguously CLEAN — those
113
- // dominated archive volume in production.
124
+ // numeric score of 0 with no triggered rules is unambiguously CLEAN.
114
125
  const score = (scanResult && typeof scanResult.score === 'number') ? scanResult.score : 0;
115
126
  const rules = (scanResult && Array.isArray(scanResult.rulesTriggered)) ? scanResult.rulesTriggered : [];
116
127
  if (score === 0 && rules.length === 0) {
117
128
  return false;
118
129
  }
119
130
 
131
+ // Tarballs dominate archive volume (~439MB/day of .tgz vs ~3.6MB/day of JSON).
132
+ // Keep the cheap JSON metadata for EVERY suspect (audit trail + GT-promotion index),
133
+ // but download/retain the heavy .tgz ONLY for packages at/above the alert threshold
134
+ // (score >= MUADDIB_ARCHIVE_TGZ_MIN_SCORE, default 20 = webhook floor). This shrinks
135
+ // the archive from tens of GB to hundreds of MB without losing the record of what was seen.
136
+ const keepTarball = score >= getArchiveTgzMinScore();
137
+
120
138
  const dateStr = getArchiveDateString();
121
139
  const dayDir = path.join(ARCHIVE_DIR, dateStr);
122
140
  const safeName = sanitizeForFilename(packageName);
@@ -124,32 +142,55 @@ async function archiveSuspectTarball(packageName, version, tarballUrl, scanResul
124
142
  const tgzPath = path.join(dayDir, `${basename}.tgz`);
125
143
  const jsonPath = path.join(dayDir, `${basename}.json`);
126
144
 
127
- // Dedup: skip if already archived
128
- if (fs.existsSync(tgzPath)) {
129
- return false;
130
- }
145
+ // At/above the alert threshold: archive the full .tgz (existing behavior, unchanged).
146
+ // Below it: keep only the cheap JSON metadata (audit trail + GT-promotion index).
147
+ if (keepTarball) {
148
+ // Dedup: skip if already archived
149
+ if (fs.existsSync(tgzPath)) {
150
+ return false;
151
+ }
131
152
 
132
- // Defense layer 3: skip if disk is nearly full, even if retention is well-configured.
133
- // Prevents a burst of malicious campaigns from blowing past the 7-day budget
134
- // before the 6h periodic cleanup tick can catch up.
135
- if (!hasEnoughSpace(ARCHIVE_DIR)) {
136
- console.warn(`[Archive] Skip ${packageName}@${version}: free space below ${DEFAULT_MIN_FREE_GB}GB threshold`);
137
- return false;
138
- }
153
+ // Disk-space gate: don't let a burst of suspects run the volume to 100% between
154
+ // the periodic cleanups. Guards the heavy .tgz download.
155
+ if (!hasEnoughSpace(ARCHIVE_DIR)) {
156
+ console.warn(`[Archive] Skip ${packageName}@${version}: free space below ${DEFAULT_MIN_FREE_GB}GB threshold`);
157
+ return false;
158
+ }
139
159
 
140
- // Ensure day directory exists
141
- fs.mkdirSync(dayDir, { recursive: true });
160
+ // Ensure day directory exists
161
+ fs.mkdirSync(dayDir, { recursive: true });
142
162
 
143
- // Download with semaphore (shares concurrency with rest of pipeline)
144
- await acquireRegistrySlot();
145
- try {
146
- await downloadToFile(tarballUrl, tgzPath, ARCHIVE_TIMEOUT_MS);
147
- } finally {
148
- releaseRegistrySlot();
163
+ // Download with semaphore (shares concurrency with rest of pipeline). Download
164
+ // errors propagate to the fire-and-forget .catch() in the caller (queue.js).
165
+ await acquireRegistrySlot();
166
+ try {
167
+ await downloadToFile(tarballUrl, tgzPath, ARCHIVE_TIMEOUT_MS);
168
+ } finally {
169
+ releaseRegistrySlot();
170
+ }
171
+
172
+ const tarballSha256 = sha256File(tgzPath);
173
+ const metadata = {
174
+ package: packageName,
175
+ version,
176
+ timestamp: new Date().toISOString(),
177
+ score: scanResult.score || 0,
178
+ priority: scanResult.priority || null,
179
+ rules_triggered: scanResult.rulesTriggered || [],
180
+ llm_verdict: scanResult.llmVerdict || null,
181
+ tarball_archived: true,
182
+ tarball_sha256: tarballSha256
183
+ };
184
+ fs.writeFileSync(jsonPath, JSON.stringify(metadata, null, 2));
185
+ return true;
149
186
  }
150
187
 
151
- // Compute hash and write metadata
152
- const tarballSha256 = sha256File(tgzPath);
188
+ // Below the alert threshold — record cheap JSON metadata only, skip the tarball.
189
+ // Dedup on the JSON record so re-scans of the same package@version don't rewrite it.
190
+ if (fs.existsSync(jsonPath)) {
191
+ return false;
192
+ }
193
+ fs.mkdirSync(dayDir, { recursive: true });
153
194
  const metadata = {
154
195
  package: packageName,
155
196
  version,
@@ -158,9 +199,9 @@ async function archiveSuspectTarball(packageName, version, tarballUrl, scanResul
158
199
  priority: scanResult.priority || null,
159
200
  rules_triggered: scanResult.rulesTriggered || [],
160
201
  llm_verdict: scanResult.llmVerdict || null,
161
- tarball_sha256: tarballSha256
202
+ tarball_archived: false,
203
+ tarball_sha256: null
162
204
  };
163
-
164
205
  fs.writeFileSync(jsonPath, JSON.stringify(metadata, null, 2));
165
206
  return true;
166
207
  }
@@ -272,5 +313,6 @@ module.exports = {
272
313
  getArchiveDateString,
273
314
  getRetentionDays,
274
315
  getMinFreeBytes,
316
+ getArchiveTgzMinScore,
275
317
  parseArchiveDayDir
276
318
  };
@@ -28,7 +28,8 @@ const {
28
28
  saveState,
29
29
  loadStateRaw,
30
30
  getScansSinceLastMemoryPersist,
31
- setScansSinceLastMemoryPersist
31
+ setScansSinceLastMemoryPersist,
32
+ computeLedgerRollup
32
33
  } = require('./state.js');
33
34
  const {
34
35
  HIGH_CONFIDENCE_MALICE_TYPES,
@@ -897,7 +898,52 @@ function formatDelta(current, previous) {
897
898
  return '=0';
898
899
  }
899
900
 
900
- function buildDailyReportEmbed(stats, dailyAlerts) {
901
+ // Phase 0b: rolling window for the daily report's ledger section. The report runs
902
+ // once/day, so 24h is the natural "what happened today" view and keeps the rollup's
903
+ // distinct-key sets small (one day of scans, far below MAX_ROLLUP_KEYS). Env-tunable.
904
+ const LEDGER_ROLLUP_WINDOW_MS = (() => {
905
+ const v = parseInt(process.env.MUADDIB_LEDGER_ROLLUP_WINDOW_MS, 10);
906
+ return Number.isFinite(v) && v > 0 ? v : 24 * 60 * 60 * 1000;
907
+ })();
908
+
909
+ /**
910
+ * Compute the per-scan ledger rollup for the daily-report window. Best-effort: a
911
+ * rollup failure (corrupt ledger, I/O) must NEVER break the daily report, so this
912
+ * swallows errors and returns null. Also returns null when the ledger is empty so
913
+ * the report omits the section instead of showing a noise row of zeros.
914
+ */
915
+ function safeLedgerRollup() {
916
+ try {
917
+ const rollup = computeLedgerRollup(Date.now() - LEDGER_ROLLUP_WINDOW_MS);
918
+ return (rollup && rollup.total > 0) ? rollup : null;
919
+ } catch {
920
+ return null;
921
+ }
922
+ }
923
+
924
+ /**
925
+ * Format the ledger rollup as a Discord embed field, or null to omit it (no data).
926
+ * Surfaces operational scan coverage: scanned, alert rate (NOT a TPR — see
927
+ * computeLedgerRollup's HONEST METRIC NOTE), the dropped/vanished coverage holes,
928
+ * and a per-ecosystem split. Compact, well under Discord's 1024-char field limit.
929
+ */
930
+ function formatLedgerField(rollup) {
931
+ if (!rollup || rollup.total <= 0) return null;
932
+ const pct = rollup.alertRate != null ? (rollup.alertRate * 100).toFixed(2) : '0.00';
933
+ const lines = [`Scanned ${rollup.scanned} · Alerted ${rollup.alerted} (${pct}%)`];
934
+ if (rollup.dropped > 0) {
935
+ const vanishedNote = rollup.exactVanished ? `${rollup.vanished}` : `≥${rollup.vanished}`;
936
+ lines.push(`Dropped ${rollup.dropped} (${vanishedNote} vanished)`);
937
+ }
938
+ const ecos = Object.keys(rollup.byEcosystem)
939
+ .sort((a, b) => rollup.byEcosystem[b].total - rollup.byEcosystem[a].total);
940
+ if (ecos.length > 0) {
941
+ lines.push(ecos.slice(0, 4).map(e => `${e} ${rollup.byEcosystem[e].total}`).join(' · '));
942
+ }
943
+ return { name: 'Ledger (24h)', value: lines.join('\n'), inline: false };
944
+ }
945
+
946
+ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
901
947
  // Use in-memory stats (accumulated since last reset, restored from disk on restart)
902
948
  // instead of disk-based daily entries which can undercount due to UTC/Paris date mismatch
903
949
  const { top3: diskTop3 } = buildReportFromDisk();
@@ -1000,6 +1046,12 @@ function buildDailyReportEmbed(stats, dailyAlerts) {
1000
1046
  } catch { /* non-fatal */ }
1001
1047
  const healthText = `Up ${uptimeH}h${uptimeM}m | Heap ${heapMB}MB${jsonlInfo}`;
1002
1048
 
1049
+ // --- Phase 0b: per-scan ledger rollup (operational coverage) ---
1050
+ // Caller may pass a precomputed rollup (sendDailyReport does, to persist the same
1051
+ // numbers it displays); undefined → compute here; explicit null → omit the section.
1052
+ const ledger = ledgerRollup !== undefined ? ledgerRollup : safeLedgerRollup();
1053
+ const ledgerField = formatLedgerField(ledger);
1054
+
1003
1055
  const now = new Date();
1004
1056
  const readableTime = now.toISOString().replace('T', ' ').replace(/\.\d+Z$/, ' UTC');
1005
1057
 
@@ -1022,6 +1074,7 @@ function buildDailyReportEmbed(stats, dailyAlerts) {
1022
1074
  ? [{ name: 'Deferred Sandbox', value: `Enqueued: ${stats.sandboxDeferred || 0} | Processed: ${stats.deferredProcessed || 0} | Expired: ${stats.deferredExpired || 0}`, inline: false }]
1023
1075
  : []),
1024
1076
  { name: 'Stability', value: `Restarts (24h): ${stats.restartsToday || 0} | Temporal load-shed: ${stats.temporalLoadShed || 0} | Queue hard-drops: ${stats.queueHardDrops || 0}`, inline: false },
1077
+ ...(ledgerField ? [ledgerField] : []),
1025
1078
  { name: 'System', value: healthText, inline: false }
1026
1079
  ],
1027
1080
  footer: {
@@ -1060,7 +1113,10 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
1060
1113
  // delta. Written before the (now last) webhook so a mid-send kill can't double-count.
1061
1114
  saveLastDailyReportDate(today, captureScanStatsBaseline());
1062
1115
 
1063
- const payload = buildDailyReportEmbed(stats, dailyAlerts);
1116
+ // Phase 0b: compute the ledger rollup ONCE so the embed shows exactly the numbers
1117
+ // we persist (no double-scan, no drift between Discord and the on-disk metrics).
1118
+ const ledgerRollup = safeLedgerRollup();
1119
+ const payload = buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup);
1064
1120
 
1065
1121
  // Persist locally with full raw metrics (independent of webhook — enables trend analysis)
1066
1122
  persistDailyReport(payload, {
@@ -1081,6 +1137,7 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
1081
1137
  restartsToday: stats.restartsToday || 0,
1082
1138
  temporalLoadShed: stats.temporalLoadShed || 0,
1083
1139
  queueHardDrops: stats.queueHardDrops || 0,
1140
+ ledger: ledgerRollup || null,
1084
1141
  topSuspects: dailyAlerts.slice().sort((a, b) => (b.score || 0) - (a.score || 0) || b.findingsCount - a.findingsCount).slice(0, 10)
1085
1142
  });
1086
1143
 
@@ -1337,6 +1394,7 @@ module.exports = {
1337
1394
  buildMaintainerChangeWebhookEmbed,
1338
1395
  buildCanaryExfiltrationWebhookEmbed,
1339
1396
  buildDailyReportEmbed,
1397
+ formatLedgerField,
1340
1398
  sendDailyReport,
1341
1399
  buildReportFromDisk,
1342
1400
  buildReportEmbedFromDisk,
@@ -1001,6 +1001,10 @@ const PLAYBOOKS = {
1001
1001
  'HAUTE: binding.gyp avec script lifecycle non-standard. Code natif compile a l\'installation. ' +
1002
1002
  'Verifier le contenu de binding.gyp et les sources C/C++. Installer avec --ignore-scripts si suspect.',
1003
1003
 
1004
+ gyp_command_exec:
1005
+ 'CRITIQUE: binding.gyp utilise la command-substitution GYP <!(...) / <!@(...) — execution de code a l\'installation via node-gyp, sans script lifecycle (pattern Phantom Gyp). ' +
1006
+ 'Decoder la commande substituee. NE PAS installer : node-gyp l\'execute au build meme avec --ignore-scripts. Verifier la source officielle du package.',
1007
+
1004
1008
  string_mutation_obfuscation:
1005
1009
  'HAUTE: Chaine de .replace() reconstruisant des noms d\'API dangereuses (leet-speak). ' +
1006
1010
  'Technique d\'evasion par substitution de caracteres. Decoder la chaine finale. Supprimer si malveillant.',
@@ -2949,6 +2949,19 @@ const RULES = {
2949
2949
  ],
2950
2950
  mitre: 'T1195.002'
2951
2951
  },
2952
+ gyp_command_exec: {
2953
+ id: 'MUADDIB-PKG-023',
2954
+ name: 'GYP Command-Substitution Install Execution',
2955
+ severity: 'CRITICAL',
2956
+ confidence: 'high',
2957
+ domain: 'malware',
2958
+ description: 'binding.gyp utilise la command-substitution GYP <!(...) / <!@(...) — execution de code a l\'installation via node-gyp, sans script lifecycle package.json (pattern Phantom Gyp, juin 2026).',
2959
+ references: [
2960
+ 'https://gyp.gsrc.io/docs/InputFormatReference.md',
2961
+ 'https://attack.mitre.org/techniques/T1195.002/'
2962
+ ],
2963
+ mitre: 'T1195.002'
2964
+ },
2952
2965
  string_mutation_obfuscation: {
2953
2966
  id: 'MUADDIB-AST-074',
2954
2967
  name: 'String Mutation Obfuscation',
@@ -252,6 +252,51 @@ async function scanPackageJson(targetPath) {
252
252
  // Check if binding.gyp references C/C++ source files
253
253
  const hasNativeSources = /\.(c|cc|cpp|cxx|h|hpp)\b/.test(gypContent);
254
254
 
255
+ // Phantom Gyp (June 2026): GYP command-substitution <!(...) / <!@(...) runs a command at
256
+ // *configure* time via `node-gyp`, which npm auto-runs on install whenever a binding.gyp is
257
+ // present — NO package.json lifecycle script required, so it slips past every lifecycle-gated
258
+ // check below. Distinct from <(...) / <@(...) (plain variable expansion, benign) which MUST
259
+ // NOT fire — the required `!` gates command execution.
260
+ //
261
+ // Legit native addons use <!(...) heavily for build-env queries — `node -p process.versions`,
262
+ // `node ./util/has_lib.js`, `pkg-config ... | sed`, `node -p "require('node-addon-api').include"`
263
+ // — and a build-helper `<!(node x.js)` is statically INDISTINGUISHABLE from a payload
264
+ // `<!(node index.js)`. To honor "FPR must never increase" we flag a command-sub ONLY when it
265
+ // carries a malice-specific marker, never the bare "runs a script" shape:
266
+ // (1) GYP_DANGER — shell-level malice in the command line itself: the Phantom Gyp fake-source
267
+ // trick (`; / && / | echo <name>.c`, returning a fabricated source so node-gyp doesn't
268
+ // error), network fetch (curl/wget), pipe-to-shell (| sh, sh -c), eval/base64//dev/tcp,
269
+ // char-code obfuscation (fromCharCode/atob);
270
+ // (2) an inline interpreter payload — node|python|ruby|perl running -e/-c/-p/--eval/--print code
271
+ // that reaches the NETWORK (require/import of https|http|net|dgram|dns|tls, optional node:
272
+ // prefix; fetch; urllib/requests/httpx/http.client/urlopen; socket). Network at configure
273
+ // time is never a legit build query. We deliberately do NOT key on child_process/exec/spawn
274
+ // here — legit addons shell out to detect the toolchain (`node -e "...execSync('gcc
275
+ // --version')..."`), which would FP; an exec of curl/wget is still caught by GYP_DANGER.
276
+ // Catches `<!(node --eval require('node:https')...)`, `<!(python3 -c import requests)`.
277
+ // Honest limitation: this is a line-by-line SPEED-BUMP, not coverage. A bare `<!(node payload.js)`
278
+ // and any non-network inline payload are NOT flagged (indistinguishable from canvas/node-sass
279
+ // build helpers without false positives, FPR-first by design). Real closure needs a compound
280
+ // (configure-time sink × the run script's AST/dataflow verdict) — a separate effort.
281
+ const GYP_DANGER = /[;&|]\s*echo\s+[^|;&]*\.(?:c|cc|cpp|cxx|m|mm|cs)\b|\bcurl\b|\bwget\b|\|\s*(?:sh|bash|zsh)\b|\b(?:sh|bash|zsh)\s+-c\b|\beval\b|\bbase64\b|\/dev\/tcp|fromCharCode|\batob\b/i;
282
+ const GYP_INTERP = /\b(?:node|nodejs|python[0-9.]*|ruby|perl)\b[^|;&\n]{0,40}?\s--?(?:eval|print|e|c|p)\b/i;
283
+ const GYP_PAYLOAD_API = /(?:require|import)\s*\(\s*['"](?:node:)?(?:https?|net|dgram|dns|tls)['"]|\bfetch\s*\(|\burllib\b|\brequests\b|\bhttpx\b|http\.client|\burlopen\b|socket\.(?:socket|create_connection)/i;
284
+ let gypCommandExec = false;
285
+ const gypCmdSubRe = /<!@?\(([^\n]{0,400})/g;
286
+ let _gm;
287
+ while ((_gm = gypCmdSubRe.exec(gypContent)) !== null) {
288
+ const body = _gm[1];
289
+ if (GYP_DANGER.test(body) || (GYP_INTERP.test(body) && GYP_PAYLOAD_API.test(body))) { gypCommandExec = true; break; }
290
+ }
291
+ if (gypCommandExec) {
292
+ threats.push({
293
+ type: 'gyp_command_exec',
294
+ severity: 'CRITICAL',
295
+ message: `binding.gyp uses GYP command-substitution (<!(...) / <!@(...)) running a non-build command at install time via node-gyp, no lifecycle script required (Phantom Gyp pattern).`,
296
+ file: 'binding.gyp'
297
+ });
298
+ }
299
+
255
300
  if (hasShellActions) {
256
301
  threats.push({
257
302
  type: 'native_addon_install',
package/src/scoring.js CHANGED
@@ -130,7 +130,9 @@ const PACKAGE_LEVEL_TYPES = new Set([
130
130
  // audit DF-C1: emitted when MAX_GRAPH_NODES exceeded so cross-file blind spot is visible in scoring
131
131
  'large_package_graph_truncated',
132
132
  // audit MR-C1: informational signal that the scan target is a monorepo root (per-workspace scoring TBD)
133
- 'monorepo_detected'
133
+ 'monorepo_detected',
134
+ // Phantom Gyp: binding.gyp command-substitution is a package-level (manifest) finding
135
+ 'gyp_command_exec'
134
136
  ]);
135
137
 
136
138
  // ============================================