muaddib-scanner 2.11.75 → 2.11.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.githooks/pre-commit +18 -0
- package/README.md +15 -6
- package/package.json +1 -2
- package/{self-scan-v2.11.75.json → self-scan-v2.11.77.json} +1 -1
- package/src/commands/safe-install.js +8 -3
- package/src/monitor/daemon.js +34 -22
- package/src/monitor/ingestion.js +43 -6
- package/src/monitor/queue.js +120 -21
- package/src/monitor/scan-queue.js +100 -7
- package/src/monitor/state.js +24 -1
- package/src/monitor/webhook.js +71 -11
- package/src/scanner/temporal-analysis.js +8 -0
- package/src/scanner/temporal-ast-diff.js +5 -0
- package/.dockerignore +0 -7
- package/.env.example +0 -43
- package/ml-retrain/auto-labeler/auto_labeler.py +0 -312
- package/ml-retrain/auto-labeler/ghsa_checker.py +0 -169
- package/ml-retrain/auto-labeler/labeler.py +0 -256
- package/ml-retrain/auto-labeler/npm_checker.py +0 -228
- package/ml-retrain/auto-labeler/ossf_index.py +0 -178
- package/ml-retrain/auto-labeler/requirements.txt +0 -1
- package/ml-retrain/confusion-matrix.png +0 -0
- package/ml-retrain/model-trees-retrained.js +0 -12
- package/ml-retrain/retrain-report.json +0 -225
- package/ml-retrain/retrain.py +0 -974
- package/sbom.json +0 -0
- package/src/ml/train-bundler-detector.py +0 -725
- package/src/ml/train-xgboost.py +0 -957
- package/tools/export-model-js.py +0 -160
- package/tools/requirements-ml.txt +0 -5
- package/tools/train-classifier.py +0 -333
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# Pre-commit guard — block committing a typosquat/forbidden dependency
|
|
3
|
+
# (loadash, lodash, lodahs, …). Local mirror of the CI gate
|
|
4
|
+
# (scripts/check-deps-typosquats.js, run in the `test` job of .github/workflows/scan.yml)
|
|
5
|
+
# and defense-in-depth with the package.json `preinstall` denylist.
|
|
6
|
+
#
|
|
7
|
+
# Enable once per clone: git config core.hooksPath .githooks
|
|
8
|
+
#
|
|
9
|
+
# This repo uses NEITHER lodash NOR loadash (CLAUDE.md interdiction). The loadash
|
|
10
|
+
# typosquat has re-entered package.json repeatedly; this stops it at commit time.
|
|
11
|
+
|
|
12
|
+
node scripts/check-deps-typosquats.js
|
|
13
|
+
status=$?
|
|
14
|
+
if [ "$status" -ne 0 ]; then
|
|
15
|
+
echo "pre-commit: ABORTED — remove the typosquat dependency from package.json (see above)." >&2
|
|
16
|
+
exit 1
|
|
17
|
+
fi
|
|
18
|
+
exit 0
|
package/README.md
CHANGED
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
|
|
31
31
|
npm and PyPI supply-chain attacks are exploding. Shai-Hulud compromised 25K+ repos in 2025. Existing tools detect threats but don't help you respond.
|
|
32
32
|
|
|
33
|
-
MUAD'DIB combines **20 parallel scanners** (
|
|
33
|
+
MUAD'DIB combines **20 parallel scanners** (264 detection rules), a **deobfuscation engine**, **inter-module dataflow analysis**, **compound scoring** (17 compound rules), and a gVisor/Docker sandbox to detect known threats and suspicious behavioral patterns in npm and PyPI packages. An XGBoost classifier exists in the codebase but is **currently inactive** (see [Evaluation Metrics](#evaluation-metrics) → ML Classifier section).
|
|
34
34
|
|
|
35
35
|
---
|
|
36
36
|
|
|
@@ -202,9 +202,9 @@ muaddib replay # Ground truth validation (90/94 TPR@3, v2.11
|
|
|
202
202
|
| Python Source (PYSRC) | Import-time / install-time RCE patterns in `__init__.py` / `setup.py` (v2.11.41 — closes TrapDoor PyPI gap) |
|
|
203
203
|
| Python AST (PYAST) | Tree-sitter-Python AST with taint-aware detectors (v2.11.42+) |
|
|
204
204
|
|
|
205
|
-
###
|
|
205
|
+
### 264 detection rules
|
|
206
206
|
|
|
207
|
-
All rules (
|
|
207
|
+
All rules (259 RULES + 5 PARANOID) are mapped to MITRE ATT&CK techniques. See [SECURITY.md](SECURITY.md#detection-rules-v21176) for the complete rules reference.
|
|
208
208
|
|
|
209
209
|
### Detected campaigns
|
|
210
210
|
|
|
@@ -278,7 +278,7 @@ With pre-commit framework:
|
|
|
278
278
|
```yaml
|
|
279
279
|
repos:
|
|
280
280
|
- repo: https://github.com/DNSZLSK/muad-dib
|
|
281
|
-
rev: v2.11.
|
|
281
|
+
rev: v2.11.76
|
|
282
282
|
hooks:
|
|
283
283
|
- id: muaddib-scan
|
|
284
284
|
```
|
|
@@ -303,11 +303,20 @@ These are the numbers a user gets when running `muaddib scan` against npm or PyP
|
|
|
303
303
|
| **FPR PyPI** (v2.11.48, first honest measurement) | **9.68%** (12/124 scanned, 132 total) | **Track D fixed the PyPI downloader** — removed `pip --no-binary :all:` flag (forced compile of wheel-only packages, timed out 38% of the time) + added `.whl` extraction via `extractArchive()`. Brought 42 previously-skipped giants (numpy/pandas/django/matplotlib/scikit-learn/...) into scope. All 12 FPs cluster at score 25-35: this is the cap-PyPI-35 artifact, not new rule misfires. Lifting the cap (Track E) would drop FPR PyPI to ≈0%. 8 residual fails are >500MB packages (torch, tensorflow, scipy, opencv-python, ansible…) hitting the 30s `PACK_TIMEOUT_MS`. |
|
|
304
304
|
| **ADR** (Adversarial + Holdout, v2.11.48) | **96.26%** (103/107) | 67 adversarial + 40 holdout, global threshold=20. Stable vs v2.10.95. |
|
|
305
305
|
|
|
306
|
-
**
|
|
306
|
+
**4132 tests** across 115 files. **264 rules** (259 RULES + 5 PARANOID; v2.11.67/70 Phantom Gyp added PKG-023 + COMPOUND-017).
|
|
307
307
|
|
|
308
308
|
**Known issues (v2.11.48):**
|
|
309
309
|
- *Cap PyPI à 35/100*: Python samples plafonnent à `riskScore=35` even when `globalRiskScore=100`. Confirmed empirically — all 12 PyPI FPs at score 25-35 (flask 32, django 35, tornado 35, bottle 30, pandas 25, matplotlib 25, plotly 25, bokeh 25, pymongo 35, coverage 32, fabric 35, websockets 35). Lifting the cap will simultaneously drop FPR PyPI to ≈0% and unblock PyPI MALWARE detection at higher thresholds. Track E target.
|
|
310
310
|
|
|
311
|
+
### Operational coverage (v2.11.67-76)
|
|
312
|
+
|
|
313
|
+
The static ground-truth TPR above is measured offline. Since v2.11.67 the monitor also tracks **operational** coverage on live npm/PyPI ingestion:
|
|
314
|
+
- A per-scan **ledger** (`data/scan-ledger.jsonl`) records every scanned package's outcome; `computeLedgerRollup()` produces a 24h rollup (`alertRate`, per-ecosystem). Note: `alertRate` is a throughput signal, **not** detection TPR.
|
|
315
|
+
- An active **GHSA poller** (~15 min; npm, pypi, crates) builds an authoritative "what should we have caught" denominator (`data/ghsa-malware.jsonl`), plus a **feed-health** alarm that fires when an IOC feed silently goes dark.
|
|
316
|
+
- The Phase 5 **coverage-audit** (`scripts/coverage-audit.js`, daily 05:00 UTC) joins that denominator against ledger outcomes + the tarball archive to compute an honest GHSA-denominated **operational TPR** (`alerted / total`), and surfaces `scannedClean` misses as human-gated ground-truth candidates.
|
|
317
|
+
|
|
318
|
+
This operational TPR is the real production detection rate, distinct from the static GT TPR (which has not been re-measured since v2.11.48).
|
|
319
|
+
|
|
311
320
|
### ML Classifier (offline only)
|
|
312
321
|
|
|
313
322
|
`src/ml/classifier.js` is **not wired into `muaddib scan`**. The XGBoost model is currently exercised only by `muaddib evaluate` (offline metric replay) and `muaddib monitor` (LOG-ONLY since 2026-04-08, model collapsed pending retrain — see `src/monitor/queue.js:628`). The v2.11.48 evaluate-time replay shows the same 1.10% FPR (no additional FPs filtered) — kept as a reference for retrain validation, but the published operational FPR is the rules-only number above.
|
|
@@ -371,7 +380,7 @@ npm test
|
|
|
371
380
|
|
|
372
381
|
### Testing
|
|
373
382
|
|
|
374
|
-
- **
|
|
383
|
+
- **4132 tests** across 115 modular test files
|
|
375
384
|
- **56 fuzz tests** - Malformed inputs, ReDoS, unicode, binary
|
|
376
385
|
- **Datadog 17K benchmark** - 14,587 confirmed malware samples (in-scope)
|
|
377
386
|
- **Ground truth validation** - 96 real-world attacks (95.74% TPR@3, 88.30% TPR@20 — v2.11.48 full measure on 94 in-scope)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "muaddib-scanner",
|
|
3
|
-
"version": "2.11.
|
|
3
|
+
"version": "2.11.77",
|
|
4
4
|
"description": "Supply-chain threat detection & response for npm & PyPI/Python",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -52,7 +52,6 @@
|
|
|
52
52
|
"acorn-walk": "8.3.5",
|
|
53
53
|
"adm-zip": "0.5.17",
|
|
54
54
|
"js-yaml": "4.2.0",
|
|
55
|
-
"loadash": "^1.0.0",
|
|
56
55
|
"web-tree-sitter": "^0.26.9"
|
|
57
56
|
},
|
|
58
57
|
"devDependencies": {
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
const fs = require('fs');
|
|
2
2
|
const path = require('path');
|
|
3
|
-
|
|
3
|
+
// NB: keep the module object (cp.spawnSync), NOT a destructured `spawnSync`. Destructuring
|
|
4
|
+
// captures the original reference at load time, which makes the function impossible to mock
|
|
5
|
+
// from tests (`cp.spawnSync = ...` wouldn't be seen here) — that's exactly how the
|
|
6
|
+
// safe-install test's mock silently failed and a real `npm install loadash` ran every test
|
|
7
|
+
// run, re-contaminating package.json. Property access keeps it interceptable.
|
|
8
|
+
const cp = require('child_process');
|
|
4
9
|
const { loadCachedIOCs } = require('../ioc/updater.js');
|
|
5
10
|
const { REHABILITATED_PACKAGES, NPM_PACKAGE_REGEX } = require('../shared/constants.js');
|
|
6
11
|
|
|
@@ -172,7 +177,7 @@ async function scanPackageRecursive(pkg, depth = 0, maxDepth = 3) {
|
|
|
172
177
|
// Get the package info (uses spawnSync to avoid injection)
|
|
173
178
|
let pkgInfo;
|
|
174
179
|
try {
|
|
175
|
-
const result = spawnSync('npm', ['view', pkgName, '--json'], { encoding: 'utf8', shell: false });
|
|
180
|
+
const result = cp.spawnSync('npm', ['view', pkgName, '--json'], { encoding: 'utf8', shell: false });
|
|
176
181
|
if (result.status !== 0 || !result.stdout) {
|
|
177
182
|
if (depth === 0) console.log(`[!] Package ${pkgName} not found on npm`);
|
|
178
183
|
return { safe: false, package: pkgName, reason: 'npm_unreachable', source: 'npm-registry', description: 'Package not found on npm registry', depth };
|
|
@@ -276,7 +281,7 @@ async function safeInstall(packages, options = {}) {
|
|
|
276
281
|
if (isDev) npmArgs.push('--save-dev');
|
|
277
282
|
if (isGlobal) npmArgs.push('-g');
|
|
278
283
|
|
|
279
|
-
const result = spawnSync('npm', npmArgs, { stdio: 'inherit', shell: false });
|
|
284
|
+
const result = cp.spawnSync('npm', npmArgs, { stdio: 'inherit', shell: false });
|
|
280
285
|
|
|
281
286
|
if (result.status !== 0) {
|
|
282
287
|
console.log('');
|
package/src/monitor/daemon.js
CHANGED
|
@@ -5,14 +5,15 @@ const os = require('os');
|
|
|
5
5
|
const v8 = require('v8');
|
|
6
6
|
const { isDockerAvailable, SANDBOX_CONCURRENCY_MAX, killAllSandboxContainers } = require('../sandbox/index.js');
|
|
7
7
|
const { setVerboseMode, isSandboxEnabled, isCanaryEnabled, isLlmDetectiveEnabled, getLlmDetectiveMode, DOWNLOADS_CACHE_TTL } = require('./classify.js');
|
|
8
|
-
const { loadState, saveState, loadDailyStats, saveDailyStats, purgeTarballCache,
|
|
8
|
+
const { loadState, saveState, loadDailyStats, saveDailyStats, purgeTarballCache, isDailyReportDue, atomicWriteFileSync, saveNpmSeq, ALERTS_FILE, runStateMigrations, loadRecentlyScanned, saveRecentlyScanned } = require('./state.js');
|
|
9
9
|
const { isTemporalEnabled, isTemporalAstEnabled, isTemporalPublishEnabled, isTemporalMaintainerEnabled } = require('./temporal.js');
|
|
10
|
-
const { pendingGrouped, flushScopeGroup, sendDailyReport,
|
|
10
|
+
const { pendingGrouped, flushScopeGroup, sendDailyReport, alertedPackageRules, ALERTED_PACKAGES_MAX: MAX_ALERTED_PACKAGES } = require('./webhook.js');
|
|
11
11
|
const { poll, getPollBackoffMs } = require('./ingestion.js');
|
|
12
12
|
const { ensureWorkers, drainWorkers, getTargetConcurrency, setTargetConcurrency, getActiveWorkers, terminateAllWorkers } = require('./queue.js');
|
|
13
13
|
const { computeTarget, ADJUST_INTERVAL_MS, BASE_CONCURRENCY } = require('./adaptive-concurrency.js');
|
|
14
14
|
const { startHealthcheck } = require('./healthcheck.js');
|
|
15
15
|
const { startDeferredWorker, stopDeferredWorker, persistDeferredQueue, restoreDeferredQueue, clearDeferredQueue } = require('./deferred-sandbox.js');
|
|
16
|
+
const { evictFromScanQueueBulk } = require('./scan-queue.js');
|
|
16
17
|
const { startGhsaPoller, stopGhsaPoller } = require('../ioc/ghsa-poller.js');
|
|
17
18
|
const { cleanupOldArchives, getRetentionDays, startPeriodicCleanup } = require('./tarball-archive.js');
|
|
18
19
|
const { clearMetadataCache } = require('../scanner/temporal-analysis.js');
|
|
@@ -532,8 +533,13 @@ function pruneMemoryCaches(recentlyScanned, downloadsCache, alertedPackageRules)
|
|
|
532
533
|
* Worker spawning is gated separately in the main loop (ensureWorkers skipped at HIGH+).
|
|
533
534
|
* Ingestion is gated in ingestion.js via getMemoryPressureLevel() (skipped at CRITICAL+).
|
|
534
535
|
*/
|
|
535
|
-
function handleMemoryPressure(level, ratio, recentlyScanned, downloadsCache, scanQueue) {
|
|
536
|
+
function handleMemoryPressure(level, ratio, rssRatio, recentlyScanned, downloadsCache, scanQueue, stats) {
|
|
536
537
|
const pct = (ratio * 100).toFixed(0);
|
|
538
|
+
// Show BOTH arms: an EMERGENCY almost always fires on RSS (off-heap — gVisor containers,
|
|
539
|
+
// tarball buffers) while the heap sits low (~15%). Logging only heap made every breaker
|
|
540
|
+
// line read "heap at 15%" and hid the real cause; memPctLabel surfaces which arm tripped.
|
|
541
|
+
const rssPct = (rssRatio != null && isFinite(rssRatio)) ? (rssRatio * 100).toFixed(0) : '?';
|
|
542
|
+
const memPctLabel = `heap ${pct}% / rss ${rssPct}%`;
|
|
537
543
|
// Structured summary of what the breaker actually did this tick. Returned (the poll loop
|
|
538
544
|
// at the call site ignores it) so the reclaim is observable to callers and tests without
|
|
539
545
|
// scraping console output — CLAUDE.md §3 "Toujours logger un resume". The two kill fields
|
|
@@ -543,7 +549,7 @@ function handleMemoryPressure(level, ratio, recentlyScanned, downloadsCache, sca
|
|
|
543
549
|
|
|
544
550
|
// HIGH (85%+): clear auxiliary caches — same as old emergency prune
|
|
545
551
|
if (level >= MEMORY_PRESSURE_LEVELS.HIGH) {
|
|
546
|
-
console.error(`[MONITOR] MEMORY PRESSURE HIGH:
|
|
552
|
+
console.error(`[MONITOR] MEMORY PRESSURE HIGH: ${memPctLabel} — pruning caches, stopping new workers`);
|
|
547
553
|
recentlyScanned.clear();
|
|
548
554
|
downloadsCache.clear();
|
|
549
555
|
alertedPackageRules.clear();
|
|
@@ -552,7 +558,7 @@ function handleMemoryPressure(level, ratio, recentlyScanned, downloadsCache, sca
|
|
|
552
558
|
|
|
553
559
|
// CRITICAL (90%+): clear scanner caches, force GC
|
|
554
560
|
if (level >= MEMORY_PRESSURE_LEVELS.CRITICAL) {
|
|
555
|
-
console.error(`[MONITOR] MEMORY PRESSURE CRITICAL:
|
|
561
|
+
console.error(`[MONITOR] MEMORY PRESSURE CRITICAL: ${memPctLabel} — stopping ingestion, clearing scanner caches`);
|
|
556
562
|
// temporal-analysis._metadataCache (200 entries × full npm registry metadata)
|
|
557
563
|
try { clearMetadataCache(); } catch {}
|
|
558
564
|
// typosquat metadataCache (500 entries × npm registry metadata for typosquat scoring)
|
|
@@ -578,21 +584,30 @@ function handleMemoryPressure(level, ratio, recentlyScanned, downloadsCache, sca
|
|
|
578
584
|
if (level >= MEMORY_PRESSURE_LEVELS.EMERGENCY) {
|
|
579
585
|
const queueBefore = scanQueue.length;
|
|
580
586
|
if (queueBefore > EMERGENCY_QUEUE_KEEP) {
|
|
581
|
-
//
|
|
582
|
-
//
|
|
583
|
-
//
|
|
584
|
-
//
|
|
585
|
-
|
|
586
|
-
// splice
|
|
587
|
-
scanQueue
|
|
587
|
+
// Protected-aware bulk eviction — SINGLE SOURCE OF TRUTH with the queue-cap path
|
|
588
|
+
// (scan-queue.js evictFromScanQueueBulk / enqueueScan share _isProtected). Keeps
|
|
589
|
+
// IOC-match / burst / first-publish / ATO scans, drops the oldest UNPROTECTED items
|
|
590
|
+
// first (newest survive — most likely to still exist for re-scan), protected only as
|
|
591
|
+
// a last resort, and LEDGERS every drop. Closes the v2.10.88 gap where the raw
|
|
592
|
+
// splice(0,n) silently dropped protected scans (CLAUDE.md "ne jamais perdre de scan").
|
|
593
|
+
const { dropped, droppedProtected } = evictFromScanQueueBulk(scanQueue, EMERGENCY_QUEUE_KEEP, 'mem_emergency');
|
|
588
594
|
summary.queueDropped = dropped;
|
|
589
|
-
|
|
595
|
+
summary.queueDroppedProtected = droppedProtected;
|
|
596
|
+
if (stats) {
|
|
597
|
+
stats.queueEmergencyDrops = (stats.queueEmergencyDrops || 0) + dropped;
|
|
598
|
+
if (droppedProtected) stats.queueEmergencyProtectedDrops = (stats.queueEmergencyProtectedDrops || 0) + droppedProtected;
|
|
599
|
+
}
|
|
600
|
+
console.error(`[MONITOR] MEMORY EMERGENCY: ${memPctLabel} — truncated queue ${queueBefore} → ${scanQueue.length} (dropped ${dropped} oldest UNPROTECTED${droppedProtected ? ` + ${droppedProtected} protected as last resort` : ''}, all ledgered)`);
|
|
590
601
|
}
|
|
591
602
|
// Clear deferred sandbox queue (holds full staticResult objects)
|
|
592
603
|
const deferredDropped = clearDeferredQueue();
|
|
593
604
|
summary.deferredDropped = deferredDropped;
|
|
594
605
|
if (deferredDropped > 0) {
|
|
595
|
-
|
|
606
|
+
// Observability only (counter, NOT a ledger 'dropped' entry): the deferred queue holds
|
|
607
|
+
// post-scan sandbox ENRICHMENT for packages already statically scanned + alerted, so
|
|
608
|
+
// clearing it is not a coverage loss — ledgering them as 'dropped' would mislabel them.
|
|
609
|
+
if (stats) stats.deferredDroppedEmergency = (stats.deferredDroppedEmergency || 0) + deferredDropped;
|
|
610
|
+
console.error(`[MONITOR] MEMORY EMERGENCY: cleared ${deferredDropped} deferred sandbox items (post-scan enrichment only — primary alerts already sent)`);
|
|
596
611
|
}
|
|
597
612
|
// Free the off-heap leak that queue truncation can't touch: orphaned sandbox
|
|
598
613
|
// containers (gVisor runsc survives `docker kill`) and wedged scan workers.
|
|
@@ -642,13 +657,10 @@ function reportStats(stats) {
|
|
|
642
657
|
stats.lastReportTime = Date.now();
|
|
643
658
|
}
|
|
644
659
|
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
const { hasReportBeenSentToday } = require('./state.js');
|
|
650
|
-
return !hasReportBeenSentToday(stats);
|
|
651
|
-
}
|
|
660
|
+
// isDailyReportDue is the canonical gate in state.js (imported above) — re-exported below
|
|
661
|
+
// so monitor.js (daemonModule.isDailyReportDue) keeps resolving. The old local copy used a
|
|
662
|
+
// `hour !== 8` gate that lost a whole day whenever the daemon missed the single 08:00 minute
|
|
663
|
+
// (OOM crash-loop); state.js uses the catch-up `hour >= 8` gate instead.
|
|
652
664
|
|
|
653
665
|
// ─── P1.0 — memory-trend instrumentation ───
|
|
654
666
|
// Append one sample per memory-watchdog tick so the off-heap leak can be localised
|
|
@@ -1087,7 +1099,7 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
|
|
|
1087
1099
|
|
|
1088
1100
|
// Graduated response at HIGH+
|
|
1089
1101
|
if (pressureLevel >= MEMORY_PRESSURE_LEVELS.HIGH) {
|
|
1090
|
-
handleMemoryPressure(pressureLevel, heapRatio, recentlyScanned, downloadsCache, scanQueue);
|
|
1102
|
+
handleMemoryPressure(pressureLevel, heapRatio, rssRatio, recentlyScanned, downloadsCache, scanQueue, stats);
|
|
1091
1103
|
}
|
|
1092
1104
|
lastMemoryLogTime = Date.now();
|
|
1093
1105
|
}
|
package/src/monitor/ingestion.js
CHANGED
|
@@ -13,7 +13,7 @@ const { loadCachedIOCs } = require('../ioc/updater.js');
|
|
|
13
13
|
const { enqueueScan } = require('./scan-queue.js');
|
|
14
14
|
const {
|
|
15
15
|
saveNpmSeq, CHANGES_STREAM_URL, CHANGES_LIMIT, CHANGES_CATCHUP_MAX,
|
|
16
|
-
savePypiSerial, PYPI_XMLRPC_URL, PYPI_CATCHUP_MAX
|
|
16
|
+
savePypiSerial, PYPI_XMLRPC_URL, PYPI_CATCHUP_MAX, appendScanLedger
|
|
17
17
|
} = require('./state.js');
|
|
18
18
|
const { sendIOCPreAlert, sendCampaignPreAlert } = require('./webhook.js');
|
|
19
19
|
|
|
@@ -109,6 +109,14 @@ function httpsGet(url, timeoutMs = 30_000, deadlineMs = Math.max(timeoutMs * 2,
|
|
|
109
109
|
clearTimeout(deadline);
|
|
110
110
|
return httpsGet(location, timeoutMs, deadlineMs).then(resolve, reject);
|
|
111
111
|
}
|
|
112
|
+
if (res.statusCode === 429) {
|
|
113
|
+
res.resume();
|
|
114
|
+
// Coordinated backoff: drain the SHARED token bucket so every in-flight registry fetch
|
|
115
|
+
// slows together. This high-volume packument/changes path must signal 429 like the
|
|
116
|
+
// metadata path (npm-registry.js) does — not just acquire a slot (CLAUDE.md 429 storm).
|
|
117
|
+
try { require('../shared/http-limiter.js').signal429(); } catch { /* limiter best-effort */ }
|
|
118
|
+
return done(new Error(`HTTP 429 rate limited for ${url}`));
|
|
119
|
+
}
|
|
112
120
|
if (res.statusCode < 200 || res.statusCode >= 300) {
|
|
113
121
|
res.resume();
|
|
114
122
|
return done(new Error(`HTTP ${res.statusCode} for ${url}`));
|
|
@@ -166,6 +174,11 @@ function httpsPost(url, body, headers = {}, timeoutMs = 30_000, deadlineMs = Mat
|
|
|
166
174
|
if (err) reject(err); else resolve(value);
|
|
167
175
|
};
|
|
168
176
|
req = _deps.https.request(options, (res) => {
|
|
177
|
+
if (res.statusCode === 429) {
|
|
178
|
+
res.resume();
|
|
179
|
+
try { require('../shared/http-limiter.js').signal429(); } catch { /* limiter best-effort */ }
|
|
180
|
+
return done(new Error(`HTTP 429 rate limited for POST ${url}`));
|
|
181
|
+
}
|
|
169
182
|
if (res.statusCode < 200 || res.statusCode >= 300) {
|
|
170
183
|
res.resume();
|
|
171
184
|
return done(new Error(`HTTP ${res.statusCode} for POST ${url}`));
|
|
@@ -370,7 +383,8 @@ const RECENT_PUBLISH_MAX = 5;
|
|
|
370
383
|
* @returns {Object|null} - {
|
|
371
384
|
* version, tarball, unpackedSize, scripts, homepage, description,
|
|
372
385
|
* latestTagVersion, // dist-tags.latest (may differ from `version` under ATO)
|
|
373
|
-
* recentVersions: [{ version, tarball, unpackedSize, scripts }, ...]
|
|
386
|
+
* recentVersions: [{ version, tarball, unpackedSize, scripts }, ...], // capped at maxRecent
|
|
387
|
+
* recentWindowCount, // TRUE (uncapped) count of versions in the window (Phase 2b burst)
|
|
374
388
|
* } or null if no usable version found
|
|
375
389
|
*/
|
|
376
390
|
function selectMostRecentVersion(packument, options = {}) {
|
|
@@ -417,16 +431,28 @@ function selectMostRecentVersion(packument, options = {}) {
|
|
|
417
431
|
description: (typeof versionData.description === 'string') ? versionData.description : '',
|
|
418
432
|
latestTagVersion,
|
|
419
433
|
recentVersions: [],
|
|
434
|
+
droppedBurstVersions: [],
|
|
420
435
|
};
|
|
421
436
|
|
|
422
|
-
// Burst extras: other versions published within the recent window, excluding
|
|
423
|
-
//
|
|
424
|
-
//
|
|
437
|
+
// Burst extras: other versions published within the recent window, excluding the
|
|
438
|
+
// most-recent one. The enqueue list is bounded by maxRecent, but recentWindowCount is
|
|
439
|
+
// the TRUE (uncapped) number of versions in the window — Phase 2b burst detection uses it
|
|
440
|
+
// so a 96-version Miasma burst is distinguishable from a legit 3-5 patch-release day (the
|
|
441
|
+
// capped list alone tops out at maxRecent+1 and can't tell them apart).
|
|
442
|
+
result.recentWindowCount = 1; // includes the most-recent version itself
|
|
425
443
|
if (versionTimes.length > 1) {
|
|
426
444
|
const cutoff = versionTimes[0][1] - recentWindowMs;
|
|
427
|
-
for (let i = 1; i < versionTimes.length
|
|
445
|
+
for (let i = 1; i < versionTimes.length; i++) {
|
|
428
446
|
const [v, ts] = versionTimes[i];
|
|
429
447
|
if (ts < cutoff) break; // sorted desc, so once we cross the cutoff we're done
|
|
448
|
+
result.recentWindowCount++;
|
|
449
|
+
if (result.recentVersions.length >= maxRecent) {
|
|
450
|
+
// Burst beyond the enqueue cap: collect the version so the caller ledgers it as a
|
|
451
|
+
// coverage loss (it is never enqueued/scanned). Keeps a Miasma-style burst that
|
|
452
|
+
// outruns maxRecent visible instead of vanishing silently (CLAUDE.md "no silent caps").
|
|
453
|
+
result.droppedBurstVersions.push(v);
|
|
454
|
+
continue; // enqueue list capped; count continues
|
|
455
|
+
}
|
|
430
456
|
const vData = versions[v];
|
|
431
457
|
if (!vData) continue;
|
|
432
458
|
result.recentVersions.push({
|
|
@@ -496,6 +522,16 @@ async function getNpmLatestTarball(packageName) {
|
|
|
496
522
|
age_days: null, version_count: 0,
|
|
497
523
|
};
|
|
498
524
|
}
|
|
525
|
+
// A3: ledger burst versions dropped by the maxRecent enqueue cap — they are never scanned,
|
|
526
|
+
// so record each as a 'dropped' coverage loss (source burst_extras_cap) for the coverage
|
|
527
|
+
// audit. Best-effort; never throws. selectMostRecentVersion stays pure (it only collects).
|
|
528
|
+
if (result.droppedBurstVersions && result.droppedBurstVersions.length) {
|
|
529
|
+
for (const v of result.droppedBurstVersions) {
|
|
530
|
+
try {
|
|
531
|
+
appendScanLedger({ name: packageName, version: v, ecosystem: 'npm', outcome: 'dropped', source: 'burst_extras_cap' });
|
|
532
|
+
} catch { /* ledger is best-effort */ }
|
|
533
|
+
}
|
|
534
|
+
}
|
|
499
535
|
// Stage 2.1 — extract reputation signals from the packument we already have,
|
|
500
536
|
// so triageRisk in queue.js doesn't have to refetch metadata via
|
|
501
537
|
// getPackageMetadata. Two fields are derivable from the packument alone:
|
|
@@ -819,6 +855,7 @@ async function pollNpmChanges(state, scanQueue, stats) {
|
|
|
819
855
|
unpackedSize: docMeta ? docMeta.unpackedSize : 0,
|
|
820
856
|
registryScripts: docMeta ? docMeta.scripts : null,
|
|
821
857
|
_cacheTrigger: cacheTrigger.shouldCache ? cacheTrigger : null,
|
|
858
|
+
firstPublish: cacheTrigger.shouldCache && cacheTrigger.reason === 'first_publish',
|
|
822
859
|
isIOCMatch: isKnownIOC
|
|
823
860
|
});
|
|
824
861
|
queued++;
|
package/src/monitor/queue.js
CHANGED
|
@@ -32,8 +32,7 @@ const {
|
|
|
32
32
|
tarballCacheKey,
|
|
33
33
|
tarballCachePath,
|
|
34
34
|
appendAlert,
|
|
35
|
-
|
|
36
|
-
hasReportBeenSentToday,
|
|
35
|
+
isDailyReportDue,
|
|
37
36
|
MAX_DAILY_ALERTS,
|
|
38
37
|
loadScanMemory,
|
|
39
38
|
shouldSuppressByMemory,
|
|
@@ -57,14 +56,14 @@ const {
|
|
|
57
56
|
buildAlertData,
|
|
58
57
|
persistAlert,
|
|
59
58
|
sendIOCPreAlert,
|
|
59
|
+
sendBurstPreAlert,
|
|
60
60
|
matchVersionedIOC,
|
|
61
61
|
buildCanaryExfiltrationWebhookEmbed,
|
|
62
62
|
getWebhookUrl,
|
|
63
63
|
computeReputationFactor,
|
|
64
64
|
triageRisk,
|
|
65
65
|
sendDailyReport,
|
|
66
|
-
alertedPackageRules
|
|
67
|
-
DAILY_REPORT_HOUR
|
|
66
|
+
alertedPackageRules
|
|
68
67
|
} = require('./webhook.js');
|
|
69
68
|
|
|
70
69
|
// From ./temporal.js
|
|
@@ -98,10 +97,11 @@ let _targetConcurrency = BASE_CONCURRENCY;
|
|
|
98
97
|
const SCAN_CONCURRENCY = BASE_CONCURRENCY; // legacy export — tests check this value
|
|
99
98
|
let _activeWorkers = 0;
|
|
100
99
|
const _workerPromises = new Set();
|
|
101
|
-
// Live static-scan Worker threads
|
|
102
|
-
//
|
|
103
|
-
// ASTs)
|
|
104
|
-
|
|
100
|
+
// Live static-scan Worker threads, mapped to the {name,version,ecosystem} of the scan they
|
|
101
|
+
// run — tracked so the daemon's EMERGENCY memory handler can terminate orphaned workers
|
|
102
|
+
// (each retains its isolate heap + parsed ASTs) AND name the in-flight scans it kills.
|
|
103
|
+
// Bounded by concurrency, so it stays tiny.
|
|
104
|
+
const _liveWorkers = new Map();
|
|
105
105
|
|
|
106
106
|
function getTargetConcurrency() { return _targetConcurrency; }
|
|
107
107
|
function setTargetConcurrency(n) { _targetConcurrency = Math.max(MIN_CONCURRENCY, Math.min(MAX_CONCURRENCY, n)); }
|
|
@@ -114,10 +114,20 @@ function getActiveWorkers() { return _activeWorkers; }
|
|
|
114
114
|
*/
|
|
115
115
|
function terminateAllWorkers() {
|
|
116
116
|
let n = 0;
|
|
117
|
-
|
|
118
|
-
|
|
117
|
+
const dropped = [];
|
|
118
|
+
for (const [w, item] of Array.from(_liveWorkers.entries())) {
|
|
119
|
+
try {
|
|
120
|
+
w.terminate(); n++;
|
|
121
|
+
if (item && item.name) dropped.push(`${item.name}@${item.version || '?'}`);
|
|
122
|
+
} catch { /* already gone */ }
|
|
119
123
|
_liveWorkers.delete(w);
|
|
120
124
|
}
|
|
125
|
+
if (dropped.length) {
|
|
126
|
+
// The terminate rejects each scan's worker promise; that reject propagates to
|
|
127
|
+
// scanPackage's catch, which ledgers it (outcome:'error', source scan_error) — so these
|
|
128
|
+
// in-flight scans are NOT lost from the scan-ledger. This line names them for the operator.
|
|
129
|
+
console.error(`[MONITOR] EMERGENCY worker-terminate killed ${dropped.length} in-flight scan(s): ${dropped.slice(0, 20).join(', ')}${dropped.length > 20 ? ` (+${dropped.length - 20} more)` : ''}`);
|
|
130
|
+
}
|
|
121
131
|
return n;
|
|
122
132
|
}
|
|
123
133
|
const SCAN_TIMEOUT_MS = 300_000; // 5 minutes per package (3 sandbox runs × 90s + static scan headroom)
|
|
@@ -130,6 +140,21 @@ const RECENTLY_SCANNED_MAX = 50_000; // FIFO cap for the dedup Set (P0c — boun
|
|
|
130
140
|
const FIRST_PUBLISH_SANDBOX_MAX_QUEUE = parseInt(process.env.MUADDIB_FIRST_PUBLISH_SANDBOX_MAX_QUEUE, 10) || 10;
|
|
131
141
|
const FIRST_PUBLISH_SANDBOX_ENABLED = process.env.MUADDIB_FIRST_PUBLISH_SANDBOX !== '0';
|
|
132
142
|
|
|
143
|
+
// Phase 2b: burst (Miasma) pre-alert. A burst = >= this many versions of ONE name in the
|
|
144
|
+
// recent-publish window (the TRUE uncapped count, selectMostRecentVersion.recentWindowCount).
|
|
145
|
+
// Default 10: detection is PER-NAME, so legit multi-PLATFORM publishers (different names,
|
|
146
|
+
// e.g. @opencode-ai/cli-*-* binaries) are never caught; legit same-name release days rarely
|
|
147
|
+
// reach 10; Miasma's 96-in-72s clears it easily. Per-name + deduped + non-scoring (Discord
|
|
148
|
+
// heads-up only, no FPR impact). Env-tunable up if a feed proves noisy.
|
|
149
|
+
const BURST_PREALERT_MIN_VERSIONS = (() => {
|
|
150
|
+
const n = parseInt(process.env.MUADDIB_BURST_MIN_VERSIONS, 10);
|
|
151
|
+
return Number.isFinite(n) && n >= 2 ? n : 10;
|
|
152
|
+
})();
|
|
153
|
+
// Dedup burst pings: one per name per process window (bounded — cleared at the cap so it
|
|
154
|
+
// can never grow without limit, CLAUDE.md §2).
|
|
155
|
+
const _burstAlerted = new Set();
|
|
156
|
+
const BURST_ALERTED_MAX = 20_000;
|
|
157
|
+
|
|
133
158
|
// Stage 3 — sandbox gate. Static-score threshold below which T1b/T2 packages
|
|
134
159
|
// are NOT sandboxed (static result alone is authoritative). Tightens the prior
|
|
135
160
|
// "T1b sandbox if score >= 25 or queue < 20" to remove low-signal sandbox runs
|
|
@@ -372,7 +397,8 @@ function runScanInWorker(extractedDir, timeoutMs, scanContext = null, signal = n
|
|
|
372
397
|
const worker = new Worker(SCAN_WORKER_PATH, {
|
|
373
398
|
workerData: { extractedDir, scanContext: scanContext || {} }
|
|
374
399
|
});
|
|
375
|
-
|
|
400
|
+
const _sc = scanContext || {};
|
|
401
|
+
_liveWorkers.set(worker, { name: _sc.name, version: _sc.version, ecosystem: _sc.ecosystem });
|
|
376
402
|
|
|
377
403
|
let settled = false;
|
|
378
404
|
let timer = null;
|
|
@@ -623,6 +649,11 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
623
649
|
// deliberately hangs the parser to evade analysis would otherwise be relabelled
|
|
624
650
|
// benign. Count as inconclusive (excluded from the FP/TP denominator).
|
|
625
651
|
updateScanStats('sandbox_inconclusive');
|
|
652
|
+
// Ledger the inconclusive timeout — the 'static_timeout' outcome existed but was
|
|
653
|
+
// emitted nowhere, so a parser-hang evasion vanished from coverage. Best-effort.
|
|
654
|
+
try {
|
|
655
|
+
appendScanLedger({ name, version, ecosystem, outcome: 'static_timeout', source: 'static_timeout' });
|
|
656
|
+
} catch { /* ledger is best-effort */ }
|
|
626
657
|
return { sandboxResult: null, staticClean: false };
|
|
627
658
|
}
|
|
628
659
|
throw staticErr;
|
|
@@ -1199,6 +1230,12 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
1199
1230
|
stats.scanned++;
|
|
1200
1231
|
stats.totalTimeMs += Date.now() - startTime;
|
|
1201
1232
|
console.error(`[MONITOR] ERROR scanning ${name}@${version}: ${err.message}`);
|
|
1233
|
+
// Ledger the terminal failure so the scan-ledger never over-states coverage (an errored
|
|
1234
|
+
// package is NOT clean). Also captures EMERGENCY worker-terminate losses, whose reject
|
|
1235
|
+
// propagates here (CLAUDE.md "no silent caps"). Best-effort; never throws.
|
|
1236
|
+
try {
|
|
1237
|
+
appendScanLedger({ name, version, ecosystem, outcome: 'error', source: 'scan_error' });
|
|
1238
|
+
} catch { /* ledger is best-effort */ }
|
|
1202
1239
|
return { sandboxResult: null, staticClean: false };
|
|
1203
1240
|
} finally {
|
|
1204
1241
|
// Cleanup temp dir
|
|
@@ -1240,15 +1277,9 @@ function timeoutPromise(ms) {
|
|
|
1240
1277
|
});
|
|
1241
1278
|
}
|
|
1242
1279
|
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
*/
|
|
1247
|
-
function isDailyReportDue(stats) {
|
|
1248
|
-
const parisHour = getParisHour();
|
|
1249
|
-
if (parisHour < DAILY_REPORT_HOUR) return false;
|
|
1250
|
-
return !hasReportBeenSentToday(stats);
|
|
1251
|
-
}
|
|
1280
|
+
// isDailyReportDue is the canonical gate in state.js (imported above), called per scan in
|
|
1281
|
+
// processQueueItem below. Previously a local `parisHour < 8` copy here diverged from the
|
|
1282
|
+
// daemon's `!== 8` copy; unifying in state.js removes the divergence. Still re-exported below.
|
|
1252
1283
|
|
|
1253
1284
|
/**
|
|
1254
1285
|
* Process a single item from the scan queue.
|
|
@@ -1342,6 +1373,37 @@ function computeWorkersToSpawn(targetConcurrency, activeWorkers, queueLength) {
|
|
|
1342
1373
|
return Math.max(0, Math.min(targetConcurrency - activeWorkers, queueLength));
|
|
1343
1374
|
}
|
|
1344
1375
|
|
|
1376
|
+
// ── RSS-aware worker admission (P1 OOM durable fix) ──
|
|
1377
|
+
// The pressure breaker is reactive: it stops spawning at HIGH, but the workers already in
|
|
1378
|
+
// flight overshoot RSS by ~2GB (each isolate + gVisor sandbox ~0.55GB, draining up to
|
|
1379
|
+
// SCAN_TIMEOUT) before EMERGENCY truncates the queue + kills them. This caps the OVERSHOOT at
|
|
1380
|
+
// the source — refuse a new spawn when current RSS + one worker's footprint would breach a
|
|
1381
|
+
// soft ceiling (default 80% of the EMERGENCY RSS limit), leaving headroom for in-flight drain.
|
|
1382
|
+
const RSS_SOFT_LIMIT_MB = (() => {
|
|
1383
|
+
const parsed = parseInt(process.env.MUADDIB_RSS_SOFT_LIMIT_MB, 10);
|
|
1384
|
+
if (Number.isFinite(parsed) && parsed > 0) return parsed;
|
|
1385
|
+
const hard = parseInt(process.env.MUADDIB_RSS_LIMIT_MB, 10);
|
|
1386
|
+
const base = (Number.isFinite(hard) && hard > 0) ? hard : 8500;
|
|
1387
|
+
return Math.round(base * 0.80);
|
|
1388
|
+
})();
|
|
1389
|
+
const EST_WORKER_RSS_MB = (() => {
|
|
1390
|
+
const parsed = parseInt(process.env.MUADDIB_EST_WORKER_RSS_MB, 10);
|
|
1391
|
+
return (Number.isFinite(parsed) && parsed > 0) ? parsed : 600;
|
|
1392
|
+
})();
|
|
1393
|
+
|
|
1394
|
+
/**
|
|
1395
|
+
* Pure: how many NEW scan workers the current RSS headroom allows under the soft ceiling.
|
|
1396
|
+
* `currentRssBytes` already includes the active workers, so this answers "how many MORE fit".
|
|
1397
|
+
* Returns 0 (never negative) once RSS reaches the soft limit — existing workers are NOT killed
|
|
1398
|
+
* here, they drain and free memory; ensureWorkers keeps the queue alive with 1 worker if
|
|
1399
|
+
* nothing is running. softLimitMb / estWorkerMb are injectable for tests.
|
|
1400
|
+
*/
|
|
1401
|
+
function rssAdmissionCap(currentRssBytes, softLimitMb = RSS_SOFT_LIMIT_MB, estWorkerMb = EST_WORKER_RSS_MB) {
|
|
1402
|
+
const headroomMb = softLimitMb - (currentRssBytes / 1024 / 1024);
|
|
1403
|
+
if (headroomMb <= 0) return 0;
|
|
1404
|
+
return Math.max(0, Math.floor(headroomMb / estWorkerMb));
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1345
1407
|
/**
|
|
1346
1408
|
* Ensure the target number of workers are running. Non-blocking: spawns
|
|
1347
1409
|
* missing workers as background promises. Called from the daemon main loop
|
|
@@ -1349,7 +1411,23 @@ function computeWorkersToSpawn(targetConcurrency, activeWorkers, queueLength) {
|
|
|
1349
1411
|
*/
|
|
1350
1412
|
function ensureWorkers(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailable) {
|
|
1351
1413
|
if (scanQueue.length === 0) return;
|
|
1352
|
-
|
|
1414
|
+
let toSpawn = computeWorkersToSpawn(_targetConcurrency, _activeWorkers, scanQueue.length);
|
|
1415
|
+
if (toSpawn <= 0) return;
|
|
1416
|
+
|
|
1417
|
+
// RSS-aware admission (P1 OOM durable fix): cap NEW spawns by memory headroom so the
|
|
1418
|
+
// in-flight worker set can't overshoot the soft RSS ceiling. Never fully deadlock: if
|
|
1419
|
+
// headroom is gone AND nothing is running, allow exactly one so the queue still makes
|
|
1420
|
+
// forward progress (its completion frees memory). Bounds peak RSS BEFORE the reactive breaker.
|
|
1421
|
+
const rssNow = process.memoryUsage().rss;
|
|
1422
|
+
const rssCap = rssAdmissionCap(rssNow);
|
|
1423
|
+
if (toSpawn > rssCap) {
|
|
1424
|
+
if (rssCap === 0 && _activeWorkers === 0) {
|
|
1425
|
+
toSpawn = 1;
|
|
1426
|
+
} else {
|
|
1427
|
+
console.log(`[MONITOR] RSS admission: capping spawn ${toSpawn}->${rssCap} (rss=${Math.round(rssNow / 1024 / 1024)}MB soft=${RSS_SOFT_LIMIT_MB}MB active=${_activeWorkers})`);
|
|
1428
|
+
toSpawn = rssCap;
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1353
1431
|
if (toSpawn <= 0) return;
|
|
1354
1432
|
|
|
1355
1433
|
console.log(`[MONITOR] Spawning ${toSpawn} worker(s) (active: ${_activeWorkers}, target: ${_targetConcurrency}, queue: ${scanQueue.length})`);
|
|
@@ -1429,6 +1507,25 @@ async function resolveTarballAndScan(item, stats, dailyAlerts, recentlyScanned,
|
|
|
1429
1507
|
// only scan whichever version happened to be the most recent at resolution
|
|
1430
1508
|
// time, racing the publish stream.
|
|
1431
1509
|
const recents = Array.isArray(npmInfo.recentVersions) ? npmInfo.recentVersions : [];
|
|
1510
|
+
// Phase 2b: burst = TRUE count of versions of this name in the recent window
|
|
1511
|
+
// (uncapped recentWindowCount), NOT the capped extras list — so a 96-version Miasma
|
|
1512
|
+
// burst is distinguishable from a legit multi-version day. At/above the threshold,
|
|
1513
|
+
// flag the item (protects it + its extras from queue-cap eviction) and fire ONE
|
|
1514
|
+
// burst pre-alert per name (deduped, bounded).
|
|
1515
|
+
const burstCount = Number.isFinite(npmInfo.recentWindowCount) ? npmInfo.recentWindowCount : (recents.length + 1);
|
|
1516
|
+
const isBurst = burstCount >= BURST_PREALERT_MIN_VERSIONS;
|
|
1517
|
+
if (isBurst) {
|
|
1518
|
+
item.isBurst = true;
|
|
1519
|
+
if (!_burstAlerted.has(item.name)) {
|
|
1520
|
+
if (_burstAlerted.size >= BURST_ALERTED_MAX) _burstAlerted.clear();
|
|
1521
|
+
_burstAlerted.add(item.name);
|
|
1522
|
+
stats.burstPreAlerts = (stats.burstPreAlerts || 0) + 1;
|
|
1523
|
+
console.log(`[MONITOR] BURST PRE-ALERT: ${item.name} — ${burstCount} versions in the recent window`);
|
|
1524
|
+
sendBurstPreAlert(item.name, burstCount, item.ecosystem).catch(err => {
|
|
1525
|
+
console.error(`[MONITOR] burst pre-alert webhook failed for ${item.name}: ${err.message}`);
|
|
1526
|
+
});
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1432
1529
|
for (const recent of recents) {
|
|
1433
1530
|
if (!recent || !recent.tarball || !recent.version) continue;
|
|
1434
1531
|
const dedupeKey = `${item.name}@${recent.version}`;
|
|
@@ -1441,6 +1538,7 @@ async function resolveTarballAndScan(item, stats, dailyAlerts, recentlyScanned,
|
|
|
1441
1538
|
unpackedSize: recent.unpackedSize || 0,
|
|
1442
1539
|
registryScripts: recent.scripts || null,
|
|
1443
1540
|
atoSignal: item.atoSignal === true,
|
|
1541
|
+
isBurst,
|
|
1444
1542
|
isATOBurstExtra: true,
|
|
1445
1543
|
}, stats);
|
|
1446
1544
|
}
|
|
@@ -1721,6 +1819,7 @@ module.exports = {
|
|
|
1721
1819
|
getActiveWorkers,
|
|
1722
1820
|
terminateAllWorkers,
|
|
1723
1821
|
computeWorkersToSpawn,
|
|
1822
|
+
rssAdmissionCap,
|
|
1724
1823
|
ensureWorkers,
|
|
1725
1824
|
drainWorkers,
|
|
1726
1825
|
|