muaddib-scanner 2.11.52 → 2.11.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/muaddib.js +1 -1
- package/package.json +4 -4
- package/{self-scan-v2.11.52.json → self-scan-v2.11.57.json} +2 -2
- package/src/commands/safe-install.js +0 -1
- package/src/index.js +1 -1
- package/src/integrations/maintainer-change.js +0 -1
- package/src/integrations/webhook.js +1 -3
- package/src/ioc/scraper.js +8 -69
- package/src/ml/classifier.js +3 -2
- package/src/ml/feature-extractor.js +1 -1
- package/src/ml/llm-detective.js +2 -2
- package/src/monitor/daemon.js +65 -15
- package/src/monitor/deferred-sandbox.js +8 -1
- package/src/monitor/ingestion.js +4 -4
- package/src/monitor/queue.js +133 -73
- package/src/monitor/state.js +2 -2
- package/src/monitor/webhook.js +9 -10
- package/src/output/cyclonedx.js +1 -1
- package/src/output/report.js +1 -1
- package/src/output/sarif.js +1 -1
- package/src/pipeline/executor.js +2 -2
- package/src/pipeline/processor.js +2 -2
- package/src/runtime/monitor-feed.js +0 -3
- package/src/sandbox/compound-triggers.js +2 -2
- package/src/sandbox/index.js +219 -104
- package/src/scanner/ai-config.js +60 -5
- package/src/scanner/ast-detectors/constants.js +12 -0
- package/src/scanner/ast-detectors/handle-assignment-expression.js +0 -1
- package/src/scanner/ast-detectors/handle-call-expression.js +88 -7
- package/src/scanner/ast-detectors/handle-variable-declarator.js +2 -2
- package/src/scanner/ast.js +2 -3
- package/src/scanner/dataflow.js +1 -2
- package/src/scanner/deobfuscate.js +1 -2
- package/src/scanner/entropy.js +0 -1
- package/src/scanner/github-actions.js +1 -1
- package/src/scanner/hash.js +1 -1
- package/src/scanner/module-graph/annotate-sinks.js +1 -2
- package/src/scanner/module-graph/annotate-tainted.js +1 -2
- package/src/scanner/module-graph/detect-callback-flows.js +0 -1
- package/src/scanner/module-graph/detect-cross-file.js +1 -1
- package/src/scanner/module-graph/detect-event-flows.js +1 -1
- package/src/scanner/module-graph/parse-utils.js +1 -2
- package/src/scanner/npm-registry.js +1 -4
- package/src/scanner/obfuscation.js +0 -1
- package/src/scanner/package.js +1 -1
- package/src/scanner/python-ast-detectors/handle-setup-call.js +0 -1
- package/src/scanner/reachability.js +1 -1
- package/src/scanner/shell.js +1 -1
- package/src/scanner/temporal-ast-diff.js +1 -2
- package/src/scanner/typosquat.js +3 -3
- package/src/scoring.js +1 -1
- package/src/shared/constants.js +1 -1
- package/src/shared/download.js +1 -0
- package/src/utils.js +1 -1
package/src/monitor/queue.js
CHANGED
|
@@ -10,30 +10,24 @@ const fs = require('fs');
|
|
|
10
10
|
const path = require('path');
|
|
11
11
|
const os = require('os');
|
|
12
12
|
const { Worker } = require('worker_threads');
|
|
13
|
-
const {
|
|
14
|
-
const { runSandbox, isDockerAvailable, tryAcquireSandboxSlot, SANDBOX_CONCURRENCY_MAX } = require('../sandbox/index.js');
|
|
13
|
+
const { runSandbox, tryAcquireSandboxSlot } = require('../sandbox/index.js');
|
|
15
14
|
const { sendWebhook } = require('../webhook.js');
|
|
16
|
-
const { downloadToFile,
|
|
15
|
+
const { downloadToFile, extractArchive, sanitizePackageName } = require('../shared/download.js');
|
|
17
16
|
const { MAX_TARBALL_SIZE } = require('../shared/constants.js');
|
|
18
17
|
const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
|
|
19
18
|
const { loadCachedIOCs } = require('../ioc/updater.js');
|
|
20
19
|
const { scanPackageJson } = require('../scanner/package.js');
|
|
21
20
|
const { scanShellScripts } = require('../scanner/shell.js');
|
|
22
21
|
const { buildTrainingRecord } = require('../ml/feature-extractor.js');
|
|
23
|
-
const { appendRecord: appendTrainingRecord, relabelRecords
|
|
22
|
+
const { appendRecord: appendTrainingRecord, relabelRecords } = require('../ml/jsonl-writer.js');
|
|
24
23
|
|
|
25
24
|
// From ./state.js
|
|
26
25
|
const {
|
|
27
26
|
cacheTarball,
|
|
28
27
|
updateScanStats,
|
|
29
28
|
appendDetection,
|
|
30
|
-
saveScanMemory,
|
|
31
29
|
maybePersistDailyStats,
|
|
32
|
-
loadNpmSeq,
|
|
33
|
-
saveNpmSeq,
|
|
34
|
-
getParisDateString,
|
|
35
30
|
appendTemporalDetection,
|
|
36
|
-
atomicWriteFileSync,
|
|
37
31
|
tarballCacheKey,
|
|
38
32
|
tarballCachePath,
|
|
39
33
|
appendAlert,
|
|
@@ -46,21 +40,11 @@ const {
|
|
|
46
40
|
const {
|
|
47
41
|
isSuspectClassification,
|
|
48
42
|
hasHighConfidenceThreat,
|
|
49
|
-
hasIOCMatch,
|
|
50
|
-
hasTyposquat,
|
|
51
|
-
hasLifecycleWithIntent,
|
|
52
43
|
isSandboxEnabled,
|
|
53
44
|
isCanaryEnabled,
|
|
54
45
|
recordError,
|
|
55
|
-
classifyError,
|
|
56
46
|
formatFindings,
|
|
57
|
-
|
|
58
|
-
isFirstPublishHighRisk,
|
|
59
|
-
DOWNLOADS_CACHE_TTL,
|
|
60
|
-
HIGH_CONFIDENCE_MALICE_TYPES,
|
|
61
|
-
IOC_MATCH_TYPES,
|
|
62
|
-
TIER1_TYPES,
|
|
63
|
-
hasHighOrCritical
|
|
47
|
+
isFirstPublishHighRisk
|
|
64
48
|
} = require('./classify.js');
|
|
65
49
|
|
|
66
50
|
// From ./webhook.js
|
|
@@ -74,7 +58,6 @@ const {
|
|
|
74
58
|
getWebhookUrl,
|
|
75
59
|
computeReputationFactor,
|
|
76
60
|
triageRisk,
|
|
77
|
-
computeRiskLevel,
|
|
78
61
|
sendDailyReport,
|
|
79
62
|
alertedPackageRules,
|
|
80
63
|
DAILY_REPORT_HOUR
|
|
@@ -82,16 +65,11 @@ const {
|
|
|
82
65
|
|
|
83
66
|
// From ./temporal.js
|
|
84
67
|
const {
|
|
85
|
-
isTemporalEnabled,
|
|
86
68
|
runTemporalCheck,
|
|
87
|
-
isTemporalAstEnabled,
|
|
88
69
|
runTemporalAstCheck,
|
|
89
|
-
isTemporalPublishEnabled,
|
|
90
70
|
runTemporalPublishCheck,
|
|
91
|
-
isTemporalMaintainerEnabled,
|
|
92
71
|
runTemporalMaintainerCheck,
|
|
93
72
|
getTemporalMaxSeverity,
|
|
94
|
-
isPublishAnomalyOnly,
|
|
95
73
|
tryTemporalAlert,
|
|
96
74
|
tryTemporalAstAlert,
|
|
97
75
|
isSafeLifecycleScript
|
|
@@ -115,13 +93,32 @@ let _targetConcurrency = BASE_CONCURRENCY;
|
|
|
115
93
|
const SCAN_CONCURRENCY = BASE_CONCURRENCY; // legacy export — tests check this value
|
|
116
94
|
let _activeWorkers = 0;
|
|
117
95
|
const _workerPromises = new Set();
|
|
96
|
+
// Live static-scan Worker threads — tracked so the daemon's EMERGENCY memory
|
|
97
|
+
// handler can terminate orphaned workers (each retains its isolate heap + parsed
|
|
98
|
+
// ASTs). Bounded by concurrency, so it stays tiny.
|
|
99
|
+
const _liveWorkers = new Set();
|
|
118
100
|
|
|
119
101
|
function getTargetConcurrency() { return _targetConcurrency; }
|
|
120
102
|
function setTargetConcurrency(n) { _targetConcurrency = Math.max(MIN_CONCURRENCY, Math.min(MAX_CONCURRENCY, n)); }
|
|
121
103
|
function getActiveWorkers() { return _activeWorkers; }
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Terminate every live static-scan Worker (best-effort). Returns the count.
|
|
107
|
+
* Called by daemon.js handleMemoryPressure() at EMERGENCY — a wedged/slow worker
|
|
108
|
+
* holds its isolate heap and parsed ASTs, part of the off-heap RSS leak.
|
|
109
|
+
*/
|
|
110
|
+
function terminateAllWorkers() {
|
|
111
|
+
let n = 0;
|
|
112
|
+
for (const w of Array.from(_liveWorkers)) {
|
|
113
|
+
try { w.terminate(); n++; } catch { /* already gone */ }
|
|
114
|
+
_liveWorkers.delete(w);
|
|
115
|
+
}
|
|
116
|
+
return n;
|
|
117
|
+
}
|
|
122
118
|
const SCAN_TIMEOUT_MS = 300_000; // 5 minutes per package (3 sandbox runs × 90s + static scan headroom)
|
|
123
119
|
const STATIC_SCAN_TIMEOUT_MS = 45_000; // 45s for static analysis only
|
|
124
120
|
const LARGE_PACKAGE_SIZE = 10 * 1024 * 1024; // 10MB
|
|
121
|
+
const RECENTLY_SCANNED_MAX = 50_000; // FIFO cap for the dedup Set (P0c — bounded resource)
|
|
125
122
|
|
|
126
123
|
// First-publish sandbox: max pending sandbox items before deferring first-publish clean scans
|
|
127
124
|
// Prevents starving T1a sandbox capacity when many first-publish packages arrive at once
|
|
@@ -246,56 +243,58 @@ function countPackageFiles(dir) {
|
|
|
246
243
|
* and a monitorMode flag to perform registry queries.
|
|
247
244
|
* @returns {Promise<object>} Scan result (same shape as run(_, {_capture:true}))
|
|
248
245
|
*/
|
|
249
|
-
function runScanInWorker(extractedDir, timeoutMs, scanContext = null) {
|
|
246
|
+
function runScanInWorker(extractedDir, timeoutMs, scanContext = null, signal = null) {
|
|
250
247
|
return new Promise((resolve, reject) => {
|
|
251
248
|
const worker = new Worker(SCAN_WORKER_PATH, {
|
|
252
249
|
workerData: { extractedDir, scanContext: scanContext || {} }
|
|
253
250
|
});
|
|
251
|
+
_liveWorkers.add(worker);
|
|
254
252
|
|
|
255
253
|
let settled = false;
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
settled = true;
|
|
259
|
-
worker.terminate().then(() => {
|
|
260
|
-
reject(new Error(`Static scan timeout after ${timeoutMs / 1000}s (worker terminated)`));
|
|
261
|
-
}).catch(() => {
|
|
262
|
-
reject(new Error(`Static scan timeout after ${timeoutMs / 1000}s (worker terminate failed)`));
|
|
263
|
-
});
|
|
264
|
-
}
|
|
265
|
-
}, timeoutMs);
|
|
266
|
-
|
|
267
|
-
worker.on('message', (msg) => {
|
|
254
|
+
let timer = null;
|
|
255
|
+
const done = (fn) => {
|
|
268
256
|
if (settled) return;
|
|
269
257
|
settled = true;
|
|
270
258
|
clearTimeout(timer);
|
|
271
|
-
if (
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
259
|
+
if (signal) { try { signal.removeEventListener('abort', onAbort); } catch { /* not added */ } }
|
|
260
|
+
_liveWorkers.delete(worker);
|
|
261
|
+
fn();
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
// Outer-timeout abort (processQueueItem's SCAN_TIMEOUT_MS): terminate the worker
|
|
265
|
+
// so a static scan that outlives the whole-package budget cannot keep its isolate
|
|
266
|
+
// (heap + ASTs) alive in the background.
|
|
267
|
+
const onAbort = () => done(() => {
|
|
268
|
+
worker.terminate().finally(() => reject(new Error('Static scan aborted (outer timeout — worker terminated)')));
|
|
276
269
|
});
|
|
277
270
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
});
|
|
271
|
+
timer = setTimeout(() => done(() => {
|
|
272
|
+
worker.terminate()
|
|
273
|
+
.then(() => reject(new Error(`Static scan timeout after ${timeoutMs / 1000}s (worker terminated)`)))
|
|
274
|
+
.catch(() => reject(new Error(`Static scan timeout after ${timeoutMs / 1000}s (worker terminate failed)`)));
|
|
275
|
+
}), timeoutMs);
|
|
284
276
|
|
|
285
|
-
|
|
286
|
-
if (
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
277
|
+
if (signal) {
|
|
278
|
+
if (signal.aborted) onAbort();
|
|
279
|
+
else signal.addEventListener('abort', onAbort, { once: true });
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
worker.on('message', (msg) => done(() => {
|
|
283
|
+
if (msg.type === 'result') resolve(msg.data);
|
|
284
|
+
else if (msg.type === 'error') reject(new Error(msg.message));
|
|
285
|
+
}));
|
|
286
|
+
|
|
287
|
+
worker.on('error', (err) => done(() => reject(err)));
|
|
288
|
+
|
|
289
|
+
worker.on('exit', (code) => done(() => {
|
|
290
|
+
if (code !== 0) reject(new Error(`Worker exited with code ${code}`));
|
|
291
|
+
}));
|
|
293
292
|
});
|
|
294
293
|
}
|
|
295
294
|
|
|
296
295
|
// --- Package scanning ---
|
|
297
296
|
|
|
298
|
-
async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, stats, dailyAlerts, recentlyScanned, downloadsCache, scanQueue, sandboxAvailable) {
|
|
297
|
+
async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, stats, dailyAlerts, recentlyScanned, downloadsCache, scanQueue, sandboxAvailable, signal = null) {
|
|
299
298
|
const startTime = Date.now();
|
|
300
299
|
const tmpBase = path.join(os.tmpdir(), 'muaddib-monitor');
|
|
301
300
|
if (!fs.existsSync(tmpBase)) fs.mkdirSync(tmpBase, { recursive: true });
|
|
@@ -375,7 +374,6 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
375
374
|
// Malware payloads are tiny (<1MB); 10MB has 10x safety margin.
|
|
376
375
|
// Quick scan: extract + check package.json + shell scripts for lifecycle threats.
|
|
377
376
|
const unpackedSize = meta.unpackedSize || 0;
|
|
378
|
-
let alreadyExtracted = false;
|
|
379
377
|
let extractedDir = null;
|
|
380
378
|
|
|
381
379
|
if (unpackedSize > LARGE_PACKAGE_SIZE || meta.fastTrack) {
|
|
@@ -394,7 +392,6 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
394
392
|
// Validates actual tarball contents (not just registry metadata).
|
|
395
393
|
let bypassQuickScan = false;
|
|
396
394
|
try {
|
|
397
|
-
alreadyExtracted = true;
|
|
398
395
|
extractedDir = extractArchive(tgzPath, tmpDir);
|
|
399
396
|
|
|
400
397
|
const [pkgThreats, shellThreats] = await Promise.all([
|
|
@@ -417,9 +414,8 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
417
414
|
updateScanStats('clean');
|
|
418
415
|
return;
|
|
419
416
|
}
|
|
420
|
-
} catch
|
|
417
|
+
} catch {
|
|
421
418
|
// Extract/quick scan failed — fallback to registry metadata check
|
|
422
|
-
alreadyExtracted = false;
|
|
423
419
|
extractedDir = null;
|
|
424
420
|
const scripts = meta.registryScripts || {};
|
|
425
421
|
const DANGEROUS_LIFECYCLE = ['preinstall', 'install', 'postinstall'];
|
|
@@ -467,19 +463,34 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
467
463
|
// the full 20-scanner pipeline (unchanged behaviour).
|
|
468
464
|
scanMode: (meta && meta.scanMode) || 'full'
|
|
469
465
|
};
|
|
470
|
-
result = await runScanInWorker(extractedDir, STATIC_SCAN_TIMEOUT_MS, scanContext);
|
|
466
|
+
result = await runScanInWorker(extractedDir, STATIC_SCAN_TIMEOUT_MS, scanContext, signal);
|
|
471
467
|
} catch (staticErr) {
|
|
472
468
|
if (/static scan timeout/i.test(staticErr.message)) {
|
|
473
|
-
console.error(`[MONITOR] STATIC_TIMEOUT: ${name}@${version} — exceeded ${STATIC_SCAN_TIMEOUT_MS / 1000}s (worker terminated)`);
|
|
469
|
+
console.error(`[MONITOR] STATIC_TIMEOUT: ${name}@${version} — exceeded ${STATIC_SCAN_TIMEOUT_MS / 1000}s (worker terminated, kept INCONCLUSIVE not clean)`);
|
|
474
470
|
recordError(staticErr, stats);
|
|
475
471
|
stats.scanned++;
|
|
476
472
|
stats.totalTimeMs += Date.now() - startTime;
|
|
477
|
-
|
|
473
|
+
// Garde-fou: a static-scan timeout must NOT count as clean — a package that
|
|
474
|
+
// deliberately hangs the parser to evade analysis would otherwise be relabelled
|
|
475
|
+
// benign. Count as inconclusive (excluded from the FP/TP denominator).
|
|
476
|
+
updateScanStats('sandbox_inconclusive');
|
|
478
477
|
return { sandboxResult: null, staticClean: false };
|
|
479
478
|
}
|
|
480
479
|
throw staticErr;
|
|
481
480
|
}
|
|
482
481
|
|
|
482
|
+
// Phase 3 signal — agent-supply-chain lens. Pure observability, no scoring impact.
|
|
483
|
+
// Cisco AI Defense / SkillSieve / Snyk Agent Scan EVO scan skill marketplaces;
|
|
484
|
+
// they don't monitor the npm/PyPI firehose. Tracking which packages bundle a
|
|
485
|
+
// SKILL.md is our unique intersection (npm-package-bundling-malicious-skill).
|
|
486
|
+
try {
|
|
487
|
+
const det = detectSkillMdBundled(extractedDir, result && result.threats);
|
|
488
|
+
if (det.bundled) {
|
|
489
|
+
stats.skillMdBundled = (stats.skillMdBundled || 0) + 1;
|
|
490
|
+
console.log(`[MONITOR] SKILL_MD_BUNDLED: ${name}@${version} (${ecosystem}) — ${det.count} file(s)`);
|
|
491
|
+
}
|
|
492
|
+
} catch { /* observability signal — never let it break the scan */ }
|
|
493
|
+
|
|
483
494
|
// First-publish detection: used for sandbox priority below
|
|
484
495
|
const isFirstPublish = cacheTrigger && cacheTrigger.reason === 'first_publish';
|
|
485
496
|
|
|
@@ -518,7 +529,7 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
518
529
|
try {
|
|
519
530
|
const canary = isCanaryEnabled();
|
|
520
531
|
console.log(`[MONITOR] SANDBOX (first-publish): launching for ${name}@${version}${canary ? ' (canary: on)' : ''}...`);
|
|
521
|
-
sandboxResult = await runSandbox(name, { canary });
|
|
532
|
+
sandboxResult = await runSandbox(name, { canary, signal });
|
|
522
533
|
console.log(`[MONITOR] SANDBOX: ${name}@${version} → score: ${sandboxResult.score}, severity: ${sandboxResult.severity}`);
|
|
523
534
|
} catch (err) {
|
|
524
535
|
console.error(`[MONITOR] SANDBOX ERROR: ${name}@${version} — ${err.message}`);
|
|
@@ -678,7 +689,8 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
678
689
|
// LOG-ONLY: record ML prediction for retraining data but do NOT filter.
|
|
679
690
|
// When model is retrained and validated, remove the 'true ||' guard below.
|
|
680
691
|
console.log(`[MONITOR] ML LOG-ONLY: ${name}@${version} (prediction=${mlResult.prediction}, p=${mlResult.probability}, score=${riskScore})`);
|
|
681
|
-
|
|
692
|
+
const ML_FILTER_ENABLED = false;
|
|
693
|
+
if (ML_FILTER_ENABLED && mlResult.prediction === 'clean') {
|
|
682
694
|
// DISABLED: model collapsed (p≈0.002 for all inputs). Re-enable after retrain.
|
|
683
695
|
console.log(`[MONITOR] ML CLEAN: ${name}@${version} (p=${mlResult.probability}, score=${riskScore})`);
|
|
684
696
|
stats.mlFiltered++;
|
|
@@ -774,12 +786,12 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
774
786
|
if (tier === '1a') {
|
|
775
787
|
// T1a: mandatory sandbox — block-wait (high-confidence threats MUST get sandbox)
|
|
776
788
|
console.log(`[MONITOR] SANDBOX: launching for ${name}@${version}${canary ? ' (canary: on)' : ''}...`);
|
|
777
|
-
sandboxResult = await runSandbox(name, { canary, maxRuns });
|
|
789
|
+
sandboxResult = await runSandbox(name, { canary, maxRuns, signal });
|
|
778
790
|
} else if (tryAcquireSandboxSlot()) {
|
|
779
791
|
// T1b/T2: non-blocking — slot acquired atomically, run with skipSemaphore
|
|
780
792
|
const reason = tier === 2 ? ' (T2, queue low)' : ' (T1b, conditional)';
|
|
781
793
|
console.log(`[MONITOR] SANDBOX${reason}: launching for ${name}@${version}${canary ? ' (canary: on)' : ''}...`);
|
|
782
|
-
sandboxResult = await runSandbox(name, { canary, maxRuns, skipSemaphore: true });
|
|
794
|
+
sandboxResult = await runSandbox(name, { canary, maxRuns, skipSemaphore: true, signal });
|
|
783
795
|
} else {
|
|
784
796
|
// T1b/T2: all sandbox slots busy — defer instead of blocking worker
|
|
785
797
|
console.log(`[MONITOR] SANDBOX DEFER (slots full): ${name}@${version} (tier=${tier}, score=${riskScore})`);
|
|
@@ -950,7 +962,7 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
950
962
|
// LLM Detective: AI-powered analysis for T1a/T1b suspects
|
|
951
963
|
// Skip for fast-track (large boring packages — LLM analysis adds 10-30s for no value)
|
|
952
964
|
let llmResult = null;
|
|
953
|
-
if (!meta.fastTrack && (tier === '1a' || tier === '1b') && (adjustedResult.summary.riskScore || 0) >= 25) {
|
|
965
|
+
if (!meta.fastTrack && (tier === '1a' || tier === '1b') && (adjustedResult.summary.riskScore || 0) >= 25 && !(signal && signal.aborted)) {
|
|
954
966
|
try {
|
|
955
967
|
const { investigatePackage, isLlmEnabled, getLlmMode } = require('../ml/llm-detective.js');
|
|
956
968
|
if (isLlmEnabled()) {
|
|
@@ -1004,6 +1016,34 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
|
|
|
1004
1016
|
}
|
|
1005
1017
|
}
|
|
1006
1018
|
|
|
1019
|
+
/**
|
|
1020
|
+
* Detect whether a package bundles a SKILL.md (Anthropic Agent Skills spec).
|
|
1021
|
+
* Pure observability — drives the `stats.skillMdBundled` counter, no scoring effect.
|
|
1022
|
+
*
|
|
1023
|
+
* Two-pass check: (1) inspect emitted threats for SKILL.md filenames so we catch
|
|
1024
|
+
* cases the scanner already touched without re-walking the tree; (2) fall back to
|
|
1025
|
+
* a bounded findFiles walk (maxDepth 4, maxFiles 5) for packages where no scanner
|
|
1026
|
+
* has flagged anything.
|
|
1027
|
+
*
|
|
1028
|
+
* @param {string|null} extractedDir - Unpacked tarball root, or null if unknown.
|
|
1029
|
+
* @param {Array<{file?:string}>|null} threats - Threats array from the scan result.
|
|
1030
|
+
* @returns {{bundled: boolean, count: number}}
|
|
1031
|
+
*/
|
|
1032
|
+
function detectSkillMdBundled(extractedDir, threats) {
|
|
1033
|
+
const fromThreats = Array.isArray(threats) && threats.some(
|
|
1034
|
+
t => /(?:^|[\\/])SKILL\.md$/i.test((t && t.file) || '')
|
|
1035
|
+
);
|
|
1036
|
+
if (fromThreats) return { bundled: true, count: 1 };
|
|
1037
|
+
if (!extractedDir) return { bundled: false, count: 0 };
|
|
1038
|
+
try {
|
|
1039
|
+
const { findFiles } = require('../utils.js');
|
|
1040
|
+
const found = findFiles(extractedDir, { extensions: ['SKILL.md'], maxDepth: 4, maxFiles: 5 });
|
|
1041
|
+
return { bundled: found.length > 0, count: found.length };
|
|
1042
|
+
} catch {
|
|
1043
|
+
return { bundled: false, count: 0 };
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1007
1047
|
function timeoutPromise(ms) {
|
|
1008
1048
|
return new Promise((_, reject) => {
|
|
1009
1049
|
setTimeout(() => reject(new Error(`Scan timeout after ${ms / 1000}s`)), ms);
|
|
@@ -1102,6 +1142,16 @@ async function _spawnWorker(scanQueue, stats, dailyAlerts, recentlyScanned, down
|
|
|
1102
1142
|
}
|
|
1103
1143
|
}
|
|
1104
1144
|
|
|
1145
|
+
/**
|
|
1146
|
+
* Pure spawn-count decision: how many workers to start given the target concurrency, the number
|
|
1147
|
+
* already active, and the queue depth. Never negative (active can exceed target during scale-down)
|
|
1148
|
+
* and never more than the backlog. Extracted so the adaptive worker-pool math is unit-testable
|
|
1149
|
+
* without spawning real (network-bound) workers.
|
|
1150
|
+
*/
|
|
1151
|
+
function computeWorkersToSpawn(targetConcurrency, activeWorkers, queueLength) {
|
|
1152
|
+
return Math.max(0, Math.min(targetConcurrency - activeWorkers, queueLength));
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1105
1155
|
/**
|
|
1106
1156
|
* Ensure the target number of workers are running. Non-blocking: spawns
|
|
1107
1157
|
* missing workers as background promises. Called from the daemon main loop
|
|
@@ -1109,7 +1159,7 @@ async function _spawnWorker(scanQueue, stats, dailyAlerts, recentlyScanned, down
|
|
|
1109
1159
|
*/
|
|
1110
1160
|
function ensureWorkers(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailable) {
|
|
1111
1161
|
if (scanQueue.length === 0) return;
|
|
1112
|
-
const toSpawn =
|
|
1162
|
+
const toSpawn = computeWorkersToSpawn(_targetConcurrency, _activeWorkers, scanQueue.length);
|
|
1113
1163
|
if (toSpawn <= 0) return;
|
|
1114
1164
|
|
|
1115
1165
|
console.log(`[MONITOR] Spawning ${toSpawn} worker(s) (active: ${_activeWorkers}, target: ${_targetConcurrency}, queue: ${scanQueue.length})`);
|
|
@@ -1274,6 +1324,13 @@ async function resolveTarballAndScan(item, stats, dailyAlerts, recentlyScanned,
|
|
|
1274
1324
|
return;
|
|
1275
1325
|
}
|
|
1276
1326
|
recentlyScanned.add(dedupeKey);
|
|
1327
|
+
// FIFO eviction (P0c — bounded resource): a Set preserves insertion order, so the
|
|
1328
|
+
// first value is the oldest. Evicting at most one per insert pins the size at the
|
|
1329
|
+
// cap and prevents unbounded heap growth within an uptime (the hourly clear in
|
|
1330
|
+
// daemon.js pruneMemoryCaches remains a coarse backstop).
|
|
1331
|
+
if (recentlyScanned.size > RECENTLY_SCANNED_MAX) {
|
|
1332
|
+
recentlyScanned.delete(recentlyScanned.values().next().value);
|
|
1333
|
+
}
|
|
1277
1334
|
// Coverage numerator: one count per unique (ecosystem, name, version) that
|
|
1278
1335
|
// reaches a scan attempt. Excludes ATO burst extras that lose the dedup
|
|
1279
1336
|
// race, retries, size-cap rejections — those inflate stats.scanned but
|
|
@@ -1360,7 +1417,7 @@ async function resolveTarballAndScan(item, stats, dailyAlerts, recentlyScanned,
|
|
|
1360
1417
|
_cacheTrigger: item._cacheTrigger || null,
|
|
1361
1418
|
fastTrack: item.fastTrack || false,
|
|
1362
1419
|
scanMode: effectiveScanMode
|
|
1363
|
-
}, stats, dailyAlerts, recentlyScanned, downloadsCache, scanQueue, sandboxAvailable);
|
|
1420
|
+
}, stats, dailyAlerts, recentlyScanned, downloadsCache, scanQueue, sandboxAvailable, signal);
|
|
1364
1421
|
const sandboxResult = scanResult && scanResult.sandboxResult;
|
|
1365
1422
|
const staticClean = scanResult && scanResult.staticClean;
|
|
1366
1423
|
|
|
@@ -1470,6 +1527,8 @@ module.exports = {
|
|
|
1470
1527
|
getTargetConcurrency,
|
|
1471
1528
|
setTargetConcurrency,
|
|
1472
1529
|
getActiveWorkers,
|
|
1530
|
+
terminateAllWorkers,
|
|
1531
|
+
computeWorkersToSpawn,
|
|
1473
1532
|
ensureWorkers,
|
|
1474
1533
|
drainWorkers,
|
|
1475
1534
|
|
|
@@ -1480,6 +1539,7 @@ module.exports = {
|
|
|
1480
1539
|
runScanInWorker,
|
|
1481
1540
|
scanPackage,
|
|
1482
1541
|
timeoutPromise,
|
|
1542
|
+
detectSkillMdBundled,
|
|
1483
1543
|
isDailyReportDue,
|
|
1484
1544
|
processQueueItem,
|
|
1485
1545
|
processQueue,
|
package/src/monitor/state.js
CHANGED
|
@@ -165,12 +165,12 @@ function atomicWriteFileSync(filePath, data) {
|
|
|
165
165
|
} catch (err) {
|
|
166
166
|
if (err.code === 'EROFS' || err.code === 'EACCES' || err.code === 'EPERM') {
|
|
167
167
|
console.warn(`[MONITOR] Cannot write ${path.basename(filePath)} (${err.code}) — skipping`);
|
|
168
|
-
try { fs.unlinkSync(tmpFile); } catch
|
|
168
|
+
try { fs.unlinkSync(tmpFile); } catch { /* ignore */ }
|
|
169
169
|
return;
|
|
170
170
|
}
|
|
171
171
|
if (err.code === 'ENOSPC') {
|
|
172
172
|
console.warn(`[MONITOR] WARNING: disk full (ENOSPC) — cannot write ${path.basename(filePath)}. Free space in /tmp and data/ immediately.`);
|
|
173
|
-
try { fs.unlinkSync(tmpFile); } catch
|
|
173
|
+
try { fs.unlinkSync(tmpFile); } catch { /* ignore */ }
|
|
174
174
|
return;
|
|
175
175
|
}
|
|
176
176
|
throw err;
|
package/src/monitor/webhook.js
CHANGED
|
@@ -18,28 +18,20 @@ const {
|
|
|
18
18
|
getParisHour,
|
|
19
19
|
loadScanStats,
|
|
20
20
|
loadDetections,
|
|
21
|
-
loadLastDailyReportDate,
|
|
22
21
|
saveLastDailyReportDate,
|
|
23
|
-
saveDailyStats,
|
|
24
22
|
resetDailyStats,
|
|
25
23
|
saveScanMemory,
|
|
26
24
|
shouldSuppressByMemory,
|
|
27
25
|
recordScanMemory,
|
|
28
|
-
loadState,
|
|
29
26
|
saveState,
|
|
30
27
|
loadStateRaw,
|
|
31
28
|
getScansSinceLastMemoryPersist,
|
|
32
|
-
setScansSinceLastMemoryPersist
|
|
33
|
-
STATE_FILE
|
|
29
|
+
setScansSinceLastMemoryPersist
|
|
34
30
|
} = require('./state.js');
|
|
35
31
|
const {
|
|
36
32
|
HIGH_CONFIDENCE_MALICE_TYPES,
|
|
37
33
|
hasIOCMatch,
|
|
38
34
|
hasHighOrCritical,
|
|
39
|
-
hasHighConfidenceThreat,
|
|
40
|
-
hasTyposquat,
|
|
41
|
-
hasLifecycleWithIntent,
|
|
42
|
-
isSuspectClassification,
|
|
43
35
|
formatErrorBreakdown
|
|
44
36
|
} = require('./classify.js');
|
|
45
37
|
|
|
@@ -48,6 +40,7 @@ const {
|
|
|
48
40
|
// Webhook dedup: track alerted packages by name -> Set<rule_ids> (cleared with daily report).
|
|
49
41
|
// If a new version triggers the same rules, skip the webhook. If new rules appear, let it through.
|
|
50
42
|
const alertedPackageRules = new Map();
|
|
43
|
+
const ALERTED_PACKAGES_MAX = 5_000; // single source of truth for the alerted-packages cap (daemon.js imports this for pruneMemoryCaches)
|
|
51
44
|
|
|
52
45
|
// Scope grouping: buffer scoped npm packages for grouped webhooks (monorepo noise reduction).
|
|
53
46
|
// @scope -> { packages[], timer, maxScore, ecosystem }
|
|
@@ -358,7 +351,7 @@ function triageRisk(item, meta) {
|
|
|
358
351
|
reasons.push('no_metadata');
|
|
359
352
|
} else if (ecosystem === 'npm') {
|
|
360
353
|
const factor = computeReputationFactor(meta);
|
|
361
|
-
if (factor >= 1.
|
|
354
|
+
if (factor >= 1.2) reasons.push(`reputation_factor=${factor.toFixed(2)}`);
|
|
362
355
|
} else if (ecosystem === 'pypi') {
|
|
363
356
|
// PyPI has no weekly_downloads source today, so we cannot reuse
|
|
364
357
|
// computeReputationFactor as-is. Use direct signals instead.
|
|
@@ -559,6 +552,11 @@ async function trySendWebhook(name, version, ecosystem, result, sandboxResult, m
|
|
|
559
552
|
for (const r of currentRules) previousRules.add(r);
|
|
560
553
|
} else {
|
|
561
554
|
alertedPackageRules.set(name, new Set(currentRules));
|
|
555
|
+
// FIFO size cap (bounded resource): evict the oldest tracked package when over
|
|
556
|
+
// the cap. A Map preserves insertion order, so the first key is the oldest.
|
|
557
|
+
if (alertedPackageRules.size > ALERTED_PACKAGES_MAX) {
|
|
558
|
+
alertedPackageRules.delete(alertedPackageRules.keys().next().value);
|
|
559
|
+
}
|
|
562
560
|
}
|
|
563
561
|
|
|
564
562
|
// Scope grouping: buffer scoped npm packages for grouped webhook
|
|
@@ -1291,6 +1289,7 @@ module.exports = {
|
|
|
1291
1289
|
// Constants
|
|
1292
1290
|
HIGH_INTENT_TYPES,
|
|
1293
1291
|
DAILY_REPORT_HOUR,
|
|
1292
|
+
ALERTED_PACKAGES_MAX,
|
|
1294
1293
|
|
|
1295
1294
|
// Functions
|
|
1296
1295
|
getWebhookUrl,
|
package/src/output/cyclonedx.js
CHANGED
|
@@ -189,7 +189,7 @@ function saveCycloneDX(results, outputPath) {
|
|
|
189
189
|
try {
|
|
190
190
|
fs.writeFileSync(outputPath, JSON.stringify(bom, null, 2));
|
|
191
191
|
} catch (e) {
|
|
192
|
-
throw new Error('Failed to write CycloneDX report to ' + outputPath + ': ' + e.message);
|
|
192
|
+
throw new Error('Failed to write CycloneDX report to ' + outputPath + ': ' + e.message, { cause: e });
|
|
193
193
|
}
|
|
194
194
|
return outputPath;
|
|
195
195
|
}
|
package/src/output/report.js
CHANGED
|
@@ -262,7 +262,7 @@ function saveReport(results, outputPath) {
|
|
|
262
262
|
try {
|
|
263
263
|
fs.writeFileSync(outputPath, html);
|
|
264
264
|
} catch (e) {
|
|
265
|
-
throw new Error(`Failed to write HTML report to ${outputPath}: ${e.message}
|
|
265
|
+
throw new Error(`Failed to write HTML report to ${outputPath}: ${e.message}`, { cause: e });
|
|
266
266
|
}
|
|
267
267
|
return outputPath;
|
|
268
268
|
}
|
package/src/output/sarif.js
CHANGED
|
@@ -99,7 +99,7 @@ function saveSARIF(results, outputPath) {
|
|
|
99
99
|
try {
|
|
100
100
|
fs.writeFileSync(outputPath, JSON.stringify(sarif, null, 2));
|
|
101
101
|
} catch (e) {
|
|
102
|
-
throw new Error(`Failed to write SARIF report to ${outputPath}: ${e.message}
|
|
102
|
+
throw new Error(`Failed to write SARIF report to ${outputPath}: ${e.message}`, { cause: e });
|
|
103
103
|
}
|
|
104
104
|
return outputPath;
|
|
105
105
|
}
|
package/src/pipeline/executor.js
CHANGED
|
@@ -18,8 +18,8 @@ const { scanEntropy } = require('../scanner/entropy.js');
|
|
|
18
18
|
const { scanAIConfig } = require('../scanner/ai-config.js');
|
|
19
19
|
const { deobfuscate } = require('../scanner/deobfuscate.js');
|
|
20
20
|
const { buildModuleGraph, annotateTaintedExports, detectCrossFileFlows, annotateSinkExports, detectCallbackCrossFileFlows, detectEventEmitterFlows } = require('../scanner/module-graph');
|
|
21
|
-
const { loadCachedIOCs
|
|
22
|
-
const {
|
|
21
|
+
const { loadCachedIOCs } = require('../ioc/updater.js');
|
|
22
|
+
const { normalizePythonName } = require('../scanner/python.js');
|
|
23
23
|
const { scanPythonSource } = require('../scanner/python-source.js');
|
|
24
24
|
const { initPythonParser, scanPythonAST } = require('../scanner/python-ast.js');
|
|
25
25
|
const { Spinner, listInstalledPackages, wasFilesCapped, getOverflowFiles, debugLog } = require('../utils.js');
|
|
@@ -5,7 +5,7 @@ const { getPlaybook } = require('../response/playbooks.js');
|
|
|
5
5
|
const { computeReachableFiles, computeReachableFunctions } = require('../scanner/reachability.js');
|
|
6
6
|
const { applyFPReductions, applyCompoundBoosts, calculateRiskScore, getSeverityWeights, applyContextualFPCaps, applySingleFireCriticalFloor, applyReputationFactor, applyMatureStableCap, applySandboxVerdict, applyDeltaMultiplier } = require('../scoring.js');
|
|
7
7
|
const { loadPriorVersionSignatures, computeSignatures, saveCachedSignatures } = require('../scoring/delta-multiplier.js');
|
|
8
|
-
const { annotateConfidenceTiers
|
|
8
|
+
const { annotateConfidenceTiers } = require('../rules/confidence-tiers.js');
|
|
9
9
|
const { buildIntentPairs } = require('../intent-graph.js');
|
|
10
10
|
const { debugLog } = require('../utils.js');
|
|
11
11
|
const { getPackageMetadata } = require('../scanner/npm-registry.js');
|
|
@@ -472,7 +472,7 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
|
|
|
472
472
|
// Per-file max scoring (v2.2.11) with intent graph bonus
|
|
473
473
|
const {
|
|
474
474
|
riskScore, riskLevel, globalRiskScore,
|
|
475
|
-
maxFileScore, packageScore,
|
|
475
|
+
maxFileScore, packageScore, mostSuspiciousFile, fileScores,
|
|
476
476
|
criticalCount, highCount, mediumCount, lowCount
|
|
477
477
|
} = calculateRiskScore(deduped, intentResult);
|
|
478
478
|
|
|
@@ -206,10 +206,10 @@ function evaluateSandboxTrigger(threats, score, fileSizes) {
|
|
|
206
206
|
return { shouldRun: false, compound: null, watchpoints: [], reason: 'score above window' };
|
|
207
207
|
}
|
|
208
208
|
for (const trigger of TRIGGERS) {
|
|
209
|
-
let matched
|
|
209
|
+
let matched;
|
|
210
210
|
try {
|
|
211
211
|
matched = trigger.matches(threats, fileSizes || {});
|
|
212
|
-
} catch
|
|
212
|
+
} catch {
|
|
213
213
|
matched = false;
|
|
214
214
|
}
|
|
215
215
|
if (matched) {
|