muaddib-scanner 2.10.2 → 2.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/bin/muaddib.js +6 -1
- package/package.json +1 -1
- package/scripts/analyze-score0.js +190 -0
- package/scripts/cleanup-fp-labels.js +81 -0
- package/src/canary-tokens.js +52 -0
- package/src/index.js +29 -0
- package/src/ml/classifier.js +109 -7
- package/src/ml/feature-extractor.js +2 -1
- package/src/ml/jsonl-writer.js +19 -2
- package/src/ml/model-bundler.js +11 -0
- package/src/ml/model-trees.js +7 -9
- package/src/ml/train-bundler-detector.py +704 -0
- package/src/ml/train-xgboost.py +733 -0
- package/src/response/playbooks.js +20 -0
- package/src/rules/index.js +49 -0
- package/src/sandbox/index.js +11 -0
- package/src/scanner/ast-detectors.js +136 -8
- package/src/scanner/ast.js +3 -1
- package/src/scoring.js +64 -5
- package/src/webhook.js +46 -14
package/README.md
CHANGED
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
|
|
31
31
|
npm and PyPI supply-chain attacks are exploding. Shai-Hulud compromised 25K+ repos in 2025. Existing tools detect threats but don't help you respond.
|
|
32
32
|
|
|
33
|
-
MUAD'DIB combines **14 parallel scanners** (158 detection rules), a **deobfuscation engine**, **inter-module dataflow analysis**, **
|
|
33
|
+
MUAD'DIB combines **14 parallel scanners** (158 detection rules), a **deobfuscation engine**, **inter-module dataflow analysis**, **compound scoring**, and Docker sandbox to detect known threats and suspicious behavioral patterns in npm and PyPI packages.
|
|
34
34
|
|
|
35
35
|
---
|
|
36
36
|
|
|
@@ -345,6 +345,8 @@ npm test
|
|
|
345
345
|
|
|
346
346
|
## Documentation
|
|
347
347
|
|
|
348
|
+
- [Blog](https://dnszlsk.github.io/muad-dib/blog/) - Technical articles on supply-chain threat detection
|
|
349
|
+
- [Carnet de bord](docs/CARNET_DE_BORD_MUADDIB.md) - Development journal (in French)
|
|
348
350
|
- [Documentation Index](docs/INDEX.md) - All documentation in one place
|
|
349
351
|
- [Evaluation Methodology](docs/EVALUATION_METHODOLOGY.md) - Experimental protocol, holdout scores
|
|
350
352
|
- [Threat Model](docs/threat-model.md) - What MUAD'DIB detects and doesn't detect
|
package/bin/muaddib.js
CHANGED
|
@@ -34,6 +34,7 @@ let noDeobfuscate = false;
|
|
|
34
34
|
let noModuleGraph = false;
|
|
35
35
|
let noReachability = false;
|
|
36
36
|
let configPath = null;
|
|
37
|
+
let autoSandbox = false;
|
|
37
38
|
let feedLimit = null;
|
|
38
39
|
let feedSeverity = null;
|
|
39
40
|
let feedSince = null;
|
|
@@ -137,6 +138,8 @@ for (let i = 0; i < options.length; i++) {
|
|
|
137
138
|
}
|
|
138
139
|
configPath = cfgPath;
|
|
139
140
|
i++;
|
|
141
|
+
} else if (options[i] === '--auto-sandbox') {
|
|
142
|
+
autoSandbox = true;
|
|
140
143
|
} else if (options[i] === '--temporal') {
|
|
141
144
|
temporalMode = true;
|
|
142
145
|
} else if (options[i] === '--limit') {
|
|
@@ -429,6 +432,7 @@ const helpText = `
|
|
|
429
432
|
--temporal-publish Detect publish frequency anomalies (bursts, dormant spikes)
|
|
430
433
|
--temporal-maintainer Detect maintainer changes (new maintainer, account takeover)
|
|
431
434
|
--temporal-full All temporal analyses (lifecycle + AST + publish + maintainer)
|
|
435
|
+
--auto-sandbox Auto-trigger sandbox when static scan score >= 20 (requires Docker)
|
|
432
436
|
--no-canary Disable honey token injection in sandbox
|
|
433
437
|
--no-deobfuscate Disable deobfuscation pre-processing
|
|
434
438
|
--no-module-graph Disable cross-file dataflow analysis
|
|
@@ -482,7 +486,8 @@ if (command === 'version' || command === '--version' || command === '-v') {
|
|
|
482
486
|
noDeobfuscate: noDeobfuscate,
|
|
483
487
|
noModuleGraph: noModuleGraph,
|
|
484
488
|
noReachability: noReachability,
|
|
485
|
-
configPath: configPath
|
|
489
|
+
configPath: configPath,
|
|
490
|
+
autoSandbox: autoSandbox
|
|
486
491
|
}).then(exitCode => {
|
|
487
492
|
process.exit(exitCode);
|
|
488
493
|
}).catch(err => {
|
package/package.json
CHANGED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* analyze-score0.js — Diagnostic script for score-0 malware investigation.
|
|
6
|
+
*
|
|
7
|
+
* Analyzes packages from the Datadog benchmark that scored 0 (zero threats detected).
|
|
8
|
+
* Categorizes each package to identify blind spots vs expected non-detections.
|
|
9
|
+
*
|
|
10
|
+
* Categories:
|
|
11
|
+
* - empty_package: no code files at all
|
|
12
|
+
* - ts_only: only .ts files (no .js)
|
|
13
|
+
* - binary_only: only .wasm/.node/.dll/.so
|
|
14
|
+
* - non_code_assets: CSS/images/fonts/markdown only
|
|
15
|
+
* - minimum_viable: package.json + README only
|
|
16
|
+
* - python_in_npm: .py files in an npm package
|
|
17
|
+
* - unknown: has .js but 0 detections — TRUE BLIND SPOT
|
|
18
|
+
*
|
|
19
|
+
* Usage:
|
|
20
|
+
* node scripts/analyze-score0.js --benchmark data/datadog-benchmark.jsonl
|
|
21
|
+
* node scripts/analyze-score0.js --benchmark data/datadog-benchmark.jsonl --csv report.csv
|
|
22
|
+
* node scripts/analyze-score0.js --dir .muaddib-cache/datadog-tarballs/
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
const fs = require('fs');
|
|
26
|
+
const path = require('path');
|
|
27
|
+
|
|
28
|
+
const CODE_EXTENSIONS = new Set(['.js', '.cjs', '.mjs', '.jsx']);
|
|
29
|
+
const TS_EXTENSIONS = new Set(['.ts', '.tsx', '.cts', '.mts']);
|
|
30
|
+
const BINARY_EXTENSIONS = new Set(['.wasm', '.node', '.dll', '.so', '.dylib', '.exe']);
|
|
31
|
+
const ASSET_EXTENSIONS = new Set(['.css', '.scss', '.less', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico',
|
|
32
|
+
'.woff', '.woff2', '.ttf', '.eot', '.otf', '.md', '.txt', '.html', '.htm', '.map']);
|
|
33
|
+
const PY_EXTENSIONS = new Set(['.py', '.pyx', '.pyi']);
|
|
34
|
+
|
|
35
|
+
function categorizePackage(packageDir) {
|
|
36
|
+
if (!fs.existsSync(packageDir)) return 'missing';
|
|
37
|
+
|
|
38
|
+
const files = [];
|
|
39
|
+
function walk(dir, depth) {
|
|
40
|
+
if (depth > 5) return; // Limit depth
|
|
41
|
+
try {
|
|
42
|
+
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
43
|
+
for (const entry of entries) {
|
|
44
|
+
if (entry.name === 'node_modules' || entry.name === '.git') continue;
|
|
45
|
+
const full = path.join(dir, entry.name);
|
|
46
|
+
if (entry.isDirectory()) {
|
|
47
|
+
walk(full, depth + 1);
|
|
48
|
+
} else if (entry.isFile()) {
|
|
49
|
+
files.push(entry.name);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
} catch { /* skip permission errors */ }
|
|
53
|
+
}
|
|
54
|
+
walk(packageDir, 0);
|
|
55
|
+
|
|
56
|
+
if (files.length === 0) return 'empty_package';
|
|
57
|
+
|
|
58
|
+
const extensions = files.map(f => path.extname(f).toLowerCase());
|
|
59
|
+
const hasCode = extensions.some(e => CODE_EXTENSIONS.has(e));
|
|
60
|
+
const hasTs = extensions.some(e => TS_EXTENSIONS.has(e));
|
|
61
|
+
const hasBinary = extensions.some(e => BINARY_EXTENSIONS.has(e));
|
|
62
|
+
const hasPython = extensions.some(e => PY_EXTENSIONS.has(e));
|
|
63
|
+
const hasAssets = extensions.some(e => ASSET_EXTENSIONS.has(e));
|
|
64
|
+
|
|
65
|
+
// Only package.json + README
|
|
66
|
+
const nonMeta = files.filter(f => !['package.json', 'readme.md', 'readme', 'license', 'license.md', 'changelog.md'].includes(f.toLowerCase()));
|
|
67
|
+
if (nonMeta.length === 0) return 'minimum_viable';
|
|
68
|
+
|
|
69
|
+
if (hasCode) return 'unknown'; // TRUE BLIND SPOT: has JS but 0 detections
|
|
70
|
+
|
|
71
|
+
if (hasTs && !hasCode) return 'ts_only';
|
|
72
|
+
if (hasBinary && !hasCode && !hasTs) return 'binary_only';
|
|
73
|
+
if (hasPython && !hasCode) return 'python_in_npm';
|
|
74
|
+
if (hasAssets && !hasCode && !hasTs && !hasBinary) return 'non_code_assets';
|
|
75
|
+
|
|
76
|
+
return 'unknown'; // Fallback
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function loadBenchmarkResults(filepath) {
|
|
80
|
+
if (!fs.existsSync(filepath)) {
|
|
81
|
+
console.error(`[SCORE0] File not found: ${filepath}`);
|
|
82
|
+
process.exit(1);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const content = fs.readFileSync(filepath, 'utf8');
|
|
86
|
+
const records = [];
|
|
87
|
+
for (const line of content.split('\n')) {
|
|
88
|
+
if (!line.trim()) continue;
|
|
89
|
+
try {
|
|
90
|
+
const record = JSON.parse(line);
|
|
91
|
+
if (record.score === 0 && record.threat_count === 0) {
|
|
92
|
+
records.push(record);
|
|
93
|
+
}
|
|
94
|
+
} catch { /* skip malformed */ }
|
|
95
|
+
}
|
|
96
|
+
return records;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function main() {
|
|
100
|
+
const args = process.argv.slice(2);
|
|
101
|
+
const benchmarkIdx = args.indexOf('--benchmark');
|
|
102
|
+
const dirIdx = args.indexOf('--dir');
|
|
103
|
+
const csvIdx = args.indexOf('--csv');
|
|
104
|
+
|
|
105
|
+
const benchmarkFile = benchmarkIdx >= 0 ? args[benchmarkIdx + 1] : null;
|
|
106
|
+
const tarballDir = dirIdx >= 0 ? args[dirIdx + 1] : null;
|
|
107
|
+
const csvFile = csvIdx >= 0 ? args[csvIdx + 1] : null;
|
|
108
|
+
|
|
109
|
+
if (!benchmarkFile && !tarballDir) {
|
|
110
|
+
console.log('Usage:');
|
|
111
|
+
console.log(' node scripts/analyze-score0.js --benchmark data/datadog-benchmark.jsonl');
|
|
112
|
+
console.log(' node scripts/analyze-score0.js --dir .muaddib-cache/datadog-tarballs/');
|
|
113
|
+
console.log(' node scripts/analyze-score0.js --benchmark data/datadog-benchmark.jsonl --csv report.csv');
|
|
114
|
+
process.exit(0);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
let packages = [];
|
|
118
|
+
|
|
119
|
+
if (benchmarkFile) {
|
|
120
|
+
const records = loadBenchmarkResults(benchmarkFile);
|
|
121
|
+
console.log(`[SCORE0] Loaded ${records.length} score-0 packages from benchmark`);
|
|
122
|
+
packages = records.map(r => ({
|
|
123
|
+
name: r.name || r.package || 'unknown',
|
|
124
|
+
version: r.version || '',
|
|
125
|
+
dir: tarballDir ? path.join(tarballDir, r.name || r.package || 'unknown') : null
|
|
126
|
+
}));
|
|
127
|
+
} else if (tarballDir) {
|
|
128
|
+
// Direct directory scan mode
|
|
129
|
+
if (!fs.existsSync(tarballDir)) {
|
|
130
|
+
console.error(`[SCORE0] Directory not found: ${tarballDir}`);
|
|
131
|
+
process.exit(1);
|
|
132
|
+
}
|
|
133
|
+
const entries = fs.readdirSync(tarballDir, { withFileTypes: true });
|
|
134
|
+
packages = entries
|
|
135
|
+
.filter(e => e.isDirectory())
|
|
136
|
+
.map(e => ({ name: e.name, version: '', dir: path.join(tarballDir, e.name) }));
|
|
137
|
+
console.log(`[SCORE0] Found ${packages.length} package directories`);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Categorize
|
|
141
|
+
const categories = {};
|
|
142
|
+
const results = [];
|
|
143
|
+
|
|
144
|
+
for (const pkg of packages) {
|
|
145
|
+
let category = 'no_dir';
|
|
146
|
+
if (pkg.dir && fs.existsSync(pkg.dir)) {
|
|
147
|
+
category = categorizePackage(pkg.dir);
|
|
148
|
+
}
|
|
149
|
+
categories[category] = (categories[category] || 0) + 1;
|
|
150
|
+
results.push({ name: pkg.name, version: pkg.version, category });
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Summary
|
|
154
|
+
console.log('\n=== SCORE 0 INVESTIGATION REPORT ===\n');
|
|
155
|
+
console.log(`Total score-0 packages: ${packages.length}\n`);
|
|
156
|
+
|
|
157
|
+
const sortedCategories = Object.entries(categories).sort((a, b) => b[1] - a[1]);
|
|
158
|
+
for (const [cat, count] of sortedCategories) {
|
|
159
|
+
const pct = ((count / packages.length) * 100).toFixed(1);
|
|
160
|
+
const label = cat === 'unknown' ? `${cat} *** BLIND SPOT ***` : cat;
|
|
161
|
+
console.log(` ${label}: ${count} (${pct}%)`);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const unknownCount = categories.unknown || 0;
|
|
165
|
+
console.log(`\n Actionable blind spots: ${unknownCount} packages with JS code but 0 detections`);
|
|
166
|
+
|
|
167
|
+
// CSV output
|
|
168
|
+
if (csvFile) {
|
|
169
|
+
const csvLines = ['name,version,category'];
|
|
170
|
+
for (const r of results) {
|
|
171
|
+
csvLines.push(`${r.name},${r.version},${r.category}`);
|
|
172
|
+
}
|
|
173
|
+
fs.writeFileSync(csvFile, csvLines.join('\n'), 'utf8');
|
|
174
|
+
console.log(`\n CSV report written to: ${csvFile}`);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// List unknown packages (first 20)
|
|
178
|
+
const unknowns = results.filter(r => r.category === 'unknown');
|
|
179
|
+
if (unknowns.length > 0) {
|
|
180
|
+
console.log('\n First 20 "unknown" (blind spot) packages:');
|
|
181
|
+
for (const u of unknowns.slice(0, 20)) {
|
|
182
|
+
console.log(` - ${u.name}@${u.version}`);
|
|
183
|
+
}
|
|
184
|
+
if (unknowns.length > 20) {
|
|
185
|
+
console.log(` ... and ${unknowns.length - 20} more`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
main();
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* cleanup-fp-labels.js — One-shot script to convert contaminated 'fp' labels to 'unconfirmed'.
|
|
6
|
+
*
|
|
7
|
+
* Context: During 3 months of monitoring, sandbox score === 0 was automatically relabeled
|
|
8
|
+
* as 'fp' (false positive). Without honey tokens, sandbox clean ≠ false positive.
|
|
9
|
+
* This script converts all automated 'fp' labels to 'unconfirmed' so they are excluded
|
|
10
|
+
* from ML training (neither positive nor negative).
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* node scripts/cleanup-fp-labels.js # Dry-run (default)
|
|
14
|
+
* node scripts/cleanup-fp-labels.js --apply # Write changes
|
|
15
|
+
* node scripts/cleanup-fp-labels.js --file path # Custom JSONL path
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const fs = require('fs');
|
|
19
|
+
const path = require('path');
|
|
20
|
+
|
|
21
|
+
const DEFAULT_FILE = path.join(__dirname, '..', 'data', 'ml-training.jsonl');
|
|
22
|
+
|
|
23
|
+
function main() {
|
|
24
|
+
const args = process.argv.slice(2);
|
|
25
|
+
const apply = args.includes('--apply');
|
|
26
|
+
const fileIdx = args.indexOf('--file');
|
|
27
|
+
const filePath = fileIdx >= 0 && args[fileIdx + 1] ? args[fileIdx + 1] : DEFAULT_FILE;
|
|
28
|
+
|
|
29
|
+
if (!fs.existsSync(filePath)) {
|
|
30
|
+
console.log(`[CLEANUP] File not found: ${filePath}`);
|
|
31
|
+
process.exit(1);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const content = fs.readFileSync(filePath, 'utf8');
|
|
35
|
+
const lines = content.split('\n');
|
|
36
|
+
|
|
37
|
+
let totalRecords = 0;
|
|
38
|
+
let fpCount = 0;
|
|
39
|
+
let convertedLines = [];
|
|
40
|
+
|
|
41
|
+
for (const line of lines) {
|
|
42
|
+
if (!line.trim()) {
|
|
43
|
+
convertedLines.push(line);
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
try {
|
|
48
|
+
const record = JSON.parse(line);
|
|
49
|
+
totalRecords++;
|
|
50
|
+
|
|
51
|
+
if (record.label === 'fp') {
|
|
52
|
+
fpCount++;
|
|
53
|
+
if (apply) {
|
|
54
|
+
record.label = 'unconfirmed';
|
|
55
|
+
convertedLines.push(JSON.stringify(record));
|
|
56
|
+
} else {
|
|
57
|
+
convertedLines.push(line);
|
|
58
|
+
}
|
|
59
|
+
} else {
|
|
60
|
+
convertedLines.push(line);
|
|
61
|
+
}
|
|
62
|
+
} catch {
|
|
63
|
+
convertedLines.push(line); // Keep malformed lines as-is
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
console.log(`[CLEANUP] File: ${filePath}`);
|
|
68
|
+
console.log(`[CLEANUP] Total records: ${totalRecords}`);
|
|
69
|
+
console.log(`[CLEANUP] Records with label 'fp': ${fpCount}`);
|
|
70
|
+
|
|
71
|
+
if (apply && fpCount > 0) {
|
|
72
|
+
fs.writeFileSync(filePath, convertedLines.join('\n'), 'utf8');
|
|
73
|
+
console.log(`[CLEANUP] APPLIED: Converted ${fpCount} 'fp' labels to 'unconfirmed'`);
|
|
74
|
+
} else if (!apply && fpCount > 0) {
|
|
75
|
+
console.log(`[CLEANUP] DRY-RUN: Would convert ${fpCount} labels. Use --apply to write.`);
|
|
76
|
+
} else {
|
|
77
|
+
console.log(`[CLEANUP] No 'fp' labels found. Nothing to do.`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
main();
|
package/src/canary-tokens.js
CHANGED
|
@@ -71,6 +71,55 @@ function createCanaryNpmrc(tokens) {
|
|
|
71
71
|
return `//registry.npmjs.org/:_authToken=${tokens.NPM_AUTH_TOKEN}\n`;
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
+
/**
|
|
75
|
+
* Generate fake AWS credentials file content.
|
|
76
|
+
* Format matches ~/.aws/credentials (INI format, format-valid key IDs).
|
|
77
|
+
* @param {Record<string, string>} tokens - The token map from generateCanaryTokens()
|
|
78
|
+
* @returns {string} AWS credentials file content
|
|
79
|
+
*/
|
|
80
|
+
function createCanaryAwsCredentials(tokens) {
|
|
81
|
+
return [
|
|
82
|
+
'[default]',
|
|
83
|
+
`aws_access_key_id = ${tokens.AWS_ACCESS_KEY_ID}`,
|
|
84
|
+
`aws_secret_access_key = ${tokens.AWS_SECRET_ACCESS_KEY}`,
|
|
85
|
+
'region = us-east-1',
|
|
86
|
+
''
|
|
87
|
+
].join('\n');
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Generate a fake SSH private key (Ed25519 format).
|
|
92
|
+
* The key is structurally valid PEM but cryptographically meaningless.
|
|
93
|
+
* Malware that reads ~/.ssh/id_rsa or id_ed25519 will exfiltrate this.
|
|
94
|
+
* @returns {string} Fake SSH private key content
|
|
95
|
+
*/
|
|
96
|
+
function createCanarySshKey() {
|
|
97
|
+
const fakeKeyData = crypto.randomBytes(64).toString('base64');
|
|
98
|
+
return [
|
|
99
|
+
'-----BEGIN OPENSSH PRIVATE KEY-----',
|
|
100
|
+
fakeKeyData.substring(0, 70),
|
|
101
|
+
fakeKeyData.substring(0, 70),
|
|
102
|
+
'-----END OPENSSH PRIVATE KEY-----',
|
|
103
|
+
''
|
|
104
|
+
].join('\n');
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Generate a fake .gitconfig with user identity.
|
|
109
|
+
* Malware fingerprinting the developer will exfiltrate this.
|
|
110
|
+
* @returns {string} Fake .gitconfig content
|
|
111
|
+
*/
|
|
112
|
+
function createCanaryGitconfig() {
|
|
113
|
+
return [
|
|
114
|
+
'[user]',
|
|
115
|
+
'\tname = John Developer',
|
|
116
|
+
'\temail = john.dev@company-internal.example.com',
|
|
117
|
+
'[credential]',
|
|
118
|
+
'\thelper = store',
|
|
119
|
+
''
|
|
120
|
+
].join('\n');
|
|
121
|
+
}
|
|
122
|
+
|
|
74
123
|
/**
|
|
75
124
|
* Search for canary tokens in network logs from sandbox.
|
|
76
125
|
* Network log structure matches sandbox.js report.network:
|
|
@@ -199,6 +248,9 @@ module.exports = {
|
|
|
199
248
|
generateCanaryTokens,
|
|
200
249
|
createCanaryEnvFile,
|
|
201
250
|
createCanaryNpmrc,
|
|
251
|
+
createCanaryAwsCredentials,
|
|
252
|
+
createCanarySshKey,
|
|
253
|
+
createCanaryGitconfig,
|
|
202
254
|
detectCanaryExfiltration,
|
|
203
255
|
detectCanaryInOutput
|
|
204
256
|
};
|
package/src/index.js
CHANGED
|
@@ -523,6 +523,35 @@ async function run(targetPath, options = {}) {
|
|
|
523
523
|
threats.push(...temporalThreats);
|
|
524
524
|
}
|
|
525
525
|
|
|
526
|
+
// Auto-sandbox: trigger sandbox analysis when static scan detects threats.
|
|
527
|
+
// Preliminary score estimate: count CRITICAL/HIGH threats as a quick heuristic.
|
|
528
|
+
// Only when --auto-sandbox flag is set, no explicit sandboxResult, and Docker available.
|
|
529
|
+
if (options.autoSandbox && !options.sandboxResult) {
|
|
530
|
+
const critCount = threats.filter(t => t.severity === 'CRITICAL').length;
|
|
531
|
+
const highCount = threats.filter(t => t.severity === 'HIGH').length;
|
|
532
|
+
const prelimScore = Math.min(100, critCount * 25 + highCount * 10);
|
|
533
|
+
if (prelimScore >= 20) {
|
|
534
|
+
try {
|
|
535
|
+
const { isDockerAvailable, buildSandboxImage, runSandbox } = require('./sandbox/index.js');
|
|
536
|
+
if (isDockerAvailable()) {
|
|
537
|
+
console.log(`\n[AUTO-SANDBOX] Preliminary score ~${prelimScore} >= 20 — triggering sandbox analysis...`);
|
|
538
|
+
const built = await buildSandboxImage();
|
|
539
|
+
if (built) {
|
|
540
|
+
const sbResult = await runSandbox(targetPath, { local: true, strict: false });
|
|
541
|
+
if (sbResult && Array.isArray(sbResult.findings)) {
|
|
542
|
+
options.sandboxResult = sbResult;
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
} else {
|
|
546
|
+
debugLog('[AUTO-SANDBOX] Docker not available — skipping sandbox');
|
|
547
|
+
}
|
|
548
|
+
} catch (e) {
|
|
549
|
+
debugLog('[AUTO-SANDBOX] Error:', e && e.message);
|
|
550
|
+
// Graceful fallback — sandbox is best-effort
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
526
555
|
// Sandbox integration
|
|
527
556
|
let sandboxData = null;
|
|
528
557
|
if (options.sandboxResult && Array.isArray(options.sandboxResult.findings)) {
|
package/src/ml/classifier.js
CHANGED
|
@@ -8,15 +8,19 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Guard rails:
|
|
10
10
|
* - score < 20 → clean (below T1 threshold)
|
|
11
|
-
* - score >= 35
|
|
12
|
-
*
|
|
13
|
-
*
|
|
11
|
+
* - score >= 35:
|
|
12
|
+
* 1. HC_TYPES present → bypass (never suppress)
|
|
13
|
+
* 2. Bundler model available → bundler model decides (fp_bundler or bypass)
|
|
14
|
+
* 3. Bundler model absent → bypass (unchanged)
|
|
15
|
+
* - model absent → bypass (T1 zone)
|
|
16
|
+
* - high-confidence threat types → bypass (never suppress HC types, T1 zone)
|
|
14
17
|
*/
|
|
15
18
|
|
|
16
19
|
const { extractFeatures } = require('./feature-extractor.js');
|
|
17
20
|
|
|
18
|
-
// Lazy-loaded
|
|
21
|
+
// Lazy-loaded models (allows resetModel for testing)
|
|
19
22
|
let _model = undefined; // undefined = not yet loaded, null = absent
|
|
23
|
+
let _bundlerModel = undefined; // undefined = not yet loaded, null = absent
|
|
20
24
|
|
|
21
25
|
// High-confidence malice types that must NEVER be suppressed by ML
|
|
22
26
|
const HC_TYPES = new Set([
|
|
@@ -59,6 +63,37 @@ function resetModel() {
|
|
|
59
63
|
_model = undefined;
|
|
60
64
|
}
|
|
61
65
|
|
|
66
|
+
// --- Bundler detector model (ML2) ---
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Load the bundler detector model from model-bundler.js. Returns the model object or null.
|
|
70
|
+
*/
|
|
71
|
+
function loadBundlerModel() {
|
|
72
|
+
if (_bundlerModel !== undefined) return _bundlerModel;
|
|
73
|
+
try {
|
|
74
|
+
const trees = require('./model-bundler.js');
|
|
75
|
+
_bundlerModel = trees || null;
|
|
76
|
+
} catch {
|
|
77
|
+
_bundlerModel = null;
|
|
78
|
+
}
|
|
79
|
+
return _bundlerModel;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Check if a trained bundler model is available.
|
|
84
|
+
* @returns {boolean}
|
|
85
|
+
*/
|
|
86
|
+
function isBundlerModelAvailable() {
|
|
87
|
+
return loadBundlerModel() !== null;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Reset bundler model cache (for testing isolation).
|
|
92
|
+
*/
|
|
93
|
+
function resetBundlerModel() {
|
|
94
|
+
_bundlerModel = undefined;
|
|
95
|
+
}
|
|
96
|
+
|
|
62
97
|
/**
|
|
63
98
|
* Sigmoid function: maps raw margin to probability [0, 1].
|
|
64
99
|
* @param {number} x - raw margin (sum of tree outputs)
|
|
@@ -134,6 +169,43 @@ function buildFeatureVector(result, meta) {
|
|
|
134
169
|
return values;
|
|
135
170
|
}
|
|
136
171
|
|
|
172
|
+
/**
|
|
173
|
+
* Build ordered feature vector for the bundler model from scan result and metadata.
|
|
174
|
+
* @param {Object} result - scan result from run()
|
|
175
|
+
* @param {Object} meta - enriched metadata
|
|
176
|
+
* @returns {Array<number>} ordered feature values
|
|
177
|
+
*/
|
|
178
|
+
function buildBundlerFeatureVector(result, meta) {
|
|
179
|
+
const model = loadBundlerModel();
|
|
180
|
+
if (!model) return [];
|
|
181
|
+
|
|
182
|
+
const features = extractFeatures(result, meta || {});
|
|
183
|
+
const values = new Array(model.features.length);
|
|
184
|
+
for (let i = 0; i < model.features.length; i++) {
|
|
185
|
+
values[i] = features[model.features[i]] || 0;
|
|
186
|
+
}
|
|
187
|
+
return values;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Run bundler model prediction on ordered feature values.
|
|
192
|
+
* @param {Array<number>} featureValues - ordered feature values matching bundler model features
|
|
193
|
+
* @returns {{ probability: number, prediction: string }}
|
|
194
|
+
*/
|
|
195
|
+
function predictBundler(featureValues) {
|
|
196
|
+
const model = loadBundlerModel();
|
|
197
|
+
if (!model) return { probability: 0.5, prediction: 'bypass' };
|
|
198
|
+
|
|
199
|
+
let margin = 0;
|
|
200
|
+
for (const tree of model.trees) {
|
|
201
|
+
margin += traverseTree(tree, featureValues);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const probability = sigmoid(margin);
|
|
205
|
+
const prediction = probability >= model.threshold ? 'malicious' : 'clean';
|
|
206
|
+
return { probability, prediction };
|
|
207
|
+
}
|
|
208
|
+
|
|
137
209
|
/**
|
|
138
210
|
* Check if result contains any high-confidence threat types.
|
|
139
211
|
* @param {Object} result - scan result
|
|
@@ -150,7 +222,7 @@ function hasHighConfidenceThreat(result) {
|
|
|
150
222
|
* @param {Object} result - scan result from run() with { threats, summary }
|
|
151
223
|
* @param {Object} meta - enriched metadata for feature extraction
|
|
152
224
|
* @returns {{ prediction: string, probability: number, reason: string }}
|
|
153
|
-
* prediction: 'clean' | 'malicious' | 'bypass'
|
|
225
|
+
* prediction: 'clean' | 'malicious' | 'bypass' | 'fp_bundler'
|
|
154
226
|
* reason: explains why this prediction was made
|
|
155
227
|
*/
|
|
156
228
|
function classifyPackage(result, meta) {
|
|
@@ -161,8 +233,32 @@ function classifyPackage(result, meta) {
|
|
|
161
233
|
return { prediction: 'clean', probability: 0, reason: 'below_t1' };
|
|
162
234
|
}
|
|
163
235
|
|
|
164
|
-
// Guard rail 2: above T1 zone —
|
|
236
|
+
// Guard rail 2: above T1 zone — bundler model or bypass
|
|
165
237
|
if (score >= 35) {
|
|
238
|
+
// Guard rail 2a: HC types present → always bypass (never suppress)
|
|
239
|
+
if (hasHighConfidenceThreat(result)) {
|
|
240
|
+
return { prediction: 'bypass', probability: 1, reason: 'high_confidence_threat' };
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// Guard rail 2b: bundler model available → let it decide
|
|
244
|
+
if (isBundlerModelAvailable()) {
|
|
245
|
+
const bundlerVec = buildBundlerFeatureVector(result, meta);
|
|
246
|
+
const bundlerResult = predictBundler(bundlerVec);
|
|
247
|
+
if (bundlerResult.prediction === 'clean') {
|
|
248
|
+
return {
|
|
249
|
+
prediction: 'fp_bundler',
|
|
250
|
+
probability: Math.round(bundlerResult.probability * 1000) / 1000,
|
|
251
|
+
reason: 'ml_bundler_clean'
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
return {
|
|
255
|
+
prediction: 'bypass',
|
|
256
|
+
probability: Math.round(bundlerResult.probability * 1000) / 1000,
|
|
257
|
+
reason: 'ml_bundler_malicious'
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// Guard rail 2c: bundler model absent → bypass
|
|
166
262
|
return { prediction: 'bypass', probability: 1, reason: 'score_above_threshold' };
|
|
167
263
|
}
|
|
168
264
|
|
|
@@ -196,5 +292,11 @@ module.exports = {
|
|
|
196
292
|
traverseTree,
|
|
197
293
|
sigmoid,
|
|
198
294
|
buildFeatureVector,
|
|
199
|
-
hasHighConfidenceThreat
|
|
295
|
+
hasHighConfidenceThreat,
|
|
296
|
+
// Bundler detector (ML2)
|
|
297
|
+
isBundlerModelAvailable,
|
|
298
|
+
resetBundlerModel,
|
|
299
|
+
loadBundlerModel,
|
|
300
|
+
predictBundler,
|
|
301
|
+
buildBundlerFeatureVector
|
|
200
302
|
};
|
|
@@ -205,8 +205,9 @@ function buildTrainingRecord(result, params) {
|
|
|
205
205
|
// --- Label ---
|
|
206
206
|
// 'clean' = no findings or T3 only
|
|
207
207
|
// 'suspect' = T1/T2 (pending manual review)
|
|
208
|
+
// 'unconfirmed' = sandbox clean, not manually reviewed (default for automated relabeling)
|
|
208
209
|
// 'confirmed' = manually confirmed malicious
|
|
209
|
-
// 'fp' = manually confirmed false positive
|
|
210
|
+
// 'fp' = manually confirmed false positive (requires manualReview=true)
|
|
210
211
|
record.label = label || 'suspect';
|
|
211
212
|
record.tier = tier || null;
|
|
212
213
|
|
package/src/ml/jsonl-writer.js
CHANGED
|
@@ -128,16 +128,33 @@ function getStats() {
|
|
|
128
128
|
}
|
|
129
129
|
}
|
|
130
130
|
|
|
131
|
+
// Valid labels for ML training records
|
|
132
|
+
const VALID_LABELS = new Set(['fp', 'confirmed', 'unconfirmed']);
|
|
133
|
+
|
|
131
134
|
/**
|
|
132
135
|
* Update the label of records matching a given package name.
|
|
133
136
|
* Used when manual confirmation (fp/confirmed) is applied retroactively.
|
|
134
137
|
*
|
|
135
138
|
* @param {string} packageName - package name to relabel
|
|
136
|
-
* @param {string} newLabel - 'fp' or '
|
|
139
|
+
* @param {string} newLabel - 'fp', 'confirmed', or 'unconfirmed'
|
|
137
140
|
* @param {number} [sandboxFindingCount] - number of sandbox findings (defense-in-depth for 'confirmed')
|
|
141
|
+
* @param {boolean} [manualReview] - required for 'fp' label (prevents automated contamination)
|
|
138
142
|
* @returns {number} number of records updated
|
|
139
143
|
*/
|
|
140
|
-
function relabelRecords(packageName, newLabel, sandboxFindingCount) {
|
|
144
|
+
function relabelRecords(packageName, newLabel, sandboxFindingCount, manualReview) {
|
|
145
|
+
// Validate label
|
|
146
|
+
if (!VALID_LABELS.has(newLabel)) {
|
|
147
|
+
console.warn(`[ML] BLOCKED relabel to '${newLabel}' for ${packageName}: invalid label (valid: ${[...VALID_LABELS].join(', ')})`);
|
|
148
|
+
return 0;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Defense-in-depth: 'fp' requires explicit manual review flag to prevent
|
|
152
|
+
// automated sandbox-clean → fp contamination (8176 records in 3 months)
|
|
153
|
+
if (newLabel === 'fp' && manualReview !== true) {
|
|
154
|
+
console.warn(`[ML] BLOCKED relabel to 'fp' for ${packageName}: manualReview required (use 'unconfirmed' for automated relabeling)`);
|
|
155
|
+
return 0;
|
|
156
|
+
}
|
|
157
|
+
|
|
141
158
|
// Defense-in-depth: never write 'confirmed' without real sandbox findings
|
|
142
159
|
if (newLabel === 'confirmed' && (!sandboxFindingCount || sandboxFindingCount === 0)) {
|
|
143
160
|
console.warn(`[ML] BLOCKED relabel to 'confirmed' for ${packageName}: sandbox_finding_count=${sandboxFindingCount || 0}`);
|