muaddib-scanner 2.11.174 → 2.11.177
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +4 -3
- package/{self-scan-v2.11.174.json → self-scan-v2.11.177.json} +1 -1
- package/src/commands/hooks-init.js +30 -17
- package/src/commands/safe-install.js +25 -14
- package/src/ioc/bootstrap.js +20 -0
- package/src/ioc/scraper.js +49 -114
- package/src/ioc/updater.js +11 -12
- package/src/ml/llm-detective.js +4 -1
- package/src/monitor/queue.js +94 -37
- package/src/monitor/webhook.js +21 -10
- package/src/pipeline/executor.js +45 -41
- package/src/runtime/serve.js +32 -4
- package/src/scanner/ai-config.js +4 -0
- package/src/scanner/anti-scanner-injection.js +4 -0
- package/src/scanner/ast-detectors/helpers.js +3 -19
- package/src/scanner/ast.js +1 -6
- package/src/scanner/dependencies.js +5 -0
- package/src/scanner/entropy.js +13 -25
- package/src/scanner/paranoid.js +4 -0
- package/src/scanner/python.js +18 -7
- package/src/scoring/delta-multiplier.js +4 -1
- package/src/shared/entropy.js +45 -0
- package/src/shared/registry-auth.js +5 -2
- package/src/utils.js +15 -1
- package/stats.json +2 -2
package/README.md
CHANGED
|
@@ -214,7 +214,7 @@ muaddib replay # Ground truth validation (90/94 TPR@3, v2.11
|
|
|
214
214
|
|
|
215
215
|
### <!--stat:rulesTotal-->277<!--/stat:rulesTotal--> detection rules
|
|
216
216
|
|
|
217
|
-
All rules (<!--stat:rulesCore-->272<!--/stat:rulesCore--> RULES + <!--stat:rulesParanoid-->5<!--/stat:rulesParanoid--> PARANOID) are mapped to MITRE ATT&CK techniques. See [SECURITY.md](SECURITY.md#detection-rules
|
|
217
|
+
All rules (<!--stat:rulesCore-->272<!--/stat:rulesCore--> RULES + <!--stat:rulesParanoid-->5<!--/stat:rulesParanoid--> PARANOID) are mapped to MITRE ATT&CK techniques. See [SECURITY.md](SECURITY.md#detection-rules) for the complete rules reference.
|
|
218
218
|
|
|
219
219
|
### Detected campaigns
|
|
220
220
|
|
|
@@ -345,7 +345,7 @@ npm test
|
|
|
345
345
|
|
|
346
346
|
### Testing
|
|
347
347
|
|
|
348
|
-
- **<!--stat:tests-->
|
|
348
|
+
- **<!--stat:tests-->4539<!--/stat:tests--> tests** across <!--stat:testFiles-->152<!--/stat:testFiles--> modular test files
|
|
349
349
|
- **56 fuzz tests** - Malformed inputs, ReDoS, unicode, binary
|
|
350
350
|
- **Datadog 17K benchmark** - 14,587 confirmed malware samples (in-scope)
|
|
351
351
|
- **Ground truth validation** - 96 real-world attacks (95.74% TPR@3, 88.30% TPR@20 — v2.11.48 full measure on 94 in-scope)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "muaddib-scanner",
|
|
3
|
-
"version": "2.11.
|
|
3
|
+
"version": "2.11.177",
|
|
4
4
|
"description": "Supply-chain threat detection & response for npm & PyPI/Python",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
"scan": "node bin/muaddib.js scan .",
|
|
15
15
|
"update": "node bin/muaddib.js update",
|
|
16
16
|
"lint": "eslint src bin --ext .js",
|
|
17
|
+
"ci:local": "node scripts/check-deps-typosquats.js && node scripts/check-no-tracked-node-modules.js && npm audit --audit-level=high && npm run docs:stats:check && npm test",
|
|
17
18
|
"docs:stats": "node scripts/collect-stats.js && node scripts/sync-stats.js",
|
|
18
19
|
"docs:stats:tests": "node scripts/collect-stats.js --with-tests && node scripts/sync-stats.js",
|
|
19
20
|
"docs:stats:check": "node scripts/collect-stats.js --check && node scripts/sync-stats.js --check",
|
|
@@ -54,8 +55,8 @@
|
|
|
54
55
|
"@inquirer/prompts": "8.5.2",
|
|
55
56
|
"acorn": "8.17.0",
|
|
56
57
|
"acorn-walk": "8.3.5",
|
|
57
|
-
"adm-zip": "0.
|
|
58
|
-
"js-yaml": "5.1
|
|
58
|
+
"adm-zip": "0.6.0",
|
|
59
|
+
"js-yaml": "5.2.1",
|
|
59
60
|
"web-tree-sitter": "^0.26.9"
|
|
60
61
|
},
|
|
61
62
|
"devDependencies": {
|
|
@@ -54,6 +54,32 @@ const HOOK_COMMANDS = {
|
|
|
54
54
|
diff: 'npx muaddib diff HEAD --fail-on high'
|
|
55
55
|
};
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Back up an existing pre-commit hook before overwriting it, keeping the 3 most recent
|
|
59
|
+
* timestamped backups. Shared by the husky and native-git hook writers so neither silently
|
|
60
|
+
* destroys a user's existing hook (lint-staged, tests, etc.).
|
|
61
|
+
* @param {string} hookPath - Absolute path to the pre-commit hook to be written
|
|
62
|
+
*/
|
|
63
|
+
function backupExistingHook(hookPath) {
|
|
64
|
+
if (!fs.existsSync(hookPath)) return;
|
|
65
|
+
const backup = `${hookPath}.backup.${Date.now()}`;
|
|
66
|
+
fs.copyFileSync(hookPath, backup);
|
|
67
|
+
console.log(`[INFO] Backed up existing hook to ${backup}`);
|
|
68
|
+
|
|
69
|
+
// Cleanup old backups, keep only 3 most recent
|
|
70
|
+
try {
|
|
71
|
+
const hooksDir = path.dirname(hookPath);
|
|
72
|
+
const base = path.basename(hookPath);
|
|
73
|
+
const backups = fs.readdirSync(hooksDir)
|
|
74
|
+
.filter(f => f.startsWith(base + '.backup.'))
|
|
75
|
+
.sort()
|
|
76
|
+
.reverse();
|
|
77
|
+
for (const old of backups.slice(3)) {
|
|
78
|
+
fs.unlinkSync(path.join(hooksDir, old));
|
|
79
|
+
}
|
|
80
|
+
} catch { /* ignore cleanup errors */ }
|
|
81
|
+
}
|
|
82
|
+
|
|
57
83
|
async function initHooks(targetPath, options = {}) {
|
|
58
84
|
const resolvedPath = path.resolve(targetPath);
|
|
59
85
|
const hookType = options.type || 'auto';
|
|
@@ -138,6 +164,9 @@ echo "[MUADDIB] Running security check..."
|
|
|
138
164
|
${command}
|
|
139
165
|
`;
|
|
140
166
|
|
|
167
|
+
// Back up any existing hook before overwriting (same policy as initGitHook): a project may
|
|
168
|
+
// already have a husky pre-commit (lint-staged, tests…) — clobbering it silently loses config.
|
|
169
|
+
backupExistingHook(preCommitPath);
|
|
141
170
|
fs.writeFileSync(preCommitPath, hookContent, { mode: 0o755 });
|
|
142
171
|
if (process.platform !== 'win32') {
|
|
143
172
|
fs.chmodSync(preCommitPath, 0o755);
|
|
@@ -219,23 +248,7 @@ exit 0
|
|
|
219
248
|
`;
|
|
220
249
|
|
|
221
250
|
// Backup existing hook (limit to 3 backups)
|
|
222
|
-
|
|
223
|
-
const backup = `${preCommitPath}.backup.${Date.now()}`;
|
|
224
|
-
fs.copyFileSync(preCommitPath, backup);
|
|
225
|
-
console.log(`[INFO] Backed up existing hook to ${backup}`);
|
|
226
|
-
|
|
227
|
-
// Cleanup old backups, keep only 3 most recent
|
|
228
|
-
try {
|
|
229
|
-
const hooksDir = path.dirname(preCommitPath);
|
|
230
|
-
const backups = fs.readdirSync(hooksDir)
|
|
231
|
-
.filter(f => f.startsWith('pre-commit.backup.'))
|
|
232
|
-
.sort()
|
|
233
|
-
.reverse();
|
|
234
|
-
for (const old of backups.slice(3)) {
|
|
235
|
-
fs.unlinkSync(path.join(hooksDir, old));
|
|
236
|
-
}
|
|
237
|
-
} catch { /* ignore cleanup errors */ }
|
|
238
|
-
}
|
|
251
|
+
backupExistingHook(preCommitPath);
|
|
239
252
|
|
|
240
253
|
fs.writeFileSync(preCommitPath, hookContent, { mode: 0o755 });
|
|
241
254
|
if (process.platform !== 'win32') {
|
|
@@ -133,31 +133,42 @@ function checkIOCs(pkg, pkgName, pkgVersion) {
|
|
|
133
133
|
return malicious || null;
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
-
//
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
136
|
+
// Split a package spec into { pkgName, pkgVersion }.
|
|
137
|
+
// Handles scoped packages (@scope/name) and version specs (@scope/name@spec or name@spec).
|
|
138
|
+
// Transitive deps arrive as `name@^1.2.3` (range specs from dependency maps): the spec must
|
|
139
|
+
// be split off the name even when it is not an exact version, otherwise pkgName keeps the
|
|
140
|
+
// full spec and the IOC map lookup (keyed on bare names) never matches — wildcard IOCs
|
|
141
|
+
// included. Range/tag specs leave pkgVersion null so only wildcard IOCs match (SFI-003).
|
|
142
|
+
function parsePackageSpec(pkg) {
|
|
141
143
|
let pkgName = pkg;
|
|
142
144
|
let pkgVersion = null;
|
|
143
|
-
|
|
144
|
-
// Handle scoped packages (@scope/name) and versions (@scope/name@version or name@version)
|
|
145
145
|
if (pkg.startsWith('@')) {
|
|
146
146
|
// Scoped package
|
|
147
147
|
const parts = pkg.slice(1).split('@');
|
|
148
|
-
if (parts.length >= 2
|
|
149
|
-
|
|
148
|
+
if (parts.length >= 2) {
|
|
149
|
+
const spec = parts.pop();
|
|
150
150
|
pkgName = '@' + parts.join('@');
|
|
151
|
+
if (spec.match(/^\d/)) pkgVersion = spec;
|
|
151
152
|
}
|
|
152
153
|
} else {
|
|
153
154
|
const parts = pkg.split('@');
|
|
154
|
-
if (parts.length >= 2
|
|
155
|
-
|
|
155
|
+
if (parts.length >= 2) {
|
|
156
|
+
const spec = parts.pop();
|
|
156
157
|
pkgName = parts.join('@');
|
|
158
|
+
if (spec.match(/^\d/)) pkgVersion = spec;
|
|
157
159
|
}
|
|
158
160
|
}
|
|
159
|
-
|
|
160
|
-
|
|
161
|
+
return { pkgName, pkgVersion };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Scan a package and its dependencies recursively
|
|
165
|
+
async function scanPackageRecursive(pkg, depth = 0, maxDepth = 3) {
|
|
166
|
+
const indent = ' '.repeat(depth);
|
|
167
|
+
|
|
168
|
+
// Extract name and version of the package
|
|
169
|
+
const { pkgName, pkgVersion } = parsePackageSpec(pkg);
|
|
170
|
+
|
|
171
|
+
|
|
161
172
|
// Avoid infinite loops
|
|
162
173
|
if (scannedPackages.has(pkgName)) {
|
|
163
174
|
return { safe: true };
|
|
@@ -309,4 +320,4 @@ async function safeInstall(packages, options = {}) {
|
|
|
309
320
|
return { blocked: false };
|
|
310
321
|
}
|
|
311
322
|
|
|
312
|
-
module.exports = { safeInstall, scanPackageRecursive, REHABILITATED_PACKAGES, checkRehabilitated, isValidPackageName, checkIOCs };
|
|
323
|
+
module.exports = { safeInstall, scanPackageRecursive, parsePackageSpec, REHABILITATED_PACKAGES, checkRehabilitated, isValidPackageName, checkIOCs };
|
package/src/ioc/bootstrap.js
CHANGED
|
@@ -51,6 +51,13 @@ const DOWNLOAD_TIMEOUT = 60_000;
|
|
|
51
51
|
// Max redirects to follow
|
|
52
52
|
const MAX_REDIRECTS = 5;
|
|
53
53
|
|
|
54
|
+
// Decompression bomb guard: cap the DECOMPRESSED byte count streamed to disk. A compromised
|
|
55
|
+
// release asset (exactly this project's threat model — maintainer account takeover) could
|
|
56
|
+
// otherwise fill the disk: this was the only unbounded decompressor in the repo (download.js
|
|
57
|
+
// has the M4 gzip hint, scraper.js has MAX_ENTRY/TOTAL_UNCOMPRESSED). Env-tunable like the
|
|
58
|
+
// scraper caps; default leaves ample headroom over the ~112-223MB iocs.json.
|
|
59
|
+
const MAX_DECOMPRESSED_SIZE = parseInt(process.env.MUADDIB_BOOTSTRAP_MAX_DECOMPRESSED || '', 10) || 1024 * 1024 * 1024; // 1 GB
|
|
60
|
+
|
|
54
61
|
// Allowed redirect domains (SSRF protection)
|
|
55
62
|
const ALLOWED_REDIRECT_DOMAINS = [
|
|
56
63
|
'github.com',
|
|
@@ -115,6 +122,19 @@ function downloadAndDecompress(url, destPath) {
|
|
|
115
122
|
const gunzip = zlib.createGunzip();
|
|
116
123
|
const fileStream = fs.createWriteStream(tmpPath);
|
|
117
124
|
|
|
125
|
+
// Decompression bomb guard: count decompressed bytes, abort past the cap.
|
|
126
|
+
let decompressedBytes = 0;
|
|
127
|
+
gunzip.on('data', (chunk) => {
|
|
128
|
+
decompressedBytes += chunk.length;
|
|
129
|
+
if (decompressedBytes > MAX_DECOMPRESSED_SIZE) {
|
|
130
|
+
res.destroy();
|
|
131
|
+
gunzip.destroy();
|
|
132
|
+
fileStream.destroy();
|
|
133
|
+
try { fs.unlinkSync(tmpPath); } catch (e) { debugLog('cleanup failed:', e.message); }
|
|
134
|
+
reject(new Error('Decompressed size exceeds cap (' + MAX_DECOMPRESSED_SIZE + ' bytes) — aborting'));
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
|
|
118
138
|
gunzip.on('error', (err) => {
|
|
119
139
|
fileStream.destroy();
|
|
120
140
|
try { fs.unlinkSync(tmpPath); } catch (e) { debugLog('cleanup failed:', e.message); }
|
package/src/ioc/scraper.js
CHANGED
|
@@ -733,6 +733,7 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
|
|
|
733
733
|
fetchSpinner.start('Fetching OSSF entries... 0/' + toFetch.length);
|
|
734
734
|
}
|
|
735
735
|
|
|
736
|
+
let failedFetches = 0;
|
|
736
737
|
for (let i = 0; i < toFetch.length; i += BATCH_SIZE) {
|
|
737
738
|
const batch = toFetch.slice(i, i + BATCH_SIZE);
|
|
738
739
|
const results = await Promise.all(batch.map(function(entry) {
|
|
@@ -741,7 +742,7 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
|
|
|
741
742
|
}));
|
|
742
743
|
|
|
743
744
|
for (const result of results) {
|
|
744
|
-
if (!result || result.status !== 200 || !result.data) continue;
|
|
745
|
+
if (!result || result.status !== 200 || !result.data) { failedFetches++; continue; }
|
|
745
746
|
const parsed = parseOSVEntry(result.data, 'ossf-malicious');
|
|
746
747
|
for (const p of parsed) packages.push(p);
|
|
747
748
|
}
|
|
@@ -760,8 +761,16 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
|
|
|
760
761
|
fetchSpinner.succeed('Fetched OSSF entries: ' + packages.length + ' packages');
|
|
761
762
|
}
|
|
762
763
|
|
|
763
|
-
// Save tree SHA for next incremental run
|
|
764
|
-
|
|
764
|
+
// Save tree SHA for next incremental run — only if every entry was fetched.
|
|
765
|
+
// Persisting the SHA after partial fetches (e.g. GitHub rate-limiting the raw batches)
|
|
766
|
+
// marks the failed entries as "done": subsequent runs skip everything until the next
|
|
767
|
+
// OSSF commit, silently losing IOCs. On partial failure, keep the old SHA so the next
|
|
768
|
+
// run retries the delta.
|
|
769
|
+
if (failedFetches === 0) {
|
|
770
|
+
try { fs.writeFileSync(shaFile, treeSha); } catch {}
|
|
771
|
+
} else {
|
|
772
|
+
console.log('[SCRAPER] ' + failedFetches + ' OSSF fetches failed — tree SHA not persisted, next run will retry');
|
|
773
|
+
}
|
|
765
774
|
} catch (e) {
|
|
766
775
|
console.log('[SCRAPER] Error: ' + e.message);
|
|
767
776
|
}
|
|
@@ -921,65 +930,14 @@ async function scrapeOSVPyPIDataDump() {
|
|
|
921
930
|
return packages;
|
|
922
931
|
}
|
|
923
932
|
|
|
924
|
-
//
|
|
925
|
-
//
|
|
926
|
-
//
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
const resp = await fetchJSON('https://api.osv.dev/v1/query', {
|
|
933
|
-
method: 'POST',
|
|
934
|
-
headers: { 'Content-Type': 'application/json' },
|
|
935
|
-
body: { package: { ecosystem: 'npm' } }
|
|
936
|
-
});
|
|
937
|
-
|
|
938
|
-
if (resp.status === 200 && resp.data && resp.data.vulns) {
|
|
939
|
-
for (const vuln of resp.data.vulns) {
|
|
940
|
-
// Filter GHSA with malware mention
|
|
941
|
-
if (vuln.id && vuln.id.startsWith('GHSA-')) {
|
|
942
|
-
const summary = (vuln.summary || '').toLowerCase();
|
|
943
|
-
const details = (vuln.details || '').toLowerCase();
|
|
944
|
-
const isMalware = summary.includes('malware') ||
|
|
945
|
-
summary.includes('malicious') ||
|
|
946
|
-
details.includes('malware') ||
|
|
947
|
-
details.includes('malicious') ||
|
|
948
|
-
summary.includes('backdoor') ||
|
|
949
|
-
summary.includes('trojan');
|
|
950
|
-
|
|
951
|
-
if (isMalware) {
|
|
952
|
-
for (const affected of vuln.affected || []) {
|
|
953
|
-
if (affected.package && affected.package.ecosystem === 'npm') {
|
|
954
|
-
const versions = extractVersions(affected);
|
|
955
|
-
for (const ver of versions) {
|
|
956
|
-
packages.push({
|
|
957
|
-
id: vuln.id,
|
|
958
|
-
name: affected.package.name,
|
|
959
|
-
version: ver,
|
|
960
|
-
severity: 'critical',
|
|
961
|
-
confidence: 'high',
|
|
962
|
-
source: 'github-advisory',
|
|
963
|
-
description: (vuln.summary || 'Malicious package').slice(0, 200),
|
|
964
|
-
references: ['https://github.com/advisories/' + vuln.id],
|
|
965
|
-
mitre: 'T1195.002',
|
|
966
|
-
freshness: createFreshness('github-advisory', 'high')
|
|
967
|
-
});
|
|
968
|
-
}
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
}
|
|
972
|
-
}
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
|
|
976
|
-
console.log(`[SCRAPER] ${packages.length} packages`);
|
|
977
|
-
} catch (e) {
|
|
978
|
-
console.log(`[SCRAPER] Error: ${e.message}`);
|
|
979
|
-
}
|
|
980
|
-
|
|
981
|
-
return packages;
|
|
982
|
-
}
|
|
933
|
+
// SOURCE 4 (GitHub Advisory Database) removed 2026-07 (GAP-2): it POSTed
|
|
934
|
+
// api.osv.dev/v1/query with `{ package: { ecosystem: 'npm' } }` — no package name,
|
|
935
|
+
// which the OSV /v1/query API rejects with HTTP 400 on every call. It therefore
|
|
936
|
+
// returned 0 packages permanently while logging as a normal completion (fictional
|
|
937
|
+
// coverage). Empirically verified that the GHSA-malware it targeted is already
|
|
938
|
+
// 100% covered by the OSV MAL- dump (153/153 pre-snapshot npm GHSA advisories
|
|
939
|
+
// present in the IOC store), so its removal loses no coverage. The GHSA poller
|
|
940
|
+
// (src/ioc/ghsa-poller.js) tracks the coverage denominator independently.
|
|
983
941
|
|
|
984
942
|
// ============================================
|
|
985
943
|
// MAIN SCRAPER
|
|
@@ -1063,7 +1021,6 @@ async function runScraper() {
|
|
|
1063
1021
|
scrapeShaiHuludDetector(),
|
|
1064
1022
|
scrapeDatadogIOCs(),
|
|
1065
1023
|
scrapeOSSFMaliciousPackages(osvResult.knownIds),
|
|
1066
|
-
scrapeGitHubAdvisory(),
|
|
1067
1024
|
scrapeOSVPyPIDataDump(),
|
|
1068
1025
|
scrapeAikidoMalwareFeed(),
|
|
1069
1026
|
scrapeOSMQueryLatest()
|
|
@@ -1072,10 +1029,9 @@ async function runScraper() {
|
|
|
1072
1029
|
const shaiHuludResult = results[0];
|
|
1073
1030
|
const datadogResult = results[1];
|
|
1074
1031
|
const ossfPackages = results[2];
|
|
1075
|
-
const
|
|
1076
|
-
const
|
|
1077
|
-
const
|
|
1078
|
-
const osmResult = results[6];
|
|
1032
|
+
const pypiPackages = results[3];
|
|
1033
|
+
const aikidoResult = results[4];
|
|
1034
|
+
const osmResult = results[5];
|
|
1079
1035
|
|
|
1080
1036
|
// Log aggregated warnings
|
|
1081
1037
|
if (_noVersionSkipCount > 0) {
|
|
@@ -1100,7 +1056,6 @@ async function runScraper() {
|
|
|
1100
1056
|
...shaiHuludResult.packages,
|
|
1101
1057
|
...datadogResult.packages,
|
|
1102
1058
|
...ossfPackages,
|
|
1103
|
-
...githubPackages,
|
|
1104
1059
|
...aikidoResult.packages,
|
|
1105
1060
|
...osmResult.packages
|
|
1106
1061
|
];
|
|
@@ -1284,11 +1239,21 @@ async function runScraper() {
|
|
|
1284
1239
|
'npm-removed'
|
|
1285
1240
|
];
|
|
1286
1241
|
|
|
1287
|
-
// Save enriched (full) IOCs — atomic write via .tmp + rename
|
|
1242
|
+
// Save enriched (full) IOCs — atomic write via .tmp + rename.
|
|
1243
|
+
// Stringify ONCE: the pretty-printed DB is a giant string (~2x the 112MB+ file); a second
|
|
1244
|
+
// stringify for the home copy used to double the peak on top of the zip buffers already in
|
|
1245
|
+
// flight. The HMAC is computed here from the same string, then the string is released and
|
|
1246
|
+
// the home copy reuses the file on disk (copyFileSync — no second serialization).
|
|
1288
1247
|
const saveSpinner = new Spinner();
|
|
1289
1248
|
saveSpinner.start('Saving IOCs...');
|
|
1290
1249
|
const tmpIOCFile = IOC_FILE + '.tmp';
|
|
1291
|
-
|
|
1250
|
+
let iocHmac;
|
|
1251
|
+
{
|
|
1252
|
+
const iocJsonData = JSON.stringify(existingIOCs, null, 2);
|
|
1253
|
+
fs.writeFileSync(tmpIOCFile, iocJsonData);
|
|
1254
|
+
const { generateIOCHMAC } = require('./updater.js');
|
|
1255
|
+
iocHmac = generateIOCHMAC(iocJsonData);
|
|
1256
|
+
}
|
|
1292
1257
|
fs.renameSync(tmpIOCFile, IOC_FILE);
|
|
1293
1258
|
|
|
1294
1259
|
// Save compact IOCs (lightweight, shipped in npm) — atomic write
|
|
@@ -1311,12 +1276,11 @@ async function runScraper() {
|
|
|
1311
1276
|
}
|
|
1312
1277
|
try {
|
|
1313
1278
|
const tmpHomeFile = HOME_IOC_FILE + '.tmp';
|
|
1314
|
-
|
|
1315
|
-
fs.
|
|
1316
|
-
// Write HMAC before rename for consistency with updater.js
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
fs.writeFileSync(HOME_IOC_FILE + '.hmac', homeHmac);
|
|
1279
|
+
// Byte-identical copy of the file already written locally — no second stringify peak.
|
|
1280
|
+
fs.copyFileSync(IOC_FILE, tmpHomeFile);
|
|
1281
|
+
// Write HMAC before rename for consistency with updater.js (computed once above from
|
|
1282
|
+
// the same serialized string the local file was written from).
|
|
1283
|
+
fs.writeFileSync(HOME_IOC_FILE + '.hmac', iocHmac);
|
|
1320
1284
|
fs.renameSync(tmpHomeFile, HOME_IOC_FILE);
|
|
1321
1285
|
// Mark HMAC as initialized
|
|
1322
1286
|
const hmacMarker = path.join(homeDir, '.hmac-initialized');
|
|
@@ -1579,43 +1543,14 @@ async function scrapeOSMQueryLatest() {
|
|
|
1579
1543
|
return { packages: npmPackages, pypi_packages: pypiPackages };
|
|
1580
1544
|
}
|
|
1581
1545
|
|
|
1582
|
-
//
|
|
1583
|
-
// SOURCE
|
|
1584
|
-
//
|
|
1585
|
-
//
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
* @returns {Promise<Array>} Parsed IOC package entries
|
|
1591
|
-
*/
|
|
1592
|
-
async function scrapeOSVLightweightAPI() {
|
|
1593
|
-
console.log('[SCRAPER] OSV.dev lightweight API...');
|
|
1594
|
-
const packages = [];
|
|
1595
|
-
|
|
1596
|
-
try {
|
|
1597
|
-
const resp = await fetchJSON('https://api.osv.dev/v1/query', {
|
|
1598
|
-
method: 'POST',
|
|
1599
|
-
headers: { 'Content-Type': 'application/json' },
|
|
1600
|
-
body: { package: { ecosystem: 'npm' } }
|
|
1601
|
-
});
|
|
1602
|
-
|
|
1603
|
-
if (resp.status === 200 && resp.data && resp.data.vulns) {
|
|
1604
|
-
for (const vuln of resp.data.vulns) {
|
|
1605
|
-
if (vuln.id && vuln.id.startsWith('MAL-')) {
|
|
1606
|
-
const parsed = parseOSVEntry(vuln, 'osv-api');
|
|
1607
|
-
for (const p of parsed) packages.push(p);
|
|
1608
|
-
}
|
|
1609
|
-
}
|
|
1610
|
-
}
|
|
1611
|
-
|
|
1612
|
-
console.log('[SCRAPER] ' + packages.length + ' MAL-* packages from OSV API');
|
|
1613
|
-
} catch (e) {
|
|
1614
|
-
console.log('[SCRAPER] OSV API error: ' + e.message);
|
|
1615
|
-
}
|
|
1616
|
-
|
|
1617
|
-
return packages;
|
|
1618
|
-
}
|
|
1546
|
+
// SOURCE 5 (OSV.dev Lightweight API) removed 2026-07 (GAP-2): same defect as
|
|
1547
|
+
// SOURCE 4 — it POSTed api.osv.dev/v1/query with no package name (`{ package:
|
|
1548
|
+
// { ecosystem: 'npm' } }`), which OSV rejects with HTTP 400, so it returned 0
|
|
1549
|
+
// MAL-* packages permanently while `muaddib update` logged "+0 OSV API" as normal.
|
|
1550
|
+
// It was also 100% redundant with the OSV MAL- zip dump (scrapeOSVDataDump), which
|
|
1551
|
+
// pulls the same MAL- npm set. The `muaddib update` light path keeps its recent-
|
|
1552
|
+
// malware coverage via GenSecAI + DataDog + OSM; the authoritative OSV MAL- set
|
|
1553
|
+
// comes from the deep `muaddib scrape`.
|
|
1619
1554
|
|
|
1620
1555
|
/**
|
|
1621
1556
|
* Batch query OSV.dev for specific package names.
|
|
@@ -1700,7 +1635,7 @@ module.exports = {
|
|
|
1700
1635
|
runScraper, scrapeShaiHuludDetector, scrapeDatadogIOCs,
|
|
1701
1636
|
scrapeAikidoMalwareFeed,
|
|
1702
1637
|
scrapeOSMQueryLatest,
|
|
1703
|
-
|
|
1638
|
+
queryOSVBatch,
|
|
1704
1639
|
getSourceConfidence,
|
|
1705
1640
|
// Pure utility functions (exported for testing)
|
|
1706
1641
|
parseCSVLine, parseCSV, extractVersions, parseOSVEntry,
|
package/src/ioc/updater.js
CHANGED
|
@@ -66,28 +66,29 @@ async function updateIOCs() {
|
|
|
66
66
|
mergeIOCs(baseIOCs, yamlStandard);
|
|
67
67
|
console.log('[2/4] YAML IOCs: ' + yamlStandard.packages.length + ' packages, ' + yamlStandard.hashes.length + ' hashes');
|
|
68
68
|
|
|
69
|
-
// Step 3: Download additional IOCs
|
|
69
|
+
// Step 3: Download additional IOCs (GenSecAI + DataDog + OSM)
|
|
70
70
|
// Light path: JSON/REST only, NO heavy zip dumps. Designed to be safe at 15min cadence.
|
|
71
|
-
//
|
|
72
|
-
|
|
73
|
-
|
|
71
|
+
// The OSV MAL- set comes from the deep refresh (`muaddib scrape`, OSV zip dumps + OSSF +
|
|
72
|
+
// Aikido). The former OSV lightweight source was removed 2026-07 (GAP-2): it POSTed a
|
|
73
|
+
// nameless OSV /v1/query (HTTP 400 → always 0) and was redundant with the OSV zip dump.
|
|
74
|
+
const { scrapeShaiHuludDetector, scrapeDatadogIOCs, scrapeOSMQueryLatest } = require('./scraper.js');
|
|
75
|
+
console.log('[3/4] Downloading GenSecAI + DataDog + OSM IOCs...');
|
|
74
76
|
|
|
75
|
-
const [shaiHulud, datadog,
|
|
77
|
+
const [shaiHulud, datadog, osmResult] = await Promise.all([
|
|
76
78
|
scrapeShaiHuludDetector(),
|
|
77
79
|
scrapeDatadogIOCs(),
|
|
78
|
-
scrapeOSVLightweightAPI(),
|
|
79
80
|
scrapeOSMQueryLatest()
|
|
80
81
|
]);
|
|
81
82
|
|
|
82
83
|
const githubIOCs = {
|
|
83
|
-
packages: [].concat(shaiHulud.packages, datadog.packages,
|
|
84
|
+
packages: [].concat(shaiHulud.packages, datadog.packages, osmResult.packages),
|
|
84
85
|
pypi_packages: (osmResult.pypi_packages || []).slice(),
|
|
85
86
|
hashes: [].concat(shaiHulud.hashes || [], datadog.hashes || []),
|
|
86
87
|
markers: [],
|
|
87
88
|
files: []
|
|
88
89
|
};
|
|
89
90
|
mergeIOCs(baseIOCs, githubIOCs);
|
|
90
|
-
console.log(' +' + shaiHulud.packages.length + ' GenSecAI, +' + datadog.packages.length + ' DataDog, +' +
|
|
91
|
+
console.log(' +' + shaiHulud.packages.length + ' GenSecAI, +' + datadog.packages.length + ' DataDog, +' + osmResult.packages.length + ' OSM npm, +' + (osmResult.pypi_packages || []).length + ' OSM PyPI');
|
|
91
92
|
|
|
92
93
|
// Phase 2c (feed health): a feed that previously returned data but now returns 0 is the
|
|
93
94
|
// silent failure mode that froze the OSM feed and collapsed coverage. Raise a one-shot
|
|
@@ -95,13 +96,11 @@ async function updateIOCs() {
|
|
|
95
96
|
// Only feeds that were actually ATTEMPTED are health-checked: OSM is token-gated, so without
|
|
96
97
|
// OSM_API_TOKEN it is SKIPPED (not down) — counting it would raise a false "OSM went dark"
|
|
97
98
|
// alarm in any no-token context (e.g. an ad-hoc `muaddib update`) against the monitor-seeded
|
|
98
|
-
// baseline.
|
|
99
|
-
// so the engine naturally never establishes an alarm-able baseline for it.
|
|
99
|
+
// baseline.
|
|
100
100
|
try {
|
|
101
101
|
const feedCounts = {
|
|
102
102
|
'GenSecAI': shaiHulud.packages.length,
|
|
103
|
-
'DataDog': datadog.packages.length
|
|
104
|
-
'OSV-API': osvApi.length
|
|
103
|
+
'DataDog': datadog.packages.length
|
|
105
104
|
};
|
|
106
105
|
if (process.env.OSM_API_TOKEN) {
|
|
107
106
|
feedCounts['OSM'] = osmResult.packages.length + (osmResult.pypi_packages || []).length;
|
package/src/ml/llm-detective.js
CHANGED
|
@@ -46,7 +46,10 @@ const LLM_CONCURRENCY_MAX = 2; // max simultaneous API calls
|
|
|
46
46
|
const LLM_DAILY_LIMIT_DEFAULT = 100;
|
|
47
47
|
const MAX_SINGLE_FILE_BYTES = 512 * 1024; // skip individual files > 512KB
|
|
48
48
|
|
|
49
|
-
// Extensions to collect from packages
|
|
49
|
+
// Extensions to collect from packages for LLM context (NOT a detection scanner — this feeds
|
|
50
|
+
// the monitor's opt-in LLM investigation). `.json` is intentionally included (manifest/config
|
|
51
|
+
// context for the model); `.jsx/.tsx` are out of scope here by design. Deliberately distinct
|
|
52
|
+
// from the scanner source-extension lists — do not sync.
|
|
50
53
|
const SOURCE_EXTENSIONS = ['.js', '.mjs', '.cjs', '.ts', '.json', '.py'];
|
|
51
54
|
|
|
52
55
|
// ── Semaphore (pattern: src/shared/http-limiter.js) ──
|