muaddib-scanner 2.11.174 → 2.11.177

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -214,7 +214,7 @@ muaddib replay # Ground truth validation (90/94 TPR@3, v2.11
214
214
 
215
215
  ### <!--stat:rulesTotal-->277<!--/stat:rulesTotal--> detection rules
216
216
 
217
- All rules (<!--stat:rulesCore-->272<!--/stat:rulesCore--> RULES + <!--stat:rulesParanoid-->5<!--/stat:rulesParanoid--> PARANOID) are mapped to MITRE ATT&CK techniques. See [SECURITY.md](SECURITY.md#detection-rules-v211139) for the complete rules reference.
217
+ All rules (<!--stat:rulesCore-->272<!--/stat:rulesCore--> RULES + <!--stat:rulesParanoid-->5<!--/stat:rulesParanoid--> PARANOID) are mapped to MITRE ATT&CK techniques. See [SECURITY.md](SECURITY.md#detection-rules) for the complete rules reference.
218
218
 
219
219
  ### Detected campaigns
220
220
 
@@ -345,7 +345,7 @@ npm test
345
345
 
346
346
  ### Testing
347
347
 
348
- - **<!--stat:tests-->4561<!--/stat:tests--> tests** across <!--stat:testFiles-->152<!--/stat:testFiles--> modular test files
348
+ - **<!--stat:tests-->4539<!--/stat:tests--> tests** across <!--stat:testFiles-->152<!--/stat:testFiles--> modular test files
349
349
  - **56 fuzz tests** - Malformed inputs, ReDoS, unicode, binary
350
350
  - **Datadog 17K benchmark** - 14,587 confirmed malware samples (in-scope)
351
351
  - **Ground truth validation** - 96 real-world attacks (95.74% TPR@3, 88.30% TPR@20 — v2.11.48 full measure on 94 in-scope)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "muaddib-scanner",
3
- "version": "2.11.174",
3
+ "version": "2.11.177",
4
4
  "description": "Supply-chain threat detection & response for npm & PyPI/Python",
5
5
  "main": "src/index.js",
6
6
  "bin": {
@@ -14,6 +14,7 @@
14
14
  "scan": "node bin/muaddib.js scan .",
15
15
  "update": "node bin/muaddib.js update",
16
16
  "lint": "eslint src bin --ext .js",
17
+ "ci:local": "node scripts/check-deps-typosquats.js && node scripts/check-no-tracked-node-modules.js && npm audit --audit-level=high && npm run docs:stats:check && npm test",
17
18
  "docs:stats": "node scripts/collect-stats.js && node scripts/sync-stats.js",
18
19
  "docs:stats:tests": "node scripts/collect-stats.js --with-tests && node scripts/sync-stats.js",
19
20
  "docs:stats:check": "node scripts/collect-stats.js --check && node scripts/sync-stats.js --check",
@@ -54,8 +55,8 @@
54
55
  "@inquirer/prompts": "8.5.2",
55
56
  "acorn": "8.17.0",
56
57
  "acorn-walk": "8.3.5",
57
- "adm-zip": "0.5.18",
58
- "js-yaml": "5.1.0",
58
+ "adm-zip": "0.6.0",
59
+ "js-yaml": "5.2.1",
59
60
  "web-tree-sitter": "^0.26.9"
60
61
  },
61
62
  "devDependencies": {
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "target": "node_modules",
3
- "timestamp": "2026-07-17T09:25:49.894Z",
3
+ "timestamp": "2026-07-22T22:32:17.558Z",
4
4
  "threats": [
5
5
  {
6
6
  "type": "string_mutation_obfuscation",
@@ -54,6 +54,32 @@ const HOOK_COMMANDS = {
54
54
  diff: 'npx muaddib diff HEAD --fail-on high'
55
55
  };
56
56
 
57
+ /**
58
+ * Back up an existing pre-commit hook before overwriting it, keeping the 3 most recent
59
+ * timestamped backups. Shared by the husky and native-git hook writers so neither silently
60
+ * destroys a user's existing hook (lint-staged, tests, etc.).
61
+ * @param {string} hookPath - Absolute path to the pre-commit hook to be written
62
+ */
63
+ function backupExistingHook(hookPath) {
64
+ if (!fs.existsSync(hookPath)) return;
65
+ const backup = `${hookPath}.backup.${Date.now()}`;
66
+ fs.copyFileSync(hookPath, backup);
67
+ console.log(`[INFO] Backed up existing hook to ${backup}`);
68
+
69
+ // Cleanup old backups, keep only 3 most recent
70
+ try {
71
+ const hooksDir = path.dirname(hookPath);
72
+ const base = path.basename(hookPath);
73
+ const backups = fs.readdirSync(hooksDir)
74
+ .filter(f => f.startsWith(base + '.backup.'))
75
+ .sort()
76
+ .reverse();
77
+ for (const old of backups.slice(3)) {
78
+ fs.unlinkSync(path.join(hooksDir, old));
79
+ }
80
+ } catch { /* ignore cleanup errors */ }
81
+ }
82
+
57
83
  async function initHooks(targetPath, options = {}) {
58
84
  const resolvedPath = path.resolve(targetPath);
59
85
  const hookType = options.type || 'auto';
@@ -138,6 +164,9 @@ echo "[MUADDIB] Running security check..."
138
164
  ${command}
139
165
  `;
140
166
 
167
+ // Back up any existing hook before overwriting (same policy as initGitHook): a project may
168
+ // already have a husky pre-commit (lint-staged, tests…) — clobbering it silently loses config.
169
+ backupExistingHook(preCommitPath);
141
170
  fs.writeFileSync(preCommitPath, hookContent, { mode: 0o755 });
142
171
  if (process.platform !== 'win32') {
143
172
  fs.chmodSync(preCommitPath, 0o755);
@@ -219,23 +248,7 @@ exit 0
219
248
  `;
220
249
 
221
250
  // Backup existing hook (limit to 3 backups)
222
- if (fs.existsSync(preCommitPath)) {
223
- const backup = `${preCommitPath}.backup.${Date.now()}`;
224
- fs.copyFileSync(preCommitPath, backup);
225
- console.log(`[INFO] Backed up existing hook to ${backup}`);
226
-
227
- // Cleanup old backups, keep only 3 most recent
228
- try {
229
- const hooksDir = path.dirname(preCommitPath);
230
- const backups = fs.readdirSync(hooksDir)
231
- .filter(f => f.startsWith('pre-commit.backup.'))
232
- .sort()
233
- .reverse();
234
- for (const old of backups.slice(3)) {
235
- fs.unlinkSync(path.join(hooksDir, old));
236
- }
237
- } catch { /* ignore cleanup errors */ }
238
- }
251
+ backupExistingHook(preCommitPath);
239
252
 
240
253
  fs.writeFileSync(preCommitPath, hookContent, { mode: 0o755 });
241
254
  if (process.platform !== 'win32') {
@@ -133,31 +133,42 @@ function checkIOCs(pkg, pkgName, pkgVersion) {
133
133
  return malicious || null;
134
134
  }
135
135
 
136
- // Scan a package and its dependencies recursively
137
- async function scanPackageRecursive(pkg, depth = 0, maxDepth = 3) {
138
- const indent = ' '.repeat(depth);
139
-
140
- // Extract name and version of the package
136
+ // Split a package spec into { pkgName, pkgVersion }.
137
+ // Handles scoped packages (@scope/name) and version specs (@scope/name@spec or name@spec).
138
+ // Transitive deps arrive as `name@^1.2.3` (range specs from dependency maps): the spec must
139
+ // be split off the name even when it is not an exact version, otherwise pkgName keeps the
140
+ // full spec and the IOC map lookup (keyed on bare names) never matches — wildcard IOCs
141
+ // included. Range/tag specs leave pkgVersion null so only wildcard IOCs match (SFI-003).
142
+ function parsePackageSpec(pkg) {
141
143
  let pkgName = pkg;
142
144
  let pkgVersion = null;
143
-
144
- // Handle scoped packages (@scope/name) and versions (@scope/name@version or name@version)
145
145
  if (pkg.startsWith('@')) {
146
146
  // Scoped package
147
147
  const parts = pkg.slice(1).split('@');
148
- if (parts.length >= 2 && parts[parts.length - 1].match(/^\d/)) {
149
- pkgVersion = parts.pop();
148
+ if (parts.length >= 2) {
149
+ const spec = parts.pop();
150
150
  pkgName = '@' + parts.join('@');
151
+ if (spec.match(/^\d/)) pkgVersion = spec;
151
152
  }
152
153
  } else {
153
154
  const parts = pkg.split('@');
154
- if (parts.length >= 2 && parts[parts.length - 1].match(/^\d/)) {
155
- pkgVersion = parts.pop();
155
+ if (parts.length >= 2) {
156
+ const spec = parts.pop();
156
157
  pkgName = parts.join('@');
158
+ if (spec.match(/^\d/)) pkgVersion = spec;
157
159
  }
158
160
  }
159
-
160
-
161
+ return { pkgName, pkgVersion };
162
+ }
163
+
164
+ // Scan a package and its dependencies recursively
165
+ async function scanPackageRecursive(pkg, depth = 0, maxDepth = 3) {
166
+ const indent = ' '.repeat(depth);
167
+
168
+ // Extract name and version of the package
169
+ const { pkgName, pkgVersion } = parsePackageSpec(pkg);
170
+
171
+
161
172
  // Avoid infinite loops
162
173
  if (scannedPackages.has(pkgName)) {
163
174
  return { safe: true };
@@ -309,4 +320,4 @@ async function safeInstall(packages, options = {}) {
309
320
  return { blocked: false };
310
321
  }
311
322
 
312
- module.exports = { safeInstall, scanPackageRecursive, REHABILITATED_PACKAGES, checkRehabilitated, isValidPackageName, checkIOCs };
323
+ module.exports = { safeInstall, scanPackageRecursive, parsePackageSpec, REHABILITATED_PACKAGES, checkRehabilitated, isValidPackageName, checkIOCs };
@@ -51,6 +51,13 @@ const DOWNLOAD_TIMEOUT = 60_000;
51
51
  // Max redirects to follow
52
52
  const MAX_REDIRECTS = 5;
53
53
 
54
+ // Decompression bomb guard: cap the DECOMPRESSED byte count streamed to disk. A compromised
55
+ // release asset (exactly this project's threat model — maintainer account takeover) could
56
+ // otherwise fill the disk: this was the only unbounded decompressor in the repo (download.js
57
+ // has the M4 gzip hint, scraper.js has MAX_ENTRY/TOTAL_UNCOMPRESSED). Env-tunable like the
58
+ // scraper caps; default leaves ample headroom over the ~112-223MB iocs.json.
59
+ const MAX_DECOMPRESSED_SIZE = parseInt(process.env.MUADDIB_BOOTSTRAP_MAX_DECOMPRESSED || '', 10) || 1024 * 1024 * 1024; // 1 GB
60
+
54
61
  // Allowed redirect domains (SSRF protection)
55
62
  const ALLOWED_REDIRECT_DOMAINS = [
56
63
  'github.com',
@@ -115,6 +122,19 @@ function downloadAndDecompress(url, destPath) {
115
122
  const gunzip = zlib.createGunzip();
116
123
  const fileStream = fs.createWriteStream(tmpPath);
117
124
 
125
+ // Decompression bomb guard: count decompressed bytes, abort past the cap.
126
+ let decompressedBytes = 0;
127
+ gunzip.on('data', (chunk) => {
128
+ decompressedBytes += chunk.length;
129
+ if (decompressedBytes > MAX_DECOMPRESSED_SIZE) {
130
+ res.destroy();
131
+ gunzip.destroy();
132
+ fileStream.destroy();
133
+ try { fs.unlinkSync(tmpPath); } catch (e) { debugLog('cleanup failed:', e.message); }
134
+ reject(new Error('Decompressed size exceeds cap (' + MAX_DECOMPRESSED_SIZE + ' bytes) — aborting'));
135
+ }
136
+ });
137
+
118
138
  gunzip.on('error', (err) => {
119
139
  fileStream.destroy();
120
140
  try { fs.unlinkSync(tmpPath); } catch (e) { debugLog('cleanup failed:', e.message); }
@@ -733,6 +733,7 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
733
733
  fetchSpinner.start('Fetching OSSF entries... 0/' + toFetch.length);
734
734
  }
735
735
 
736
+ let failedFetches = 0;
736
737
  for (let i = 0; i < toFetch.length; i += BATCH_SIZE) {
737
738
  const batch = toFetch.slice(i, i + BATCH_SIZE);
738
739
  const results = await Promise.all(batch.map(function(entry) {
@@ -741,7 +742,7 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
741
742
  }));
742
743
 
743
744
  for (const result of results) {
744
- if (!result || result.status !== 200 || !result.data) continue;
745
+ if (!result || result.status !== 200 || !result.data) { failedFetches++; continue; }
745
746
  const parsed = parseOSVEntry(result.data, 'ossf-malicious');
746
747
  for (const p of parsed) packages.push(p);
747
748
  }
@@ -760,8 +761,16 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
760
761
  fetchSpinner.succeed('Fetched OSSF entries: ' + packages.length + ' packages');
761
762
  }
762
763
 
763
- // Save tree SHA for next incremental run
764
- try { fs.writeFileSync(shaFile, treeSha); } catch {}
764
+ // Save tree SHA for next incremental run — only if every entry was fetched.
765
+ // Persisting the SHA after partial fetches (e.g. GitHub rate-limiting the raw batches)
766
+ // marks the failed entries as "done": subsequent runs skip everything until the next
767
+ // OSSF commit, silently losing IOCs. On partial failure, keep the old SHA so the next
768
+ // run retries the delta.
769
+ if (failedFetches === 0) {
770
+ try { fs.writeFileSync(shaFile, treeSha); } catch {}
771
+ } else {
772
+ console.log('[SCRAPER] ' + failedFetches + ' OSSF fetches failed — tree SHA not persisted, next run will retry');
773
+ }
765
774
  } catch (e) {
766
775
  console.log('[SCRAPER] Error: ' + e.message);
767
776
  }
@@ -921,65 +930,14 @@ async function scrapeOSVPyPIDataDump() {
921
930
  return packages;
922
931
  }
923
932
 
924
- // ============================================
925
- // SOURCE 4: GitHub Advisory Database (Malware)
926
- // ============================================
927
- async function scrapeGitHubAdvisory() {
928
- console.log('[SCRAPER] GitHub Advisory Database (malware)...');
929
- const packages = [];
930
-
931
- try {
932
- const resp = await fetchJSON('https://api.osv.dev/v1/query', {
933
- method: 'POST',
934
- headers: { 'Content-Type': 'application/json' },
935
- body: { package: { ecosystem: 'npm' } }
936
- });
937
-
938
- if (resp.status === 200 && resp.data && resp.data.vulns) {
939
- for (const vuln of resp.data.vulns) {
940
- // Filter GHSA with malware mention
941
- if (vuln.id && vuln.id.startsWith('GHSA-')) {
942
- const summary = (vuln.summary || '').toLowerCase();
943
- const details = (vuln.details || '').toLowerCase();
944
- const isMalware = summary.includes('malware') ||
945
- summary.includes('malicious') ||
946
- details.includes('malware') ||
947
- details.includes('malicious') ||
948
- summary.includes('backdoor') ||
949
- summary.includes('trojan');
950
-
951
- if (isMalware) {
952
- for (const affected of vuln.affected || []) {
953
- if (affected.package && affected.package.ecosystem === 'npm') {
954
- const versions = extractVersions(affected);
955
- for (const ver of versions) {
956
- packages.push({
957
- id: vuln.id,
958
- name: affected.package.name,
959
- version: ver,
960
- severity: 'critical',
961
- confidence: 'high',
962
- source: 'github-advisory',
963
- description: (vuln.summary || 'Malicious package').slice(0, 200),
964
- references: ['https://github.com/advisories/' + vuln.id],
965
- mitre: 'T1195.002',
966
- freshness: createFreshness('github-advisory', 'high')
967
- });
968
- }
969
- }
970
- }
971
- }
972
- }
973
- }
974
- }
975
-
976
- console.log(`[SCRAPER] ${packages.length} packages`);
977
- } catch (e) {
978
- console.log(`[SCRAPER] Error: ${e.message}`);
979
- }
980
-
981
- return packages;
982
- }
933
+ // SOURCE 4 (GitHub Advisory Database) removed 2026-07 (GAP-2): it POSTed
934
+ // api.osv.dev/v1/query with `{ package: { ecosystem: 'npm' } }` — no package name,
935
+ // which the OSV /v1/query API rejects with HTTP 400 on every call. It therefore
936
+ // returned 0 packages permanently while logging as a normal completion (fictional
937
+ // coverage). Empirically verified that the GHSA-malware it targeted is already
938
+ // 100% covered by the OSV MAL- dump (153/153 pre-snapshot npm GHSA advisories
939
+ // present in the IOC store), so its removal loses no coverage. The GHSA poller
940
+ // (src/ioc/ghsa-poller.js) tracks the coverage denominator independently.
983
941
 
984
942
  // ============================================
985
943
  // MAIN SCRAPER
@@ -1063,7 +1021,6 @@ async function runScraper() {
1063
1021
  scrapeShaiHuludDetector(),
1064
1022
  scrapeDatadogIOCs(),
1065
1023
  scrapeOSSFMaliciousPackages(osvResult.knownIds),
1066
- scrapeGitHubAdvisory(),
1067
1024
  scrapeOSVPyPIDataDump(),
1068
1025
  scrapeAikidoMalwareFeed(),
1069
1026
  scrapeOSMQueryLatest()
@@ -1072,10 +1029,9 @@ async function runScraper() {
1072
1029
  const shaiHuludResult = results[0];
1073
1030
  const datadogResult = results[1];
1074
1031
  const ossfPackages = results[2];
1075
- const githubPackages = results[3];
1076
- const pypiPackages = results[4];
1077
- const aikidoResult = results[5];
1078
- const osmResult = results[6];
1032
+ const pypiPackages = results[3];
1033
+ const aikidoResult = results[4];
1034
+ const osmResult = results[5];
1079
1035
 
1080
1036
  // Log aggregated warnings
1081
1037
  if (_noVersionSkipCount > 0) {
@@ -1100,7 +1056,6 @@ async function runScraper() {
1100
1056
  ...shaiHuludResult.packages,
1101
1057
  ...datadogResult.packages,
1102
1058
  ...ossfPackages,
1103
- ...githubPackages,
1104
1059
  ...aikidoResult.packages,
1105
1060
  ...osmResult.packages
1106
1061
  ];
@@ -1284,11 +1239,21 @@ async function runScraper() {
1284
1239
  'npm-removed'
1285
1240
  ];
1286
1241
 
1287
- // Save enriched (full) IOCs — atomic write via .tmp + rename
1242
+ // Save enriched (full) IOCs — atomic write via .tmp + rename.
1243
+ // Stringify ONCE: the pretty-printed DB is a giant string (~2x the 112MB+ file); a second
1244
+ // stringify for the home copy used to double the peak on top of the zip buffers already in
1245
+ // flight. The HMAC is computed here from the same string, then the string is released and
1246
+ // the home copy reuses the file on disk (copyFileSync — no second serialization).
1288
1247
  const saveSpinner = new Spinner();
1289
1248
  saveSpinner.start('Saving IOCs...');
1290
1249
  const tmpIOCFile = IOC_FILE + '.tmp';
1291
- fs.writeFileSync(tmpIOCFile, JSON.stringify(existingIOCs, null, 2));
1250
+ let iocHmac;
1251
+ {
1252
+ const iocJsonData = JSON.stringify(existingIOCs, null, 2);
1253
+ fs.writeFileSync(tmpIOCFile, iocJsonData);
1254
+ const { generateIOCHMAC } = require('./updater.js');
1255
+ iocHmac = generateIOCHMAC(iocJsonData);
1256
+ }
1292
1257
  fs.renameSync(tmpIOCFile, IOC_FILE);
1293
1258
 
1294
1259
  // Save compact IOCs (lightweight, shipped in npm) — atomic write
@@ -1311,12 +1276,11 @@ async function runScraper() {
1311
1276
  }
1312
1277
  try {
1313
1278
  const tmpHomeFile = HOME_IOC_FILE + '.tmp';
1314
- const homeJsonData = JSON.stringify(existingIOCs, null, 2);
1315
- fs.writeFileSync(tmpHomeFile, homeJsonData);
1316
- // Write HMAC before rename for consistency with updater.js
1317
- const { generateIOCHMAC } = require('./updater.js');
1318
- const homeHmac = generateIOCHMAC(homeJsonData);
1319
- fs.writeFileSync(HOME_IOC_FILE + '.hmac', homeHmac);
1279
+ // Byte-identical copy of the file already written locally — no second stringify peak.
1280
+ fs.copyFileSync(IOC_FILE, tmpHomeFile);
1281
+ // Write HMAC before rename for consistency with updater.js (computed once above from
1282
+ // the same serialized string the local file was written from).
1283
+ fs.writeFileSync(HOME_IOC_FILE + '.hmac', iocHmac);
1320
1284
  fs.renameSync(tmpHomeFile, HOME_IOC_FILE);
1321
1285
  // Mark HMAC as initialized
1322
1286
  const hmacMarker = path.join(homeDir, '.hmac-initialized');
@@ -1579,43 +1543,14 @@ async function scrapeOSMQueryLatest() {
1579
1543
  return { packages: npmPackages, pypi_packages: pypiPackages };
1580
1544
  }
1581
1545
 
1582
- // ============================================
1583
- // SOURCE 5: OSV.dev Lightweight API
1584
- // Used by `muaddib update` (fast, no zip download)
1585
- // ============================================
1586
-
1587
- /**
1588
- * Lightweight OSV.dev query fetches recent npm MAL-* entries via REST API.
1589
- * Used by `muaddib update` as a fast complement to the full zip scrape.
1590
- * @returns {Promise<Array>} Parsed IOC package entries
1591
- */
1592
- async function scrapeOSVLightweightAPI() {
1593
- console.log('[SCRAPER] OSV.dev lightweight API...');
1594
- const packages = [];
1595
-
1596
- try {
1597
- const resp = await fetchJSON('https://api.osv.dev/v1/query', {
1598
- method: 'POST',
1599
- headers: { 'Content-Type': 'application/json' },
1600
- body: { package: { ecosystem: 'npm' } }
1601
- });
1602
-
1603
- if (resp.status === 200 && resp.data && resp.data.vulns) {
1604
- for (const vuln of resp.data.vulns) {
1605
- if (vuln.id && vuln.id.startsWith('MAL-')) {
1606
- const parsed = parseOSVEntry(vuln, 'osv-api');
1607
- for (const p of parsed) packages.push(p);
1608
- }
1609
- }
1610
- }
1611
-
1612
- console.log('[SCRAPER] ' + packages.length + ' MAL-* packages from OSV API');
1613
- } catch (e) {
1614
- console.log('[SCRAPER] OSV API error: ' + e.message);
1615
- }
1616
-
1617
- return packages;
1618
- }
1546
+ // SOURCE 5 (OSV.dev Lightweight API) removed 2026-07 (GAP-2): same defect as
1547
+ // SOURCE 4 — it POSTed api.osv.dev/v1/query with no package name (`{ package:
1548
+ // { ecosystem: 'npm' } }`), which OSV rejects with HTTP 400, so it returned 0
1549
+ // MAL-* packages permanently while `muaddib update` logged "+0 OSV API" as normal.
1550
+ // It was also 100% redundant with the OSV MAL- zip dump (scrapeOSVDataDump), which
1551
+ // pulls the same MAL- npm set. The `muaddib update` light path keeps its recent-
1552
+ // malware coverage via GenSecAI + DataDog + OSM; the authoritative OSV MAL- set
1553
+ // comes from the deep `muaddib scrape`.
1619
1554
 
1620
1555
  /**
1621
1556
  * Batch query OSV.dev for specific package names.
@@ -1700,7 +1635,7 @@ module.exports = {
1700
1635
  runScraper, scrapeShaiHuludDetector, scrapeDatadogIOCs,
1701
1636
  scrapeAikidoMalwareFeed,
1702
1637
  scrapeOSMQueryLatest,
1703
- scrapeOSVLightweightAPI, queryOSVBatch,
1638
+ queryOSVBatch,
1704
1639
  getSourceConfidence,
1705
1640
  // Pure utility functions (exported for testing)
1706
1641
  parseCSVLine, parseCSV, extractVersions, parseOSVEntry,
@@ -66,28 +66,29 @@ async function updateIOCs() {
66
66
  mergeIOCs(baseIOCs, yamlStandard);
67
67
  console.log('[2/4] YAML IOCs: ' + yamlStandard.packages.length + ' packages, ' + yamlStandard.hashes.length + ' hashes');
68
68
 
69
- // Step 3: Download additional IOCs from GitHub + OSV API (GenSecAI + DataDog + OSV lightweight + OSM)
69
+ // Step 3: Download additional IOCs (GenSecAI + DataDog + OSM)
70
70
  // Light path: JSON/REST only, NO heavy zip dumps. Designed to be safe at 15min cadence.
71
- // For the deep refresh (OSV zip dumps + OSSF + Aikido + GitHub Advisory), use `muaddib scrape` (~5min).
72
- const { scrapeShaiHuludDetector, scrapeDatadogIOCs, scrapeOSVLightweightAPI, scrapeOSMQueryLatest } = require('./scraper.js');
73
- console.log('[3/4] Downloading GitHub + OSV API + OSM IOCs...');
71
+ // The OSV MAL- set comes from the deep refresh (`muaddib scrape`, OSV zip dumps + OSSF +
72
+ // Aikido). The former OSV lightweight source was removed 2026-07 (GAP-2): it POSTed a
73
+ // nameless OSV /v1/query (HTTP 400 always 0) and was redundant with the OSV zip dump.
74
+ const { scrapeShaiHuludDetector, scrapeDatadogIOCs, scrapeOSMQueryLatest } = require('./scraper.js');
75
+ console.log('[3/4] Downloading GenSecAI + DataDog + OSM IOCs...');
74
76
 
75
- const [shaiHulud, datadog, osvApi, osmResult] = await Promise.all([
77
+ const [shaiHulud, datadog, osmResult] = await Promise.all([
76
78
  scrapeShaiHuludDetector(),
77
79
  scrapeDatadogIOCs(),
78
- scrapeOSVLightweightAPI(),
79
80
  scrapeOSMQueryLatest()
80
81
  ]);
81
82
 
82
83
  const githubIOCs = {
83
- packages: [].concat(shaiHulud.packages, datadog.packages, osvApi, osmResult.packages),
84
+ packages: [].concat(shaiHulud.packages, datadog.packages, osmResult.packages),
84
85
  pypi_packages: (osmResult.pypi_packages || []).slice(),
85
86
  hashes: [].concat(shaiHulud.hashes || [], datadog.hashes || []),
86
87
  markers: [],
87
88
  files: []
88
89
  };
89
90
  mergeIOCs(baseIOCs, githubIOCs);
90
- console.log(' +' + shaiHulud.packages.length + ' GenSecAI, +' + datadog.packages.length + ' DataDog, +' + osvApi.length + ' OSV API, +' + osmResult.packages.length + ' OSM npm, +' + (osmResult.pypi_packages || []).length + ' OSM PyPI');
91
+ console.log(' +' + shaiHulud.packages.length + ' GenSecAI, +' + datadog.packages.length + ' DataDog, +' + osmResult.packages.length + ' OSM npm, +' + (osmResult.pypi_packages || []).length + ' OSM PyPI');
91
92
 
92
93
  // Phase 2c (feed health): a feed that previously returned data but now returns 0 is the
93
94
  // silent failure mode that froze the OSM feed and collapsed coverage. Raise a one-shot
@@ -95,13 +96,11 @@ async function updateIOCs() {
95
96
  // Only feeds that were actually ATTEMPTED are health-checked: OSM is token-gated, so without
96
97
  // OSM_API_TOKEN it is SKIPPED (not down) — counting it would raise a false "OSM went dark"
97
98
  // alarm in any no-token context (e.g. an ad-hoc `muaddib update`) against the monitor-seeded
98
- // baseline. OSV-API is public and volatile; its small counts rarely cross MIN_HEALTHY_BASELINE,
99
- // so the engine naturally never establishes an alarm-able baseline for it.
99
+ // baseline.
100
100
  try {
101
101
  const feedCounts = {
102
102
  'GenSecAI': shaiHulud.packages.length,
103
- 'DataDog': datadog.packages.length,
104
- 'OSV-API': osvApi.length
103
+ 'DataDog': datadog.packages.length
105
104
  };
106
105
  if (process.env.OSM_API_TOKEN) {
107
106
  feedCounts['OSM'] = osmResult.packages.length + (osmResult.pypi_packages || []).length;
@@ -46,7 +46,10 @@ const LLM_CONCURRENCY_MAX = 2; // max simultaneous API calls
46
46
  const LLM_DAILY_LIMIT_DEFAULT = 100;
47
47
  const MAX_SINGLE_FILE_BYTES = 512 * 1024; // skip individual files > 512KB
48
48
 
49
- // Extensions to collect from packages
49
+ // Extensions to collect from packages for LLM context (NOT a detection scanner — this feeds
50
+ // the monitor's opt-in LLM investigation). `.json` is intentionally included (manifest/config
51
+ // context for the model); `.jsx/.tsx` are out of scope here by design. Deliberately distinct
52
+ // from the scanner source-extension lists — do not sync.
50
53
  const SOURCE_EXTENSIONS = ['.js', '.mjs', '.cjs', '.ts', '.json', '.py'];
51
54
 
52
55
  // ── Semaphore (pattern: src/shared/http-limiter.js) ──