@paa1997/metho 1.0.0

package/src/cli.js

@@ -0,0 +1,76 @@
+import { Command } from 'commander';
+import { resolve } from 'path';
+import { existsSync } from 'fs';
+import { runPipeline } from './engine.js';
+import { createLogger } from './logger.js';
+
+export function run() {
+  const program = new Command();
+
+  program
+    .name('metho')
+    .description('Automated recon pipeline: subfinder → gau → filter → katana → findsomething')
+    .version('1.0.0')
+    .option('-d, --domain <domain>', 'Single target domain')
+    .option('-dL, --domain-list <file>', 'File with list of domains/subdomains')
+    .option('-o, --output <dir>', 'Output directory', './metho-results')
+    .option('--skip-subfinder', 'Skip subdomain enumeration')
+    .option('--skip-subdomain-probe', 'Skip httpx subdomain liveness check')
+    .option('--skip-gau', 'Skip passive URL crawling')
+    .option('--skip-filter', 'Skip URL filtering/liveness check')
+    .option('--skip-katana', 'Skip active crawling')
+    .option('--skip-findsomething', 'Skip secret scanning')
+    .option('--katana-chunk-size <n>', 'Max URLs per katana batch', '1000')
+    .option('--katana-depth <n>', 'Katana crawl depth', '2')
+    .option('--findsomething-path <path>', 'Path to findsomething_secrets.py')
+    .option('--debug', 'Verbose debug logging')
+    .option('--gui', 'Launch web GUI in browser')
+    .option('--port <n>', 'GUI server port', '3000');
+
+  program.parse();
+  const opts = program.opts();
+
+  // GUI mode
+  if (opts.gui) {
+    import('./server.js').then(({ startServer }) => {
+      startServer(parseInt(opts.port, 10));
+    });
+    return;
+  }
+
+  // CLI mode validation
+  if (!opts.domain && !opts.domainList) {
+    console.error('Error: Must provide either -d <domain> or -dL <file>, or use --gui for web interface');
+    process.exit(1);
+  }
+  if (opts.domain && opts.domainList) {
+    console.error('Error: Cannot use both -d and -dL together');
+    process.exit(1);
+  }
+  if (opts.domainList && !existsSync(opts.domainList)) {
+    console.error(`Error: Domain list file not found: ${opts.domainList}`);
+    process.exit(1);
+  }
+
+  const config = {
+    domain: opts.domain || null,
+    domainList: opts.domainList ? resolve(opts.domainList) : null,
+    outputDir: resolve(opts.output),
+    skipSubfinder: opts.skipSubfinder || false,
+    skipSubdomainProbe: opts.skipSubdomainProbe || false,
+    skipGau: opts.skipGau || false,
+    skipFilter: opts.skipFilter || false,
+    skipKatana: opts.skipKatana || false,
+    skipFindsomething: opts.skipFindsomething || false,
+    katanaChunkSize: parseInt(opts.katanaChunkSize, 10),
+    katanaDepth: parseInt(opts.katanaDepth, 10),
+    findsomethingPath: opts.findsomethingPath || '',
+    debug: opts.debug || false,
+  };
+
+  const logger = createLogger(config.debug);
+  runPipeline(config, logger).catch((err) => {
+    logger.error(`Pipeline failed: ${err.message}`);
+    process.exit(1);
+  });
+}

package/src/constants.js

@@ -0,0 +1,20 @@
+export const BLOCKED_EXTENSIONS = new Set([
+  // Images
+  'png', 'jpg', 'jpeg', 'gif', 'svg', 'ico', 'bmp', 'webp', 'tiff', 'tif', 'avif',
+  // Fonts
+  'ttf', 'woff', 'woff2', 'eot', 'otf',
+  // Media
+  'mp4', 'mp3', 'avi', 'mov', 'wmv', 'flv', 'webm', 'mkv', 'ogg', 'wav', 'flac',
+  // Documents / archives
+  'pdf', 'zip', 'tar', 'gz', 'rar', '7z', 'bz2', 'xz',
+  // Stylesheets / maps
+  'css', 'map',
+  // Binary / misc
+  'exe', 'dll', 'so', 'dylib', 'bin', 'dmg', 'iso', 'img', 'apk', 'deb', 'rpm',
+  'swf',
+]);
+
+export const DEFAULT_KATANA_CHUNK_SIZE = 1000;
+export const DEFAULT_KATANA_DEPTH = 2;
+
+export const HTTPX_STATUS_CODES = '200,201,202,204,301,302,307,308,401,403';

package/src/engine.js

@@ -0,0 +1,180 @@
+import { mkdirSync, writeFileSync, copyFileSync, readFileSync, existsSync } from 'fs';
+import { join, basename } from 'path';
+import { setupSignalHandlers, getGlobalContext, createRunContext } from './signals.js';
+import { ensureTools } from './tools/manager.js';
+import { SubfinderStep } from './steps/subfinder.js';
+import { SubdomainProbeStep } from './steps/subdomain-probe.js';
+import { GauStep } from './steps/gau.js';
+import { FilterStep } from './steps/filter.js';
+import { KatanaStep } from './steps/katana.js';
+import { FindSomethingStep } from './steps/findsomething.js';
+
+function makeRunDir(outputDir, domain, domainList) {
+  const label = domain || basename(domainList, '.txt');
+  const now = new Date();
+  const ts = now.toISOString().replace(/[-:T]/g, '').replace(/\..+/, '').replace(/(\d{8})(\d{6})/, '$1-$2');
+  const dirName = `${label}-${ts}`;
+  const runDir = join(outputDir, dirName);
+  mkdirSync(runDir, { recursive: true });
+  return runDir;
+}
+
+/**
+ * Run the full recon pipeline.
+ * @param {object} config - pipeline configuration
+ * @param {object} logger - logger instance
+ * @param {RunContext} [ctx] - optional run context (created automatically if not provided)
+ */
+export async function runPipeline(config, logger, ctx) {
+  // Use provided context (server mode) or global context (CLI mode)
+  if (!ctx) {
+    ctx = getGlobalContext();
+    setupSignalHandlers(logger);
+  }
+
+  logger.banner('METHO - Automated Recon Pipeline');
+
+  // Create output directory
+  const runDir = makeRunDir(config.outputDir, config.domain, config.domainList);
+  logger.info(`Output directory: ${runDir}`);
+
+  // Set up log file
+  const logFile = join(runDir, 'metho.log');
+  logger.setLogFile(logFile);
+  logger.debug(`Config: ${JSON.stringify(config, null, 2)}`);
+
+  // Prepare input file
+  const inputFile = join(runDir, '00-input-domains.txt');
+  if (config.domain) {
+    writeFileSync(inputFile, config.domain + '\n');
+  } else {
+    copyFileSync(config.domainList, inputFile);
+  }
+  ctx.trackFile(inputFile, 'Input domains');
+  logger.info(`Input: ${config.domain || config.domainList}`);
+
+  // Detect tools
+  logger.info('Checking required tools...');
+  await ensureTools(config, logger);
+
+  // Track results for final summary
+  const results = [];
+  let currentInput = inputFile;
+  let stepNum = 0;
+
+  // --- Step 1: Subfinder ---
+  stepNum++;
+  const subdomainsFile = join(runDir, '01-subdomains.txt');
+  if (config.skipSubfinder) {
+    logger.step(stepNum, 'Subfinder (subdomain enumeration)', 'skip');
+  } else {
+    logger.step(stepNum, 'Subfinder (subdomain enumeration)', 'start');
+    const subfinder = new SubfinderStep(logger, ctx);
+    const res = await subfinder.execute(config, currentInput, subdomainsFile);
+    ctx.trackFile(subdomainsFile, 'Subdomains');
+    results.push({ step: 'Subfinder', file: subdomainsFile, lines: res.lineCount });
+    logger.step(stepNum, `Subfinder → ${res.lineCount} subdomains`, 'done');
+    logger.result('Subfinder', subdomainsFile, res.lineCount);
+    currentInput = subdomainsFile;
+  }
+  if (ctx.isInterrupted()) return printSummary(results, runDir, logger);
+
+  // --- Step 2: Subdomain Probe (httpx) ---
+  stepNum++;
+  const liveSubsFile = join(runDir, '02-live-subdomains.txt');
+  if (config.skipSubdomainProbe) {
+    logger.step(stepNum, 'Subdomain Probe (httpx)', 'skip');
+  } else {
+    logger.step(stepNum, 'Subdomain Probe (httpx on subdomains)', 'start');
+    const probe = new SubdomainProbeStep(logger, ctx);
+    const res = await probe.execute(config, currentInput, liveSubsFile);
+    ctx.trackFile(liveSubsFile, 'Live Subdomains');
+    results.push({ step: 'Subdomain Probe', file: liveSubsFile, lines: res.lineCount });
+    logger.step(stepNum, `Subdomain Probe → ${res.lineCount} live subdomains`, 'done');
+    logger.result('Subdomain Probe', liveSubsFile, res.lineCount);
+    currentInput = liveSubsFile;
+  }
+  if (ctx.isInterrupted()) return printSummary(results, runDir, logger);
+
+  // --- Step 3: GAU ---
+  stepNum++;
+  const gauFile = join(runDir, '03-passive-urls.txt');
+  if (config.skipGau) {
+    logger.step(stepNum, 'GAU (passive URL crawling)', 'skip');
+  } else {
+    logger.step(stepNum, 'GAU (passive URL crawling)', 'start');
+    const gau = new GauStep(logger, ctx);
+    const res = await gau.execute(config, currentInput, gauFile);
+    ctx.trackFile(gauFile, 'Passive URLs');
+    results.push({ step: 'GAU', file: gauFile, lines: res.lineCount });
+    logger.step(stepNum, `GAU → ${res.lineCount} URLs`, 'done');
+    logger.result('GAU', gauFile, res.lineCount);
+    currentInput = gauFile;
+  }
+  if (ctx.isInterrupted()) return printSummary(results, runDir, logger);
+
+  // --- Step 4: Filter ---
+  stepNum++;
+  const liveUrlsFile = join(runDir, '04-live-urls.txt');
+  if (config.skipFilter) {
+    logger.step(stepNum, 'Filter (extension + liveness)', 'skip');
+  } else {
+    logger.step(stepNum, 'Filter (extension + liveness)', 'start');
+    const filter = new FilterStep(logger, ctx);
+    const res = await filter.execute(config, currentInput, liveUrlsFile);
+    ctx.trackFile(liveUrlsFile, 'Live URLs');
+    results.push({ step: 'Filter', file: liveUrlsFile, lines: res.lineCount });
+    logger.step(stepNum, `Filter → ${res.lineCount} live URLs`, 'done');
+    logger.result('Filter', liveUrlsFile, res.lineCount);
+    currentInput = liveUrlsFile;
+  }
+  if (ctx.isInterrupted()) return printSummary(results, runDir, logger);
+
+  // --- Step 5: Katana ---
+  stepNum++;
+  const crawledFile = join(runDir, '05-crawled-urls.txt');
+  if (config.skipKatana) {
+    logger.step(stepNum, 'Katana (active crawling)', 'skip');
+  } else {
+    logger.step(stepNum, 'Katana (active crawling)', 'start');
+    const katana = new KatanaStep(logger, ctx);
+    const res = await katana.execute(config, currentInput, crawledFile);
+    ctx.trackFile(crawledFile, 'Crawled URLs');
+    results.push({ step: 'Katana', file: crawledFile, lines: res.lineCount });
+    logger.step(stepNum, `Katana → ${res.lineCount} URLs`, 'done');
+    logger.result('Katana', crawledFile, res.lineCount);
+    currentInput = crawledFile;
+  }
+  if (ctx.isInterrupted()) return printSummary(results, runDir, logger);
+
+  // --- Step 6: FindSomething ---
+  stepNum++;
+  const secretsFile = join(runDir, '06-secrets.txt');
+  if (config.skipFindsomething) {
+    logger.step(stepNum, 'FindSomething (secret scanning)', 'skip');
+  } else {
+    logger.step(stepNum, 'FindSomething (secret scanning)', 'start');
+    const fs = new FindSomethingStep(logger, ctx);
+    const res = await fs.execute(config, currentInput, secretsFile);
+    ctx.trackFile(secretsFile, 'Secrets');
+    results.push({ step: 'FindSomething', file: secretsFile, lines: res.lineCount });
+    logger.step(stepNum, `FindSomething → ${res.lineCount} findings`, 'done');
+    logger.result('FindSomething', secretsFile, res.lineCount);
+  }
+
+  printSummary(results, runDir, logger);
+  return { results, runDir };
+}
+
+function printSummary(results, runDir, logger) {
+  logger.banner('Pipeline Complete');
+  logger.info(`Results directory: ${runDir}`);
+  if (results.length > 0) {
+    logger.info('Summary:');
+    for (const r of results) {
+      logger.info(`  ${r.step}: ${r.lines} lines → ${r.file}`);
+    }
+  }
+  logger.info(`Full log: ${join(runDir, 'metho.log')}`);
+  return { results, runDir };
+}