@paa1997/metho 1.0.0

package/src/logger.js

@@ -0,0 +1,95 @@
+import chalk from 'chalk';
+import { appendFileSync, mkdirSync } from 'fs';
+import { dirname } from 'path';
+
+export function createLogger(debugMode = false) {
+  let logFilePath = null;
+  let eventCallback = null;
+
+  function timestamp() {
+    return new Date().toISOString().replace('T', ' ').replace('Z', '');
+  }
+
+  function writeToFile(level, msg) {
+    if (!logFilePath) return;
+    try {
+      appendFileSync(logFilePath, `[${timestamp()}] [${level}] ${msg}\n`);
+    } catch { /* ignore file write errors */ }
+  }
+
+  function emit(event) {
+    if (eventCallback) {
+      event.timestamp = timestamp();
+      eventCallback(event);
+    }
+  }
+
+  return {
+    setLogFile(filePath) {
+      logFilePath = filePath;
+      mkdirSync(dirname(filePath), { recursive: true });
+    },
+
+    onEvent(callback) {
+      eventCallback = callback;
+    },
+
+    info(msg) {
+      console.log(chalk.blue('ℹ'), msg);
+      writeToFile('INFO', msg);
+      emit({ type: 'log', level: 'info', message: msg });
+    },
+
+    debug(msg) {
+      writeToFile('DEBUG', msg);
+      if (debugMode) {
+        console.log(chalk.gray(`  [debug] ${msg}`));
+      }
+      emit({ type: 'log', level: 'debug', message: msg });
+    },
+
+    success(msg) {
+      console.log(chalk.green('✔'), msg);
+      writeToFile('SUCCESS', msg);
+      emit({ type: 'log', level: 'success', message: msg });
+    },
+
+    error(msg) {
+      console.error(chalk.red('✖'), msg);
+      writeToFile('ERROR', msg);
+      emit({ type: 'log', level: 'error', message: msg });
+    },
+
+    warn(msg) {
+      console.log(chalk.yellow('⚠'), msg);
+      writeToFile('WARN', msg);
+      emit({ type: 'log', level: 'warn', message: msg });
+    },
+
+    step(stepNum, name, status) {
+      const prefix = chalk.cyan(`[Step ${stepNum}]`);
+      if (status === 'start') {
+        console.log(`\n${prefix} ${chalk.bold(name)} ${chalk.gray('starting...')}`);
+      } else if (status === 'done') {
+        console.log(`${prefix} ${chalk.bold(name)} ${chalk.green('done')}`);
+      } else if (status === 'skip') {
+        console.log(`${prefix} ${chalk.bold(name)} ${chalk.yellow('skipped')}`);
+      }
+      writeToFile('STEP', `[Step ${stepNum}] ${name} - ${status}`);
+      emit({ type: 'step', stepNum, name, status });
+    },
+
+    result(stepName, file, lines) {
+      writeToFile('RESULT', `${stepName}: ${lines} lines → ${file}`);
+      emit({ type: 'result', step: stepName, file, lines });
+    },
+
+    banner(msg) {
+      console.log(chalk.bold.magenta(`\n${'═'.repeat(60)}`));
+      console.log(chalk.bold.magenta(`  ${msg}`));
+      console.log(chalk.bold.magenta(`${'═'.repeat(60)}\n`));
+      writeToFile('BANNER', msg);
+      emit({ type: 'banner', message: msg });
+    },
+  };
+}

package/src/server.js

@@ -0,0 +1,261 @@
+import { createServer } from 'http';
+import { readFileSync } from 'fs';
+import { resolve, dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+import { exec } from 'child_process';
+import { createLogger } from './logger.js';
+import { runPipeline } from './engine.js';
+import { createRunContext } from './signals.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export function startServer(port = 3000) {
+  let sseClients = [];
+  let runCounter = 0;
+  // Track active runs: runId → { ctx, logger, label, config }
+  const activeRuns = new Map();
+
+  // Read HTML at startup
+  let guiHtml;
+  try {
+    guiHtml = readFileSync(join(__dirname, 'gui.html'), 'utf-8');
+  } catch (err) {
+    console.error('Failed to read gui.html:', err.message);
+    process.exit(1);
+  }
+
+  function broadcast(event) {
+    const data = `data: ${JSON.stringify(event)}\n\n`;
+    const dead = [];
+    for (let i = 0; i < sseClients.length; i++) {
+      try {
+        sseClients[i].write(data);
+      } catch {
+        dead.push(i);
+      }
+    }
+    for (let i = dead.length - 1; i >= 0; i--) {
+      sseClients.splice(dead[i], 1);
+    }
+  }
+
+  function readBody(req) {
+    return new Promise((resolve, reject) => {
+      let body = '';
+      req.on('data', (chunk) => { body += chunk; });
+      req.on('end', () => resolve(body));
+      req.on('error', reject);
+    });
+  }
+
+  const server = createServer(async (req, res) => {
+    const url = req.url.split('?')[0];
+    const method = req.method;
+
+    console.log(`[server] ${method} ${url}`);
+
+    try {
+      res.setHeader('Access-Control-Allow-Origin', '*');
+      res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+      res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+
+      if (method === 'OPTIONS') {
+        res.writeHead(204);
+        return res.end();
+      }
+
+      // GET / → serve GUI
+      if (method === 'GET' && url === '/') {
+        res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
+        return res.end(guiHtml);
+      }
+
+      // GET /status → active runs info
+      if (method === 'GET' && url === '/status') {
+        const runs = [];
+        for (const [id, run] of activeRuns) {
+          runs.push({ runId: id, label: run.label });
+        }
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        return res.end(JSON.stringify({ running: activeRuns.size, runs }));
+      }
+
+      // GET /events → SSE stream
+      if (method === 'GET' && url === '/events') {
+        res.writeHead(200, {
+          'Content-Type': 'text/event-stream',
+          'Cache-Control': 'no-cache',
+          'Connection': 'keep-alive',
+          'X-Accel-Buffering': 'no',
+        });
+        res.write('data: {"type":"connected"}\n\n');
+        sseClients.push(res);
+        req.on('close', () => {
+          sseClients = sseClients.filter(c => c !== res);
+        });
+        return;
+      }
+
+      // POST /run → start a new pipeline run
+      if (method === 'POST' && url === '/run') {
+        const body = await readBody(req);
+        let params;
+        try {
+          params = JSON.parse(body);
+        } catch {
+          res.writeHead(400, { 'Content-Type': 'application/json' });
+          return res.end(JSON.stringify({ error: 'Invalid JSON body' }));
+        }
+
+        if (!params.domain && !params.domainList) {
+          res.writeHead(400, { 'Content-Type': 'application/json' });
+          return res.end(JSON.stringify({ error: 'Must provide domain or domainList' }));
+        }
+
+        runCounter++;
+        const runId = 'run-' + runCounter;
+        const label = params.domain || params.domainList;
+
+        const config = {
+          domain: params.domain || null,
+          domainList: params.domainList ? resolve(params.domainList) : null,
+          outputDir: resolve(params.outputDir || './metho-results'),
+          skipSubfinder: !!params.skipSubfinder,
+          skipSubdomainProbe: !!params.skipSubdomainProbe,
+          skipGau: !!params.skipGau,
+          skipFilter: !!params.skipFilter,
+          skipKatana: !!params.skipKatana,
+          skipFindsomething: !!params.skipFindsomething,
+          katanaChunkSize: parseInt(params.katanaChunkSize, 10) || 1000,
+          katanaDepth: parseInt(params.katanaDepth, 10) || 2,
+          findsomethingPath: params.findsomethingPath || '',
+          debug: true,
+        };
+
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ ok: true, runId, label }));
+        console.log(`[server] Starting ${runId} (${label})`);
+
+        setImmediate(() => startPipelineRun(runId, label, config));
+        return;
+      }
+
+      // POST /stop → stop a specific run or all runs
+      if (method === 'POST' && url === '/stop') {
+        const body = await readBody(req);
+        let params = {};
+        try { params = JSON.parse(body); } catch { /* empty body = stop all */ }
+
+        if (params.runId) {
+          // Stop a specific run
+          const run = activeRuns.get(params.runId);
+          if (run) {
+            run.ctx.triggerInterrupt(run.logger);
+            console.log(`[server] Stopping ${params.runId}`);
+          }
+        } else {
+          // Stop all runs
+          for (const [id, run] of activeRuns) {
+            run.ctx.triggerInterrupt(run.logger);
+            console.log(`[server] Stopping ${id}`);
+          }
+        }
+
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        return res.end(JSON.stringify({ ok: true }));
+      }
+
+      // GET /file?path=... → read result file
+      if (method === 'GET' && url === '/file') {
+        const fullUrl = new URL(req.url, `http://localhost:${port}`);
+        const filePath = fullUrl.searchParams.get('path');
+        if (!filePath) {
+          res.writeHead(400, { 'Content-Type': 'text/plain' });
+          return res.end('Missing path parameter');
+        }
+        try {
+          const content = readFileSync(filePath, 'utf-8');
+          res.writeHead(200, { 'Content-Type': 'text/plain; charset=utf-8' });
+          return res.end(content);
+        } catch (err) {
+          res.writeHead(404, { 'Content-Type': 'text/plain' });
+          return res.end('File not found: ' + err.message);
+        }
+      }
+
+      // 404
+      res.writeHead(404, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: 'Not found' }));
+    } catch (err) {
+      console.error('[server] Request handler error:', err);
+      if (!res.headersSent) {
+        res.writeHead(500, { 'Content-Type': 'application/json' });
+      }
+      res.end(JSON.stringify({ error: 'Internal server error' }));
+    }
+  });
+
+  async function startPipelineRun(runId, label, config) {
+    const ctx = createRunContext();
+    const logger = createLogger(true);
+
+    activeRuns.set(runId, { ctx, logger, label, config });
+
+    // Prefix all SSE events with runId
+    logger.onEvent((event) => {
+      broadcast({ ...event, runId });
+    });
+
+    // Notify GUI of new run
+    broadcast({ type: 'run-started', runId, label });
+
+    try {
+      const result = await runPipeline(config, logger, ctx);
+      const results = result?.results || [];
+      const runDir = result?.runDir || '';
+      broadcast({ type: 'complete', runId, results, runDir });
+    } catch (err) {
+      console.error(`[server] Pipeline ${runId} error:`, err);
+      broadcast({ type: 'error', runId, message: err.message });
+    } finally {
+      activeRuns.delete(runId);
+      broadcast({ type: 'run-ended', runId, activeRuns: activeRuns.size });
+
+      // Close SSE clients only when ALL runs are done
+      if (activeRuns.size === 0) {
+        setTimeout(() => {
+          if (activeRuns.size === 0) {
+            for (const client of sseClients) {
+              try { client.end(); } catch { /* ignore */ }
+            }
+            sseClients = [];
+          }
+        }, 3000);
+      }
+    }
+  }
+
+  server.on('error', (err) => {
+    if (err.code === 'EADDRINUSE') {
+      console.error(`Port ${port} is already in use. Try: metho --gui --port ${port + 1}`);
+    } else {
+      console.error('Server error:', err);
+    }
+    process.exit(1);
+  });
+
+  server.listen(port, () => {
+    const url = `http://localhost:${port}`;
+    console.log(`\n  Metho GUI running at ${url}\n`);
+    openBrowser(url);
+  });
+
+  return server;
+}
+
+function openBrowser(url) {
+  const cmd = process.platform === 'win32' ? `start "" "${url}"`
+    : process.platform === 'darwin' ? `open "${url}"`
+    : `xdg-open "${url}"`;
+  exec(cmd, () => {});
+}

package/src/signals.js

@@ -0,0 +1,101 @@
+import { exec } from 'child_process';
+
+const isWin = process.platform === 'win32';
+
+function forceKillChildren(children, logger) {
+  for (const child of children) {
+    const pid = child.pid;
+    if (!pid) continue;
+    if (logger) logger.debug(`Force-killing PID ${pid}`);
+    try {
+      if (isWin) {
+        exec(`taskkill /F /T /PID ${pid}`, () => {});
+      } else {
+        try { process.kill(-pid, 'SIGKILL'); } catch {}
+        try { child.kill('SIGKILL'); } catch {}
+      }
+    } catch {
+      try { child.kill('SIGKILL'); } catch {}
+    }
+  }
+  children.clear();
+}
+
+/**
+ * Per-run context — isolates children, interrupt flag, and tracked files for one pipeline run.
+ */
+export class RunContext {
+  constructor() {
+    this.children = new Set();
+    this.interrupted = false;
+    this.files = [];
+  }
+
+  registerChild(child) {
+    this.children.add(child);
+    child.on('close', () => this.children.delete(child));
+  }
+
+  clearChild(child) {
+    if (child) this.children.delete(child);
+  }
+
+  trackFile(filePath, description) {
+    this.files.push({ path: filePath, description });
+  }
+
+  isInterrupted() {
+    return this.interrupted;
+  }
+
+  triggerInterrupt(logger) {
+    if (this.interrupted) return;
+    this.interrupted = true;
+    if (logger) logger.warn('Stop requested. Killing active processes...');
+    forceKillChildren(this.children, logger);
+  }
+}
+
+/**
+ * Create a new isolated run context.
+ */
+export function createRunContext() {
+  return new RunContext();
+}
+
+// ---- Global context (used by CLI mode + SIGINT/SIGTERM handler) ----
+
+let globalCtx = new RunContext();
+
+/** Set up process signal handlers that interrupt the global context. */
+export function setupSignalHandlers(logger) {
+  const handler = (signal) => {
+    if (globalCtx.interrupted) {
+      logger.warn('Force exit.');
+      process.exit(1);
+    }
+    globalCtx.interrupted = true;
+    logger.warn(`\n${signal} received. Stopping...`);
+    forceKillChildren(globalCtx.children, logger);
+
+    if (globalCtx.files.length > 0) {
+      logger.info('Partial results saved:');
+      for (const f of globalCtx.files) {
+        logger.info(`  ${f.description}: ${f.path}`);
+      }
+    }
+  };
+
+  process.on('SIGINT', () => handler('SIGINT'));
+  process.on('SIGTERM', () => handler('SIGTERM'));
+}
+
+/** Get the global context (for CLI single-run mode). */
+export function getGlobalContext() {
+  return globalCtx;
+}
+
+/** Reset the global context (for CLI mode restarts). */
+export function resetGlobalContext() {
+  globalCtx = new RunContext();
+}

package/src/steps/base-step.js

@@ -0,0 +1,111 @@
+import { spawn } from 'child_process';
+import { createReadStream } from 'fs';
+import { stat } from 'fs/promises';
+
+const isWin = process.platform === 'win32';
+
+export class BaseStep {
+  constructor(name, logger, ctx) {
+    this.name = name;
+    this.logger = logger;
+    this.ctx = ctx;
+  }
+
+  /**
+   * Spawn a child process with streaming output.
+   * Returns a promise that resolves with { code, lineCount }.
+   */
+  runCommand(cmd, args, options = {}) {
+    return new Promise((resolve, reject) => {
+      if (this.ctx.isInterrupted()) {
+        return reject(new Error('Pipeline interrupted'));
+      }
+
+      this.logger.debug(`${this.name}: ${cmd} ${args.join(' ')}`);
+
+      const spawnOpts = {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        shell: isWin,
+      };
+      if (options.cwd) spawnOpts.cwd = options.cwd;
+
+      const child = spawn(cmd, args, spawnOpts);
+      this.ctx.registerChild(child);
+
+      let stderrBuf = '';
+
+      child.stdout.on('data', (data) => {
+        const lines = data.toString().trim();
+        if (lines) this.logger.debug(`${this.name} stdout: ${lines}`);
+      });
+
+      child.stderr.on('data', (data) => {
+        const text = data.toString().trim();
+        if (text) {
+          stderrBuf += text + '\n';
+          this.logger.debug(`${this.name} stderr: ${text}`);
+        }
+      });
+
+      // Pipe stdin from file if requested
+      if (options.stdinFile) {
+        const stdinStream = createReadStream(options.stdinFile);
+        stdinStream.pipe(child.stdin);
+        stdinStream.on('error', (err) => {
+          this.logger.debug(`${this.name} stdin stream error: ${err.message}`);
+        });
+      } else {
+        child.stdin.end();
+      }
+
+      child.on('error', (err) => {
+        this.ctx.clearChild(child);
+        reject(new Error(`${this.name}: Failed to spawn ${cmd}: ${err.message}`));
+      });
+
+      child.on('close', async (code) => {
+        this.ctx.clearChild(child);
+
+        if (this.ctx.isInterrupted()) {
+          return reject(new Error('Pipeline interrupted'));
+        }
+
+        let lineCount = 0;
+        if (options.outputFile) {
+          lineCount = await this.countLines(options.outputFile);
+        }
+
+        if (code !== 0) {
+          if (lineCount > 0) {
+            this.logger.warn(`${this.name} exited with code ${code} but produced ${lineCount} results`);
+            return resolve({ code, lineCount });
+          }
+          return reject(new Error(`${this.name}: ${cmd} exited with code ${code}\n${stderrBuf}`));
+        }
+
+        resolve({ code, lineCount });
+      });
+    });
+  }
+
+  async countLines(filePath) {
+    try {
+      const stats = await stat(filePath);
+      if (stats.size === 0) return 0;
+
+      return new Promise((resolve) => {
+        let count = 0;
+        const stream = createReadStream(filePath, { encoding: 'utf-8' });
+        stream.on('data', (chunk) => {
+          for (let i = 0; i < chunk.length; i++) {
+            if (chunk[i] === '\n') count++;
+          }
+        });
+        stream.on('end', () => resolve(count));
+        stream.on('error', () => resolve(0));
+      });
+    } catch {
+      return 0;
+    }
+  }
+}

package/src/steps/filter.js

@@ -0,0 +1,43 @@
+import { BaseStep } from './base-step.js';
+import { getToolPath } from '../tools/manager.js';
+import { filterExtensions } from '../utils/filter-extensions.js';
+import { join, dirname } from 'path';
+import { HTTPX_STATUS_CODES } from '../constants.js';
+
+export class FilterStep extends BaseStep {
+  constructor(logger, ctx) {
+    super('filter', logger, ctx);
+  }
+
+  async execute(config, inputFile, outputFile) {
+    const filteredTmpFile = join(dirname(outputFile), 'tmp-extension-filtered.txt');
+    this.logger.debug('Filter Phase A: Removing blocked extensions...');
+
+    const { kept, removed } = await filterExtensions(inputFile, filteredTmpFile);
+    this.logger.info(`Extension filter: kept ${kept}, removed ${removed} URLs`);
+
+    if (kept === 0) {
+      this.logger.warn('No URLs remaining after extension filtering');
+      const { writeFileSync } = await import('fs');
+      writeFileSync(outputFile, '');
+      return { code: 0, lineCount: 0 };
+    }
+
+    this.logger.debug('Filter Phase B: httpx liveness check...');
+    const args = [
+      '-l', filteredTmpFile,
+      '-o', outputFile,
+      '-mc', HTTPX_STATUS_CODES,
+      '-silent',
+    ];
+
+    const result = await this.runCommand(getToolPath('httpx'), args, { outputFile });
+
+    try {
+      const { unlinkSync } = await import('fs');
+      unlinkSync(filteredTmpFile);
+    } catch { /* ignore */ }
+
+    return result;
+  }
+}

package/src/steps/findsomething.js

@@ -0,0 +1,42 @@
+import { BaseStep } from './base-step.js';
+import { existsSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const isWin = process.platform === 'win32';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Bundled script path: metho/scripts/findsomething_secrets.py
+const BUNDLED_SCRIPT = join(__dirname, '..', '..', 'scripts', 'findsomething_secrets.py');
+
+export class FindSomethingStep extends BaseStep {
+  constructor(logger, ctx) {
+    super('findsomething', logger, ctx);
+  }
+
+  async execute(config, inputFile, outputFile) {
+    // Priority: explicit config > env var > bundled script
+    let scriptPath = config.findsomethingPath;
+    if (!scriptPath || !existsSync(scriptPath)) {
+      scriptPath = process.env.FINDSOMETHING_PATH;
+    }
+    if (!scriptPath || !existsSync(scriptPath)) {
+      scriptPath = BUNDLED_SCRIPT;
+    }
+
+    if (!existsSync(scriptPath)) {
+      throw new Error(
+        `FindSomething script not found.\n` +
+        `Checked: ${BUNDLED_SCRIPT}\n` +
+        `Set the path with --findsomething-path <path> or FINDSOMETHING_PATH env var`
+      );
+    }
+
+    this.logger.debug(`Using findsomething script: ${scriptPath}`);
+    const pythonBin = isWin ? 'python' : 'python3';
+    const args = [scriptPath, '-l', inputFile, '-o', outputFile];
+
+    const result = await this.runCommand(pythonBin, args, { outputFile });
+    return result;
+  }
+}