@paa1997/metho 1.0.0

package/src/steps/gau.js

@@ -0,0 +1,86 @@
+import { BaseStep } from './base-step.js';
+import { spawn } from 'child_process';
+import { createReadStream, createWriteStream, unlinkSync } from 'fs';
+import { dedup } from '../utils/dedup.js';
+
+const isWin = process.platform === 'win32';
+
+export class GauStep extends BaseStep {
+  constructor(logger, ctx) {
+    super('gau', logger, ctx);
+  }
+
+  async execute(config, inputFile, outputFile) {
+    const rawFile = outputFile + '.raw';
+
+    await this._runGau(inputFile, rawFile);
+
+    const rawCount = await this.countLines(rawFile);
+    this.logger.info(`GAU raw: ${rawCount} URLs, deduplicating...`);
+
+    const uniqueCount = await dedup(rawFile, outputFile);
+    this.logger.info(`GAU deduped: ${uniqueCount} unique URLs (removed ${rawCount - uniqueCount} duplicates)`);
+
+    try { unlinkSync(rawFile); } catch { /* ignore */ }
+
+    return { code: 0, lineCount: uniqueCount };
+  }
+
+  _runGau(inputFile, outputFile) {
+    return new Promise((resolve, reject) => {
+      if (this.ctx.isInterrupted()) return reject(new Error('Pipeline interrupted'));
+
+      const args = ['--subs'];
+      this.logger.debug(`gau: gau ${args.join(' ')} < ${inputFile} > ${outputFile}`);
+
+      const child = spawn('gau', args, {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        shell: isWin,
+      });
+      this.ctx.registerChild(child);
+
+      const outStream = createWriteStream(outputFile);
+      const stdinStream = createReadStream(inputFile);
+
+      stdinStream.pipe(child.stdin);
+      stdinStream.on('error', (err) => {
+        this.logger.debug(`gau stdin error: ${err.message}`);
+      });
+
+      child.stdout.pipe(outStream);
+
+      child.stdout.on('data', (data) => {
+        this.logger.debug(`gau stdout chunk: ${data.toString().split('\n').length - 1} lines`);
+      });
+
+      let stderrBuf = '';
+      child.stderr.on('data', (data) => {
+        const text = data.toString().trim();
+        if (text) {
+          stderrBuf += text + '\n';
+          this.logger.debug(`gau stderr: ${text}`);
+        }
+      });
+
+      child.on('error', (err) => {
+        this.ctx.clearChild(child);
+        reject(new Error(`gau: Failed to spawn: ${err.message}`));
+      });
+
+      child.on('close', async (code) => {
+        this.ctx.clearChild(child);
+        outStream.end();
+
+        if (this.ctx.isInterrupted()) return reject(new Error('Pipeline interrupted'));
+
+        const lineCount = await this.countLines(outputFile);
+
+        if (code !== 0 && lineCount === 0) {
+          return reject(new Error(`gau exited with code ${code}\n${stderrBuf}`));
+        }
+
+        resolve({ code, lineCount });
+      });
+    });
+  }
+}

package/src/steps/katana.js

@@ -0,0 +1,61 @@
+import { BaseStep } from './base-step.js';
+import { splitFile } from '../utils/file-splitter.js';
+import { dedup } from '../utils/dedup.js';
+import { join, dirname } from 'path';
+import { unlinkSync } from 'fs';
+
+export class KatanaStep extends BaseStep {
+  constructor(logger, ctx) {
+    super('katana', logger, ctx);
+  }
+
+  async execute(config, inputFile, outputFile) {
+    const lineCount = await this.countLines(inputFile);
+    const chunkSize = config.katanaChunkSize;
+    const depth = config.katanaDepth;
+
+    if (lineCount <= chunkSize) {
+      // Small enough to run in one shot
+      this.logger.debug(`Katana: ${lineCount} URLs, running single batch`);
+      const args = ['-list', inputFile, '-d', String(depth), '-o', outputFile];
+      return this.runCommand('katana', args, { outputFile });
+    }
+
+    // Split into chunks and run sequentially
+    this.logger.info(`Katana: ${lineCount} URLs, splitting into chunks of ${chunkSize}`);
+    const tmpDir = dirname(outputFile);
+    const chunks = await splitFile(inputFile, chunkSize, tmpDir, 'katana-chunk');
+    this.logger.info(`Katana: ${chunks.length} chunks to process`);
+
+    const chunkOutputs = [];
+
+    for (let i = 0; i < chunks.length; i++) {
+      this.logger.info(`Katana chunk ${i + 1}/${chunks.length}...`);
+      const chunkOutput = join(tmpDir, `katana-out-${i}.txt`);
+      const args = ['-list', chunks[i], '-d', String(depth), '-o', chunkOutput];
+
+      try {
+        await this.runCommand('katana', args, { outputFile: chunkOutput });
+        chunkOutputs.push(chunkOutput);
+      } catch (err) {
+        this.logger.warn(`Katana chunk ${i + 1} failed: ${err.message}`);
+        // Continue with other chunks
+      }
+
+      // Clean up chunk input
+      try { unlinkSync(chunks[i]); } catch { /* ignore */ }
+    }
+
+    // Merge and dedup all chunk outputs
+    this.logger.debug('Katana: Merging and deduplicating chunk outputs...');
+    await dedup(chunkOutputs, outputFile);
+
+    // Clean up chunk outputs
+    for (const f of chunkOutputs) {
+      try { unlinkSync(f); } catch { /* ignore */ }
+    }
+
+    const finalCount = await this.countLines(outputFile);
+    return { code: 0, lineCount: finalCount };
+  }
+}

package/src/steps/subdomain-probe.js

@@ -0,0 +1,19 @@
+import { BaseStep } from './base-step.js';
+import { getToolPath } from '../tools/manager.js';
+
+export class SubdomainProbeStep extends BaseStep {
+  constructor(logger, ctx) {
+    super('subdomain-probe', logger, ctx);
+  }
+
+  async execute(config, inputFile, outputFile) {
+    const args = [
+      '-l', inputFile,
+      '-o', outputFile,
+      '-silent',
+    ];
+
+    const result = await this.runCommand(getToolPath('httpx'), args, { outputFile });
+    return result;
+  }
+}

package/src/steps/subfinder.js

@@ -0,0 +1,22 @@
+import { BaseStep } from './base-step.js';
+
+export class SubfinderStep extends BaseStep {
+  constructor(logger, ctx) {
+    super('subfinder', logger, ctx);
+  }
+
+  async execute(config, inputFile, outputFile) {
+    const args = [];
+
+    if (config.domain && !config.domainList) {
+      args.push('-d', config.domain);
+    } else {
+      args.push('-dL', inputFile);
+    }
+
+    args.push('-all', '-o', outputFile);
+
+    const result = await this.runCommand('subfinder', args, { outputFile });
+    return result;
+  }
+}

package/src/tools/installer.js

@@ -0,0 +1,111 @@
+import { execSync } from 'child_process';
+import { existsSync, mkdirSync } from 'fs';
+import { join } from 'path';
+
+const isWin = process.platform === 'win32';
+
+/**
+ * Check if a binary is available on PATH.
+ */
+export function isBinaryAvailable(binary) {
+  try {
+    const cmd = isWin ? `where ${binary}` : `which ${binary}`;
+    execSync(cmd, { stdio: 'pipe' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Check if httpx on PATH is ProjectDiscovery's version (not Python's httpx CLI).
+ */
+export function isProjectDiscoveryHttpx() {
+  try {
+    const output = execSync('httpx -version', { stdio: 'pipe', timeout: 5000 }).toString();
+    return true;
+  } catch (err) {
+    const stderr = err.stderr ? err.stderr.toString() : '';
+    const stdout = err.stdout ? err.stdout.toString() : '';
+    const combined = stdout + stderr;
+    if (combined.includes('projectdiscovery') || combined.includes('Current Version:')) {
+      return true;
+    }
+    if (combined.includes('[OPTIONS] URL') || combined.includes('No such option')) {
+      return false;
+    }
+    return false;
+  }
+}
+
+/**
+ * Attempt to install a tool using its install command (globally via go install).
+ * Returns true if install succeeded, false otherwise.
+ */
+export function attemptInstall(toolDef, logger) {
+  if (!toolDef.installCmd) {
+    return false;
+  }
+
+  if (toolDef.installCmd.startsWith('go install')) {
+    if (!isBinaryAvailable('go')) {
+      logger.warn('Go is not installed. Cannot auto-install Go tools.');
+      logger.warn('Install Go from https://go.dev/dl/ and ensure GOPATH/bin is in PATH.');
+      return false;
+    }
+  }
+
+  logger.info(`Installing ${toolDef.binary}: ${toolDef.installCmd}`);
+  try {
+    execSync(toolDef.installCmd, {
+      stdio: 'pipe',
+      timeout: 120000,
+      env: { ...process.env },
+    });
+    return isBinaryAvailable(toolDef.binary);
+  } catch (err) {
+    logger.debug(`Install failed for ${toolDef.binary}: ${err.message}`);
+    return false;
+  }
+}
+
+/**
+ * Install a Go tool to a local directory instead of GOPATH/bin.
+ * Returns the full path to the installed binary, or null on failure.
+ */
+export function installToLocal(toolDef, localBinDir, logger) {
+  if (!toolDef.installCmd || !toolDef.installCmd.startsWith('go install')) {
+    return null;
+  }
+
+  if (!isBinaryAvailable('go')) {
+    logger.warn('Go is not installed. Cannot auto-install Go tools.');
+    logger.warn('Install Go from https://go.dev/dl/ and ensure GOPATH/bin is in PATH.');
+    return null;
+  }
+
+  mkdirSync(localBinDir, { recursive: true });
+
+  const binaryName = toolDef.binary + (isWin ? '.exe' : '');
+  const binaryPath = join(localBinDir, binaryName);
+
+  logger.info(`Installing ${toolDef.binary} to ${localBinDir}...`);
+  try {
+    execSync(toolDef.installCmd, {
+      stdio: 'pipe',
+      timeout: 120000,
+      env: { ...process.env, GOBIN: localBinDir },
+    });
+
+    if (existsSync(binaryPath)) {
+      logger.debug(`${toolDef.binary} installed at ${binaryPath}`);
+      return binaryPath;
+    }
+
+    logger.debug(`Binary not found at expected path: ${binaryPath}`);
+    return null;
+  } catch (err) {
+    logger.debug(`Local install failed for ${toolDef.binary}: ${err.message}`);
+    return null;
+  }
+}

package/src/tools/manager.js

@@ -0,0 +1,114 @@
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+import { TOOLS, STEP_TOOLS } from './registry.js';
+import { isBinaryAvailable, attemptInstall, installToLocal, isProjectDiscoveryHttpx } from './installer.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const LOCAL_BIN_DIR = join(__dirname, '..', '..', 'bin', 'tools');
+
+// Resolved binary paths — maps tool name to full path (or just the name if on PATH)
+const resolvedPaths = {};
+
+/**
+ * Get the resolved binary path for a tool.
+ * Returns the local path if installed locally, otherwise the binary name (for PATH lookup).
+ */
+export function getToolPath(toolName) {
+  return resolvedPaths[toolName] || TOOLS[toolName]?.binary || toolName;
+}
+
+/**
+ * Detect and install all tools required by non-skipped steps.
+ * Automatically installs any missing or incorrect tools.
+ */
+export async function ensureTools(config, logger) {
+  const stepsToCheck = [];
+  if (!config.skipSubfinder) stepsToCheck.push('subfinder');
+  if (!config.skipGau) stepsToCheck.push('gau');
+  if (!config.skipSubdomainProbe) stepsToCheck.push('subdomainProbe');
+  if (!config.skipFilter) stepsToCheck.push('filter');
+  if (!config.skipKatana) stepsToCheck.push('katana');
+  if (!config.skipFindsomething) stepsToCheck.push('findsomething');
+
+  // Collect unique tools needed
+  const toolsNeeded = new Set();
+  for (const step of stepsToCheck) {
+    for (const tool of STEP_TOOLS[step] || []) {
+      toolsNeeded.add(tool);
+    }
+  }
+
+  const missing = [];
+
+  for (const toolName of toolsNeeded) {
+    const toolDef = TOOLS[toolName];
+    if (!toolDef) {
+      missing.push({ name: toolName, reason: 'Unknown tool (not in registry)' });
+      continue;
+    }
+
+    logger.debug(`Checking for ${toolName} (${toolDef.binary})...`);
+
+    if (isBinaryAvailable(toolDef.binary)) {
+      // Special check: verify httpx is ProjectDiscovery's, not Python's
+      if (toolName === 'httpx' && !isProjectDiscoveryHttpx()) {
+        logger.warn('Found Python\'s httpx on PATH (not ProjectDiscovery\'s)');
+        logger.info('Installing ProjectDiscovery httpx locally to avoid conflict...');
+
+        const localPath = installToLocal(toolDef, LOCAL_BIN_DIR, logger);
+        if (localPath) {
+          resolvedPaths[toolName] = localPath;
+          logger.success(`ProjectDiscovery httpx installed at ${localPath}`);
+          continue;
+        }
+
+        missing.push({
+          name: toolName,
+          binary: toolDef.binary,
+          hint: 'Could not install locally. Run: ' + toolDef.installCmd,
+        });
+        continue;
+      }
+
+      resolvedPaths[toolName] = toolDef.binary;
+      logger.debug(`${toolName} found on PATH`);
+      continue;
+    }
+
+    // Tool not found at all — try global install first
+    logger.warn(`${toolName} not found, installing...`);
+    const installed = attemptInstall(toolDef, logger);
+
+    if (installed) {
+      resolvedPaths[toolName] = toolDef.binary;
+      logger.success(`${toolName} installed successfully`);
+    } else {
+      // Try local install as fallback
+      if (toolDef.installCmd && toolDef.installCmd.startsWith('go install')) {
+        logger.info(`Global install failed, trying local install to ${LOCAL_BIN_DIR}...`);
+        const localPath = installToLocal(toolDef, LOCAL_BIN_DIR, logger);
+        if (localPath) {
+          resolvedPaths[toolName] = localPath;
+          logger.success(`${toolName} installed locally at ${localPath}`);
+          continue;
+        }
+      }
+
+      let hint = `Install manually: ${toolDef.installCmd || 'see tool documentation'}`;
+      if (toolName === 'python') {
+        hint = 'Install Python from https://python.org/downloads/';
+      }
+      missing.push({ name: toolName, binary: toolDef.binary, hint });
+    }
+  }
+
+  if (missing.length > 0) {
+    logger.error('Missing required tools:');
+    for (const m of missing) {
+      logger.error(`  - ${m.name} (${m.binary || '?'}): ${m.hint || m.reason}`);
+    }
+    throw new Error(`Cannot proceed: ${missing.length} required tool(s) missing. See above for install instructions.`);
+  }
+
+  logger.success('All required tools detected');
+}

package/src/tools/registry.js

@@ -0,0 +1,45 @@
+// Tool definitions: binary name, install command, verify command
+const isWin = process.platform === 'win32';
+
+export const TOOLS = {
+  subfinder: {
+    binary: 'subfinder',
+    installCmd: 'go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest',
+    verifyCmd: 'subfinder -version',
+    description: 'Subdomain enumeration tool (ProjectDiscovery)',
+  },
+  gau: {
+    binary: 'gau',
+    installCmd: 'go install github.com/lc/gau/v2/cmd/gau@latest',
+    verifyCmd: 'gau --version',
+    description: 'Fetch known URLs from AlienVault OTX, Wayback Machine, and Common Crawl',
+  },
+  httpx: {
+    binary: 'httpx',
+    installCmd: 'go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest',
+    verifyCmd: 'httpx -version',
+    description: 'HTTP probe tool (ProjectDiscovery)',
+  },
+  katana: {
+    binary: 'katana',
+    installCmd: 'go install github.com/projectdiscovery/katana/cmd/katana@latest',
+    verifyCmd: 'katana -version',
+    description: 'Active web crawler (ProjectDiscovery)',
+  },
+  python: {
+    binary: isWin ? 'python' : 'python3',
+    installCmd: null, // Cannot auto-install python
+    verifyCmd: isWin ? 'python --version' : 'python3 --version',
+    description: 'Python interpreter (for findsomething)',
+  },
+};
+
+// Map step names to required tools
+export const STEP_TOOLS = {
+  subfinder: ['subfinder'],
+  subdomainProbe: ['httpx'],
+  gau: ['gau'],
+  filter: ['httpx'],
+  katana: ['katana'],
+  findsomething: ['python'],
+};

package/src/utils/dedup.js

@@ -0,0 +1,37 @@
+import { createReadStream, createWriteStream } from 'fs';
+import { createInterface } from 'readline';
+
+/**
+ * Deduplicate lines from one or more input files, write unique lines to output.
+ * @param {string|string[]} inputFiles - single file path or array of file paths
+ * @param {string} outputFile - output file path
+ * @returns {Promise<number>} - number of unique lines written
+ */
+export async function dedup(inputFiles, outputFile) {
+  const files = Array.isArray(inputFiles) ? inputFiles : [inputFiles];
+  const seen = new Set();
+  const output = createWriteStream(outputFile);
+
+  for (const file of files) {
+    await new Promise((resolve, reject) => {
+      const input = createReadStream(file, { encoding: 'utf-8' });
+      const rl = createInterface({ input, crlfDelay: Infinity });
+
+      rl.on('line', (line) => {
+        const trimmed = line.trim();
+        if (trimmed && !seen.has(trimmed)) {
+          seen.add(trimmed);
+          output.write(trimmed + '\n');
+        }
+      });
+
+      rl.on('close', resolve);
+      rl.on('error', reject);
+      input.on('error', reject);
+    });
+  }
+
+  return new Promise((resolve) => {
+    output.end(() => resolve(seen.size));
+  });
+}

package/src/utils/file-splitter.js

@@ -0,0 +1,48 @@
+import { createReadStream, createWriteStream } from 'fs';
+import { createInterface } from 'readline';
+import { join } from 'path';
+
+/**
+ * Split a file into chunks of N lines each.
+ * Returns an array of chunk file paths.
+ */
+export async function splitFile(inputFile, chunkSize, outputDir, prefix = 'chunk') {
+  return new Promise((resolve, reject) => {
+    const input = createReadStream(inputFile, { encoding: 'utf-8' });
+    const rl = createInterface({ input, crlfDelay: Infinity });
+
+    const chunks = [];
+    let currentChunk = null;
+    let lineCount = 0;
+    let chunkIndex = 0;
+
+    function newChunk() {
+      if (currentChunk) currentChunk.end();
+      const chunkPath = join(outputDir, `${prefix}-${chunkIndex}.txt`);
+      chunks.push(chunkPath);
+      currentChunk = createWriteStream(chunkPath);
+      chunkIndex++;
+      lineCount = 0;
+    }
+
+    rl.on('line', (line) => {
+      const trimmed = line.trim();
+      if (!trimmed) return;
+
+      if (lineCount === 0 || lineCount >= chunkSize) {
+        newChunk();
+      }
+
+      currentChunk.write(trimmed + '\n');
+      lineCount++;
+    });
+
+    rl.on('close', () => {
+      if (currentChunk) currentChunk.end();
+      resolve(chunks);
+    });
+
+    rl.on('error', reject);
+    input.on('error', reject);
+  });
+}

package/src/utils/filter-extensions.js

@@ -0,0 +1,52 @@
+import { createReadStream, createWriteStream } from 'fs';
+import { createInterface } from 'readline';
+import { BLOCKED_EXTENSIONS } from '../constants.js';
+
+/**
+ * Filter URLs by extension - removes URLs ending in blocked extensions.
+ * Returns { kept, removed } counts.
+ */
+export async function filterExtensions(inputFile, outputFile) {
+  return new Promise((resolve, reject) => {
+    const input = createReadStream(inputFile, { encoding: 'utf-8' });
+    const output = createWriteStream(outputFile);
+    const rl = createInterface({ input, crlfDelay: Infinity });
+
+    let kept = 0;
+    let removed = 0;
+
+    rl.on('line', (line) => {
+      const url = line.trim();
+      if (!url) return;
+
+      if (hasBlockedExtension(url)) {
+        removed++;
+      } else {
+        output.write(url + '\n');
+        kept++;
+      }
+    });
+
+    rl.on('close', () => {
+      output.end(() => resolve({ kept, removed }));
+    });
+
+    rl.on('error', reject);
+    input.on('error', reject);
+  });
+}
+
+function hasBlockedExtension(url) {
+  try {
+    // Strip query string and fragment
+    let path = url.split('?')[0].split('#')[0];
+    // Get the last path segment
+    const lastSegment = path.split('/').pop() || '';
+    const dotIndex = lastSegment.lastIndexOf('.');
+    if (dotIndex === -1) return false;
+    const ext = lastSegment.slice(dotIndex + 1).toLowerCase();
+    return BLOCKED_EXTENSIONS.has(ext);
+  } catch {
+    return false;
+  }
+}