@paa1997/metho 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.js +27 -2
- package/src/engine.js +165 -11
- package/src/gui.html +240 -42
- package/src/server.js +37 -14
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { resolve } from 'path';
|
|
3
3
|
import { existsSync } from 'fs';
|
|
4
|
-
import {
|
|
4
|
+
import { createInterface } from 'readline';
|
|
5
|
+
import { runPipeline, findIncompleteRun } from './engine.js';
|
|
5
6
|
import { createLogger } from './logger.js';
|
|
6
7
|
|
|
7
|
-
export function run() {
|
|
8
|
+
export async function run() {
|
|
8
9
|
const program = new Command();
|
|
9
10
|
|
|
10
11
|
program
|
|
@@ -66,11 +67,35 @@ export function run() {
|
|
|
66
67
|
katanaDepth: parseInt(opts.katanaDepth, 10),
|
|
67
68
|
findsomethingPath: opts.findsomethingPath || '',
|
|
68
69
|
debug: opts.debug || false,
|
|
70
|
+
resumeDir: null,
|
|
69
71
|
};
|
|
70
72
|
|
|
73
|
+
// Check for incomplete previous run
|
|
74
|
+
const incomplete = findIncompleteRun(config.outputDir, config.domain, config.domainList);
|
|
75
|
+
if (incomplete) {
|
|
76
|
+
const stepNames = incomplete.completedSteps.map(s => s.name).join(', ');
|
|
77
|
+
console.log(`\n Previous incomplete run found: ${incomplete.runDir}`);
|
|
78
|
+
console.log(` Completed steps: ${stepNames}`);
|
|
79
|
+
|
|
80
|
+
const answer = await askUser(' Resume this run? [Y/n] ');
|
|
81
|
+
if (answer.toLowerCase() !== 'n') {
|
|
82
|
+
config.resumeDir = incomplete.runDir;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
71
86
|
const logger = createLogger(config.debug);
|
|
72
87
|
runPipeline(config, logger).catch((err) => {
|
|
73
88
|
logger.error(`Pipeline failed: ${err.message}`);
|
|
74
89
|
process.exit(1);
|
|
75
90
|
});
|
|
76
91
|
}
|
|
92
|
+
|
|
93
|
+
function askUser(question) {
|
|
94
|
+
return new Promise((resolve) => {
|
|
95
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
96
|
+
rl.question(question, (answer) => {
|
|
97
|
+
rl.close();
|
|
98
|
+
resolve(answer.trim());
|
|
99
|
+
});
|
|
100
|
+
});
|
|
101
|
+
}
|
package/src/engine.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { mkdirSync, writeFileSync, copyFileSync, readFileSync, existsSync } from 'fs';
|
|
2
|
-
import { join, basename } from 'path';
|
|
1
|
+
import { mkdirSync, writeFileSync, copyFileSync, readFileSync, existsSync, readdirSync, statSync, rmSync } from 'fs';
|
|
2
|
+
import { join, basename, resolve } from 'path';
|
|
3
3
|
import { setupSignalHandlers, getGlobalContext, createRunContext } from './signals.js';
|
|
4
4
|
import { ensureTools } from './tools/manager.js';
|
|
5
5
|
import { SubfinderStep } from './steps/subfinder.js';
|
|
@@ -9,8 +9,22 @@ import { FilterStep } from './steps/filter.js';
|
|
|
9
9
|
import { KatanaStep } from './steps/katana.js';
|
|
10
10
|
import { FindSomethingStep } from './steps/findsomething.js';
|
|
11
11
|
|
|
12
|
+
// Step output files in order
|
|
13
|
+
const STEP_FILES = [
|
|
14
|
+
{ num: 1, file: '01-subdomains.txt', name: 'Subfinder' },
|
|
15
|
+
{ num: 2, file: '02-live-subdomains.txt', name: 'Subdomain Probe' },
|
|
16
|
+
{ num: 3, file: '03-passive-urls.txt', name: 'GAU' },
|
|
17
|
+
{ num: 4, file: '04-live-urls.txt', name: 'Filter' },
|
|
18
|
+
{ num: 5, file: '05-crawled-urls.txt', name: 'Katana' },
|
|
19
|
+
{ num: 6, file: '06-secrets.txt', name: 'FindSomething' },
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
function getRunLabel(domain, domainList) {
|
|
23
|
+
return domain || basename(domainList, '.txt');
|
|
24
|
+
}
|
|
25
|
+
|
|
12
26
|
function makeRunDir(outputDir, domain, domainList) {
|
|
13
|
-
const label = domain
|
|
27
|
+
const label = getRunLabel(domain, domainList);
|
|
14
28
|
const now = new Date();
|
|
15
29
|
const ts = now.toISOString().replace(/[-:T]/g, '').replace(/\..+/, '').replace(/(\d{8})(\d{6})/, '$1-$2');
|
|
16
30
|
const dirName = `${label}-${ts}`;
|
|
@@ -19,6 +33,82 @@ function makeRunDir(outputDir, domain, domainList) {
|
|
|
19
33
|
return runDir;
|
|
20
34
|
}
|
|
21
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Check if a file exists and has content (>0 bytes).
|
|
38
|
+
*/
|
|
39
|
+
function fileHasContent(filePath) {
|
|
40
|
+
try {
|
|
41
|
+
return existsSync(filePath) && statSync(filePath).size > 0;
|
|
42
|
+
} catch { return false; }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Find the most recent incomplete run for a given domain/label.
|
|
47
|
+
* Returns { runDir, completedSteps, lastCompletedFile } or null.
|
|
48
|
+
*/
|
|
49
|
+
export function findIncompleteRun(outputDir, domain, domainList) {
|
|
50
|
+
const label = getRunLabel(domain, domainList);
|
|
51
|
+
const absOutputDir = resolve(outputDir);
|
|
52
|
+
|
|
53
|
+
if (!existsSync(absOutputDir)) return null;
|
|
54
|
+
|
|
55
|
+
let dirs;
|
|
56
|
+
try {
|
|
57
|
+
dirs = readdirSync(absOutputDir)
|
|
58
|
+
.filter(d => d.startsWith(label + '-'))
|
|
59
|
+
.filter(d => {
|
|
60
|
+
try { return statSync(join(absOutputDir, d)).isDirectory(); } catch { return false; }
|
|
61
|
+
})
|
|
62
|
+
.sort()
|
|
63
|
+
.reverse(); // most recent first
|
|
64
|
+
} catch { return null; }
|
|
65
|
+
|
|
66
|
+
for (const dir of dirs) {
|
|
67
|
+
const runDir = join(absOutputDir, dir);
|
|
68
|
+
const completed = [];
|
|
69
|
+
let lastFile = null;
|
|
70
|
+
|
|
71
|
+
for (const step of STEP_FILES) {
|
|
72
|
+
const filePath = join(runDir, step.file);
|
|
73
|
+
if (fileHasContent(filePath)) {
|
|
74
|
+
completed.push({ num: step.num, name: step.name, file: step.file });
|
|
75
|
+
lastFile = filePath;
|
|
76
|
+
} else {
|
|
77
|
+
break; // steps must be sequential
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Incomplete = has input file + at least 1 completed step but not all 6
|
|
82
|
+
const hasInput = fileHasContent(join(runDir, '00-input-domains.txt'));
|
|
83
|
+
if (hasInput && completed.length > 0 && completed.length < STEP_FILES.length) {
|
|
84
|
+
return { runDir, completedSteps: completed, lastCompletedFile: lastFile, label };
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Delete all run directories in the output directory.
|
|
93
|
+
*/
|
|
94
|
+
export function deleteAllRuns(outputDir) {
|
|
95
|
+
const absOutputDir = resolve(outputDir);
|
|
96
|
+
if (!existsSync(absOutputDir)) return 0;
|
|
97
|
+
|
|
98
|
+
let count = 0;
|
|
99
|
+
const entries = readdirSync(absOutputDir);
|
|
100
|
+
for (const entry of entries) {
|
|
101
|
+
const fullPath = join(absOutputDir, entry);
|
|
102
|
+
try {
|
|
103
|
+
if (statSync(fullPath).isDirectory()) {
|
|
104
|
+
rmSync(fullPath, { recursive: true, force: true });
|
|
105
|
+
count++;
|
|
106
|
+
}
|
|
107
|
+
} catch { /* ignore */ }
|
|
108
|
+
}
|
|
109
|
+
return count;
|
|
110
|
+
}
|
|
111
|
+
|
|
22
112
|
/**
|
|
23
113
|
* Run the full recon pipeline.
|
|
24
114
|
* @param {object} config - pipeline configuration
|
|
@@ -34,21 +124,41 @@ export async function runPipeline(config, logger, ctx) {
|
|
|
34
124
|
|
|
35
125
|
logger.banner('METHO - Automated Recon Pipeline');
|
|
36
126
|
|
|
37
|
-
//
|
|
38
|
-
|
|
39
|
-
|
|
127
|
+
// Determine run directory (resume existing or create new)
|
|
128
|
+
let runDir;
|
|
129
|
+
let resumeFrom = 0; // 0 = no resume, N = resume after step N
|
|
130
|
+
|
|
131
|
+
if (config.resumeDir && existsSync(config.resumeDir)) {
|
|
132
|
+
runDir = config.resumeDir;
|
|
133
|
+
logger.info(`Resuming run in: ${runDir}`);
|
|
134
|
+
|
|
135
|
+
// Detect which steps are already complete
|
|
136
|
+
for (const step of STEP_FILES) {
|
|
137
|
+
if (fileHasContent(join(runDir, step.file))) {
|
|
138
|
+
resumeFrom = step.num;
|
|
139
|
+
} else {
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
logger.info(`Resuming after step ${resumeFrom} (${STEP_FILES[resumeFrom - 1]?.name || 'none'})`);
|
|
144
|
+
} else {
|
|
145
|
+
runDir = makeRunDir(config.outputDir, config.domain, config.domainList);
|
|
146
|
+
logger.info(`Output directory: ${runDir}`);
|
|
147
|
+
}
|
|
40
148
|
|
|
41
|
-
// Set up log file
|
|
149
|
+
// Set up log file (append if resuming)
|
|
42
150
|
const logFile = join(runDir, 'metho.log');
|
|
43
151
|
logger.setLogFile(logFile);
|
|
44
152
|
logger.debug(`Config: ${JSON.stringify(config, null, 2)}`);
|
|
45
153
|
|
|
46
154
|
// Prepare input file
|
|
47
155
|
const inputFile = join(runDir, '00-input-domains.txt');
|
|
48
|
-
if (config.
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
156
|
+
if (!config.resumeDir) {
|
|
157
|
+
if (config.domain) {
|
|
158
|
+
writeFileSync(inputFile, config.domain + '\n');
|
|
159
|
+
} else {
|
|
160
|
+
copyFileSync(config.domainList, inputFile);
|
|
161
|
+
}
|
|
52
162
|
}
|
|
53
163
|
ctx.trackFile(inputFile, 'Input domains');
|
|
54
164
|
logger.info(`Input: ${config.domain || config.domainList}`);
|
|
@@ -59,14 +169,29 @@ export async function runPipeline(config, logger, ctx) {
|
|
|
59
169
|
|
|
60
170
|
// Track results for final summary
|
|
61
171
|
const results = [];
|
|
172
|
+
|
|
173
|
+
// If resuming, set currentInput to the last completed step's output
|
|
62
174
|
let currentInput = inputFile;
|
|
175
|
+
if (resumeFrom > 0) {
|
|
176
|
+
const lastFile = join(runDir, STEP_FILES[resumeFrom - 1].file);
|
|
177
|
+
if (fileHasContent(lastFile)) currentInput = lastFile;
|
|
178
|
+
}
|
|
63
179
|
let stepNum = 0;
|
|
64
180
|
|
|
181
|
+
// Helper: check if a step should be skipped because it's already done (resume)
|
|
182
|
+
function isResumeComplete(num) { return resumeFrom >= num; }
|
|
183
|
+
|
|
65
184
|
// --- Step 1: Subfinder ---
|
|
66
185
|
stepNum++;
|
|
67
186
|
const subdomainsFile = join(runDir, '01-subdomains.txt');
|
|
68
187
|
if (config.skipSubfinder) {
|
|
69
188
|
logger.step(stepNum, 'Subfinder (subdomain enumeration)', 'skip');
|
|
189
|
+
} else if (isResumeComplete(1)) {
|
|
190
|
+
const lines = readFileSync(subdomainsFile, 'utf-8').trim().split('\n').length;
|
|
191
|
+
logger.step(stepNum, `Subfinder → ${lines} subdomains (cached)`, 'done');
|
|
192
|
+
results.push({ step: 'Subfinder', file: subdomainsFile, lines });
|
|
193
|
+
logger.result('Subfinder', subdomainsFile, lines);
|
|
194
|
+
currentInput = subdomainsFile;
|
|
70
195
|
} else {
|
|
71
196
|
logger.step(stepNum, 'Subfinder (subdomain enumeration)', 'start');
|
|
72
197
|
const subfinder = new SubfinderStep(logger, ctx);
|
|
@@ -84,6 +209,12 @@ export async function runPipeline(config, logger, ctx) {
|
|
|
84
209
|
const liveSubsFile = join(runDir, '02-live-subdomains.txt');
|
|
85
210
|
if (config.skipSubdomainProbe) {
|
|
86
211
|
logger.step(stepNum, 'Subdomain Probe (httpx)', 'skip');
|
|
212
|
+
} else if (isResumeComplete(2)) {
|
|
213
|
+
const lines = readFileSync(liveSubsFile, 'utf-8').trim().split('\n').length;
|
|
214
|
+
logger.step(stepNum, `Subdomain Probe → ${lines} live subdomains (cached)`, 'done');
|
|
215
|
+
results.push({ step: 'Subdomain Probe', file: liveSubsFile, lines });
|
|
216
|
+
logger.result('Subdomain Probe', liveSubsFile, lines);
|
|
217
|
+
currentInput = liveSubsFile;
|
|
87
218
|
} else {
|
|
88
219
|
logger.step(stepNum, 'Subdomain Probe (httpx on subdomains)', 'start');
|
|
89
220
|
const probe = new SubdomainProbeStep(logger, ctx);
|
|
@@ -101,6 +232,12 @@ export async function runPipeline(config, logger, ctx) {
|
|
|
101
232
|
const gauFile = join(runDir, '03-passive-urls.txt');
|
|
102
233
|
if (config.skipGau) {
|
|
103
234
|
logger.step(stepNum, 'GAU (passive URL crawling)', 'skip');
|
|
235
|
+
} else if (isResumeComplete(3)) {
|
|
236
|
+
const lines = readFileSync(gauFile, 'utf-8').trim().split('\n').length;
|
|
237
|
+
logger.step(stepNum, `GAU → ${lines} URLs (cached)`, 'done');
|
|
238
|
+
results.push({ step: 'GAU', file: gauFile, lines });
|
|
239
|
+
logger.result('GAU', gauFile, lines);
|
|
240
|
+
currentInput = gauFile;
|
|
104
241
|
} else {
|
|
105
242
|
logger.step(stepNum, 'GAU (passive URL crawling)', 'start');
|
|
106
243
|
const gau = new GauStep(logger, ctx);
|
|
@@ -118,6 +255,12 @@ export async function runPipeline(config, logger, ctx) {
|
|
|
118
255
|
const liveUrlsFile = join(runDir, '04-live-urls.txt');
|
|
119
256
|
if (config.skipFilter) {
|
|
120
257
|
logger.step(stepNum, 'Filter (extension + liveness)', 'skip');
|
|
258
|
+
} else if (isResumeComplete(4)) {
|
|
259
|
+
const lines = readFileSync(liveUrlsFile, 'utf-8').trim().split('\n').length;
|
|
260
|
+
logger.step(stepNum, `Filter → ${lines} live URLs (cached)`, 'done');
|
|
261
|
+
results.push({ step: 'Filter', file: liveUrlsFile, lines });
|
|
262
|
+
logger.result('Filter', liveUrlsFile, lines);
|
|
263
|
+
currentInput = liveUrlsFile;
|
|
121
264
|
} else {
|
|
122
265
|
logger.step(stepNum, 'Filter (extension + liveness)', 'start');
|
|
123
266
|
const filter = new FilterStep(logger, ctx);
|
|
@@ -135,6 +278,12 @@ export async function runPipeline(config, logger, ctx) {
|
|
|
135
278
|
const crawledFile = join(runDir, '05-crawled-urls.txt');
|
|
136
279
|
if (config.skipKatana) {
|
|
137
280
|
logger.step(stepNum, 'Katana (active crawling)', 'skip');
|
|
281
|
+
} else if (isResumeComplete(5)) {
|
|
282
|
+
const lines = readFileSync(crawledFile, 'utf-8').trim().split('\n').length;
|
|
283
|
+
logger.step(stepNum, `Katana → ${lines} URLs (cached)`, 'done');
|
|
284
|
+
results.push({ step: 'Katana', file: crawledFile, lines });
|
|
285
|
+
logger.result('Katana', crawledFile, lines);
|
|
286
|
+
currentInput = crawledFile;
|
|
138
287
|
} else {
|
|
139
288
|
logger.step(stepNum, 'Katana (active crawling)', 'start');
|
|
140
289
|
const katana = new KatanaStep(logger, ctx);
|
|
@@ -152,6 +301,11 @@ export async function runPipeline(config, logger, ctx) {
|
|
|
152
301
|
const secretsFile = join(runDir, '06-secrets.txt');
|
|
153
302
|
if (config.skipFindsomething) {
|
|
154
303
|
logger.step(stepNum, 'FindSomething (secret scanning)', 'skip');
|
|
304
|
+
} else if (isResumeComplete(6)) {
|
|
305
|
+
const lines = readFileSync(secretsFile, 'utf-8').trim().split('\n').length;
|
|
306
|
+
logger.step(stepNum, `FindSomething → ${lines} findings (cached)`, 'done');
|
|
307
|
+
results.push({ step: 'FindSomething', file: secretsFile, lines });
|
|
308
|
+
logger.result('FindSomething', secretsFile, lines);
|
|
155
309
|
} else {
|
|
156
310
|
logger.step(stepNum, 'FindSomething (secret scanning)', 'start');
|
|
157
311
|
const fs = new FindSomethingStep(logger, ctx);
|
package/src/gui.html
CHANGED
|
@@ -256,6 +256,50 @@
|
|
|
256
256
|
font-family: 'Cascadia Code', 'Fira Code', monospace;
|
|
257
257
|
}
|
|
258
258
|
.run-dir-bar strong { color: #a78bfa; }
|
|
259
|
+
|
|
260
|
+
/* Modal overlay */
|
|
261
|
+
.modal-overlay {
|
|
262
|
+
display: none; position: fixed; top: 0; left: 0; right: 0; bottom: 0;
|
|
263
|
+
background: rgba(0,0,0,0.7); z-index: 1000;
|
|
264
|
+
align-items: center; justify-content: center;
|
|
265
|
+
}
|
|
266
|
+
.modal-overlay.open { display: flex; }
|
|
267
|
+
.modal {
|
|
268
|
+
background: #1a1a2e; border: 1px solid #2a2a4a; border-radius: 12px;
|
|
269
|
+
padding: 28px 32px; max-width: 500px; width: 90%; box-shadow: 0 20px 60px rgba(0,0,0,0.5);
|
|
270
|
+
}
|
|
271
|
+
.modal h3 {
|
|
272
|
+
font-size: 16px; color: #a78bfa; margin-bottom: 14px; font-weight: 700;
|
|
273
|
+
}
|
|
274
|
+
.modal p { color: #bbb; font-size: 13px; line-height: 1.6; margin-bottom: 12px; }
|
|
275
|
+
.modal .step-list {
|
|
276
|
+
background: #12121f; border: 1px solid #2a2a4a; border-radius: 6px;
|
|
277
|
+
padding: 10px 14px; margin-bottom: 16px; font-size: 12px; color: #22c55e;
|
|
278
|
+
font-family: 'Cascadia Code', 'Fira Code', monospace;
|
|
279
|
+
}
|
|
280
|
+
.modal-actions { display: flex; gap: 10px; justify-content: flex-end; }
|
|
281
|
+
.modal-actions .btn { padding: 8px 22px; font-size: 13px; }
|
|
282
|
+
.btn-secondary {
|
|
283
|
+
background: #2a2a4a; color: #ccc; border: 1px solid #3a3a5a; border-radius: 6px;
|
|
284
|
+
padding: 8px 22px; font-size: 13px; font-weight: 600; cursor: pointer;
|
|
285
|
+
transition: all 0.15s;
|
|
286
|
+
}
|
|
287
|
+
.btn-secondary:hover { background: #3a3a5a; color: #fff; }
|
|
288
|
+
.btn-danger {
|
|
289
|
+
background: #dc2626; color: #fff; border: none; border-radius: 6px;
|
|
290
|
+
padding: 8px 22px; font-size: 13px; font-weight: 600; cursor: pointer;
|
|
291
|
+
transition: all 0.15s;
|
|
292
|
+
}
|
|
293
|
+
.btn-danger:hover { background: #ef4444; }
|
|
294
|
+
|
|
295
|
+
/* Settings gear */
|
|
296
|
+
.btn-gear {
|
|
297
|
+
background: none; border: 1px solid #2a2a4a; border-radius: 6px;
|
|
298
|
+
color: #888; font-size: 18px; width: 38px; height: 38px; cursor: pointer;
|
|
299
|
+
display: flex; align-items: center; justify-content: center;
|
|
300
|
+
transition: all 0.15s;
|
|
301
|
+
}
|
|
302
|
+
.btn-gear:hover { color: #a78bfa; border-color: #7c3aed; }
|
|
259
303
|
</style>
|
|
260
304
|
</head>
|
|
261
305
|
<body>
|
|
@@ -309,6 +353,7 @@
|
|
|
309
353
|
<div class="actions">
|
|
310
354
|
<button class="btn btn-run" id="btn-run" onclick="startPipeline()">Run</button>
|
|
311
355
|
<button class="btn btn-stop" id="btn-stop-all" onclick="stopAllRuns()" disabled>Stop All</button>
|
|
356
|
+
<button class="btn-gear" onclick="openSettings()" title="Settings">⚙</button>
|
|
312
357
|
</div>
|
|
313
358
|
</div>
|
|
314
359
|
|
|
@@ -318,6 +363,51 @@
|
|
|
318
363
|
<div id="no-runs" class="no-runs-placeholder">Run the pipeline to see results here</div>
|
|
319
364
|
</div>
|
|
320
365
|
|
|
366
|
+
<!-- Resume modal -->
|
|
367
|
+
<div class="modal-overlay" id="resume-modal">
|
|
368
|
+
<div class="modal">
|
|
369
|
+
<h3>Previous Run Found</h3>
|
|
370
|
+
<p>An incomplete run was found for <strong id="resume-label"></strong></p>
|
|
371
|
+
<p>Completed steps:</p>
|
|
372
|
+
<div class="step-list" id="resume-steps"></div>
|
|
373
|
+
<p>Would you like to resume from where it left off?</p>
|
|
374
|
+
<div class="modal-actions">
|
|
375
|
+
<button class="btn-secondary" onclick="dismissResume()">Start New</button>
|
|
376
|
+
<button class="btn btn-run" onclick="confirmResume()">Resume</button>
|
|
377
|
+
</div>
|
|
378
|
+
</div>
|
|
379
|
+
</div>
|
|
380
|
+
|
|
381
|
+
<!-- Settings modal -->
|
|
382
|
+
<div class="modal-overlay" id="settings-modal">
|
|
383
|
+
<div class="modal">
|
|
384
|
+
<h3>Settings</h3>
|
|
385
|
+
<p>Manage your pipeline data and preferences.</p>
|
|
386
|
+
<div style="margin-top: 16px;">
|
|
387
|
+
<button class="btn-danger" onclick="confirmDeleteLogs()">Delete All Run Logs</button>
|
|
388
|
+
<p style="font-size: 11px; color: #666; margin-top: 8px;">
|
|
389
|
+
Removes all previous results and prevents resuming any past runs.
|
|
390
|
+
</p>
|
|
391
|
+
</div>
|
|
392
|
+
<div class="modal-actions" style="margin-top: 24px;">
|
|
393
|
+
<button class="btn-secondary" onclick="closeSettings()">Close</button>
|
|
394
|
+
</div>
|
|
395
|
+
</div>
|
|
396
|
+
</div>
|
|
397
|
+
|
|
398
|
+
<!-- Delete confirmation modal -->
|
|
399
|
+
<div class="modal-overlay" id="delete-modal">
|
|
400
|
+
<div class="modal">
|
|
401
|
+
<h3>Confirm Deletion</h3>
|
|
402
|
+
<p>Are you sure you want to delete <strong>all</strong> run logs and results?</p>
|
|
403
|
+
<p style="color: #f59e0b;">This action cannot be undone and will prevent you from resuming any previous runs.</p>
|
|
404
|
+
<div class="modal-actions">
|
|
405
|
+
<button class="btn-secondary" onclick="closeDeleteModal()">Cancel</button>
|
|
406
|
+
<button class="btn-danger" onclick="executeDeleteLogs()">Delete Everything</button>
|
|
407
|
+
</div>
|
|
408
|
+
</div>
|
|
409
|
+
</div>
|
|
410
|
+
|
|
321
411
|
<!-- Template for a run panel (cloned per run) -->
|
|
322
412
|
<template id="run-panel-template">
|
|
323
413
|
<div class="run-panel">
|
|
@@ -359,6 +449,7 @@
|
|
|
359
449
|
let eventSource = null;
|
|
360
450
|
let sseConnected = false;
|
|
361
451
|
let activeRunId = null; // currently viewed tab
|
|
452
|
+
let sseRetryTimer = null;
|
|
362
453
|
|
|
363
454
|
// Per-run state: runId → { label, status, logLines, resultCount, el, tabEl }
|
|
364
455
|
const runs = {};
|
|
@@ -390,55 +481,60 @@ function esc(str) {
|
|
|
390
481
|
return d.innerHTML;
|
|
391
482
|
}
|
|
392
483
|
|
|
393
|
-
// ---- SSE Connection ----
|
|
484
|
+
// ---- SSE Connection (with auto-reconnect) ----
|
|
485
|
+
|
|
486
|
+
function connectSSE() {
|
|
487
|
+
if (eventSource) { eventSource.close(); eventSource = null; }
|
|
488
|
+
sseConnected = false;
|
|
489
|
+
if (sseRetryTimer) { clearTimeout(sseRetryTimer); sseRetryTimer = null; }
|
|
490
|
+
|
|
491
|
+
const es = new EventSource('/events');
|
|
492
|
+
|
|
493
|
+
es.onmessage = (e) => {
|
|
494
|
+
try {
|
|
495
|
+
const data = JSON.parse(e.data);
|
|
496
|
+
if (data.type === 'connected') {
|
|
497
|
+
sseConnected = true;
|
|
498
|
+
eventSource = es;
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
handleEvent(data);
|
|
502
|
+
} catch { /* ignore */ }
|
|
503
|
+
};
|
|
504
|
+
|
|
505
|
+
es.onerror = () => {
|
|
506
|
+
sseConnected = false;
|
|
507
|
+
eventSource = null;
|
|
508
|
+
es.close();
|
|
509
|
+
// Auto-reconnect after 2 seconds if there are running runs
|
|
510
|
+
const hasRunning = Object.values(runs).some(r => r.status === 'running');
|
|
511
|
+
if (hasRunning) {
|
|
512
|
+
sseRetryTimer = setTimeout(() => connectSSE(), 2000);
|
|
513
|
+
}
|
|
514
|
+
};
|
|
515
|
+
}
|
|
394
516
|
|
|
395
517
|
function ensureSSE() {
|
|
396
518
|
if (eventSource && sseConnected) return Promise.resolve();
|
|
397
519
|
return new Promise((resolve, reject) => {
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
const
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
try {
|
|
406
|
-
const data = JSON.parse(e.data);
|
|
407
|
-
if (data.type === 'connected' && !sseConnected) {
|
|
408
|
-
sseConnected = true;
|
|
409
|
-
clearTimeout(timeout);
|
|
410
|
-
eventSource = es;
|
|
411
|
-
resolve();
|
|
412
|
-
return;
|
|
413
|
-
}
|
|
414
|
-
handleEvent(data);
|
|
415
|
-
} catch { /* ignore */ }
|
|
416
|
-
};
|
|
417
|
-
|
|
418
|
-
es.onerror = () => {
|
|
419
|
-
if (!sseConnected) {
|
|
420
|
-
clearTimeout(timeout);
|
|
421
|
-
es.close();
|
|
422
|
-
reject(new Error('SSE connection failed'));
|
|
423
|
-
} else {
|
|
424
|
-
sseConnected = false;
|
|
425
|
-
eventSource = null;
|
|
426
|
-
}
|
|
427
|
-
};
|
|
520
|
+
connectSSE();
|
|
521
|
+
// Wait for connection
|
|
522
|
+
let attempts = 0;
|
|
523
|
+
const check = setInterval(() => {
|
|
524
|
+
if (sseConnected) { clearInterval(check); resolve(); }
|
|
525
|
+
else if (++attempts > 25) { clearInterval(check); reject(new Error('SSE timeout')); }
|
|
526
|
+
}, 200);
|
|
428
527
|
});
|
|
429
528
|
}
|
|
430
529
|
|
|
431
530
|
// ---- Start / Stop ----
|
|
432
531
|
|
|
433
|
-
|
|
434
|
-
const domain = document.getElementById('domain').value.trim();
|
|
435
|
-
const domainList = document.getElementById('domainList').value.trim();
|
|
436
|
-
if (!domain && !domainList) { alert('Enter a domain or domain list file path'); return; }
|
|
437
|
-
if (domain && domainList) { alert('Use either domain or domain list, not both'); return; }
|
|
532
|
+
let pendingResumeDir = null; // set when user chooses resume
|
|
438
533
|
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
534
|
+
function buildConfig() {
|
|
535
|
+
return {
|
|
536
|
+
domain: document.getElementById('domain').value.trim() || null,
|
|
537
|
+
domainList: document.getElementById('domainList').value.trim() || null,
|
|
442
538
|
outputDir: document.getElementById('outputDir').value.trim() || './metho-results',
|
|
443
539
|
findsomethingPath: document.getElementById('findsomethingPath').value.trim() || '',
|
|
444
540
|
katanaDepth: parseInt(document.getElementById('katanaDepth').value) || 2,
|
|
@@ -450,6 +546,43 @@ async function startPipeline() {
|
|
|
450
546
|
skipKatana: document.getElementById('skipKatana').checked,
|
|
451
547
|
skipFindsomething: document.getElementById('skipFindsomething').checked,
|
|
452
548
|
};
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
async function startPipeline() {
|
|
552
|
+
const domain = document.getElementById('domain').value.trim();
|
|
553
|
+
const domainList = document.getElementById('domainList').value.trim();
|
|
554
|
+
if (!domain && !domainList) { alert('Enter a domain or domain list file path'); return; }
|
|
555
|
+
if (domain && domainList) { alert('Use either domain or domain list, not both'); return; }
|
|
556
|
+
|
|
557
|
+
// Check for incomplete previous run
|
|
558
|
+
try {
|
|
559
|
+
const checkResp = await fetch('/check-resume', {
|
|
560
|
+
method: 'POST',
|
|
561
|
+
headers: { 'Content-Type': 'application/json' },
|
|
562
|
+
body: JSON.stringify({
|
|
563
|
+
domain: domain || null,
|
|
564
|
+
domainList: domainList || null,
|
|
565
|
+
outputDir: document.getElementById('outputDir').value.trim() || './metho-results',
|
|
566
|
+
}),
|
|
567
|
+
});
|
|
568
|
+
const checkData = await checkResp.json();
|
|
569
|
+
if (checkData.hasIncomplete) {
|
|
570
|
+
// Show resume modal
|
|
571
|
+
pendingResumeDir = checkData.runDir;
|
|
572
|
+
document.getElementById('resume-label').textContent = checkData.label;
|
|
573
|
+
document.getElementById('resume-steps').textContent =
|
|
574
|
+
checkData.completedSteps.map(s => s.name).join(' -> ');
|
|
575
|
+
document.getElementById('resume-modal').classList.add('open');
|
|
576
|
+
return;
|
|
577
|
+
}
|
|
578
|
+
} catch { /* check failed, proceed with new run */ }
|
|
579
|
+
|
|
580
|
+
launchRun(null);
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
async function launchRun(resumeDir) {
|
|
584
|
+
const config = buildConfig();
|
|
585
|
+
if (resumeDir) config.resumeDir = resumeDir;
|
|
453
586
|
|
|
454
587
|
try {
|
|
455
588
|
await ensureSSE();
|
|
@@ -472,6 +605,53 @@ async function startPipeline() {
|
|
|
472
605
|
}
|
|
473
606
|
}
|
|
474
607
|
|
|
608
|
+
function confirmResume() {
|
|
609
|
+
document.getElementById('resume-modal').classList.remove('open');
|
|
610
|
+
launchRun(pendingResumeDir);
|
|
611
|
+
pendingResumeDir = null;
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
function dismissResume() {
|
|
615
|
+
document.getElementById('resume-modal').classList.remove('open');
|
|
616
|
+
launchRun(null);
|
|
617
|
+
pendingResumeDir = null;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
// ---- Settings ----
|
|
621
|
+
|
|
622
|
+
function openSettings() {
|
|
623
|
+
document.getElementById('settings-modal').classList.add('open');
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
function closeSettings() {
|
|
627
|
+
document.getElementById('settings-modal').classList.remove('open');
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
function confirmDeleteLogs() {
|
|
631
|
+
document.getElementById('settings-modal').classList.remove('open');
|
|
632
|
+
document.getElementById('delete-modal').classList.add('open');
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
function closeDeleteModal() {
|
|
636
|
+
document.getElementById('delete-modal').classList.remove('open');
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
async function executeDeleteLogs() {
|
|
640
|
+
document.getElementById('delete-modal').classList.remove('open');
|
|
641
|
+
const outputDir = document.getElementById('outputDir').value.trim() || './metho-results';
|
|
642
|
+
try {
|
|
643
|
+
const resp = await fetch('/delete-logs', {
|
|
644
|
+
method: 'POST',
|
|
645
|
+
headers: { 'Content-Type': 'application/json' },
|
|
646
|
+
body: JSON.stringify({ outputDir }),
|
|
647
|
+
});
|
|
648
|
+
const data = await resp.json();
|
|
649
|
+
alert('Deleted ' + data.deleted + ' run(s).');
|
|
650
|
+
} catch (err) {
|
|
651
|
+
alert('Failed to delete: ' + err.message);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
|
|
475
655
|
function stopAllRuns() {
|
|
476
656
|
fetch('/stop', { method: 'POST', body: '{}' }).catch(() => {});
|
|
477
657
|
for (const runId of Object.keys(runs)) {
|
|
@@ -619,9 +799,6 @@ function handleEvent(data) {
|
|
|
619
799
|
if (runs[runId] && runs[runId].status === 'running') {
|
|
620
800
|
setRunStatus(runId, 'stopped');
|
|
621
801
|
}
|
|
622
|
-
if (data.activeRuns === 0 && eventSource) {
|
|
623
|
-
eventSource.close(); eventSource = null; sseConnected = false;
|
|
624
|
-
}
|
|
625
802
|
break;
|
|
626
803
|
}
|
|
627
804
|
}
|
|
@@ -761,6 +938,27 @@ async function loadResultFile(filePath, runId, tabId) {
|
|
|
761
938
|
}
|
|
762
939
|
}
|
|
763
940
|
|
|
941
|
+
// ---- Restore active runs on page load ----
|
|
942
|
+
|
|
943
|
+
async function restoreActiveRuns() {
|
|
944
|
+
try {
|
|
945
|
+
const resp = await fetch('/status');
|
|
946
|
+
const data = await resp.json();
|
|
947
|
+
if (data.running > 0) {
|
|
948
|
+
// There are active runs — reconnect SSE and create placeholder tabs
|
|
949
|
+
for (const run of data.runs) {
|
|
950
|
+
if (!runs[run.runId]) {
|
|
951
|
+
createRunTab(run.runId, run.label);
|
|
952
|
+
addRunLog(run.runId, 'warn', 'Reconnected — earlier log history was lost');
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
connectSSE();
|
|
956
|
+
}
|
|
957
|
+
} catch { /* server not reachable, ignore */ }
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
window.addEventListener('load', restoreActiveRuns);
|
|
961
|
+
|
|
764
962
|
async function copyTab(tabId) {
|
|
765
963
|
const data = tabFileData[tabId];
|
|
766
964
|
if (!data) return;
|
package/src/server.js
CHANGED
|
@@ -4,7 +4,7 @@ import { resolve, dirname, join } from 'path';
|
|
|
4
4
|
import { fileURLToPath } from 'url';
|
|
5
5
|
import { exec } from 'child_process';
|
|
6
6
|
import { createLogger } from './logger.js';
|
|
7
|
-
import { runPipeline } from './engine.js';
|
|
7
|
+
import { runPipeline, findIncompleteRun, deleteAllRuns } from './engine.js';
|
|
8
8
|
import { createRunContext } from './signals.js';
|
|
9
9
|
|
|
10
10
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
@@ -129,6 +129,7 @@ export function startServer(port = 3000) {
|
|
|
129
129
|
katanaChunkSize: parseInt(params.katanaChunkSize, 10) || 1000,
|
|
130
130
|
katanaDepth: parseInt(params.katanaDepth, 10) || 2,
|
|
131
131
|
findsomethingPath: params.findsomethingPath || '',
|
|
132
|
+
resumeDir: params.resumeDir || null,
|
|
132
133
|
debug: true,
|
|
133
134
|
};
|
|
134
135
|
|
|
@@ -195,6 +196,39 @@ export function startServer(port = 3000) {
|
|
|
195
196
|
}
|
|
196
197
|
}
|
|
197
198
|
|
|
199
|
+
// POST /check-resume → check for incomplete runs
|
|
200
|
+
if (method === 'POST' && url === '/check-resume') {
|
|
201
|
+
const body = await readBody(req);
|
|
202
|
+
let params;
|
|
203
|
+
try { params = JSON.parse(body); } catch {
|
|
204
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
205
|
+
return res.end(JSON.stringify({ error: 'Invalid JSON' }));
|
|
206
|
+
}
|
|
207
|
+
const outputDir = resolve(params.outputDir || './metho-results');
|
|
208
|
+
const result = findIncompleteRun(outputDir, params.domain || null, params.domainList || null);
|
|
209
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
210
|
+
if (result) {
|
|
211
|
+
return res.end(JSON.stringify({
|
|
212
|
+
hasIncomplete: true,
|
|
213
|
+
runDir: result.runDir,
|
|
214
|
+
completedSteps: result.completedSteps,
|
|
215
|
+
label: result.label,
|
|
216
|
+
}));
|
|
217
|
+
}
|
|
218
|
+
return res.end(JSON.stringify({ hasIncomplete: false }));
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// POST /delete-logs → delete all run directories
|
|
222
|
+
if (method === 'POST' && url === '/delete-logs') {
|
|
223
|
+
const body = await readBody(req);
|
|
224
|
+
let params = {};
|
|
225
|
+
try { params = JSON.parse(body); } catch { /* use defaults */ }
|
|
226
|
+
const outputDir = resolve(params.outputDir || './metho-results');
|
|
227
|
+
const count = deleteAllRuns(outputDir);
|
|
228
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
229
|
+
return res.end(JSON.stringify({ ok: true, deleted: count }));
|
|
230
|
+
}
|
|
231
|
+
|
|
198
232
|
// 404
|
|
199
233
|
res.writeHead(404, { 'Content-Type': 'application/json' });
|
|
200
234
|
res.end(JSON.stringify({ error: 'Not found' }));
|
|
@@ -232,18 +266,6 @@ export function startServer(port = 3000) {
|
|
|
232
266
|
} finally {
|
|
233
267
|
activeRuns.delete(runId);
|
|
234
268
|
broadcast({ type: 'run-ended', runId, activeRuns: activeRuns.size });
|
|
235
|
-
|
|
236
|
-
// Close SSE clients only when ALL runs are done
|
|
237
|
-
if (activeRuns.size === 0) {
|
|
238
|
-
setTimeout(() => {
|
|
239
|
-
if (activeRuns.size === 0) {
|
|
240
|
-
for (const client of sseClients) {
|
|
241
|
-
try { client.end(); } catch { /* ignore */ }
|
|
242
|
-
}
|
|
243
|
-
sseClients = [];
|
|
244
|
-
}
|
|
245
|
-
}, 3000);
|
|
246
|
-
}
|
|
247
269
|
}
|
|
248
270
|
}
|
|
249
271
|
|
|
@@ -276,7 +298,8 @@ function openFolderDialog() {
|
|
|
276
298
|
return new Promise((resolve, reject) => {
|
|
277
299
|
let cmd;
|
|
278
300
|
if (process.platform === 'win32') {
|
|
279
|
-
|
|
301
|
+
// Use Shell.Application COM object — more reliable than FolderBrowserDialog via CLI
|
|
302
|
+
cmd = 'powershell -NoProfile -Command "$shell = New-Object -ComObject Shell.Application; $folder = $shell.BrowseForFolder(0, \'Select output directory\', 0, 0); if ($folder) { $folder.Self.Path }"';
|
|
280
303
|
} else if (process.platform === 'darwin') {
|
|
281
304
|
cmd = "osascript -e 'POSIX path of (choose folder with prompt \"Select output directory\")'";
|
|
282
305
|
} else {
|