@akotliar/sitemap-qa 1.0.0-alpha.0 → 1.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/index.cjs +1985 -71
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1962 -71
- package/dist/index.js.map +1 -1
- package/package.json +65 -65
package/dist/index.js
CHANGED
|
@@ -1,27 +1,1596 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
2
|
+
|
|
3
|
+
// src/index.ts
|
|
4
|
+
import "dotenv/config";
|
|
5
|
+
import { Command as Command2 } from "commander";
|
|
6
|
+
|
|
7
|
+
// src/commands/analyze.ts
|
|
8
|
+
import { Command } from "commander";
|
|
9
|
+
import { promises as fs2 } from "fs";
|
|
10
|
+
import ora from "ora";
|
|
11
|
+
import chalk from "chalk";
|
|
12
|
+
import cliProgress from "cli-progress";
|
|
13
|
+
import os2 from "os";
|
|
14
|
+
|
|
15
|
+
// src/config/config-loader.ts
|
|
16
|
+
import { readFile } from "fs/promises";
|
|
17
|
+
import { existsSync } from "fs";
|
|
18
|
+
import { join } from "path";
|
|
19
|
+
import { homedir } from "os";
|
|
20
|
+
|
|
21
|
+
// src/types/config.ts
|
|
22
|
+
var DEFAULT_CONFIG = {
|
|
23
|
+
timeout: 30,
|
|
24
|
+
concurrency: 10,
|
|
25
|
+
parsingConcurrency: 50,
|
|
26
|
+
// Optimized for network-bound parallel parsing
|
|
27
|
+
discoveryConcurrency: 50,
|
|
28
|
+
// Optimized for recursive sitemap index discovery
|
|
29
|
+
outputFormat: "html",
|
|
30
|
+
outputDir: "./sitemap-qa/report",
|
|
31
|
+
verbose: false,
|
|
32
|
+
baseUrl: "https://example.com",
|
|
33
|
+
// Default for tests
|
|
34
|
+
acceptedPatterns: [],
|
|
35
|
+
riskDetectionBatchSize: 1e4,
|
|
36
|
+
riskDetectionConcurrency: void 0,
|
|
37
|
+
// Auto-detect in risk-detector.ts
|
|
38
|
+
progressBar: void 0,
|
|
39
|
+
// Auto-detect TTY
|
|
40
|
+
silent: false,
|
|
41
|
+
benchmark: false
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
// src/config/config-loader.ts
|
|
45
|
+
async function loadConfig(cliOptions) {
|
|
46
|
+
let config = { ...DEFAULT_CONFIG };
|
|
47
|
+
const globalConfigPath = join(homedir(), ".sitemap-qa", "config.json");
|
|
48
|
+
if (existsSync(globalConfigPath)) {
|
|
49
|
+
try {
|
|
50
|
+
const globalConfig = JSON.parse(await readFile(globalConfigPath, "utf-8"));
|
|
51
|
+
config = { ...config, ...globalConfig };
|
|
52
|
+
} catch (error) {
|
|
53
|
+
console.warn(`Warning: Failed to load global config: ${error}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
const projectConfigPath = join(process.cwd(), ".sitemap-qa.config.json");
|
|
57
|
+
if (existsSync(projectConfigPath)) {
|
|
58
|
+
try {
|
|
59
|
+
const projectConfig = JSON.parse(await readFile(projectConfigPath, "utf-8"));
|
|
60
|
+
config = { ...config, ...projectConfig };
|
|
61
|
+
} catch (error) {
|
|
62
|
+
console.warn(`Warning: Failed to load project config: ${error}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const envConfig = loadFromEnv();
|
|
66
|
+
config = { ...config, ...envConfig };
|
|
67
|
+
config = mergeCliOptions(config, cliOptions);
|
|
68
|
+
if (cliOptions.baseUrl) {
|
|
69
|
+
config.baseUrl = cliOptions.baseUrl;
|
|
70
|
+
}
|
|
71
|
+
validateConfig(config);
|
|
72
|
+
return config;
|
|
73
|
+
}
|
|
74
|
+
function loadFromEnv() {
|
|
75
|
+
const env = {};
|
|
76
|
+
if (process.env.SITEMAP_VERIFY_TIMEOUT) {
|
|
77
|
+
env.timeout = parseInt(process.env.SITEMAP_VERIFY_TIMEOUT, 10);
|
|
78
|
+
}
|
|
79
|
+
return env;
|
|
80
|
+
}
|
|
81
|
+
function mergeCliOptions(config, cliOptions) {
|
|
82
|
+
const merged = { ...config };
|
|
83
|
+
if (cliOptions.timeout && cliOptions.timeout !== "30") {
|
|
84
|
+
merged.timeout = parseInt(cliOptions.timeout, 10);
|
|
85
|
+
}
|
|
86
|
+
if (cliOptions.output) {
|
|
87
|
+
merged.outputFormat = cliOptions.output;
|
|
88
|
+
}
|
|
89
|
+
if (cliOptions.outputDir) {
|
|
90
|
+
merged.outputDir = cliOptions.outputDir;
|
|
91
|
+
}
|
|
92
|
+
if (cliOptions.verbose === true) {
|
|
93
|
+
merged.verbose = true;
|
|
94
|
+
}
|
|
95
|
+
if (cliOptions.acceptedPatterns) {
|
|
96
|
+
merged.acceptedPatterns = cliOptions.acceptedPatterns.split(",").map((p) => p.trim()).filter(Boolean);
|
|
97
|
+
}
|
|
98
|
+
return merged;
|
|
99
|
+
}
|
|
100
|
+
function validateConfig(config) {
|
|
101
|
+
if (config.timeout < 1 || config.timeout > 300) {
|
|
102
|
+
throw new Error("Timeout must be between 1 and 300 seconds");
|
|
103
|
+
}
|
|
104
|
+
if (!["json", "html"].includes(config.outputFormat)) {
|
|
105
|
+
throw new Error("Output format must be json or html");
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// src/errors/network-errors.ts
|
|
110
|
+
var NetworkError = class extends Error {
|
|
111
|
+
constructor(url, originalError) {
|
|
112
|
+
super(`Network request failed for ${url}: ${originalError.message}`);
|
|
113
|
+
this.url = url;
|
|
114
|
+
this.originalError = originalError;
|
|
115
|
+
this.name = "NetworkError";
|
|
116
|
+
}
|
|
117
|
+
code = "NETWORK_ERROR";
|
|
118
|
+
};
|
|
119
|
+
var HttpError = class extends Error {
|
|
120
|
+
constructor(url, statusCode, statusText) {
|
|
121
|
+
let message = `HTTP ${statusCode} error for ${url}`;
|
|
122
|
+
if (statusCode === 403) {
|
|
123
|
+
message += "\n Note: 403 Forbidden often indicates bot protection (Cloudflare, etc.) or access restrictions";
|
|
124
|
+
}
|
|
125
|
+
super(message);
|
|
126
|
+
this.url = url;
|
|
127
|
+
this.statusCode = statusCode;
|
|
128
|
+
this.statusText = statusText;
|
|
129
|
+
this.name = "HttpError";
|
|
130
|
+
}
|
|
131
|
+
code = "HTTP_ERROR";
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
// src/utils/http-client.ts
|
|
135
|
+
import { chromium } from "playwright";
|
|
136
|
+
import axios from "axios";
|
|
137
|
+
import { Agent as HttpAgent } from "http";
|
|
138
|
+
import { Agent as HttpsAgent } from "https";
|
|
139
|
+
var httpAgent = new HttpAgent({
|
|
140
|
+
keepAlive: true,
|
|
141
|
+
maxSockets: 200,
|
|
142
|
+
// Allow many concurrent connections
|
|
143
|
+
maxFreeSockets: 50,
|
|
144
|
+
timeout: 15e3
|
|
145
|
+
});
|
|
146
|
+
var httpsAgent = new HttpsAgent({
|
|
147
|
+
keepAlive: true,
|
|
148
|
+
maxSockets: 200,
|
|
149
|
+
maxFreeSockets: 50,
|
|
150
|
+
timeout: 15e3
|
|
151
|
+
});
|
|
152
|
+
var axiosInstance = axios.create({
|
|
153
|
+
httpAgent,
|
|
154
|
+
httpsAgent,
|
|
155
|
+
maxRedirects: 5,
|
|
156
|
+
validateStatus: () => true
|
|
157
|
+
// Don't throw on any status code
|
|
158
|
+
});
|
|
159
|
+
async function fetchUrlWithBrowser(url, timeout) {
|
|
160
|
+
let browser;
|
|
161
|
+
try {
|
|
162
|
+
browser = await chromium.launch({
|
|
163
|
+
headless: true,
|
|
164
|
+
args: [
|
|
165
|
+
"--disable-blink-features=AutomationControlled",
|
|
166
|
+
// Hide automation flags
|
|
167
|
+
"--disable-dev-shm-usage",
|
|
168
|
+
"--no-sandbox"
|
|
169
|
+
]
|
|
170
|
+
});
|
|
171
|
+
const context = await browser.newContext({
|
|
172
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
173
|
+
viewport: { width: 1920, height: 1080 },
|
|
174
|
+
locale: "en-US",
|
|
175
|
+
timezoneId: "America/New_York",
|
|
176
|
+
extraHTTPHeaders: {
|
|
177
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
178
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
179
|
+
"Accept-Encoding": "gzip, deflate, br",
|
|
180
|
+
"DNT": "1",
|
|
181
|
+
"Connection": "keep-alive",
|
|
182
|
+
"Upgrade-Insecure-Requests": "1"
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
const page = await context.newPage();
|
|
186
|
+
await page.addInitScript(() => {
|
|
187
|
+
Object.defineProperty(navigator, "webdriver", {
|
|
188
|
+
get: () => false
|
|
189
|
+
});
|
|
190
|
+
window.chrome = {
|
|
191
|
+
runtime: {}
|
|
192
|
+
};
|
|
193
|
+
const originalQuery = window.navigator.permissions.query;
|
|
194
|
+
window.navigator.permissions.query = (parameters) => parameters.name === "notifications" ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters);
|
|
195
|
+
});
|
|
196
|
+
page.setDefaultTimeout(timeout * 1e3);
|
|
197
|
+
const response = await page.goto(url, {
|
|
198
|
+
waitUntil: "domcontentloaded",
|
|
199
|
+
// Changed from networkidle - faster for simple XML
|
|
200
|
+
timeout: timeout * 1e3
|
|
201
|
+
});
|
|
202
|
+
if (!response) {
|
|
203
|
+
throw new Error("No response received from page");
|
|
204
|
+
}
|
|
205
|
+
const statusCode = response.status();
|
|
206
|
+
const content = await page.content();
|
|
207
|
+
const finalUrl = page.url();
|
|
208
|
+
await browser.close();
|
|
209
|
+
if (statusCode >= 200 && statusCode < 300) {
|
|
210
|
+
return {
|
|
211
|
+
content,
|
|
212
|
+
statusCode,
|
|
213
|
+
url: finalUrl
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
throw new HttpError(finalUrl, statusCode);
|
|
217
|
+
} catch (error) {
|
|
218
|
+
if (browser) {
|
|
219
|
+
await browser.close();
|
|
220
|
+
}
|
|
221
|
+
if (error.code === "HTTP_ERROR") {
|
|
222
|
+
throw error;
|
|
223
|
+
}
|
|
224
|
+
throw new NetworkError(url, error);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
async function fetchUrl(url, options = {}) {
|
|
228
|
+
const {
|
|
229
|
+
timeout = 30,
|
|
230
|
+
maxRetries = 3,
|
|
231
|
+
retryDelay = 1e3,
|
|
232
|
+
useBrowser = false,
|
|
233
|
+
disableBrowserFallback = false
|
|
234
|
+
} = options;
|
|
235
|
+
new URL(url);
|
|
236
|
+
const retryableStatuses = [408, 429, 500, 502, 503, 504];
|
|
237
|
+
let lastError = null;
|
|
238
|
+
let attemptedBrowser = false;
|
|
239
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
240
|
+
try {
|
|
241
|
+
if (useBrowser || attemptedBrowser) {
|
|
242
|
+
return await fetchUrlWithBrowser(url, timeout);
|
|
243
|
+
}
|
|
244
|
+
const response = await axiosInstance.get(url, {
|
|
245
|
+
timeout: timeout * 1e3,
|
|
246
|
+
headers: {
|
|
247
|
+
"User-Agent": "sitemap-qa/1.0.0 (compatible; +https://github.com/Akotliar/sitemap-qa)",
|
|
248
|
+
"Accept": "text/xml,application/xml,text/plain,*/*",
|
|
249
|
+
"Accept-Encoding": "gzip, deflate",
|
|
250
|
+
"Connection": "keep-alive"
|
|
251
|
+
}
|
|
252
|
+
});
|
|
253
|
+
const statusCode = response.status;
|
|
254
|
+
const body = response.data;
|
|
255
|
+
if (statusCode >= 200 && statusCode < 300) {
|
|
256
|
+
return {
|
|
257
|
+
content: typeof body === "string" ? body : JSON.stringify(body),
|
|
258
|
+
statusCode,
|
|
259
|
+
url: response.request?.res?.responseUrl || url
|
|
260
|
+
// Final URL after redirects
|
|
261
|
+
};
|
|
262
|
+
}
|
|
263
|
+
if (statusCode === 403 && !attemptedBrowser && !disableBrowserFallback) {
|
|
264
|
+
attemptedBrowser = true;
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
if (!retryableStatuses.includes(statusCode)) {
|
|
268
|
+
throw new HttpError(response.request?.res?.responseUrl || url, statusCode);
|
|
269
|
+
}
|
|
270
|
+
lastError = new HttpError(response.request?.res?.responseUrl || url, statusCode);
|
|
271
|
+
} catch (error) {
|
|
272
|
+
if (error.code === "HTTP_ERROR") {
|
|
273
|
+
const httpError = error;
|
|
274
|
+
if (!retryableStatuses.includes(httpError.statusCode)) {
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
277
|
+
lastError = error;
|
|
278
|
+
} else {
|
|
279
|
+
lastError = new NetworkError(url, error);
|
|
280
|
+
}
|
|
281
|
+
if (attempt === maxRetries) break;
|
|
282
|
+
}
|
|
283
|
+
if (attempt < maxRetries) {
|
|
284
|
+
const delay = retryDelay * Math.pow(2, attempt);
|
|
285
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
throw lastError;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// src/core/discovery.ts
|
|
292
|
+
async function tryStandardPaths(baseUrl, config) {
|
|
293
|
+
const baseDomain = new URL(baseUrl).origin;
|
|
294
|
+
const accessIssues = [];
|
|
295
|
+
const standardPaths = [
|
|
296
|
+
"/sitemap.xml",
|
|
297
|
+
"/sitemap_index.xml",
|
|
298
|
+
"/sitemap-index.xml"
|
|
299
|
+
];
|
|
300
|
+
const results = await Promise.allSettled(
|
|
301
|
+
standardPaths.map(async (path) => {
|
|
302
|
+
const sitemapUrl = `${baseDomain}${path}`;
|
|
303
|
+
try {
|
|
304
|
+
const result = await fetchUrl(sitemapUrl, {
|
|
305
|
+
timeout: config.timeout,
|
|
306
|
+
maxRetries: 0
|
|
307
|
+
// Don't retry on standard paths - fail fast
|
|
308
|
+
});
|
|
309
|
+
if (result.statusCode === 200) {
|
|
310
|
+
if (config.verbose) {
|
|
311
|
+
console.log(`\u2713 Found sitemap at: ${sitemapUrl}`);
|
|
312
|
+
}
|
|
313
|
+
return { found: true, url: sitemapUrl };
|
|
314
|
+
}
|
|
315
|
+
return { found: false };
|
|
316
|
+
} catch (error) {
|
|
317
|
+
if (error instanceof HttpError) {
|
|
318
|
+
if (error.statusCode === 401 || error.statusCode === 403) {
|
|
319
|
+
accessIssues.push({
|
|
320
|
+
url: sitemapUrl,
|
|
321
|
+
statusCode: error.statusCode,
|
|
322
|
+
error: error.statusCode === 401 ? "Unauthorized" : "Access Denied"
|
|
323
|
+
});
|
|
324
|
+
if (config.verbose) {
|
|
325
|
+
console.log(`\u26A0 Access denied: ${sitemapUrl} (${error.statusCode})`);
|
|
326
|
+
}
|
|
327
|
+
} else if (config.verbose) {
|
|
328
|
+
console.log(`\u2717 Not found: ${sitemapUrl} (${error.statusCode})`);
|
|
329
|
+
}
|
|
330
|
+
} else if (config.verbose) {
|
|
331
|
+
console.log(`\u2717 Not found: ${sitemapUrl}`);
|
|
332
|
+
}
|
|
333
|
+
return { found: false };
|
|
334
|
+
}
|
|
335
|
+
})
|
|
336
|
+
);
|
|
337
|
+
for (const result of results) {
|
|
338
|
+
if (result.status === "fulfilled" && result.value.found) {
|
|
339
|
+
return { sitemaps: [result.value.url], issues: accessIssues };
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
if (config.verbose) {
|
|
343
|
+
console.log("No sitemap found at standard paths");
|
|
344
|
+
}
|
|
345
|
+
return { sitemaps: [], issues: accessIssues };
|
|
346
|
+
}
|
|
347
|
+
async function parseRobotsTxt(baseUrl, config) {
|
|
348
|
+
const robotsUrl = `${new URL(baseUrl).origin}/robots.txt`;
|
|
349
|
+
try {
|
|
350
|
+
const result = await fetchUrl(robotsUrl, {
|
|
351
|
+
timeout: config.timeout,
|
|
352
|
+
maxRetries: 1
|
|
353
|
+
});
|
|
354
|
+
const lines = result.content.split("\n");
|
|
355
|
+
const sitemaps = [];
|
|
356
|
+
for (const line of lines) {
|
|
357
|
+
const match = line.match(/^Sitemap:\s*(.+)$/i);
|
|
358
|
+
if (match) {
|
|
359
|
+
const sitemapUrl = match[1].trim();
|
|
360
|
+
try {
|
|
361
|
+
new URL(sitemapUrl);
|
|
362
|
+
sitemaps.push(sitemapUrl);
|
|
363
|
+
} catch {
|
|
364
|
+
if (config.verbose) {
|
|
365
|
+
console.warn(`Invalid sitemap URL in robots.txt: ${sitemapUrl}`);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
if (config.verbose && sitemaps.length > 0) {
|
|
371
|
+
console.log(`Found ${sitemaps.length} sitemap(s) in robots.txt`);
|
|
372
|
+
}
|
|
373
|
+
return sitemaps;
|
|
374
|
+
} catch (error) {
|
|
375
|
+
if (config.verbose) {
|
|
376
|
+
console.log(`No robots.txt found at ${robotsUrl}`);
|
|
377
|
+
}
|
|
378
|
+
return [];
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
function isSitemapIndex(xmlContent) {
|
|
382
|
+
if (xmlContent.includes("<sitemapindex")) {
|
|
383
|
+
return true;
|
|
384
|
+
}
|
|
385
|
+
if (xmlContent.includes("<urlset")) {
|
|
386
|
+
const urlBlockRegex = /<url[^>]*>.*?<loc>([^<]+)<\/loc>.*?<\/url>/gs;
|
|
387
|
+
const matches = Array.from(xmlContent.matchAll(urlBlockRegex));
|
|
388
|
+
const samplesToCheck = Math.min(5, matches.length);
|
|
389
|
+
let sitemapLikeCount = 0;
|
|
390
|
+
for (let i = 0; i < samplesToCheck; i++) {
|
|
391
|
+
const url = matches[i][1].trim().toLowerCase();
|
|
392
|
+
if (url.includes("sitemap") || url.endsWith(".xml")) {
|
|
393
|
+
sitemapLikeCount++;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
return sitemapLikeCount > samplesToCheck / 2;
|
|
397
|
+
}
|
|
398
|
+
return false;
|
|
399
|
+
}
|
|
400
|
+
function extractSitemapIndexUrls(xmlContent) {
|
|
401
|
+
const urls = [];
|
|
402
|
+
if (xmlContent.includes("<sitemapindex")) {
|
|
403
|
+
const sitemapBlockRegex = /<sitemap[^>]*>(.*?)<\/sitemap>/gs;
|
|
404
|
+
let sitemapMatch;
|
|
405
|
+
while ((sitemapMatch = sitemapBlockRegex.exec(xmlContent)) !== null) {
|
|
406
|
+
const locMatch = /<loc>([^<]+)<\/loc>/i.exec(sitemapMatch[1]);
|
|
407
|
+
if (locMatch) {
|
|
408
|
+
const url = locMatch[1].trim();
|
|
409
|
+
try {
|
|
410
|
+
new URL(url);
|
|
411
|
+
urls.push(url);
|
|
412
|
+
} catch {
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
} else {
|
|
417
|
+
const urlBlockRegex = /<url[^>]*>(.*?)<\/url>/gs;
|
|
418
|
+
let urlMatch;
|
|
419
|
+
while ((urlMatch = urlBlockRegex.exec(xmlContent)) !== null) {
|
|
420
|
+
const locMatch = /<loc>([^<]+)<\/loc>/i.exec(urlMatch[1]);
|
|
421
|
+
if (locMatch) {
|
|
422
|
+
const url = locMatch[1].trim();
|
|
423
|
+
if (url.toLowerCase().includes("sitemap") || url.toLowerCase().endsWith(".xml")) {
|
|
424
|
+
try {
|
|
425
|
+
new URL(url);
|
|
426
|
+
urls.push(url);
|
|
427
|
+
} catch {
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
return urls;
|
|
434
|
+
}
|
|
435
|
+
async function discoverAllSitemaps(initialSitemaps, config) {
|
|
436
|
+
const finalSitemaps = [];
|
|
437
|
+
const toProcess = [...initialSitemaps];
|
|
438
|
+
const processed = /* @__PURE__ */ new Set();
|
|
439
|
+
const inaccessible = /* @__PURE__ */ new Set();
|
|
440
|
+
const BATCH_SIZE = config.discoveryConcurrency || 50;
|
|
441
|
+
while (toProcess.length > 0) {
|
|
442
|
+
const batch = toProcess.splice(0, Math.min(BATCH_SIZE, toProcess.length));
|
|
443
|
+
const batchResults = await Promise.all(batch.map(async (sitemapUrl) => {
|
|
444
|
+
if (processed.has(sitemapUrl)) {
|
|
445
|
+
if (config.verbose) {
|
|
446
|
+
console.warn(`Skipping duplicate sitemap: ${sitemapUrl}`);
|
|
447
|
+
}
|
|
448
|
+
return { type: "skip" };
|
|
449
|
+
}
|
|
450
|
+
processed.add(sitemapUrl);
|
|
451
|
+
try {
|
|
452
|
+
const result = await fetchUrl(sitemapUrl, {
|
|
453
|
+
timeout: config.timeout,
|
|
454
|
+
maxRetries: 2
|
|
455
|
+
});
|
|
456
|
+
if (isSitemapIndex(result.content)) {
|
|
457
|
+
if (config.verbose) {
|
|
458
|
+
console.log(`Found sitemap index: ${sitemapUrl}`);
|
|
459
|
+
}
|
|
460
|
+
const childUrls = extractSitemapIndexUrls(result.content);
|
|
461
|
+
if (config.verbose) {
|
|
462
|
+
console.log(` \u2514\u2500 Contains ${childUrls.length} child sitemap(s)`);
|
|
463
|
+
}
|
|
464
|
+
return { type: "index", childUrls };
|
|
465
|
+
} else {
|
|
466
|
+
if (config.verbose) {
|
|
467
|
+
console.log(`\u2713 Discovered sitemap: ${sitemapUrl}`);
|
|
468
|
+
}
|
|
469
|
+
return { type: "sitemap", url: sitemapUrl };
|
|
470
|
+
}
|
|
471
|
+
} catch (error) {
|
|
472
|
+
inaccessible.add(sitemapUrl);
|
|
473
|
+
if (config.verbose) {
|
|
474
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
475
|
+
console.warn(`Failed to fetch sitemap ${sitemapUrl}: ${message}`);
|
|
476
|
+
}
|
|
477
|
+
return { type: "failed" };
|
|
478
|
+
}
|
|
479
|
+
}));
|
|
480
|
+
for (const result of batchResults) {
|
|
481
|
+
if (result.type === "index") {
|
|
482
|
+
toProcess.push(...result.childUrls);
|
|
483
|
+
} else if (result.type === "sitemap") {
|
|
484
|
+
finalSitemaps.push(result.url);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
if (processed.size > 1e3) {
|
|
488
|
+
console.warn(`\u26A0\uFE0F Processed over 1000 sitemap URLs. Stopping to prevent excessive requests.`);
|
|
489
|
+
break;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
if (finalSitemaps.length === 0 && inaccessible.size > 0) {
|
|
493
|
+
console.warn(`
|
|
494
|
+
\u26A0\uFE0F All ${inaccessible.size} sitemap(s) were inaccessible`);
|
|
495
|
+
console.warn(`Common causes: 403/404 errors, network issues, or bot protection`);
|
|
496
|
+
}
|
|
497
|
+
return finalSitemaps;
|
|
498
|
+
}
|
|
499
|
+
async function discoverSitemaps(baseUrl, config) {
|
|
500
|
+
const normalizedUrl = new URL(baseUrl).origin;
|
|
501
|
+
if (config.verbose) {
|
|
502
|
+
console.log("Checking robots.txt for sitemap directives...");
|
|
503
|
+
}
|
|
504
|
+
const robotsSitemaps = await parseRobotsTxt(normalizedUrl, config);
|
|
505
|
+
if (robotsSitemaps.length > 0) {
|
|
506
|
+
const sitemaps = await discoverAllSitemaps(robotsSitemaps, config);
|
|
507
|
+
return {
|
|
508
|
+
sitemaps,
|
|
509
|
+
source: "robots-txt",
|
|
510
|
+
accessIssues: []
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
if (config.verbose) {
|
|
514
|
+
console.log("Trying standard sitemap paths...");
|
|
515
|
+
}
|
|
516
|
+
const { sitemaps: standardSitemaps, issues } = await tryStandardPaths(normalizedUrl, config);
|
|
517
|
+
if (standardSitemaps.length > 0) {
|
|
518
|
+
const sitemaps = await discoverAllSitemaps(standardSitemaps, config);
|
|
519
|
+
if (sitemaps.length > 0) {
|
|
520
|
+
return {
|
|
521
|
+
sitemaps,
|
|
522
|
+
source: "standard-path",
|
|
523
|
+
accessIssues: []
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
return {
|
|
527
|
+
sitemaps: [],
|
|
528
|
+
source: "standard-path",
|
|
529
|
+
accessIssues: issues
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
return {
|
|
533
|
+
sitemaps: [],
|
|
534
|
+
source: "none",
|
|
535
|
+
accessIssues: issues
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// src/core/parser.ts
|
|
540
|
+
import { XMLParser, XMLValidator } from "fast-xml-parser";
|
|
541
|
+
var VALID_CHANGEFREQ = /* @__PURE__ */ new Set([
|
|
542
|
+
"always",
|
|
543
|
+
"hourly",
|
|
544
|
+
"daily",
|
|
545
|
+
"weekly",
|
|
546
|
+
"monthly",
|
|
547
|
+
"yearly",
|
|
548
|
+
"never"
|
|
549
|
+
]);
|
|
550
|
+
var parser = new XMLParser({
|
|
551
|
+
ignoreAttributes: false,
|
|
552
|
+
attributeNamePrefix: "@_",
|
|
553
|
+
textNodeName: "_text",
|
|
554
|
+
parseAttributeValue: true,
|
|
555
|
+
trimValues: true,
|
|
556
|
+
allowBooleanAttributes: true,
|
|
557
|
+
parseTagValue: false
|
|
558
|
+
// Keep values as strings for validation
|
|
559
|
+
});
|
|
560
|
+
function extractUrls(parsedXml, sitemapUrl) {
|
|
561
|
+
const urls = [];
|
|
562
|
+
if (parsedXml.urlset) {
|
|
563
|
+
const urlNodes = Array.isArray(parsedXml.urlset.url) ? parsedXml.urlset.url : [parsedXml.urlset.url];
|
|
564
|
+
for (let i = 0; i < urlNodes.length; i++) {
|
|
565
|
+
const node = urlNodes[i];
|
|
566
|
+
if (!node || !node.loc) {
|
|
567
|
+
continue;
|
|
568
|
+
}
|
|
569
|
+
urls.push({
|
|
570
|
+
loc: node.loc,
|
|
571
|
+
lastmod: node.lastmod,
|
|
572
|
+
changefreq: node.changefreq,
|
|
573
|
+
priority: node.priority ? parseFloat(node.priority) : void 0,
|
|
574
|
+
source: sitemapUrl
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
return urls;
|
|
579
|
+
}
|
|
580
|
+
async function parseSitemap(xml, sitemapUrl) {
|
|
581
|
+
const errors = [];
|
|
582
|
+
try {
|
|
583
|
+
const validationResult = XMLValidator.validate(xml);
|
|
584
|
+
if (validationResult !== true) {
|
|
585
|
+
const validationError = typeof validationResult === "object" ? validationResult.err.msg : "Invalid XML";
|
|
586
|
+
return {
|
|
587
|
+
urls: [],
|
|
588
|
+
errors: [
|
|
589
|
+
`[${sitemapUrl}] XML parsing failed: ${validationError}`
|
|
590
|
+
],
|
|
591
|
+
totalCount: 0,
|
|
592
|
+
sitemapUrl
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
const parsed = parser.parse(xml);
|
|
596
|
+
const urls = extractUrls(parsed, sitemapUrl);
|
|
597
|
+
const validUrls = [];
|
|
598
|
+
for (const entry of urls) {
|
|
599
|
+
try {
|
|
600
|
+
new URL(entry.loc);
|
|
601
|
+
if (entry.priority !== void 0) {
|
|
602
|
+
if (entry.priority < 0 || entry.priority > 1) {
|
|
603
|
+
errors.push(
|
|
604
|
+
`Invalid priority ${entry.priority} for ${entry.loc} - clamping to 0-1`
|
|
605
|
+
);
|
|
606
|
+
entry.priority = Math.max(0, Math.min(1, entry.priority));
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
if (entry.changefreq) {
|
|
610
|
+
if (!VALID_CHANGEFREQ.has(entry.changefreq.toLowerCase())) {
|
|
611
|
+
errors.push(
|
|
612
|
+
`Invalid changefreq "${entry.changefreq}" for ${entry.loc}`
|
|
613
|
+
);
|
|
614
|
+
entry.changefreq = void 0;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
validUrls.push(entry);
|
|
618
|
+
} catch (urlError) {
|
|
619
|
+
errors.push(`Invalid URL format: ${entry.loc}`);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
return {
|
|
623
|
+
urls: validUrls,
|
|
624
|
+
errors,
|
|
625
|
+
totalCount: validUrls.length,
|
|
626
|
+
sitemapUrl
|
|
627
|
+
};
|
|
628
|
+
} catch (parseError) {
|
|
629
|
+
const errorMsg = parseError instanceof Error ? parseError.message : String(parseError);
|
|
630
|
+
return {
|
|
631
|
+
urls: [],
|
|
632
|
+
errors: [
|
|
633
|
+
`[${sitemapUrl}] XML parsing failed: ${errorMsg}`
|
|
634
|
+
],
|
|
635
|
+
totalCount: 0,
|
|
636
|
+
sitemapUrl
|
|
637
|
+
};
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
// src/utils/batch-processor.ts
|
|
642
|
+
function chunkArray(array, chunkSize) {
|
|
643
|
+
const chunks = [];
|
|
644
|
+
for (let i = 0; i < array.length; i += chunkSize) {
|
|
645
|
+
chunks.push(array.slice(i, i + chunkSize));
|
|
646
|
+
}
|
|
647
|
+
return chunks;
|
|
648
|
+
}
|
|
649
|
+
async function processInBatches(items, concurrency, processor, onProgress) {
|
|
650
|
+
const results = new Array(items.length);
|
|
651
|
+
let completed = 0;
|
|
652
|
+
let currentIndex = 0;
|
|
653
|
+
const errors = [];
|
|
654
|
+
const workers = Array(Math.min(concurrency, items.length)).fill(null).map(async () => {
|
|
655
|
+
while (currentIndex < items.length) {
|
|
656
|
+
const index = currentIndex++;
|
|
657
|
+
const item = items[index];
|
|
658
|
+
try {
|
|
659
|
+
results[index] = await processor(item);
|
|
660
|
+
} catch (error) {
|
|
661
|
+
errors.push({ index, error });
|
|
662
|
+
results[index] = null;
|
|
663
|
+
}
|
|
664
|
+
completed++;
|
|
665
|
+
if (onProgress) {
|
|
666
|
+
onProgress(completed, items.length);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
});
|
|
670
|
+
await Promise.all(workers);
|
|
671
|
+
if (errors.length > 0) {
|
|
672
|
+
console.warn(`Processed ${items.length} items with ${errors.length} errors`);
|
|
673
|
+
}
|
|
674
|
+
return results;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
// src/core/extractor.ts
|
|
678
|
+
async function extractAllUrls(sitemapUrls, config, onProgress) {
|
|
679
|
+
const allUrls = [];
|
|
680
|
+
const allErrors = [];
|
|
681
|
+
let sitemapsProcessed = 0;
|
|
682
|
+
let sitemapsFailed = 0;
|
|
683
|
+
if (config.verbose) {
|
|
684
|
+
console.log(`
|
|
685
|
+
Extracting URLs from ${sitemapUrls.length} sitemap(s)...`);
|
|
686
|
+
}
|
|
687
|
+
const CONCURRENCY = config.parsingConcurrency || 50;
|
|
688
|
+
if (!config.silent && config.verbose) {
|
|
689
|
+
console.log(`Using parsing concurrency: ${CONCURRENCY}`);
|
|
690
|
+
}
|
|
691
|
+
const results = await processInBatches(
|
|
692
|
+
sitemapUrls,
|
|
693
|
+
CONCURRENCY,
|
|
694
|
+
async (sitemapUrl) => {
|
|
695
|
+
try {
|
|
696
|
+
if (config.verbose) {
|
|
697
|
+
console.log(`Extracting URLs from: ${sitemapUrl}`);
|
|
698
|
+
}
|
|
699
|
+
const response = await fetchUrl(sitemapUrl, {
|
|
700
|
+
timeout: 10,
|
|
701
|
+
// Fast timeout for sitemaps
|
|
702
|
+
maxRetries: 0,
|
|
703
|
+
// No retries - fail fast
|
|
704
|
+
disableBrowserFallback: true
|
|
705
|
+
// Don't use browser for bulk parsing
|
|
706
|
+
});
|
|
707
|
+
const parseResult = await parseSitemap(response.content, sitemapUrl);
|
|
708
|
+
const extractedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
709
|
+
parseResult.urls.forEach((url) => {
|
|
710
|
+
url.extractedAt = extractedAt;
|
|
711
|
+
});
|
|
712
|
+
if (config.verbose) {
|
|
713
|
+
console.log(` \u2713 Extracted ${parseResult.urls.length} URLs from ${sitemapUrl}`);
|
|
714
|
+
}
|
|
715
|
+
return {
|
|
716
|
+
success: true,
|
|
717
|
+
urls: parseResult.urls,
|
|
718
|
+
errors: parseResult.errors
|
|
719
|
+
};
|
|
720
|
+
} catch (error) {
|
|
721
|
+
const errorMsg = `Failed to process ${sitemapUrl}: ${error instanceof Error ? error.message : String(error)}`;
|
|
722
|
+
if (config.verbose) {
|
|
723
|
+
console.error(` \u2717 ${errorMsg}`);
|
|
724
|
+
}
|
|
725
|
+
return {
|
|
726
|
+
success: false,
|
|
727
|
+
urls: [],
|
|
728
|
+
errors: [errorMsg]
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
},
|
|
732
|
+
onProgress
|
|
733
|
+
// Pass progress callback to batch processor
|
|
734
|
+
);
|
|
735
|
+
for (const result of results) {
|
|
736
|
+
if (result.success) {
|
|
737
|
+
sitemapsProcessed++;
|
|
738
|
+
allUrls.push(...result.urls);
|
|
739
|
+
} else {
|
|
740
|
+
sitemapsFailed++;
|
|
741
|
+
}
|
|
742
|
+
allErrors.push(...result.errors);
|
|
743
|
+
}
|
|
744
|
+
if (config.verbose) {
|
|
745
|
+
console.log(`
|
|
746
|
+
Extraction complete:`);
|
|
747
|
+
console.log(` - Sitemaps processed: ${sitemapsProcessed}`);
|
|
748
|
+
console.log(` - Sitemaps failed: ${sitemapsFailed}`);
|
|
749
|
+
console.log(` - Total URLs: ${allUrls.length}`);
|
|
750
|
+
console.log(` - Errors: ${allErrors.length}`);
|
|
751
|
+
}
|
|
752
|
+
return {
|
|
753
|
+
allUrls,
|
|
754
|
+
sitemapsProcessed,
|
|
755
|
+
sitemapsFailed,
|
|
756
|
+
totalUrls: allUrls.length,
|
|
757
|
+
errors: allErrors
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
// src/core/consolidator.ts
|
|
762
|
+
function normalizeUrl(url) {
|
|
763
|
+
try {
|
|
764
|
+
const parsed = new URL(url);
|
|
765
|
+
let pathname = parsed.pathname;
|
|
766
|
+
if (pathname.endsWith("/") && pathname !== "/") {
|
|
767
|
+
pathname = pathname.slice(0, -1);
|
|
768
|
+
}
|
|
769
|
+
const params = Array.from(parsed.searchParams.entries()).sort(
|
|
770
|
+
([a], [b]) => a.localeCompare(b)
|
|
771
|
+
);
|
|
772
|
+
const sortedParams = new URLSearchParams(params);
|
|
773
|
+
return `${parsed.protocol}//${parsed.host}${pathname}${sortedParams.toString() ? "?" + sortedParams.toString() : ""}${parsed.hash}`;
|
|
774
|
+
} catch {
|
|
775
|
+
return url;
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
function mergeUrlEntries(entries) {
|
|
779
|
+
if (entries.length === 1) return entries[0];
|
|
780
|
+
const merged = { ...entries[0] };
|
|
781
|
+
const sources = entries.map((e) => e.source);
|
|
782
|
+
merged.source = sources.join(", ");
|
|
783
|
+
const lastmods = entries.map((e) => e.lastmod).filter((lm) => !!lm).map((lm) => new Date(lm).getTime()).sort((a, b) => b - a);
|
|
784
|
+
if (lastmods.length > 0) {
|
|
785
|
+
merged.lastmod = new Date(lastmods[0]).toISOString();
|
|
786
|
+
}
|
|
787
|
+
const priorities = entries.map((e) => e.priority).filter((p) => p !== void 0);
|
|
788
|
+
if (priorities.length > 0) {
|
|
789
|
+
merged.priority = Math.max(...priorities);
|
|
790
|
+
}
|
|
791
|
+
const changefreqs = entries.map((e) => e.changefreq).filter((cf) => !!cf);
|
|
792
|
+
if (changefreqs.length > 0) {
|
|
793
|
+
const counts = /* @__PURE__ */ new Map();
|
|
794
|
+
for (const cf of changefreqs) {
|
|
795
|
+
counts.set(cf, (counts.get(cf) || 0) + 1);
|
|
796
|
+
}
|
|
797
|
+
const sorted = Array.from(counts.entries()).sort((a, b) => b[1] - a[1]);
|
|
798
|
+
merged.changefreq = sorted[0][0];
|
|
799
|
+
}
|
|
800
|
+
const extractedAts = entries.map((e) => e.extractedAt).filter((ea) => !!ea).map((ea) => new Date(ea).getTime()).sort((a, b) => b - a);
|
|
801
|
+
if (extractedAts.length > 0) {
|
|
802
|
+
merged.extractedAt = new Date(extractedAts[0]).toISOString();
|
|
803
|
+
}
|
|
804
|
+
return merged;
|
|
805
|
+
}
|
|
806
|
+
function consolidateUrls(urls, verbose = false) {
|
|
807
|
+
const totalInputUrls = urls.length;
|
|
808
|
+
if (verbose) {
|
|
809
|
+
console.log(`
|
|
810
|
+
Consolidating ${urls.length} URL(s)...`);
|
|
811
|
+
}
|
|
812
|
+
const urlMap = /* @__PURE__ */ new Map();
|
|
813
|
+
for (const entry of urls) {
|
|
814
|
+
const normalized = normalizeUrl(entry.loc);
|
|
815
|
+
if (!urlMap.has(normalized)) {
|
|
816
|
+
urlMap.set(normalized, []);
|
|
817
|
+
}
|
|
818
|
+
urlMap.get(normalized).push(entry);
|
|
819
|
+
}
|
|
820
|
+
const uniqueUrls = [];
|
|
821
|
+
const duplicateGroups = [];
|
|
822
|
+
for (const [normalized, entries] of urlMap.entries()) {
|
|
823
|
+
const merged = mergeUrlEntries(entries);
|
|
824
|
+
uniqueUrls.push(merged);
|
|
825
|
+
if (entries.length > 1) {
|
|
826
|
+
duplicateGroups.push({
|
|
827
|
+
url: normalized,
|
|
828
|
+
count: entries.length,
|
|
829
|
+
sources: entries.map((e) => e.source)
|
|
830
|
+
});
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
if (verbose) {
|
|
834
|
+
console.log(`Consolidation complete:`);
|
|
835
|
+
console.log(` - Input URLs: ${totalInputUrls}`);
|
|
836
|
+
console.log(` - Unique URLs: ${uniqueUrls.length}`);
|
|
837
|
+
console.log(` - Duplicates removed: ${totalInputUrls - uniqueUrls.length}`);
|
|
838
|
+
if (duplicateGroups.length > 0) {
|
|
839
|
+
console.log(`
|
|
840
|
+
Top duplicates:`);
|
|
841
|
+
const top5 = duplicateGroups.sort((a, b) => b.count - a.count).slice(0, 5);
|
|
842
|
+
for (const group of top5) {
|
|
843
|
+
console.log(` - ${group.url} (${group.count} times)`);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
return {
|
|
848
|
+
uniqueUrls,
|
|
849
|
+
totalInputUrls,
|
|
850
|
+
duplicatesRemoved: totalInputUrls - uniqueUrls.length,
|
|
851
|
+
duplicateGroups
|
|
852
|
+
};
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
// src/core/patterns/risk-patterns.ts
|
|
856
|
+
var RISK_PATTERNS = [
|
|
857
|
+
// Sensitive Parameter Patterns (HIGH)
|
|
858
|
+
{
|
|
859
|
+
name: "Authentication Parameter",
|
|
860
|
+
category: "sensitive_params",
|
|
861
|
+
severity: "high",
|
|
862
|
+
regex: /[?&](token|auth|key|password|secret|apikey|session|credentials)=/i,
|
|
863
|
+
description: "Query parameter may contain sensitive authentication data"
|
|
864
|
+
},
|
|
865
|
+
{
|
|
866
|
+
name: "Debug Parameter",
|
|
867
|
+
category: "sensitive_params",
|
|
868
|
+
severity: "medium",
|
|
869
|
+
regex: /[?&](debug|trace|verbose|test_mode)=/i,
|
|
870
|
+
description: "Query parameter may contain debug or diagnostic flag"
|
|
871
|
+
},
|
|
872
|
+
// Protocol Inconsistency Patterns (MEDIUM)
|
|
873
|
+
{
|
|
874
|
+
name: "HTTP in HTTPS Site",
|
|
875
|
+
category: "protocol_inconsistency",
|
|
876
|
+
severity: "medium",
|
|
877
|
+
regex: /^http:\/\//,
|
|
878
|
+
description: "HTTP URL in HTTPS sitemap (potential mixed content)"
|
|
879
|
+
},
|
|
880
|
+
// Test/Unfinished Content Patterns (MEDIUM)
|
|
881
|
+
// Focuses on obvious test/placeholder patterns, avoiding false positives with legitimate content
|
|
882
|
+
{
|
|
883
|
+
name: "Test Content Path",
|
|
884
|
+
category: "test_content",
|
|
885
|
+
severity: "medium",
|
|
886
|
+
regex: /\/(?:test-|demo-|sample-|temp-|temporary-|placeholder-)|\/(test|demo|sample|temp|temporary|placeholder)(?:\/|$)/i,
|
|
887
|
+
description: "URL path suggests test, demo, or unfinished content that may not be intended for indexing"
|
|
888
|
+
}
|
|
889
|
+
];
|
|
890
|
+
|
|
891
|
+
// src/core/patterns/domain-patterns.ts
|
|
892
|
+
function escapeRegex(str) {
|
|
893
|
+
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
894
|
+
}
|
|
895
|
+
function extractRootDomain(hostname) {
|
|
896
|
+
const parts = hostname.split(".");
|
|
897
|
+
if (parts.length >= 2) {
|
|
898
|
+
return parts.slice(-2).join(".");
|
|
899
|
+
}
|
|
900
|
+
return hostname;
|
|
901
|
+
}
|
|
902
|
+
function createDomainMismatchPattern(baseUrl, options) {
|
|
903
|
+
const baseDomain = new URL(baseUrl).hostname;
|
|
904
|
+
const rootDomain = extractRootDomain(baseDomain);
|
|
905
|
+
if (options?.allowedSubdomains && options.allowedSubdomains.length > 0) {
|
|
906
|
+
const escapedRoot2 = escapeRegex(rootDomain);
|
|
907
|
+
const escapedSubdomains = options.allowedSubdomains.map(escapeRegex).join("|");
|
|
908
|
+
const pattern2 = `^https?://(?!(?:(?:${escapedSubdomains})\\.)?${escapedRoot2}(?:/|$))`;
|
|
909
|
+
return {
|
|
910
|
+
name: "Domain Mismatch",
|
|
911
|
+
category: "domain_mismatch",
|
|
912
|
+
severity: "high",
|
|
913
|
+
regex: new RegExp(pattern2),
|
|
914
|
+
description: `URL does not match expected domain or allowed subdomains`
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
const escapedRoot = escapeRegex(rootDomain);
|
|
918
|
+
const pattern = `^https?://(?!(?:www\\.)?${escapedRoot}(?:/|$))`;
|
|
919
|
+
return {
|
|
920
|
+
name: "Domain Mismatch",
|
|
921
|
+
category: "domain_mismatch",
|
|
922
|
+
severity: "high",
|
|
923
|
+
regex: new RegExp(pattern),
|
|
924
|
+
description: `URL does not match expected domain: ${rootDomain} (including www variant)`
|
|
925
|
+
};
|
|
926
|
+
}
|
|
927
|
+
var ENVIRONMENT_PATTERNS = [
|
|
928
|
+
{
|
|
929
|
+
name: "Staging Subdomain",
|
|
930
|
+
category: "environment_leakage",
|
|
931
|
+
severity: "high",
|
|
932
|
+
regex: /^https?:\/\/(staging|stg)\./i,
|
|
933
|
+
description: "URL uses staging subdomain"
|
|
934
|
+
},
|
|
935
|
+
{
|
|
936
|
+
name: "Development Subdomain",
|
|
937
|
+
category: "environment_leakage",
|
|
938
|
+
severity: "high",
|
|
939
|
+
regex: /^https?:\/\/(dev|development)\./i,
|
|
940
|
+
description: "URL uses development subdomain"
|
|
941
|
+
},
|
|
942
|
+
{
|
|
943
|
+
name: "QA/Test Subdomain",
|
|
944
|
+
category: "environment_leakage",
|
|
945
|
+
severity: "high",
|
|
946
|
+
regex: /^https?:\/\/(qa|test|uat|preprod)\./i,
|
|
947
|
+
description: "URL uses test environment subdomain"
|
|
948
|
+
},
|
|
949
|
+
{
|
|
950
|
+
name: "Localhost URL",
|
|
951
|
+
category: "environment_leakage",
|
|
952
|
+
severity: "high",
|
|
953
|
+
regex: /^https?:\/\/(localhost|127\.0\.0\.1|0\.0\.0\.0)/,
|
|
954
|
+
description: "URL points to localhost (development environment)"
|
|
955
|
+
},
|
|
956
|
+
{
|
|
957
|
+
name: "Environment in Path",
|
|
958
|
+
category: "environment_leakage",
|
|
959
|
+
severity: "high",
|
|
960
|
+
regex: /^https?:\/\/[^/]+\/(staging|dev|qa|uat|preprod)\//i,
|
|
961
|
+
description: "URL path contains environment identifier at root level"
|
|
962
|
+
}
|
|
963
|
+
];
|
|
964
|
+
|
|
965
|
+
// src/core/patterns/admin-patterns.ts
|
|
966
|
+
var ADMIN_PATH_PATTERNS = [
|
|
967
|
+
{
|
|
968
|
+
name: "Admin Path",
|
|
969
|
+
category: "admin_paths",
|
|
970
|
+
severity: "high",
|
|
971
|
+
regex: /\/(admin|administrator)(?:\/|$|\?)/i,
|
|
972
|
+
description: "URL contains /admin or /administrator as a path segment"
|
|
973
|
+
},
|
|
974
|
+
{
|
|
975
|
+
name: "Dashboard Path",
|
|
976
|
+
category: "admin_paths",
|
|
977
|
+
severity: "high",
|
|
978
|
+
regex: /\/dashboard(?:\/|$|\?)/i,
|
|
979
|
+
description: "URL contains /dashboard as a path segment"
|
|
980
|
+
},
|
|
981
|
+
{
|
|
982
|
+
name: "Config Path",
|
|
983
|
+
category: "admin_paths",
|
|
984
|
+
severity: "high",
|
|
985
|
+
regex: /\/(config|configuration)(?:\/|$|\?)/i,
|
|
986
|
+
description: "URL contains /config or /configuration as a path segment"
|
|
987
|
+
},
|
|
988
|
+
{
|
|
989
|
+
name: "Console Path",
|
|
990
|
+
category: "admin_paths",
|
|
991
|
+
severity: "high",
|
|
992
|
+
regex: /\/console(?:\/|$|\?)/i,
|
|
993
|
+
description: "URL contains /console as a path segment"
|
|
994
|
+
},
|
|
995
|
+
{
|
|
996
|
+
name: "Control Panel Path",
|
|
997
|
+
category: "admin_paths",
|
|
998
|
+
severity: "high",
|
|
999
|
+
regex: /\/(cpanel|control-panel)(?:\/|$|\?)/i,
|
|
1000
|
+
description: "URL contains control panel as a path segment"
|
|
1001
|
+
}
|
|
1002
|
+
];
|
|
1003
|
+
var INTERNAL_CONTENT_PATTERNS = [
|
|
1004
|
+
{
|
|
1005
|
+
name: "Internal Content Path",
|
|
1006
|
+
category: "internal_content",
|
|
1007
|
+
severity: "medium",
|
|
1008
|
+
regex: /\/internal\b/i,
|
|
1009
|
+
description: "URL contains /internal path segment - may be internal-only content not intended for public indexing"
|
|
1010
|
+
}
|
|
1011
|
+
];
|
|
1012
|
+
var SENSITIVE_PARAM_PATTERNS = [
|
|
1013
|
+
{
|
|
1014
|
+
name: "Authentication Token Parameter",
|
|
1015
|
+
category: "sensitive_params",
|
|
1016
|
+
severity: "high",
|
|
1017
|
+
regex: /[?&](token|auth_token|access_token|api_token)=/i,
|
|
1018
|
+
description: "Query parameter may contain authentication token"
|
|
1019
|
+
},
|
|
1020
|
+
{
|
|
1021
|
+
name: "API Key Parameter",
|
|
1022
|
+
category: "sensitive_params",
|
|
1023
|
+
severity: "high",
|
|
1024
|
+
regex: /[?&](apikey|api_key|key)=/i,
|
|
1025
|
+
description: "Query parameter may contain API key"
|
|
1026
|
+
},
|
|
1027
|
+
{
|
|
1028
|
+
name: "Password Parameter",
|
|
1029
|
+
category: "sensitive_params",
|
|
1030
|
+
severity: "high",
|
|
1031
|
+
regex: /[?&](password|passwd|pwd)=/i,
|
|
1032
|
+
description: "Query parameter may contain password"
|
|
1033
|
+
},
|
|
1034
|
+
{
|
|
1035
|
+
name: "Secret Parameter",
|
|
1036
|
+
category: "sensitive_params",
|
|
1037
|
+
severity: "high",
|
|
1038
|
+
regex: /[?&](secret|client_secret)=/i,
|
|
1039
|
+
description: "Query parameter may contain secret value"
|
|
1040
|
+
},
|
|
1041
|
+
{
|
|
1042
|
+
name: "Session Parameter",
|
|
1043
|
+
category: "sensitive_params",
|
|
1044
|
+
severity: "high",
|
|
1045
|
+
regex: /[?&](session|sessionid|sid)=/i,
|
|
1046
|
+
description: "Query parameter may contain session identifier"
|
|
1047
|
+
},
|
|
1048
|
+
{
|
|
1049
|
+
name: "Credentials Parameter",
|
|
1050
|
+
category: "sensitive_params",
|
|
1051
|
+
severity: "high",
|
|
1052
|
+
regex: /[?&]credentials=/i,
|
|
1053
|
+
description: "Query parameter may contain credentials"
|
|
1054
|
+
},
|
|
1055
|
+
{
|
|
1056
|
+
name: "Debug Parameter",
|
|
1057
|
+
category: "sensitive_params",
|
|
1058
|
+
severity: "medium",
|
|
1059
|
+
regex: /[?&](debug|trace|verbose)=/i,
|
|
1060
|
+
description: "Query parameter contains debug or diagnostic flag"
|
|
1061
|
+
},
|
|
1062
|
+
{
|
|
1063
|
+
name: "Test Mode Parameter",
|
|
1064
|
+
category: "sensitive_params",
|
|
1065
|
+
severity: "medium",
|
|
1066
|
+
regex: /[?&](test_mode|test|testing)=/i,
|
|
1067
|
+
description: "Query parameter indicates test mode"
|
|
1068
|
+
}
|
|
1069
|
+
];
|
|
1070
|
+
|
|
1071
|
+
// src/utils/sanitizer.ts
|
|
1072
|
+
function sanitizeUrl(url) {
|
|
1073
|
+
try {
|
|
1074
|
+
const parsed = new URL(url);
|
|
1075
|
+
const sensitiveParams = [
|
|
1076
|
+
"token",
|
|
1077
|
+
"auth",
|
|
1078
|
+
"auth_token",
|
|
1079
|
+
"access_token",
|
|
1080
|
+
"api_token",
|
|
1081
|
+
"apikey",
|
|
1082
|
+
"api_key",
|
|
1083
|
+
"key",
|
|
1084
|
+
"password",
|
|
1085
|
+
"passwd",
|
|
1086
|
+
"pwd",
|
|
1087
|
+
"secret",
|
|
1088
|
+
"client_secret",
|
|
1089
|
+
"session",
|
|
1090
|
+
"sessionid",
|
|
1091
|
+
"sid",
|
|
1092
|
+
"credentials"
|
|
1093
|
+
];
|
|
1094
|
+
for (const param of sensitiveParams) {
|
|
1095
|
+
if (parsed.searchParams.has(param)) {
|
|
1096
|
+
parsed.searchParams.set(param, "[REDACTED]");
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
return parsed.toString();
|
|
1100
|
+
} catch {
|
|
1101
|
+
return url;
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
// src/core/risk-grouper.ts
|
|
1106
|
+
function generateRecommendation(category, _severity, count) {
|
|
1107
|
+
switch (category) {
|
|
1108
|
+
case "environment_leakage":
|
|
1109
|
+
return {
|
|
1110
|
+
rationale: `Production sitemap contains ${count} URL(s) from non-production environments (staging, dev, QA, test). This indicates configuration errors or environment leakage.`,
|
|
1111
|
+
recommendedAction: "Verify sitemap generation excludes non-production environments. Review deployment configuration and environment filtering rules."
|
|
1112
|
+
};
|
|
1113
|
+
case "admin_paths":
|
|
1114
|
+
return {
|
|
1115
|
+
rationale: `${count} administrative path(s) detected in public sitemap (admin, dashboard, config). These paths may expose privileged access points.`,
|
|
1116
|
+
recommendedAction: "Confirm if admin paths should be publicly indexed. Consider excluding via robots.txt or removing from sitemap. Verify access controls."
|
|
1117
|
+
};
|
|
1118
|
+
case "internal_content":
|
|
1119
|
+
return {
|
|
1120
|
+
rationale: `${count} URL(s) contain "internal" in the path. These may be internal-facing content not intended for public indexing.`,
|
|
1121
|
+
recommendedAction: "Review URLs to determine if they should be publicly accessible. Consider excluding internal content from sitemap or adding noindex meta tags."
|
|
1122
|
+
};
|
|
1123
|
+
case "test_content":
|
|
1124
|
+
return {
|
|
1125
|
+
rationale: `${count} URL(s) contain test/demo/sample identifiers. These may be placeholder or unfinished content not intended for indexing.`,
|
|
1126
|
+
recommendedAction: "Review and remove test content from production sitemaps. Verify content is production-ready before including in sitemap."
|
|
1127
|
+
};
|
|
1128
|
+
case "sensitive_params":
|
|
1129
|
+
return {
|
|
1130
|
+
rationale: `${count} URL(s) contain sensitive query parameters (token, auth, key, password, session). This may expose authentication credentials or debugging flags.`,
|
|
1131
|
+
recommendedAction: "Review why sensitive parameters are in sitemap URLs. Remove authentication tokens from URLs. Consider POST requests for sensitive data."
|
|
1132
|
+
};
|
|
1133
|
+
case "protocol_inconsistency":
|
|
1134
|
+
return {
|
|
1135
|
+
rationale: `${count} URL(s) use HTTP protocol in HTTPS sitemap. This creates mixed content warnings and potential security issues.`,
|
|
1136
|
+
recommendedAction: "Update URLs to use HTTPS consistently. Verify SSL certificate coverage. Check for hardcoded HTTP URLs in content."
|
|
1137
|
+
};
|
|
1138
|
+
case "domain_mismatch":
|
|
1139
|
+
return {
|
|
1140
|
+
rationale: `${count} URL(s) do not match expected base domain. This may indicate external links, CDN URLs, or configuration errors.`,
|
|
1141
|
+
recommendedAction: "Verify if external domains are intentional. Review sitemap generation logic. Confirm CDN or subdomain configuration is correct."
|
|
1142
|
+
};
|
|
1143
|
+
default:
|
|
1144
|
+
return {
|
|
1145
|
+
rationale: `${count} URL(s) flagged in category: ${category}`,
|
|
1146
|
+
recommendedAction: "Review flagged URLs and determine appropriate action."
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
function groupRiskFindings(findings, maxSampleUrls = 5) {
|
|
1151
|
+
const categoryMap = /* @__PURE__ */ new Map();
|
|
1152
|
+
for (const finding of findings) {
|
|
1153
|
+
if (!categoryMap.has(finding.category)) {
|
|
1154
|
+
categoryMap.set(finding.category, []);
|
|
1155
|
+
}
|
|
1156
|
+
categoryMap.get(finding.category).push(finding);
|
|
1157
|
+
}
|
|
1158
|
+
const groups = [];
|
|
1159
|
+
for (const [category, categoryFindings] of categoryMap.entries()) {
|
|
1160
|
+
const uniqueUrls = Array.from(new Set(categoryFindings.map((f) => f.url)));
|
|
1161
|
+
const severity = categoryFindings.reduce((highest, finding) => {
|
|
1162
|
+
const severityOrder = ["low", "medium", "high"];
|
|
1163
|
+
return severityOrder.indexOf(finding.severity) > severityOrder.indexOf(highest) ? finding.severity : highest;
|
|
1164
|
+
}, "low");
|
|
1165
|
+
const sampleUrls = uniqueUrls.slice(0, maxSampleUrls);
|
|
1166
|
+
const { rationale, recommendedAction } = generateRecommendation(category, severity, uniqueUrls.length);
|
|
1167
|
+
groups.push({
|
|
1168
|
+
category,
|
|
1169
|
+
severity,
|
|
1170
|
+
count: uniqueUrls.length,
|
|
1171
|
+
rationale,
|
|
1172
|
+
sampleUrls,
|
|
1173
|
+
recommendedAction,
|
|
1174
|
+
allUrls: uniqueUrls
|
|
1175
|
+
});
|
|
1176
|
+
}
|
|
1177
|
+
groups.sort((a, b) => {
|
|
1178
|
+
const severityOrder = ["high", "medium", "low"];
|
|
1179
|
+
return severityOrder.indexOf(a.severity) - severityOrder.indexOf(b.severity);
|
|
1180
|
+
});
|
|
1181
|
+
const totalRiskUrls = new Set(findings.map((f) => f.url)).size;
|
|
1182
|
+
const highSeverityCount = groups.filter((g) => g.severity === "high").reduce((sum, g) => sum + g.count, 0);
|
|
1183
|
+
const mediumSeverityCount = groups.filter((g) => g.severity === "medium").reduce((sum, g) => sum + g.count, 0);
|
|
1184
|
+
const lowSeverityCount = groups.filter((g) => g.severity === "low").reduce((sum, g) => sum + g.count, 0);
|
|
1185
|
+
return {
|
|
1186
|
+
groups,
|
|
1187
|
+
totalRiskUrls,
|
|
1188
|
+
highSeverityCount,
|
|
1189
|
+
mediumSeverityCount,
|
|
1190
|
+
lowSeverityCount
|
|
1191
|
+
};
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
// src/core/risk-detector.ts
|
|
1195
|
+
import os from "os";
|
|
1196
|
+
function compileAcceptedPatterns(config) {
|
|
1197
|
+
const patterns = [];
|
|
1198
|
+
if (config.acceptedPatterns && config.acceptedPatterns.length > 0) {
|
|
1199
|
+
for (const pattern of config.acceptedPatterns) {
|
|
1200
|
+
try {
|
|
1201
|
+
let regexPattern = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, "[^/]*");
|
|
1202
|
+
if (!regexPattern.endsWith("$") && !regexPattern.includes("(?:")) {
|
|
1203
|
+
regexPattern = regexPattern + "(?:/|$|\\?|#)";
|
|
1204
|
+
}
|
|
1205
|
+
patterns.push(new RegExp(regexPattern, "i"));
|
|
1206
|
+
} catch (error) {
|
|
1207
|
+
if (config.verbose) {
|
|
1208
|
+
console.warn(`Invalid accepted pattern: ${pattern}`);
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
return patterns;
|
|
1214
|
+
}
|
|
1215
|
+
async function detectRisksInBatch(urls, allPatterns, acceptedPatterns, expectedProtocol, verbose) {
|
|
1216
|
+
const findings = [];
|
|
1217
|
+
for (const urlEntry of urls) {
|
|
1218
|
+
const url = urlEntry.loc;
|
|
1219
|
+
let isAccepted = false;
|
|
1220
|
+
for (const acceptedPattern of acceptedPatterns) {
|
|
1221
|
+
if (acceptedPattern.test(url)) {
|
|
1222
|
+
isAccepted = true;
|
|
1223
|
+
break;
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
if (isAccepted) continue;
|
|
1227
|
+
for (const pattern of allPatterns) {
|
|
1228
|
+
if (pattern.category === "protocol_inconsistency") {
|
|
1229
|
+
try {
|
|
1230
|
+
const urlProtocol = new URL(url).protocol;
|
|
1231
|
+
if (expectedProtocol === "https:" && urlProtocol === "http:") {
|
|
1232
|
+
findings.push({
|
|
1233
|
+
url,
|
|
1234
|
+
category: pattern.category,
|
|
1235
|
+
severity: pattern.severity,
|
|
1236
|
+
pattern: pattern.name,
|
|
1237
|
+
rationale: pattern.description,
|
|
1238
|
+
matchedValue: "http://"
|
|
1239
|
+
});
|
|
1240
|
+
}
|
|
1241
|
+
} catch (error) {
|
|
1242
|
+
continue;
|
|
1243
|
+
}
|
|
1244
|
+
} else {
|
|
1245
|
+
try {
|
|
1246
|
+
const match = url.match(pattern.regex);
|
|
1247
|
+
if (match) {
|
|
1248
|
+
findings.push({
|
|
1249
|
+
url: pattern.category === "sensitive_params" ? sanitizeUrl(url) : url,
|
|
1250
|
+
category: pattern.category,
|
|
1251
|
+
severity: pattern.severity,
|
|
1252
|
+
pattern: pattern.name,
|
|
1253
|
+
rationale: pattern.description,
|
|
1254
|
+
matchedValue: match[0]
|
|
1255
|
+
});
|
|
1256
|
+
}
|
|
1257
|
+
} catch (error) {
|
|
1258
|
+
if (verbose) {
|
|
1259
|
+
console.error(`Pattern matching failed for ${pattern.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1260
|
+
}
|
|
1261
|
+
continue;
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
return { findings, urlsProcessed: urls.length };
|
|
1267
|
+
}
|
|
1268
|
+
async function detectRisks(urls, baseUrl, config) {
|
|
1269
|
+
const startTime = Date.now();
|
|
1270
|
+
const domainPattern = createDomainMismatchPattern(baseUrl);
|
|
1271
|
+
const allPatterns = [
|
|
1272
|
+
...RISK_PATTERNS,
|
|
1273
|
+
...ENVIRONMENT_PATTERNS,
|
|
1274
|
+
...ADMIN_PATH_PATTERNS,
|
|
1275
|
+
...SENSITIVE_PARAM_PATTERNS,
|
|
1276
|
+
...INTERNAL_CONTENT_PATTERNS,
|
|
1277
|
+
domainPattern
|
|
1278
|
+
];
|
|
1279
|
+
const acceptedPatterns = compileAcceptedPatterns(config);
|
|
1280
|
+
let expectedProtocol;
|
|
1281
|
+
try {
|
|
1282
|
+
expectedProtocol = new URL(baseUrl).protocol;
|
|
1283
|
+
} catch (error) {
|
|
1284
|
+
if (config.verbose) {
|
|
1285
|
+
console.warn(`Invalid base URL: ${baseUrl}, defaulting to https:`);
|
|
1286
|
+
}
|
|
1287
|
+
expectedProtocol = "https:";
|
|
1288
|
+
}
|
|
1289
|
+
const BATCH_SIZE = config.riskDetectionBatchSize || 1e4;
|
|
1290
|
+
const CONCURRENCY = config.riskDetectionConcurrency || Math.max(2, os.cpus().length - 1);
|
|
1291
|
+
const batches = chunkArray(urls, BATCH_SIZE);
|
|
1292
|
+
if (config.verbose) {
|
|
1293
|
+
console.log(`
|
|
1294
|
+
Risk Detection Configuration:`);
|
|
1295
|
+
console.log(` - Total URLs: ${urls.length.toLocaleString()}`);
|
|
1296
|
+
console.log(` - Batch size: ${BATCH_SIZE.toLocaleString()}`);
|
|
1297
|
+
console.log(` - Concurrency: ${CONCURRENCY}`);
|
|
1298
|
+
console.log(` - Total batches: ${batches.length}`);
|
|
1299
|
+
try {
|
|
1300
|
+
console.log(` - Base domain: ${new URL(baseUrl).hostname}`);
|
|
1301
|
+
} catch (error) {
|
|
1302
|
+
console.log(` - Base URL: ${baseUrl}`);
|
|
1303
|
+
}
|
|
1304
|
+
if (acceptedPatterns.length > 0) {
|
|
1305
|
+
console.log(` - Accepted patterns: ${acceptedPatterns.length}`);
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
let completedBatches = 0;
|
|
1309
|
+
const totalBatches = batches.length;
|
|
1310
|
+
const batchStartTime = Date.now();
|
|
1311
|
+
const batchResults = await processInBatches(
|
|
1312
|
+
batches,
|
|
1313
|
+
CONCURRENCY,
|
|
1314
|
+
(batch) => detectRisksInBatch(batch, allPatterns, acceptedPatterns, expectedProtocol, config.verbose),
|
|
1315
|
+
(completed) => {
|
|
1316
|
+
completedBatches = completed;
|
|
1317
|
+
const pct = (completed / totalBatches * 100).toFixed(1);
|
|
1318
|
+
const elapsed = (Date.now() - batchStartTime) / 1e3;
|
|
1319
|
+
const urlsProcessed = completed * BATCH_SIZE;
|
|
1320
|
+
const speed = Math.round(urlsProcessed / elapsed);
|
|
1321
|
+
const remaining = totalBatches - completed;
|
|
1322
|
+
const eta = Math.round(remaining * BATCH_SIZE / speed);
|
|
1323
|
+
process.stdout.write(
|
|
1324
|
+
`\r\x1B[K Analyzing batch ${completed}/${totalBatches} (${pct}%) | ETA: ~${eta}s | ${speed.toLocaleString()} URLs/sec`
|
|
1325
|
+
);
|
|
1326
|
+
}
|
|
1327
|
+
);
|
|
1328
|
+
process.stdout.write("\r\x1B[K");
|
|
1329
|
+
const allFindings = batchResults.flatMap((r) => r.findings);
|
|
1330
|
+
const groupingResult = groupRiskFindings(allFindings);
|
|
1331
|
+
const processingTimeMs = Date.now() - startTime;
|
|
1332
|
+
if (config.verbose) {
|
|
1333
|
+
console.log(`
|
|
1334
|
+
Risk Detection Summary:`);
|
|
1335
|
+
console.log(` - Total URLs analyzed: ${urls.length.toLocaleString()}`);
|
|
1336
|
+
console.log(` - Risk URLs found: ${groupingResult.totalRiskUrls.toLocaleString()}`);
|
|
1337
|
+
console.log(` - HIGH severity: ${groupingResult.highSeverityCount}`);
|
|
1338
|
+
console.log(` - MEDIUM severity: ${groupingResult.mediumSeverityCount}`);
|
|
1339
|
+
console.log(` - LOW severity: ${groupingResult.lowSeverityCount}`);
|
|
1340
|
+
console.log(` - Processing time: ${(processingTimeMs / 1e3).toFixed(1)}s`);
|
|
1341
|
+
if (groupingResult.groups.length > 0) {
|
|
1342
|
+
console.log(`
|
|
1343
|
+
Risk Categories Found:`);
|
|
1344
|
+
for (const group of groupingResult.groups) {
|
|
1345
|
+
console.log(` - ${group.category}: ${group.count} URLs (${group.severity.toUpperCase()})`);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
return {
|
|
1350
|
+
findings: allFindings,
|
|
1351
|
+
groups: groupingResult.groups,
|
|
1352
|
+
totalUrlsAnalyzed: urls.length,
|
|
1353
|
+
riskUrlCount: groupingResult.totalRiskUrls,
|
|
1354
|
+
cleanUrlCount: urls.length - groupingResult.totalRiskUrls,
|
|
1355
|
+
highSeverityCount: groupingResult.highSeverityCount,
|
|
1356
|
+
mediumSeverityCount: groupingResult.mediumSeverityCount,
|
|
1357
|
+
lowSeverityCount: groupingResult.lowSeverityCount,
|
|
1358
|
+
processingTimeMs
|
|
1359
|
+
};
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
// src/summarizer.ts
|
|
1363
|
+
function summarizeRisks(request) {
|
|
1364
|
+
const severityBreakdown = {
|
|
1365
|
+
high: 0,
|
|
1366
|
+
medium: 0,
|
|
1367
|
+
low: 0
|
|
1368
|
+
};
|
|
1369
|
+
const categoryInsights = request.riskGroups.map((group) => {
|
|
1370
|
+
severityBreakdown[group.severity] += group.count;
|
|
1371
|
+
const urls = group.allUrls || group.sampleUrls;
|
|
1372
|
+
return {
|
|
1373
|
+
category: group.category,
|
|
1374
|
+
count: group.count,
|
|
1375
|
+
severity: group.severity,
|
|
1376
|
+
summary: group.rationale,
|
|
1377
|
+
examples: urls.slice(0, 3),
|
|
1378
|
+
allUrls: urls
|
|
1379
|
+
// Include all URLs for download functionality
|
|
1380
|
+
};
|
|
1381
|
+
});
|
|
1382
|
+
const totalRisks = request.riskGroups.reduce((sum, g) => sum + g.count, 0);
|
|
1383
|
+
const overview = totalRisks > 0 ? `Found ${totalRisks} potentially risky URLs across ${request.riskGroups.length} categories in ${request.totalUrls} total URLs.` : `Analyzed ${request.totalUrls} URLs. No suspicious patterns detected.`;
|
|
1384
|
+
const keyFindings = [];
|
|
1385
|
+
if (severityBreakdown.high > 0) {
|
|
1386
|
+
keyFindings.push(`${severityBreakdown.high} high-severity issues require immediate attention`);
|
|
1387
|
+
}
|
|
1388
|
+
if (severityBreakdown.medium > 0) {
|
|
1389
|
+
keyFindings.push(`${severityBreakdown.medium} medium-severity issues should be reviewed`);
|
|
1390
|
+
}
|
|
1391
|
+
if (severityBreakdown.low > 0) {
|
|
1392
|
+
keyFindings.push(`${severityBreakdown.low} low-severity items flagged for awareness`);
|
|
1393
|
+
}
|
|
1394
|
+
return {
|
|
1395
|
+
overview,
|
|
1396
|
+
keyFindings,
|
|
1397
|
+
categoryInsights,
|
|
1398
|
+
severityBreakdown,
|
|
1399
|
+
recommendations: [],
|
|
1400
|
+
generatedBy: "rule-based analysis",
|
|
1401
|
+
metadata: {
|
|
1402
|
+
tokensUsed: 0,
|
|
1403
|
+
processingTime: request.processingTime || 0,
|
|
1404
|
+
model: "pattern-matching"
|
|
1405
|
+
}
|
|
1406
|
+
};
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
// src/reporters/json-reporter.ts
|
|
1410
|
+
var TOOL_VERSION = true ? "1.0.0-alpha.2" : "dev";
|
|
1411
|
+
function generateJsonReport(summary, discoveryResult, parseResult, riskGroups, config, startTime, options = {}) {
|
|
1412
|
+
const {
|
|
1413
|
+
pretty = true,
|
|
1414
|
+
indent = 2,
|
|
1415
|
+
performanceMetrics
|
|
1416
|
+
} = options;
|
|
1417
|
+
const result = buildAnalysisResult(
|
|
1418
|
+
summary,
|
|
1419
|
+
discoveryResult,
|
|
1420
|
+
parseResult,
|
|
1421
|
+
riskGroups,
|
|
1422
|
+
config,
|
|
1423
|
+
startTime
|
|
1424
|
+
);
|
|
1425
|
+
const jsonOutput = transformToJsonOutput(result, performanceMetrics);
|
|
1426
|
+
if (pretty) {
|
|
1427
|
+
return JSON.stringify(jsonOutput, null, indent);
|
|
1428
|
+
} else {
|
|
1429
|
+
return JSON.stringify(jsonOutput);
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
function buildAnalysisResult(summary, discoveryResult, parseResult, riskGroups, config, startTime) {
|
|
1433
|
+
const metadata = buildAnalysisMetadata(
|
|
1434
|
+
config.baseUrl || "unknown",
|
|
1435
|
+
startTime,
|
|
1436
|
+
summary
|
|
1437
|
+
);
|
|
1438
|
+
const suspiciousGroups = riskGroups.map((group) => ({
|
|
1439
|
+
category: group.category,
|
|
1440
|
+
severity: group.severity,
|
|
1441
|
+
count: group.count,
|
|
1442
|
+
pattern: group.category,
|
|
1443
|
+
// Use category as pattern identifier
|
|
1444
|
+
rationale: group.rationale,
|
|
1445
|
+
sampleUrls: group.sampleUrls.slice(0, 5),
|
|
1446
|
+
// Limit to 5 samples
|
|
1447
|
+
recommendedAction: group.recommendedAction
|
|
1448
|
+
}));
|
|
1449
|
+
const summaryStats = {
|
|
1450
|
+
highSeverityCount: summary.severityBreakdown.high,
|
|
1451
|
+
mediumSeverityCount: summary.severityBreakdown.medium,
|
|
1452
|
+
lowSeverityCount: summary.severityBreakdown.low,
|
|
1453
|
+
totalRiskyUrls: riskGroups.reduce((sum, g) => sum + g.count, 0),
|
|
1454
|
+
overallStatus: determineOverallStatus(
|
|
1455
|
+
summary.severityBreakdown,
|
|
1456
|
+
parseResult.errors
|
|
1457
|
+
)
|
|
1458
|
+
};
|
|
1459
|
+
const riskSummary = {
|
|
1460
|
+
overview: summary.overview,
|
|
1461
|
+
keyFindings: summary.keyFindings,
|
|
1462
|
+
recommendations: summary.recommendations
|
|
1463
|
+
};
|
|
1464
|
+
const errors = parseResult.errors.map(transformError);
|
|
1465
|
+
return {
|
|
1466
|
+
analysisMetadata: metadata,
|
|
1467
|
+
sitemapsDiscovered: discoveryResult.sitemaps,
|
|
1468
|
+
totalUrlCount: parseResult.totalCount,
|
|
1469
|
+
urlsAnalyzed: parseResult.totalCount,
|
|
1470
|
+
suspiciousGroups,
|
|
1471
|
+
riskSummary,
|
|
1472
|
+
summary: summaryStats,
|
|
1473
|
+
errors
|
|
1474
|
+
};
|
|
1475
|
+
}
|
|
1476
|
+
function buildAnalysisMetadata(baseUrl, startTime, summary) {
|
|
1477
|
+
return {
|
|
1478
|
+
baseUrl,
|
|
1479
|
+
analysisTimestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1480
|
+
toolVersion: TOOL_VERSION,
|
|
1481
|
+
executionTimeMs: Date.now() - startTime,
|
|
1482
|
+
analysisType: summary.generatedBy
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1485
|
+
function determineOverallStatus(severityBreakdown, errors) {
|
|
1486
|
+
if (errors.length > 0) {
|
|
1487
|
+
return "errors";
|
|
1488
|
+
}
|
|
1489
|
+
const totalIssues = severityBreakdown.high + severityBreakdown.medium + severityBreakdown.low;
|
|
1490
|
+
return totalIssues > 0 ? "issues_found" : "clean";
|
|
1491
|
+
}
|
|
1492
|
+
function transformToJsonOutput(result, performanceMetrics) {
|
|
1493
|
+
const output = {
|
|
1494
|
+
analysis_metadata: transformMetadata(result.analysisMetadata),
|
|
1495
|
+
sitemaps_discovered: result.sitemapsDiscovered,
|
|
1496
|
+
total_url_count: result.totalUrlCount,
|
|
1497
|
+
urls_analyzed: result.urlsAnalyzed,
|
|
1498
|
+
suspicious_groups: result.suspiciousGroups.map(transformGroup),
|
|
1499
|
+
risk_summary: transformRiskSummary(result.riskSummary),
|
|
1500
|
+
summary: transformSummary(result.summary),
|
|
1501
|
+
errors: result.errors
|
|
1502
|
+
};
|
|
1503
|
+
if (performanceMetrics) {
|
|
1504
|
+
output.performance_metrics = {
|
|
1505
|
+
total_execution_time_ms: performanceMetrics.totalExecutionTimeMs,
|
|
1506
|
+
phase_timings: performanceMetrics.phaseTimings,
|
|
1507
|
+
throughput: performanceMetrics.throughput,
|
|
1508
|
+
resource_usage: performanceMetrics.resourceUsage
|
|
1509
|
+
};
|
|
1510
|
+
}
|
|
1511
|
+
return output;
|
|
1512
|
+
}
|
|
1513
|
+
function transformMetadata(meta) {
|
|
1514
|
+
return {
|
|
1515
|
+
base_url: meta.baseUrl,
|
|
1516
|
+
analysis_timestamp: meta.analysisTimestamp,
|
|
1517
|
+
tool_version: meta.toolVersion,
|
|
1518
|
+
execution_time_ms: meta.executionTimeMs,
|
|
1519
|
+
analysis_type: meta.analysisType
|
|
1520
|
+
};
|
|
1521
|
+
}
|
|
1522
|
+
function transformGroup(group) {
|
|
1523
|
+
return {
|
|
1524
|
+
category: group.category,
|
|
1525
|
+
severity: group.severity,
|
|
1526
|
+
count: group.count,
|
|
1527
|
+
pattern: group.pattern,
|
|
1528
|
+
rationale: group.rationale,
|
|
1529
|
+
sample_urls: group.sampleUrls,
|
|
1530
|
+
recommended_action: group.recommendedAction
|
|
1531
|
+
};
|
|
1532
|
+
}
|
|
1533
|
+
function transformRiskSummary(summary) {
|
|
1534
|
+
return {
|
|
1535
|
+
overview: summary.overview,
|
|
1536
|
+
key_findings: summary.keyFindings,
|
|
1537
|
+
recommendations: summary.recommendations
|
|
1538
|
+
};
|
|
1539
|
+
}
|
|
1540
|
+
function transformSummary(summary) {
|
|
1541
|
+
return {
|
|
1542
|
+
high_severity_count: summary.highSeverityCount,
|
|
1543
|
+
medium_severity_count: summary.mediumSeverityCount,
|
|
1544
|
+
low_severity_count: summary.lowSeverityCount,
|
|
1545
|
+
total_risky_urls: summary.totalRiskyUrls,
|
|
1546
|
+
overall_status: summary.overallStatus
|
|
1547
|
+
};
|
|
1548
|
+
}
|
|
1549
|
+
function transformError(error) {
|
|
1550
|
+
if ("code" in error) {
|
|
1551
|
+
const customError = error;
|
|
1552
|
+
const errorDetail = {
|
|
1553
|
+
code: customError.code || "UNKNOWN_ERROR",
|
|
1554
|
+
message: error.message
|
|
1555
|
+
};
|
|
1556
|
+
if ("attemptedPaths" in customError) {
|
|
1557
|
+
errorDetail.context = {
|
|
1558
|
+
attempted_paths: customError.attemptedPaths
|
|
1559
|
+
};
|
|
1560
|
+
} else if ("sitemapUrl" in customError && "lineNumber" in customError) {
|
|
1561
|
+
errorDetail.context = {
|
|
1562
|
+
sitemap_url: customError.sitemapUrl,
|
|
1563
|
+
line_number: customError.lineNumber
|
|
1564
|
+
};
|
|
1565
|
+
} else if ("url" in customError) {
|
|
1566
|
+
errorDetail.context = {
|
|
1567
|
+
url: customError.url
|
|
1568
|
+
};
|
|
1569
|
+
}
|
|
1570
|
+
return errorDetail;
|
|
1571
|
+
}
|
|
1572
|
+
return {
|
|
1573
|
+
code: "UNKNOWN_ERROR",
|
|
1574
|
+
message: error.message
|
|
1575
|
+
};
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
// src/reporters/html-reporter.ts
|
|
1579
|
+
import { promises as fs } from "fs";
|
|
1580
|
+
var TOOL_VERSION2 = "1.0.0-alpha.2";
|
|
1581
|
+
function generateHtmlReport(summary, discoveryResult, totalUrls, config, errors, options = {}) {
|
|
1582
|
+
const maxUrls = options.maxUrlsPerGroup ?? 10;
|
|
1583
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
1584
|
+
const riskyUrlCount = summary.categoryInsights.reduce((sum, g) => sum + g.count, 0);
|
|
1585
|
+
const highSeverity = summary.categoryInsights.filter((g) => g.severity === "high");
|
|
1586
|
+
const mediumSeverity = summary.categoryInsights.filter((g) => g.severity === "medium");
|
|
1587
|
+
const lowSeverity = summary.categoryInsights.filter((g) => g.severity === "low");
|
|
1588
|
+
const html = `<!DOCTYPE html>
|
|
20
1589
|
<html lang="en">
|
|
21
1590
|
<head>
|
|
22
1591
|
<meta charset="UTF-8">
|
|
23
1592
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
24
|
-
<title>Sitemap QA Report - ${
|
|
1593
|
+
<title>Sitemap QA Report - ${config.baseUrl}</title>
|
|
25
1594
|
<style>
|
|
26
1595
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
27
1596
|
body {
|
|
@@ -339,93 +1908,88 @@ Risk Categories Found:`);for(let p of l.groups)console.log(` - ${p.category}: $
|
|
|
339
1908
|
<div class="header">
|
|
340
1909
|
<h1>Sitemap Analysis</h1>
|
|
341
1910
|
<div class="meta">
|
|
342
|
-
<div>${
|
|
343
|
-
<div>${new Date(
|
|
1911
|
+
<div>${config.baseUrl}</div>
|
|
1912
|
+
<div>${new Date(timestamp).toLocaleString()}</div>
|
|
344
1913
|
</div>
|
|
345
1914
|
</div>
|
|
346
1915
|
|
|
347
1916
|
<div class="summary">
|
|
348
1917
|
<div class="summary-card">
|
|
349
1918
|
<div class="label">Sitemaps</div>
|
|
350
|
-
<div class="value">${
|
|
1919
|
+
<div class="value">${discoveryResult.sitemaps.length}</div>
|
|
351
1920
|
</div>
|
|
352
1921
|
<div class="summary-card">
|
|
353
1922
|
<div class="label">URLs Analyzed</div>
|
|
354
|
-
<div class="value">${
|
|
1923
|
+
<div class="value">${totalUrls.toLocaleString()}</div>
|
|
355
1924
|
</div>
|
|
356
1925
|
<div class="summary-card">
|
|
357
1926
|
<div class="label">Issues Found</div>
|
|
358
|
-
<div class="value" style="color: ${
|
|
1927
|
+
<div class="value" style="color: ${riskyUrlCount > 0 ? "#dc2626" : "#059669"}">${riskyUrlCount}</div>
|
|
359
1928
|
</div>
|
|
360
1929
|
<div class="summary-card">
|
|
361
1930
|
<div class="label">Scan Time</div>
|
|
362
|
-
<div class="value">${(
|
|
1931
|
+
<div class="value">${(summary.metadata.processingTime / 1e3).toFixed(1)}s</div>
|
|
363
1932
|
</div>
|
|
364
1933
|
</div>
|
|
365
1934
|
|
|
366
1935
|
<div class="content">
|
|
367
|
-
${
|
|
1936
|
+
${errors.length > 0 ? `
|
|
368
1937
|
<div class="errors-section">
|
|
369
|
-
<h3>Parsing Errors & Warnings (${
|
|
1938
|
+
<h3>Parsing Errors & Warnings (${errors.length})</h3>
|
|
370
1939
|
<ul>
|
|
371
|
-
${
|
|
372
|
-
`)}
|
|
1940
|
+
${errors.map((err) => `<li>${err.message}</li>`).join("\n ")}
|
|
373
1941
|
</ul>
|
|
374
1942
|
</div>
|
|
375
|
-
|
|
1943
|
+
` : ""}
|
|
376
1944
|
|
|
377
|
-
${
|
|
1945
|
+
${discoveryResult.sitemaps.length > 0 ? `
|
|
378
1946
|
<div class="sitemaps">
|
|
379
|
-
<h3 class="collapsed" onclick="toggleSection(this)">Sitemaps Discovered (${
|
|
1947
|
+
<h3 class="collapsed" onclick="toggleSection(this)">Sitemaps Discovered (${discoveryResult.sitemaps.length})</h3>
|
|
380
1948
|
<div class="sitemaps-content collapsed">
|
|
381
1949
|
<ul>
|
|
382
|
-
${
|
|
383
|
-
`)}
|
|
1950
|
+
${discoveryResult.sitemaps.map((s) => `<li>\u2022 ${s}</li>`).join("\n ")}
|
|
384
1951
|
</ul>
|
|
385
1952
|
</div>
|
|
386
1953
|
</div>
|
|
387
|
-
|
|
1954
|
+
` : ""}
|
|
388
1955
|
|
|
389
|
-
${
|
|
1956
|
+
${riskyUrlCount === 0 ? `
|
|
390
1957
|
<div class="status-clean">
|
|
391
1958
|
<h2>No Issues Found</h2>
|
|
392
1959
|
<p>All URLs in the sitemap passed validation checks.</p>
|
|
393
1960
|
</div>
|
|
394
|
-
|
|
1961
|
+
` : ""}
|
|
395
1962
|
|
|
396
|
-
${
|
|
1963
|
+
${highSeverity.length > 0 ? `
|
|
397
1964
|
<div class="severity-section">
|
|
398
|
-
<h2 class="severity-high" onclick="toggleSection(this)">High Severity (${
|
|
1965
|
+
<h2 class="severity-high" onclick="toggleSection(this)">High Severity (${highSeverity.reduce((sum, g) => sum + g.count, 0)} URLs)</h2>
|
|
399
1966
|
<div class="severity-content">
|
|
400
|
-
${
|
|
401
|
-
`)}
|
|
1967
|
+
${highSeverity.map((group) => renderRiskGroup(group, maxUrls)).join("\n ")}
|
|
402
1968
|
</div>
|
|
403
1969
|
</div>
|
|
404
|
-
|
|
1970
|
+
` : ""}
|
|
405
1971
|
|
|
406
|
-
${
|
|
1972
|
+
${mediumSeverity.length > 0 ? `
|
|
407
1973
|
<div class="severity-section">
|
|
408
|
-
<h2 class="severity-medium" onclick="toggleSection(this)">Medium Severity (${
|
|
1974
|
+
<h2 class="severity-medium" onclick="toggleSection(this)">Medium Severity (${mediumSeverity.reduce((sum, g) => sum + g.count, 0)} URLs)</h2>
|
|
409
1975
|
<div class="severity-content">
|
|
410
|
-
${
|
|
411
|
-
`)}
|
|
1976
|
+
${mediumSeverity.map((group) => renderRiskGroup(group, maxUrls)).join("\n ")}
|
|
412
1977
|
</div>
|
|
413
1978
|
</div>
|
|
414
|
-
|
|
1979
|
+
` : ""}
|
|
415
1980
|
|
|
416
|
-
${
|
|
1981
|
+
${lowSeverity.length > 0 ? `
|
|
417
1982
|
<div class="severity-section">
|
|
418
|
-
<h2 class="severity-low" onclick="toggleSection(this)">Low Severity (${
|
|
1983
|
+
<h2 class="severity-low" onclick="toggleSection(this)">Low Severity (${lowSeverity.reduce((sum, g) => sum + g.count, 0)} URLs)</h2>
|
|
419
1984
|
<div class="severity-content">
|
|
420
|
-
${
|
|
421
|
-
`)}
|
|
1985
|
+
${lowSeverity.map((group) => renderRiskGroup(group, maxUrls)).join("\n ")}
|
|
422
1986
|
</div>
|
|
423
1987
|
</div>
|
|
424
|
-
|
|
1988
|
+
` : ""}
|
|
425
1989
|
</div>
|
|
426
1990
|
|
|
427
1991
|
<div class="footer">
|
|
428
|
-
Generated by <strong>sitemap-qa</strong> v${
|
|
1992
|
+
Generated by <strong>sitemap-qa</strong> v${TOOL_VERSION2}
|
|
429
1993
|
</div>
|
|
430
1994
|
</div>
|
|
431
1995
|
|
|
@@ -458,27 +2022,354 @@ Risk Categories Found:`);for(let p of l.groups)console.log(` - ${p.category}: $
|
|
|
458
2022
|
}
|
|
459
2023
|
</script>
|
|
460
2024
|
</body>
|
|
461
|
-
</html
|
|
462
|
-
|
|
463
|
-
|
|
2025
|
+
</html>`;
|
|
2026
|
+
return html;
|
|
2027
|
+
}
|
|
2028
|
+
function renderRiskGroup(group, maxUrls) {
|
|
2029
|
+
const categoryTitle = group.category.split("_").map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
|
|
2030
|
+
const urlsToShow = group.examples.slice(0, maxUrls);
|
|
2031
|
+
const remaining = group.count - urlsToShow.length;
|
|
2032
|
+
const categorySlug = group.category.toLowerCase();
|
|
2033
|
+
const allUrlsJson = JSON.stringify(group.allUrls);
|
|
2034
|
+
const encodedUrls = escapeHtml(allUrlsJson);
|
|
2035
|
+
return `<div class="risk-group">
|
|
2036
|
+
<h3>${categoryTitle} <span class="count">${group.count} URLs</span></h3>
|
|
2037
|
+
<div class="impact">${group.summary}</div>
|
|
464
2038
|
<div class="urls">
|
|
465
2039
|
<h4>Sample URLs</h4>
|
|
466
2040
|
<ul>
|
|
467
|
-
${
|
|
468
|
-
`)}
|
|
2041
|
+
${urlsToShow.map((url) => `<li>${escapeHtml(url)}</li>`).join("\n ")}
|
|
469
2042
|
</ul>
|
|
470
|
-
${
|
|
471
|
-
<button class="download-btn" onclick="downloadUrls('${
|
|
2043
|
+
${remaining > 0 ? `<div class="more">... and ${remaining} more</div>` : ""}
|
|
2044
|
+
<button class="download-btn" onclick="downloadUrls('${categorySlug}', '${encodedUrls}')">\u{1F4E5} Download All ${group.count} URLs</button>
|
|
472
2045
|
</div>
|
|
473
|
-
</div
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
2046
|
+
</div>`;
|
|
2047
|
+
}
|
|
2048
|
+
function escapeHtml(text) {
|
|
2049
|
+
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
2050
|
+
}
|
|
2051
|
+
async function writeHtmlReport(summary, discoveryResult, totalUrls, config, outputPath, errors, options = {}) {
|
|
2052
|
+
const htmlContent = generateHtmlReport(summary, discoveryResult, totalUrls, config, errors, options);
|
|
2053
|
+
await fs.writeFile(outputPath, htmlContent, "utf-8");
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
// src/commands/analyze.ts
|
|
2057
|
+
var analyzeCommand = new Command("analyze").description("Analyze sitemap for QA issues").argument("<url>", "Base URL to analyze").option("--timeout <seconds>", "HTTP timeout in seconds", "30").option("--no-progress", "Disable progress bar").option("--output <format>", "Output format: html or json", "html").option("--output-dir <path>", "Output directory for reports").option("--output-file <path>", "Custom output filename").option("--accepted-patterns <patterns>", "Comma-separated regex patterns to exclude from risk detection").option("--concurrency <number>", "Number of concurrent workers for risk detection").option("--batch-size <number>", "URLs per batch for risk detection", "10000").option("--parsing-concurrency <number>", "Number of concurrent sitemap parsers", "50").option("--discovery-concurrency <number>", "Number of concurrent sitemap index fetches", "50").option("--silent", "Disable all progress output").option("--benchmark", "Save performance profile").option("--no-color", "Disable ANSI color codes in CLI output").option("--verbose", "Enable verbose logging", false).action(async (url, options) => {
|
|
2058
|
+
let config;
|
|
2059
|
+
try {
|
|
2060
|
+
validateAnalyzeOptions(options);
|
|
2061
|
+
const loadedConfig = await loadConfig({
|
|
2062
|
+
...options,
|
|
2063
|
+
baseUrl: url,
|
|
2064
|
+
outputFormat: options.output,
|
|
2065
|
+
riskDetectionConcurrency: options.concurrency ? parseInt(options.concurrency) : void 0,
|
|
2066
|
+
riskDetectionBatchSize: options.batchSize ? parseInt(options.batchSize) : void 0,
|
|
2067
|
+
parsingConcurrency: options.parsingConcurrency ? parseInt(options.parsingConcurrency) : void 0,
|
|
2068
|
+
discoveryConcurrency: options.discoveryConcurrency ? parseInt(options.discoveryConcurrency) : void 0,
|
|
2069
|
+
silent: options.silent,
|
|
2070
|
+
benchmark: options.benchmark,
|
|
2071
|
+
progressBar: options.progress
|
|
2072
|
+
});
|
|
2073
|
+
config = loadedConfig;
|
|
2074
|
+
console.log(`
|
|
2075
|
+
\u{1F50D} Analyzing ${url}...
|
|
2076
|
+
`);
|
|
2077
|
+
const result = await runAnalysisPipeline(url, config);
|
|
2078
|
+
await fs2.mkdir(config.outputDir, { recursive: true });
|
|
2079
|
+
if (options.output === "json") {
|
|
2080
|
+
const jsonReport = generateJsonReport(
|
|
2081
|
+
result.summary,
|
|
2082
|
+
result.discoveryResult,
|
|
2083
|
+
{ totalCount: result.totalUrls, uniqueUrls: [], errors: [] },
|
|
2084
|
+
result.riskGroups,
|
|
2085
|
+
config,
|
|
2086
|
+
result.executionTime,
|
|
2087
|
+
{ pretty: true, indent: 2 }
|
|
2088
|
+
);
|
|
2089
|
+
console.log("\n" + jsonReport);
|
|
2090
|
+
if (options.outputFile) {
|
|
2091
|
+
const jsonFilePath = `${config.outputDir}/${options.outputFile}`;
|
|
2092
|
+
await fs2.writeFile(jsonFilePath, jsonReport, "utf-8");
|
|
2093
|
+
console.log(`
|
|
2094
|
+
\u{1F4C4} JSON report saved to: ${chalk.cyan(jsonFilePath)}`);
|
|
2095
|
+
}
|
|
2096
|
+
} else {
|
|
2097
|
+
showCliSummary(result);
|
|
2098
|
+
const htmlFileName = options.outputFile || `sitemap-qa-report-${Date.now()}.html`;
|
|
2099
|
+
const htmlFilePath = `${config.outputDir}/${htmlFileName}`;
|
|
2100
|
+
await writeHtmlReport(
|
|
2101
|
+
result.summary,
|
|
2102
|
+
result.discoveryResult,
|
|
2103
|
+
result.totalUrls,
|
|
2104
|
+
config,
|
|
2105
|
+
htmlFilePath,
|
|
2106
|
+
result.errors,
|
|
2107
|
+
{ maxUrlsPerGroup: 10 }
|
|
2108
|
+
);
|
|
2109
|
+
console.log(`
|
|
2110
|
+
\u{1F4C4} Full report saved to: ${chalk.cyan(htmlFilePath)}`);
|
|
2111
|
+
}
|
|
2112
|
+
const exitCode = determineExitCode(result);
|
|
2113
|
+
process.exit(exitCode);
|
|
2114
|
+
} catch (error) {
|
|
2115
|
+
handleAnalysisError(error, config);
|
|
2116
|
+
process.exit(2);
|
|
2117
|
+
}
|
|
2118
|
+
});
|
|
2119
|
+
function validateAnalyzeOptions(options) {
|
|
2120
|
+
const validFormats = ["json", "html"];
|
|
2121
|
+
if (!validFormats.includes(options.output)) {
|
|
2122
|
+
throw new Error(
|
|
2123
|
+
`Invalid output format: ${options.output}. Must be one of: ${validFormats.join(", ")}`
|
|
2124
|
+
);
|
|
2125
|
+
}
|
|
2126
|
+
const timeout = parseInt(options.timeout);
|
|
2127
|
+
if (isNaN(timeout) || timeout <= 0) {
|
|
2128
|
+
throw new Error(`Invalid timeout: ${options.timeout}. Must be a positive number.`);
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
function showCliSummary(result) {
|
|
2132
|
+
const riskyUrlCount = result.summary.categoryInsights.reduce((sum, g) => sum + g.count, 0);
|
|
2133
|
+
console.log(chalk.dim("\u2500".repeat(50)));
|
|
2134
|
+
if (riskyUrlCount === 0) {
|
|
2135
|
+
console.log(chalk.green("No issues found - sitemap looks clean!"));
|
|
2136
|
+
} else {
|
|
2137
|
+
const { high, medium, low } = result.summary.severityBreakdown;
|
|
2138
|
+
const severityParts = [];
|
|
2139
|
+
if (high > 0) severityParts.push(chalk.red(`High: ${high}`));
|
|
2140
|
+
if (medium > 0) severityParts.push(chalk.yellow(`Medium: ${medium}`));
|
|
2141
|
+
if (low > 0) severityParts.push(chalk.blue(`Low: ${low}`));
|
|
2142
|
+
const severitySummary = severityParts.length > 0 ? ` (${severityParts.join(", ")})` : "";
|
|
2143
|
+
console.log(chalk.yellow(`\u26A0\uFE0F ${riskyUrlCount} risky URLs found${severitySummary}`));
|
|
2144
|
+
}
|
|
2145
|
+
console.log("");
|
|
2146
|
+
}
|
|
2147
|
+
async function runAnalysisPipeline(url, config) {
|
|
2148
|
+
const overallStartTime = Date.now();
|
|
2149
|
+
const phaseTimings = [];
|
|
2150
|
+
const errors = [];
|
|
2151
|
+
const showProgress = !config.silent && config.progressBar !== false && process.stdout.isTTY;
|
|
2152
|
+
let phaseStart = Date.now();
|
|
2153
|
+
const discoverySpinner = showProgress ? ora({ text: "Discovering sitemaps...", color: "cyan" }).start() : null;
|
|
2154
|
+
const discoveryResult = await discoverSitemaps(url, config);
|
|
2155
|
+
if (discoverySpinner) {
|
|
2156
|
+
discoverySpinner.stop();
|
|
2157
|
+
}
|
|
2158
|
+
phaseTimings.push({
|
|
2159
|
+
name: "Discovery",
|
|
2160
|
+
startTime: phaseStart,
|
|
2161
|
+
endTime: Date.now(),
|
|
2162
|
+
duration: Date.now() - phaseStart
|
|
2163
|
+
});
|
|
2164
|
+
if (discoveryResult.accessIssues.length > 0) {
|
|
2165
|
+
if (!config.silent) {
|
|
2166
|
+
console.warn(chalk.yellow(`\u26A0\uFE0F Warning: ${discoveryResult.accessIssues.length} sitemap(s) are access-blocked`));
|
|
2167
|
+
}
|
|
2168
|
+
for (const issue of discoveryResult.accessIssues) {
|
|
2169
|
+
errors.push(new Error(`Access blocked: ${issue.url} (${issue.statusCode})`));
|
|
2170
|
+
}
|
|
2171
|
+
}
|
|
2172
|
+
if (discoveryResult.sitemaps.length === 0) {
|
|
2173
|
+
throw new Error(`No sitemaps found at ${url}. Tried: /sitemap.xml, /sitemap_index.xml, /robots.txt`);
|
|
2174
|
+
}
|
|
2175
|
+
phaseStart = Date.now();
|
|
2176
|
+
let extractionResult;
|
|
2177
|
+
if (showProgress && discoveryResult.sitemaps.length > 10) {
|
|
2178
|
+
const parseBar = new cliProgress.SingleBar({
|
|
2179
|
+
format: "{bar} {percentage}% | {value}/{total} | ETA: {eta}s | {speed} sitemaps/sec",
|
|
2180
|
+
barCompleteChar: "\u2588",
|
|
2181
|
+
barIncompleteChar: "\u2591",
|
|
2182
|
+
hideCursor: true
|
|
2183
|
+
});
|
|
2184
|
+
parseBar.start(discoveryResult.sitemaps.length, 0, { speed: "0" });
|
|
2185
|
+
extractionResult = await extractAllUrls(
|
|
2186
|
+
discoveryResult.sitemaps,
|
|
2187
|
+
config,
|
|
2188
|
+
(completed, total) => {
|
|
2189
|
+
const elapsed = (Date.now() - phaseStart) / 1e3;
|
|
2190
|
+
const speed = elapsed > 0 ? (completed / elapsed).toFixed(1) : "0";
|
|
2191
|
+
parseBar.update(completed, { speed });
|
|
2192
|
+
}
|
|
2193
|
+
);
|
|
2194
|
+
parseBar.stop();
|
|
2195
|
+
} else {
|
|
2196
|
+
extractionResult = await extractAllUrls(discoveryResult.sitemaps, config);
|
|
2197
|
+
}
|
|
2198
|
+
phaseTimings.push({
|
|
2199
|
+
name: "Parsing",
|
|
2200
|
+
startTime: phaseStart,
|
|
2201
|
+
endTime: Date.now(),
|
|
2202
|
+
duration: Date.now() - phaseStart
|
|
2203
|
+
});
|
|
2204
|
+
if (extractionResult.errors.length > 0) {
|
|
2205
|
+
for (const err of extractionResult.errors) {
|
|
2206
|
+
if (typeof err === "string") {
|
|
2207
|
+
errors.push(new Error(err));
|
|
2208
|
+
} else {
|
|
2209
|
+
errors.push(err);
|
|
2210
|
+
}
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
if (extractionResult.allUrls.length === 0) {
|
|
2214
|
+
throw new Error("No URLs extracted from sitemaps");
|
|
2215
|
+
}
|
|
2216
|
+
phaseStart = Date.now();
|
|
2217
|
+
const consolidatedResult = consolidateUrls(extractionResult.allUrls);
|
|
2218
|
+
phaseTimings.push({
|
|
2219
|
+
name: "Deduplication",
|
|
2220
|
+
startTime: phaseStart,
|
|
2221
|
+
endTime: Date.now(),
|
|
2222
|
+
duration: Date.now() - phaseStart
|
|
2223
|
+
});
|
|
2224
|
+
const duplicatesRemoved = extractionResult.allUrls.length - consolidatedResult.uniqueUrls.length;
|
|
2225
|
+
const duplicatePercentage = duplicatesRemoved / extractionResult.allUrls.length * 100;
|
|
2226
|
+
if (!config.silent) {
|
|
2227
|
+
if (duplicatesRemoved > 100 || duplicatePercentage > 1) {
|
|
2228
|
+
console.log(chalk.green(`Found ${discoveryResult.sitemaps.length} sitemap(s) \u2192 ${extractionResult.allUrls.length.toLocaleString()} URLs (${consolidatedResult.uniqueUrls.length.toLocaleString()} unique)`));
|
|
2229
|
+
} else {
|
|
2230
|
+
console.log(chalk.green(`Found ${discoveryResult.sitemaps.length} sitemap(s) \u2192 ${extractionResult.allUrls.length.toLocaleString()} URLs`));
|
|
2231
|
+
}
|
|
2232
|
+
}
|
|
2233
|
+
phaseStart = Date.now();
|
|
2234
|
+
const riskResult = await detectRisks(consolidatedResult.uniqueUrls, url, config);
|
|
2235
|
+
const riskGroups = groupRiskFindings(riskResult.findings);
|
|
2236
|
+
phaseTimings.push({
|
|
2237
|
+
name: "Risk Detection",
|
|
2238
|
+
startTime: phaseStart,
|
|
2239
|
+
endTime: Date.now(),
|
|
2240
|
+
duration: Date.now() - phaseStart
|
|
2241
|
+
});
|
|
2242
|
+
phaseStart = Date.now();
|
|
2243
|
+
const executionTime = Date.now() - overallStartTime;
|
|
2244
|
+
const summary = summarizeRisks({
|
|
2245
|
+
riskGroups: riskGroups.groups,
|
|
2246
|
+
totalUrls: consolidatedResult.uniqueUrls.length,
|
|
2247
|
+
sitemapUrl: url,
|
|
2248
|
+
processingTime: executionTime
|
|
2249
|
+
});
|
|
2250
|
+
phaseTimings.push({
|
|
2251
|
+
name: "Summarization",
|
|
2252
|
+
startTime: phaseStart,
|
|
2253
|
+
endTime: Date.now(),
|
|
2254
|
+
duration: Date.now() - phaseStart
|
|
2255
|
+
});
|
|
2256
|
+
if (!config.silent && config.verbose) {
|
|
2257
|
+
displayPhaseSummary(phaseTimings, executionTime);
|
|
2258
|
+
} else if (!config.silent) {
|
|
2259
|
+
const parsingPhase = phaseTimings.find((p) => p.name === "Parsing");
|
|
2260
|
+
const sitemapsPerSec = parsingPhase ? (discoveryResult.sitemaps.length / (parsingPhase.duration / 1e3)).toFixed(1) : "0";
|
|
2261
|
+
console.log(chalk.green(`Analysis complete (${(executionTime / 1e3).toFixed(1)}s \xB7 ${sitemapsPerSec} sitemaps/sec)
|
|
2262
|
+
`));
|
|
2263
|
+
}
|
|
2264
|
+
if (config.benchmark) {
|
|
2265
|
+
await saveBenchmark(phaseTimings, url, executionTime, discoveryResult.sitemaps.length, consolidatedResult.uniqueUrls.length, config);
|
|
2266
|
+
}
|
|
2267
|
+
return {
|
|
2268
|
+
discoveryResult,
|
|
2269
|
+
totalUrls: consolidatedResult.uniqueUrls.length,
|
|
2270
|
+
riskGroups: riskGroups.groups,
|
|
2271
|
+
summary,
|
|
2272
|
+
errors,
|
|
2273
|
+
executionTime,
|
|
2274
|
+
phaseTimings
|
|
2275
|
+
};
|
|
2276
|
+
}
|
|
2277
|
+
function determineExitCode(result) {
|
|
2278
|
+
const highSeverityCount = result.summary.severityBreakdown.high;
|
|
2279
|
+
if (highSeverityCount > 0) {
|
|
2280
|
+
return 1;
|
|
2281
|
+
}
|
|
2282
|
+
return 0;
|
|
2283
|
+
}
|
|
2284
|
+
function handleAnalysisError(error, config) {
|
|
2285
|
+
console.error("\n\u274C Analysis failed\n");
|
|
2286
|
+
if (error instanceof Error) {
|
|
2287
|
+
console.error(`Error: ${error.message}`);
|
|
2288
|
+
if (config?.verbose && error.stack) {
|
|
2289
|
+
console.error("\nStack trace:");
|
|
2290
|
+
console.error(error.stack);
|
|
2291
|
+
}
|
|
2292
|
+
if (error.message.includes("No sitemaps found")) {
|
|
2293
|
+
console.error("\nSuggestions:");
|
|
2294
|
+
console.error(" \u2022 Verify the base URL is correct");
|
|
2295
|
+
console.error(" \u2022 Check if the site has a sitemap");
|
|
2296
|
+
console.error(" \u2022 Ensure the sitemap is publicly accessible");
|
|
2297
|
+
} else if (error.message.includes("Network") || error.message.includes("timeout")) {
|
|
2298
|
+
console.error("\nSuggestions:");
|
|
2299
|
+
console.error(" \u2022 Check your internet connection");
|
|
2300
|
+
console.error(" \u2022 Verify the URL is accessible");
|
|
2301
|
+
console.error(" \u2022 Try increasing the timeout with --timeout option");
|
|
2302
|
+
}
|
|
2303
|
+
} else {
|
|
2304
|
+
console.error("Unknown error occurred");
|
|
2305
|
+
console.error(String(error));
|
|
2306
|
+
}
|
|
2307
|
+
}
|
|
2308
|
+
function displayPhaseSummary(timings, totalTime) {
|
|
2309
|
+
console.log(chalk.green(`
|
|
2310
|
+
Analysis Complete (Total: ${(totalTime / 1e3).toFixed(1)}s)
|
|
2311
|
+
`));
|
|
2312
|
+
console.log(chalk.cyan("Phase Breakdown:"));
|
|
2313
|
+
for (const timing of timings) {
|
|
2314
|
+
const seconds = (timing.duration / 1e3).toFixed(1);
|
|
2315
|
+
const percentage = (timing.duration / totalTime * 100).toFixed(1);
|
|
2316
|
+
const bar = "\u2022";
|
|
2317
|
+
console.log(` ${bar} ${timing.name.padEnd(15)}: ${seconds.padStart(5)}s (${percentage.padStart(5)}%)`);
|
|
2318
|
+
}
|
|
2319
|
+
console.log("");
|
|
2320
|
+
}
|
|
2321
|
+
async function saveBenchmark(timings, url, totalTime, sitemapCount, urlCount, config) {
|
|
2322
|
+
const benchmark = {
|
|
2323
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2324
|
+
url,
|
|
2325
|
+
total_duration_ms: totalTime,
|
|
2326
|
+
phases: timings.map((t) => ({
|
|
2327
|
+
name: t.name.toLowerCase(),
|
|
2328
|
+
start_ms: t.startTime,
|
|
2329
|
+
end_ms: t.endTime,
|
|
2330
|
+
duration_ms: t.duration
|
|
2331
|
+
})),
|
|
2332
|
+
metrics: {
|
|
2333
|
+
sitemaps_processed: sitemapCount,
|
|
2334
|
+
urls_analyzed: urlCount,
|
|
2335
|
+
throughput: {
|
|
2336
|
+
urls_per_second: Math.round(urlCount / totalTime * 1e3),
|
|
2337
|
+
sitemaps_per_second: (sitemapCount / totalTime * 1e3).toFixed(2)
|
|
2338
|
+
}
|
|
2339
|
+
},
|
|
2340
|
+
system_info: {
|
|
2341
|
+
cpu_count: os2.cpus().length,
|
|
2342
|
+
node_version: process.version,
|
|
2343
|
+
platform: process.platform,
|
|
2344
|
+
memory_total_mb: Math.round(os2.totalmem() / 1024 / 1024)
|
|
2345
|
+
},
|
|
2346
|
+
config: {
|
|
2347
|
+
discovery_concurrency: config.discoveryConcurrency,
|
|
2348
|
+
parsing_concurrency: config.parsingConcurrency,
|
|
2349
|
+
risk_detection_concurrency: config.riskDetectionConcurrency,
|
|
2350
|
+
risk_detection_batch_size: config.riskDetectionBatchSize
|
|
2351
|
+
}
|
|
2352
|
+
};
|
|
2353
|
+
const filename = `performance-profile-${Date.now()}.json`;
|
|
2354
|
+
await fs2.writeFile(filename, JSON.stringify(benchmark, null, 2));
|
|
2355
|
+
console.log(chalk.blue(`\u{1F4CA} Benchmark saved to: ${filename}`));
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
// src/index.ts
|
|
2359
|
+
var program = new Command2();
|
|
2360
|
+
program.name("sitemap-qa").version("1.0.0").description("sitemap analysis for QA teams");
|
|
2361
|
+
program.addCommand(analyzeCommand);
|
|
2362
|
+
process.on("unhandledRejection", (reason, promise) => {
|
|
2363
|
+
console.error("Unhandled Rejection at:", promise, "reason:", reason);
|
|
2364
|
+
process.exit(1);
|
|
2365
|
+
});
|
|
2366
|
+
process.on("SIGINT", () => {
|
|
2367
|
+
console.log("\nGracefully shutting down...");
|
|
2368
|
+
process.exit(0);
|
|
2369
|
+
});
|
|
2370
|
+
process.on("SIGTERM", () => {
|
|
2371
|
+
console.log("\nGracefully shutting down...");
|
|
2372
|
+
process.exit(0);
|
|
2373
|
+
});
|
|
2374
|
+
program.parse();
|
|
484
2375
|
//# sourceMappingURL=index.js.map
|