@akotliar/sitemap-qa 1.0.0-alpha.0 → 1.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/index.cjs +2110 -71
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +2087 -71
- package/dist/index.js.map +1 -1
- package/package.json +67 -65
package/dist/index.cjs
CHANGED
|
@@ -1,27 +1,1744 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
"use strict";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
2
|
+
"use strict";
|
|
3
|
+
var __create = Object.create;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
var __copyProps = (to, from, except, desc) => {
|
|
10
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
11
|
+
for (let key of __getOwnPropNames(from))
|
|
12
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
13
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
14
|
+
}
|
|
15
|
+
return to;
|
|
16
|
+
};
|
|
17
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
18
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
19
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
20
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
21
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
22
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
23
|
+
mod
|
|
24
|
+
));
|
|
25
|
+
|
|
26
|
+
// src/index.ts
|
|
27
|
+
var import_config2 = require("dotenv/config");
|
|
28
|
+
var import_commander2 = require("commander");
|
|
29
|
+
|
|
30
|
+
// src/commands/analyze.ts
|
|
31
|
+
var import_commander = require("commander");
|
|
32
|
+
var import_fs3 = require("fs");
|
|
33
|
+
var import_ora = __toESM(require("ora"), 1);
|
|
34
|
+
var import_chalk = __toESM(require("chalk"), 1);
|
|
35
|
+
var import_cli_progress = __toESM(require("cli-progress"), 1);
|
|
36
|
+
var import_os3 = __toESM(require("os"), 1);
|
|
37
|
+
|
|
38
|
+
// src/config/config-loader.ts
|
|
39
|
+
var import_promises = require("fs/promises");
|
|
40
|
+
var import_fs = require("fs");
|
|
41
|
+
var import_path = require("path");
|
|
42
|
+
var import_os = require("os");
|
|
43
|
+
|
|
44
|
+
// src/types/config.ts
|
|
45
|
+
var DEFAULT_CONFIG = {
|
|
46
|
+
timeout: 30,
|
|
47
|
+
concurrency: 10,
|
|
48
|
+
parsingConcurrency: 50,
|
|
49
|
+
// Optimized for network-bound parallel parsing
|
|
50
|
+
discoveryConcurrency: 50,
|
|
51
|
+
// Optimized for recursive sitemap index discovery
|
|
52
|
+
outputFormat: "html",
|
|
53
|
+
outputDir: "./sitemap-qa/report",
|
|
54
|
+
verbose: false,
|
|
55
|
+
baseUrl: "https://example.com",
|
|
56
|
+
// Default for tests
|
|
57
|
+
acceptedPatterns: [],
|
|
58
|
+
riskDetectionBatchSize: 1e4,
|
|
59
|
+
riskDetectionConcurrency: void 0,
|
|
60
|
+
// Auto-detect in risk-detector.ts
|
|
61
|
+
progressBar: void 0,
|
|
62
|
+
// Auto-detect TTY
|
|
63
|
+
silent: false,
|
|
64
|
+
benchmark: false
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
// src/config/config-loader.ts
|
|
68
|
+
async function loadConfig(cliOptions) {
|
|
69
|
+
let config = { ...DEFAULT_CONFIG };
|
|
70
|
+
const globalConfigPath = (0, import_path.join)((0, import_os.homedir)(), ".sitemap-qa", "config.json");
|
|
71
|
+
if ((0, import_fs.existsSync)(globalConfigPath)) {
|
|
72
|
+
try {
|
|
73
|
+
const globalConfig = JSON.parse(await (0, import_promises.readFile)(globalConfigPath, "utf-8"));
|
|
74
|
+
config = { ...config, ...globalConfig };
|
|
75
|
+
} catch (error) {
|
|
76
|
+
console.warn(`Warning: Failed to load global config: ${error}`);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const projectConfigPath = (0, import_path.join)(process.cwd(), ".sitemap-qa.config.json");
|
|
80
|
+
if ((0, import_fs.existsSync)(projectConfigPath)) {
|
|
81
|
+
try {
|
|
82
|
+
const projectConfig = JSON.parse(await (0, import_promises.readFile)(projectConfigPath, "utf-8"));
|
|
83
|
+
config = { ...config, ...projectConfig };
|
|
84
|
+
} catch (error) {
|
|
85
|
+
console.warn(`Warning: Failed to load project config: ${error}`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const envConfig = loadFromEnv();
|
|
89
|
+
config = { ...config, ...envConfig };
|
|
90
|
+
config = mergeCliOptions(config, cliOptions);
|
|
91
|
+
if (cliOptions.baseUrl) {
|
|
92
|
+
config.baseUrl = cliOptions.baseUrl;
|
|
93
|
+
}
|
|
94
|
+
validateConfig(config);
|
|
95
|
+
return config;
|
|
96
|
+
}
|
|
97
|
+
function loadFromEnv() {
|
|
98
|
+
const env = {};
|
|
99
|
+
if (process.env.SITEMAP_VERIFY_TIMEOUT) {
|
|
100
|
+
env.timeout = parseInt(process.env.SITEMAP_VERIFY_TIMEOUT, 10);
|
|
101
|
+
}
|
|
102
|
+
return env;
|
|
103
|
+
}
|
|
104
|
+
function mergeCliOptions(config, cliOptions) {
|
|
105
|
+
const merged = { ...config };
|
|
106
|
+
if (cliOptions.timeout && cliOptions.timeout !== "30") {
|
|
107
|
+
merged.timeout = parseInt(cliOptions.timeout, 10);
|
|
108
|
+
}
|
|
109
|
+
if (cliOptions.output) {
|
|
110
|
+
merged.outputFormat = cliOptions.output;
|
|
111
|
+
}
|
|
112
|
+
if (cliOptions.outputDir) {
|
|
113
|
+
merged.outputDir = cliOptions.outputDir;
|
|
114
|
+
}
|
|
115
|
+
if (cliOptions.verbose === true) {
|
|
116
|
+
merged.verbose = true;
|
|
117
|
+
}
|
|
118
|
+
if (cliOptions.acceptedPatterns) {
|
|
119
|
+
merged.acceptedPatterns = cliOptions.acceptedPatterns.split(",").map((p) => p.trim()).filter(Boolean);
|
|
120
|
+
}
|
|
121
|
+
return merged;
|
|
122
|
+
}
|
|
123
|
+
function validateConfig(config) {
|
|
124
|
+
if (config.timeout < 1 || config.timeout > 300) {
|
|
125
|
+
throw new Error("Timeout must be between 1 and 300 seconds");
|
|
126
|
+
}
|
|
127
|
+
if (!["json", "html"].includes(config.outputFormat)) {
|
|
128
|
+
throw new Error("Output format must be json or html");
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// src/errors/network-errors.ts
|
|
133
|
+
var NetworkError = class extends Error {
|
|
134
|
+
constructor(url, originalError) {
|
|
135
|
+
super(`Network request failed for ${url}: ${originalError.message}`);
|
|
136
|
+
this.url = url;
|
|
137
|
+
this.originalError = originalError;
|
|
138
|
+
this.name = "NetworkError";
|
|
139
|
+
}
|
|
140
|
+
code = "NETWORK_ERROR";
|
|
141
|
+
};
|
|
142
|
+
var HttpError = class extends Error {
|
|
143
|
+
constructor(url, statusCode, statusText) {
|
|
144
|
+
let message = `HTTP ${statusCode} error for ${url}`;
|
|
145
|
+
if (statusCode === 403) {
|
|
146
|
+
message += "\n Note: 403 Forbidden often indicates bot protection (Cloudflare, etc.) or access restrictions";
|
|
147
|
+
}
|
|
148
|
+
super(message);
|
|
149
|
+
this.url = url;
|
|
150
|
+
this.statusCode = statusCode;
|
|
151
|
+
this.statusText = statusText;
|
|
152
|
+
this.name = "HttpError";
|
|
153
|
+
}
|
|
154
|
+
code = "HTTP_ERROR";
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
// src/utils/http-client.ts
|
|
158
|
+
var import_playwright = require("playwright");
|
|
159
|
+
var import_axios = __toESM(require("axios"), 1);
|
|
160
|
+
var import_http = require("http");
|
|
161
|
+
var import_https = require("https");
|
|
162
|
+
var httpAgent = new import_http.Agent({
|
|
163
|
+
keepAlive: true,
|
|
164
|
+
maxSockets: 200,
|
|
165
|
+
// Allow many concurrent connections
|
|
166
|
+
maxFreeSockets: 50,
|
|
167
|
+
timeout: 15e3
|
|
168
|
+
});
|
|
169
|
+
var httpsAgent = new import_https.Agent({
|
|
170
|
+
keepAlive: true,
|
|
171
|
+
maxSockets: 200,
|
|
172
|
+
maxFreeSockets: 50,
|
|
173
|
+
timeout: 15e3
|
|
174
|
+
});
|
|
175
|
+
var axiosInstance = import_axios.default.create({
|
|
176
|
+
httpAgent,
|
|
177
|
+
httpsAgent,
|
|
178
|
+
maxRedirects: 5,
|
|
179
|
+
validateStatus: () => true
|
|
180
|
+
// Don't throw on any status code
|
|
181
|
+
});
|
|
182
|
+
async function fetchUrlWithBrowser(url, timeout) {
|
|
183
|
+
let browser;
|
|
184
|
+
try {
|
|
185
|
+
browser = await import_playwright.chromium.launch({
|
|
186
|
+
headless: true,
|
|
187
|
+
args: [
|
|
188
|
+
"--disable-blink-features=AutomationControlled",
|
|
189
|
+
// Hide automation flags
|
|
190
|
+
"--disable-dev-shm-usage",
|
|
191
|
+
"--no-sandbox"
|
|
192
|
+
]
|
|
193
|
+
});
|
|
194
|
+
const context = await browser.newContext({
|
|
195
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
196
|
+
viewport: { width: 1920, height: 1080 },
|
|
197
|
+
locale: "en-US",
|
|
198
|
+
timezoneId: "America/New_York",
|
|
199
|
+
extraHTTPHeaders: {
|
|
200
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
201
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
202
|
+
"Accept-Encoding": "gzip, deflate, br",
|
|
203
|
+
"DNT": "1",
|
|
204
|
+
"Connection": "keep-alive",
|
|
205
|
+
"Upgrade-Insecure-Requests": "1"
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
const page = await context.newPage();
|
|
209
|
+
await page.addInitScript(() => {
|
|
210
|
+
Object.defineProperty(navigator, "webdriver", {
|
|
211
|
+
get: () => false
|
|
212
|
+
});
|
|
213
|
+
window.chrome = {
|
|
214
|
+
runtime: {}
|
|
215
|
+
};
|
|
216
|
+
const originalQuery = window.navigator.permissions.query;
|
|
217
|
+
window.navigator.permissions.query = (parameters) => parameters.name === "notifications" ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters);
|
|
218
|
+
});
|
|
219
|
+
page.setDefaultTimeout(timeout * 1e3);
|
|
220
|
+
const response = await page.goto(url, {
|
|
221
|
+
waitUntil: "domcontentloaded",
|
|
222
|
+
// Changed from networkidle - faster for simple XML
|
|
223
|
+
timeout: timeout * 1e3
|
|
224
|
+
});
|
|
225
|
+
if (!response) {
|
|
226
|
+
throw new Error("No response received from page");
|
|
227
|
+
}
|
|
228
|
+
const statusCode = response.status();
|
|
229
|
+
const content = await page.content();
|
|
230
|
+
const finalUrl = page.url();
|
|
231
|
+
await browser.close();
|
|
232
|
+
if (statusCode >= 200 && statusCode < 300) {
|
|
233
|
+
return {
|
|
234
|
+
content,
|
|
235
|
+
statusCode,
|
|
236
|
+
url: finalUrl
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
throw new HttpError(finalUrl, statusCode);
|
|
240
|
+
} catch (error) {
|
|
241
|
+
if (browser) {
|
|
242
|
+
await browser.close();
|
|
243
|
+
}
|
|
244
|
+
if (error.code === "HTTP_ERROR") {
|
|
245
|
+
throw error;
|
|
246
|
+
}
|
|
247
|
+
throw new NetworkError(url, error);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
async function fetchUrl(url, options = {}) {
|
|
251
|
+
const {
|
|
252
|
+
timeout = 30,
|
|
253
|
+
maxRetries = 3,
|
|
254
|
+
retryDelay = 1e3,
|
|
255
|
+
useBrowser = false,
|
|
256
|
+
disableBrowserFallback = false
|
|
257
|
+
} = options;
|
|
258
|
+
new URL(url);
|
|
259
|
+
const retryableStatuses = [408, 429, 500, 502, 503, 504];
|
|
260
|
+
let lastError = null;
|
|
261
|
+
let attemptedBrowser = false;
|
|
262
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
263
|
+
try {
|
|
264
|
+
if (useBrowser || attemptedBrowser) {
|
|
265
|
+
return await fetchUrlWithBrowser(url, timeout);
|
|
266
|
+
}
|
|
267
|
+
const response = await axiosInstance.get(url, {
|
|
268
|
+
timeout: timeout * 1e3,
|
|
269
|
+
headers: {
|
|
270
|
+
"User-Agent": "sitemap-qa/1.0.0 (compatible; +https://github.com/Akotliar/sitemap-qa)",
|
|
271
|
+
"Accept": "text/xml,application/xml,text/plain,*/*",
|
|
272
|
+
"Accept-Encoding": "gzip, deflate",
|
|
273
|
+
"Connection": "keep-alive"
|
|
274
|
+
}
|
|
275
|
+
});
|
|
276
|
+
const statusCode = response.status;
|
|
277
|
+
const body = response.data;
|
|
278
|
+
if (statusCode >= 200 && statusCode < 300) {
|
|
279
|
+
return {
|
|
280
|
+
content: typeof body === "string" ? body : JSON.stringify(body),
|
|
281
|
+
statusCode,
|
|
282
|
+
url: response.request?.res?.responseUrl || url
|
|
283
|
+
// Final URL after redirects
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
if (statusCode === 403 && !attemptedBrowser && !disableBrowserFallback) {
|
|
287
|
+
attemptedBrowser = true;
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
if (!retryableStatuses.includes(statusCode)) {
|
|
291
|
+
throw new HttpError(response.request?.res?.responseUrl || url, statusCode);
|
|
292
|
+
}
|
|
293
|
+
lastError = new HttpError(response.request?.res?.responseUrl || url, statusCode);
|
|
294
|
+
} catch (error) {
|
|
295
|
+
if (error.code === "HTTP_ERROR") {
|
|
296
|
+
const httpError = error;
|
|
297
|
+
if (!retryableStatuses.includes(httpError.statusCode)) {
|
|
298
|
+
throw error;
|
|
299
|
+
}
|
|
300
|
+
lastError = error;
|
|
301
|
+
} else {
|
|
302
|
+
lastError = new NetworkError(url, error);
|
|
303
|
+
}
|
|
304
|
+
if (attempt === maxRetries) break;
|
|
305
|
+
}
|
|
306
|
+
if (attempt < maxRetries) {
|
|
307
|
+
const delay = retryDelay * Math.pow(2, attempt);
|
|
308
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
throw lastError;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// src/core/discovery.ts
|
|
315
|
+
function normalizeBaseUrl(url) {
|
|
316
|
+
const parsed = new URL(url);
|
|
317
|
+
return parsed.origin;
|
|
318
|
+
}
|
|
319
|
+
async function detectCanonicalDomain(baseUrl, config) {
|
|
320
|
+
const urlObj = new URL(baseUrl);
|
|
321
|
+
const hasWww = urlObj.hostname.startsWith("www.");
|
|
322
|
+
const alternateHostname = hasWww ? urlObj.hostname.substring(4) : `www.${urlObj.hostname}`;
|
|
323
|
+
const alternateUrl = `${urlObj.protocol}//${alternateHostname}/robots.txt`;
|
|
324
|
+
try {
|
|
325
|
+
const result = await fetchUrl(alternateUrl, {
|
|
326
|
+
timeout: config.timeout,
|
|
327
|
+
maxRetries: 1
|
|
328
|
+
});
|
|
329
|
+
if (result.statusCode === 200 || result.statusCode === 404) {
|
|
330
|
+
return alternateHostname;
|
|
331
|
+
}
|
|
332
|
+
return urlObj.hostname;
|
|
333
|
+
} catch (error) {
|
|
334
|
+
if (error instanceof HttpError && error.statusCode === 301) {
|
|
335
|
+
return urlObj.hostname;
|
|
336
|
+
}
|
|
337
|
+
return urlObj.hostname;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
async function tryStandardPaths(baseUrl, config) {
|
|
341
|
+
const baseDomain = new URL(baseUrl).origin;
|
|
342
|
+
const accessIssues = [];
|
|
343
|
+
const standardPaths = [
|
|
344
|
+
"/sitemap.xml",
|
|
345
|
+
"/sitemap_index.xml",
|
|
346
|
+
"/sitemap-index.xml"
|
|
347
|
+
];
|
|
348
|
+
const results = await Promise.allSettled(
|
|
349
|
+
standardPaths.map(async (path) => {
|
|
350
|
+
const sitemapUrl = `${baseDomain}${path}`;
|
|
351
|
+
try {
|
|
352
|
+
const result = await fetchUrl(sitemapUrl, {
|
|
353
|
+
timeout: config.timeout,
|
|
354
|
+
maxRetries: 0
|
|
355
|
+
// Don't retry on standard paths - fail fast
|
|
356
|
+
});
|
|
357
|
+
if (result.statusCode === 200) {
|
|
358
|
+
if (config.verbose) {
|
|
359
|
+
console.log(`\u2713 Found sitemap at: ${sitemapUrl}`);
|
|
360
|
+
}
|
|
361
|
+
return { found: true, url: sitemapUrl };
|
|
362
|
+
}
|
|
363
|
+
return { found: false };
|
|
364
|
+
} catch (error) {
|
|
365
|
+
if (error instanceof HttpError) {
|
|
366
|
+
if (error.statusCode === 401 || error.statusCode === 403) {
|
|
367
|
+
accessIssues.push({
|
|
368
|
+
url: sitemapUrl,
|
|
369
|
+
statusCode: error.statusCode,
|
|
370
|
+
error: error.statusCode === 401 ? "Unauthorized" : "Access Denied"
|
|
371
|
+
});
|
|
372
|
+
if (config.verbose) {
|
|
373
|
+
console.log(`\u26A0 Access denied: ${sitemapUrl} (${error.statusCode})`);
|
|
374
|
+
}
|
|
375
|
+
} else if (config.verbose) {
|
|
376
|
+
console.log(`\u2717 Not found: ${sitemapUrl} (${error.statusCode})`);
|
|
377
|
+
}
|
|
378
|
+
} else if (config.verbose) {
|
|
379
|
+
console.log(`\u2717 Not found: ${sitemapUrl}`);
|
|
380
|
+
}
|
|
381
|
+
return { found: false };
|
|
382
|
+
}
|
|
383
|
+
})
|
|
384
|
+
);
|
|
385
|
+
for (const result of results) {
|
|
386
|
+
if (result.status === "fulfilled" && result.value.found) {
|
|
387
|
+
return { sitemaps: [result.value.url], issues: accessIssues };
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
if (config.verbose) {
|
|
391
|
+
console.log("No sitemap found at standard paths");
|
|
392
|
+
}
|
|
393
|
+
return { sitemaps: [], issues: accessIssues };
|
|
394
|
+
}
|
|
395
|
+
async function parseRobotsTxt(baseUrl, config) {
|
|
396
|
+
const robotsUrl = `${new URL(baseUrl).origin}/robots.txt`;
|
|
397
|
+
try {
|
|
398
|
+
const result = await fetchUrl(robotsUrl, {
|
|
399
|
+
timeout: config.timeout,
|
|
400
|
+
maxRetries: 1
|
|
401
|
+
});
|
|
402
|
+
const lines = result.content.split("\n");
|
|
403
|
+
const sitemaps = [];
|
|
404
|
+
for (const line of lines) {
|
|
405
|
+
const match = line.match(/^Sitemap:\s*(.+)$/i);
|
|
406
|
+
if (match) {
|
|
407
|
+
const sitemapUrl = match[1].trim();
|
|
408
|
+
try {
|
|
409
|
+
new URL(sitemapUrl);
|
|
410
|
+
sitemaps.push(sitemapUrl);
|
|
411
|
+
} catch {
|
|
412
|
+
if (config.verbose) {
|
|
413
|
+
console.warn(`Invalid sitemap URL in robots.txt: ${sitemapUrl}`);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
if (config.verbose && sitemaps.length > 0) {
|
|
419
|
+
console.log(`Found ${sitemaps.length} sitemap(s) in robots.txt`);
|
|
420
|
+
}
|
|
421
|
+
return sitemaps;
|
|
422
|
+
} catch (error) {
|
|
423
|
+
if (config.verbose) {
|
|
424
|
+
console.log(`No robots.txt found at ${robotsUrl}`);
|
|
425
|
+
}
|
|
426
|
+
return [];
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
function isSitemapIndex(xmlContent) {
|
|
430
|
+
if (xmlContent.includes("<sitemapindex")) {
|
|
431
|
+
return true;
|
|
432
|
+
}
|
|
433
|
+
if (xmlContent.includes("<urlset")) {
|
|
434
|
+
const urlBlockRegex = /<url[^>]*>.*?<loc>([^<]+)<\/loc>.*?<\/url>/gs;
|
|
435
|
+
const matches = Array.from(xmlContent.matchAll(urlBlockRegex));
|
|
436
|
+
const samplesToCheck = Math.min(5, matches.length);
|
|
437
|
+
let sitemapLikeCount = 0;
|
|
438
|
+
for (let i = 0; i < samplesToCheck; i++) {
|
|
439
|
+
const url = matches[i][1].trim().toLowerCase();
|
|
440
|
+
if (url.includes("sitemap") || url.endsWith(".xml")) {
|
|
441
|
+
sitemapLikeCount++;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
return sitemapLikeCount > samplesToCheck / 2;
|
|
445
|
+
}
|
|
446
|
+
return false;
|
|
447
|
+
}
|
|
448
|
+
function extractSitemapIndexUrls(xmlContent) {
|
|
449
|
+
const urls = [];
|
|
450
|
+
if (xmlContent.includes("<sitemapindex")) {
|
|
451
|
+
const sitemapBlockRegex = /<sitemap[^>]*>(.*?)<\/sitemap>/gs;
|
|
452
|
+
let sitemapMatch;
|
|
453
|
+
while ((sitemapMatch = sitemapBlockRegex.exec(xmlContent)) !== null) {
|
|
454
|
+
const locMatch = /<loc>([^<]+)<\/loc>/i.exec(sitemapMatch[1]);
|
|
455
|
+
if (locMatch) {
|
|
456
|
+
const url = locMatch[1].trim();
|
|
457
|
+
try {
|
|
458
|
+
new URL(url);
|
|
459
|
+
urls.push(url);
|
|
460
|
+
} catch {
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
} else {
|
|
465
|
+
const urlBlockRegex = /<url[^>]*>(.*?)<\/url>/gs;
|
|
466
|
+
let urlMatch;
|
|
467
|
+
while ((urlMatch = urlBlockRegex.exec(xmlContent)) !== null) {
|
|
468
|
+
const locMatch = /<loc>([^<]+)<\/loc>/i.exec(urlMatch[1]);
|
|
469
|
+
if (locMatch) {
|
|
470
|
+
const url = locMatch[1].trim();
|
|
471
|
+
if (url.toLowerCase().includes("sitemap") || url.toLowerCase().endsWith(".xml")) {
|
|
472
|
+
try {
|
|
473
|
+
new URL(url);
|
|
474
|
+
urls.push(url);
|
|
475
|
+
} catch {
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
return urls;
|
|
482
|
+
}
|
|
483
|
+
async function discoverAllSitemaps(initialSitemaps, config, baseUrl, canonicalDomain, _maxDepth = 10) {
|
|
484
|
+
const finalSitemaps = [];
|
|
485
|
+
const toProcess = [...initialSitemaps];
|
|
486
|
+
const processed = /* @__PURE__ */ new Set();
|
|
487
|
+
const failed = /* @__PURE__ */ new Set();
|
|
488
|
+
const redirected = /* @__PURE__ */ new Set();
|
|
489
|
+
let detectedCanonical = canonicalDomain;
|
|
490
|
+
const BATCH_SIZE = config.discoveryConcurrency || 50;
|
|
491
|
+
while (toProcess.length > 0) {
|
|
492
|
+
const batch = toProcess.splice(0, Math.min(BATCH_SIZE, toProcess.length));
|
|
493
|
+
const batchResults = await Promise.all(batch.map(async (sitemapUrl) => {
|
|
494
|
+
if (processed.has(sitemapUrl)) {
|
|
495
|
+
if (config.verbose) {
|
|
496
|
+
console.warn(`Skipping duplicate sitemap: ${sitemapUrl}`);
|
|
497
|
+
}
|
|
498
|
+
return { type: "skip" };
|
|
499
|
+
}
|
|
500
|
+
processed.add(sitemapUrl);
|
|
501
|
+
try {
|
|
502
|
+
const result = await fetchUrl(sitemapUrl, {
|
|
503
|
+
timeout: config.timeout,
|
|
504
|
+
maxRetries: 2
|
|
505
|
+
});
|
|
506
|
+
if (isSitemapIndex(result.content)) {
|
|
507
|
+
if (config.verbose) {
|
|
508
|
+
console.log(`Found sitemap index: ${sitemapUrl}`);
|
|
509
|
+
}
|
|
510
|
+
const childUrls = extractSitemapIndexUrls(result.content);
|
|
511
|
+
if (config.verbose) {
|
|
512
|
+
console.log(` \u2514\u2500 Contains ${childUrls.length} child sitemap(s)`);
|
|
513
|
+
}
|
|
514
|
+
return { type: "index", childUrls };
|
|
515
|
+
} else {
|
|
516
|
+
if (config.verbose) {
|
|
517
|
+
console.log(`\u2713 Discovered sitemap: ${sitemapUrl}`);
|
|
518
|
+
}
|
|
519
|
+
return { type: "sitemap", url: sitemapUrl };
|
|
520
|
+
}
|
|
521
|
+
} catch (error) {
|
|
522
|
+
if (error instanceof HttpError && error.statusCode === 301) {
|
|
523
|
+
redirected.add(sitemapUrl);
|
|
524
|
+
if (config.verbose) {
|
|
525
|
+
if (!detectedCanonical) {
|
|
526
|
+
detectedCanonical = await detectCanonicalDomain(baseUrl, config);
|
|
527
|
+
if (config.verbose) {
|
|
528
|
+
console.log(`Canonical domain detected: ${detectedCanonical}`);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
try {
|
|
532
|
+
const sitemapUrlObj = new URL(sitemapUrl);
|
|
533
|
+
if (sitemapUrlObj.hostname !== detectedCanonical) {
|
|
534
|
+
console.warn(`\u26A0\uFE0F Sitemap URL redirects (301): ${sitemapUrl}`);
|
|
535
|
+
console.warn(` Problem: The sitemap index contains a URL that redirects.`);
|
|
536
|
+
console.warn(` Likely issue: Domain mismatch - expected "${detectedCanonical}" but got "${sitemapUrlObj.hostname}"`);
|
|
537
|
+
console.warn(` Fix: Update sitemap index to use "https://${detectedCanonical}${sitemapUrlObj.pathname}"`);
|
|
538
|
+
} else {
|
|
539
|
+
console.warn(`\u26A0\uFE0F Sitemap URL redirects (301): ${sitemapUrl}`);
|
|
540
|
+
console.warn(` Fix: Update the sitemap index to reference the final URL after redirect.`);
|
|
541
|
+
}
|
|
542
|
+
} catch {
|
|
543
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
544
|
+
console.warn(`Failed to fetch sitemap ${sitemapUrl}: ${message}`);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
return { type: "redirect" };
|
|
548
|
+
} else {
|
|
549
|
+
failed.add(sitemapUrl);
|
|
550
|
+
if (config.verbose) {
|
|
551
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
552
|
+
console.warn(`Failed to fetch sitemap ${sitemapUrl}: ${message}`);
|
|
553
|
+
}
|
|
554
|
+
return { type: "failed" };
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}));
|
|
558
|
+
for (const result of batchResults) {
|
|
559
|
+
if (result.type === "index") {
|
|
560
|
+
toProcess.push(...result.childUrls);
|
|
561
|
+
} else if (result.type === "sitemap") {
|
|
562
|
+
finalSitemaps.push(result.url);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
if (processed.size > 1e3) {
|
|
566
|
+
console.warn(`\u26A0\uFE0F Processed over 1000 sitemap URLs. Stopping to prevent excessive requests.`);
|
|
567
|
+
break;
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
const totalProcessed = processed.size;
|
|
571
|
+
const totalFailed = failed.size;
|
|
572
|
+
const totalRedirected = redirected.size;
|
|
573
|
+
const sitemapIndexCount = totalProcessed - finalSitemaps.length - totalFailed - totalRedirected;
|
|
574
|
+
if (finalSitemaps.length === 0 && totalProcessed > 0) {
|
|
575
|
+
console.warn(`
|
|
576
|
+
\u26A0\uFE0F SITEMAP DISCOVERY ISSUE`);
|
|
577
|
+
if (sitemapIndexCount > 0 && (totalFailed > 0 || totalRedirected > 0)) {
|
|
578
|
+
console.warn(`Found ${sitemapIndexCount} sitemap index(es) containing ${totalFailed + totalRedirected} child sitemap(s):`);
|
|
579
|
+
if (totalRedirected > 0) {
|
|
580
|
+
console.warn(` - ${totalRedirected} sitemap(s) return 301 redirects (content not accessible without following redirect)`);
|
|
581
|
+
}
|
|
582
|
+
if (totalFailed > 0) {
|
|
583
|
+
console.warn(` - ${totalFailed} sitemap(s) returned errors (404, 403, 500, or network issues)`);
|
|
584
|
+
}
|
|
585
|
+
} else if (totalRedirected > 0) {
|
|
586
|
+
console.warn(`All ${totalRedirected} sitemap(s) return 301 redirects.`);
|
|
587
|
+
} else if (totalFailed > 0) {
|
|
588
|
+
console.warn(`All ${totalFailed} sitemap(s) returned errors.`);
|
|
589
|
+
console.warn(`
|
|
590
|
+
Common causes:`);
|
|
591
|
+
console.warn(` - 403 Forbidden: Bot protection (Cloudflare, etc.) or IP blocking`);
|
|
592
|
+
console.warn(` - 404 Not Found: Sitemaps don't exist at these URLs`);
|
|
593
|
+
console.warn(` - 500/502/503: Server errors or maintenance`);
|
|
594
|
+
console.warn(`
|
|
595
|
+
If sitemaps work in your browser but not here, the site likely has bot protection.`);
|
|
596
|
+
console.warn(`Try: Check if sitemaps load without JavaScript, or contact site administrator.`);
|
|
597
|
+
} else {
|
|
598
|
+
console.warn(`Processed ${totalProcessed} URL(s) but found no accessible sitemaps.`);
|
|
599
|
+
}
|
|
600
|
+
console.warn(`
|
|
601
|
+
Note: This tool does not follow redirects for sitemap URLs.`);
|
|
602
|
+
if (totalRedirected > 0) {
|
|
603
|
+
console.warn(`
|
|
604
|
+
Possible causes of redirects:`);
|
|
605
|
+
console.warn(` - Sitemap index uses non-canonical domain (e.g., missing 'www' or vice versa)`);
|
|
606
|
+
console.warn(` - Sitemap URLs redirect from HTTP to HTTPS`);
|
|
607
|
+
console.warn(` - Intentional redirects in your site configuration`);
|
|
608
|
+
console.warn(`
|
|
609
|
+
Recommendation: Update sitemap index URLs to match the final destination (no redirects).`);
|
|
610
|
+
}
|
|
611
|
+
console.warn(``);
|
|
612
|
+
}
|
|
613
|
+
return { sitemaps: finalSitemaps, canonicalDomain: detectedCanonical };
|
|
614
|
+
}
|
|
615
|
+
async function discoverSitemaps(baseUrl, config) {
|
|
616
|
+
const normalizedUrl = normalizeBaseUrl(baseUrl);
|
|
617
|
+
let allAccessIssues = [];
|
|
618
|
+
let canonicalDomain;
|
|
619
|
+
if (config.verbose) {
|
|
620
|
+
console.log("Strategy 1: Checking robots.txt for sitemap directives...");
|
|
621
|
+
}
|
|
622
|
+
const robotsSitemaps = await parseRobotsTxt(normalizedUrl, config);
|
|
623
|
+
if (robotsSitemaps.length > 0) {
|
|
624
|
+
const { sitemaps: allSitemaps, canonicalDomain: detected } = await discoverAllSitemaps(robotsSitemaps, config, normalizedUrl, canonicalDomain);
|
|
625
|
+
canonicalDomain = detected;
|
|
626
|
+
return {
|
|
627
|
+
sitemaps: allSitemaps,
|
|
628
|
+
source: "robots-txt",
|
|
629
|
+
accessIssues: [],
|
|
630
|
+
// Clear access issues since we found working sitemaps
|
|
631
|
+
canonicalDomain
|
|
632
|
+
};
|
|
633
|
+
}
|
|
634
|
+
if (config.verbose) {
|
|
635
|
+
console.log("Strategy 2: Trying standard sitemap paths...");
|
|
636
|
+
}
|
|
637
|
+
const { sitemaps: standardSitemaps, issues, redirectedToCanonical } = await tryStandardPaths(normalizedUrl, config);
|
|
638
|
+
allAccessIssues = issues;
|
|
639
|
+
if (standardSitemaps.length > 0) {
|
|
640
|
+
const { sitemaps: allSitemaps, canonicalDomain: detected } = await discoverAllSitemaps(standardSitemaps, config, normalizedUrl, canonicalDomain);
|
|
641
|
+
canonicalDomain = detected;
|
|
642
|
+
return {
|
|
643
|
+
sitemaps: allSitemaps,
|
|
644
|
+
source: "standard-path",
|
|
645
|
+
accessIssues: [],
|
|
646
|
+
// Clear access issues since we found working sitemaps
|
|
647
|
+
canonicalDomain
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
if (redirectedToCanonical) {
|
|
651
|
+
const canonicalUrl = `https://${redirectedToCanonical}`;
|
|
652
|
+
console.log(`
|
|
653
|
+
\u{1F4A1} All requests redirected. Retrying with canonical domain: ${redirectedToCanonical}
|
|
654
|
+
`);
|
|
655
|
+
const canonicalRobotsSitemaps = await parseRobotsTxt(canonicalUrl, config);
|
|
656
|
+
if (canonicalRobotsSitemaps.length > 0) {
|
|
657
|
+
const { sitemaps: allSitemaps, canonicalDomain: detected } = await discoverAllSitemaps(canonicalRobotsSitemaps, config, canonicalUrl, redirectedToCanonical);
|
|
658
|
+
return {
|
|
659
|
+
sitemaps: allSitemaps,
|
|
660
|
+
source: "robots-txt",
|
|
661
|
+
accessIssues: [],
|
|
662
|
+
canonicalDomain: detected || redirectedToCanonical
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
const { sitemaps: canonicalStandardSitemaps } = await tryStandardPaths(canonicalUrl, config);
|
|
666
|
+
if (canonicalStandardSitemaps.length > 0) {
|
|
667
|
+
const { sitemaps: allSitemaps, canonicalDomain: detected } = await discoverAllSitemaps(canonicalStandardSitemaps, config, canonicalUrl, redirectedToCanonical);
|
|
668
|
+
return {
|
|
669
|
+
sitemaps: allSitemaps,
|
|
670
|
+
source: "standard-path",
|
|
671
|
+
accessIssues: [],
|
|
672
|
+
canonicalDomain: detected || redirectedToCanonical
|
|
673
|
+
};
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
return {
|
|
677
|
+
sitemaps: [],
|
|
678
|
+
source: "none",
|
|
679
|
+
accessIssues: allAccessIssues,
|
|
680
|
+
canonicalDomain
|
|
681
|
+
};
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
// src/core/parser.ts
|
|
685
|
+
var import_fast_xml_parser = require("fast-xml-parser");
|
|
686
|
+
var VALID_CHANGEFREQ = /* @__PURE__ */ new Set([
|
|
687
|
+
"always",
|
|
688
|
+
"hourly",
|
|
689
|
+
"daily",
|
|
690
|
+
"weekly",
|
|
691
|
+
"monthly",
|
|
692
|
+
"yearly",
|
|
693
|
+
"never"
|
|
694
|
+
]);
|
|
695
|
+
var parser = new import_fast_xml_parser.XMLParser({
|
|
696
|
+
ignoreAttributes: false,
|
|
697
|
+
attributeNamePrefix: "@_",
|
|
698
|
+
textNodeName: "_text",
|
|
699
|
+
parseAttributeValue: true,
|
|
700
|
+
trimValues: true,
|
|
701
|
+
allowBooleanAttributes: true,
|
|
702
|
+
parseTagValue: false
|
|
703
|
+
// Keep values as strings for validation
|
|
704
|
+
});
|
|
705
|
+
function extractUrls(parsedXml, sitemapUrl) {
|
|
706
|
+
const urls = [];
|
|
707
|
+
if (parsedXml.urlset) {
|
|
708
|
+
const urlNodes = Array.isArray(parsedXml.urlset.url) ? parsedXml.urlset.url : [parsedXml.urlset.url];
|
|
709
|
+
for (let i = 0; i < urlNodes.length; i++) {
|
|
710
|
+
const node = urlNodes[i];
|
|
711
|
+
if (!node || !node.loc) {
|
|
712
|
+
continue;
|
|
713
|
+
}
|
|
714
|
+
urls.push({
|
|
715
|
+
loc: node.loc,
|
|
716
|
+
lastmod: node.lastmod,
|
|
717
|
+
changefreq: node.changefreq,
|
|
718
|
+
priority: node.priority ? parseFloat(node.priority) : void 0,
|
|
719
|
+
source: sitemapUrl
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
return urls;
|
|
724
|
+
}
|
|
725
|
+
async function parseSitemap(xml, sitemapUrl) {
|
|
726
|
+
const errors = [];
|
|
727
|
+
try {
|
|
728
|
+
const validationResult = import_fast_xml_parser.XMLValidator.validate(xml);
|
|
729
|
+
if (validationResult !== true) {
|
|
730
|
+
const validationError = typeof validationResult === "object" ? validationResult.err.msg : "Invalid XML";
|
|
731
|
+
return {
|
|
732
|
+
urls: [],
|
|
733
|
+
errors: [
|
|
734
|
+
`[${sitemapUrl}] XML parsing failed: ${validationError}`
|
|
735
|
+
],
|
|
736
|
+
totalCount: 0,
|
|
737
|
+
sitemapUrl
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
const parsed = parser.parse(xml);
|
|
741
|
+
const urls = extractUrls(parsed, sitemapUrl);
|
|
742
|
+
const validUrls = [];
|
|
743
|
+
for (const entry of urls) {
|
|
744
|
+
try {
|
|
745
|
+
new URL(entry.loc);
|
|
746
|
+
if (entry.priority !== void 0) {
|
|
747
|
+
if (entry.priority < 0 || entry.priority > 1) {
|
|
748
|
+
errors.push(
|
|
749
|
+
`Invalid priority ${entry.priority} for ${entry.loc} - clamping to 0-1`
|
|
750
|
+
);
|
|
751
|
+
entry.priority = Math.max(0, Math.min(1, entry.priority));
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
if (entry.changefreq) {
|
|
755
|
+
if (!VALID_CHANGEFREQ.has(entry.changefreq.toLowerCase())) {
|
|
756
|
+
errors.push(
|
|
757
|
+
`Invalid changefreq "${entry.changefreq}" for ${entry.loc}`
|
|
758
|
+
);
|
|
759
|
+
entry.changefreq = void 0;
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
validUrls.push(entry);
|
|
763
|
+
} catch (urlError) {
|
|
764
|
+
errors.push(`Invalid URL format: ${entry.loc}`);
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
return {
|
|
768
|
+
urls: validUrls,
|
|
769
|
+
errors,
|
|
770
|
+
totalCount: validUrls.length,
|
|
771
|
+
sitemapUrl
|
|
772
|
+
};
|
|
773
|
+
} catch (parseError) {
|
|
774
|
+
const errorMsg = parseError instanceof Error ? parseError.message : String(parseError);
|
|
775
|
+
return {
|
|
776
|
+
urls: [],
|
|
777
|
+
errors: [
|
|
778
|
+
`[${sitemapUrl}] XML parsing failed: ${errorMsg}`
|
|
779
|
+
],
|
|
780
|
+
totalCount: 0,
|
|
781
|
+
sitemapUrl
|
|
782
|
+
};
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
// src/utils/batch-processor.ts
|
|
787
|
+
function chunkArray(array, chunkSize) {
|
|
788
|
+
const chunks = [];
|
|
789
|
+
for (let i = 0; i < array.length; i += chunkSize) {
|
|
790
|
+
chunks.push(array.slice(i, i + chunkSize));
|
|
791
|
+
}
|
|
792
|
+
return chunks;
|
|
793
|
+
}
|
|
794
|
+
async function processInBatches(items, concurrency, processor, onProgress) {
|
|
795
|
+
const results = new Array(items.length);
|
|
796
|
+
let completed = 0;
|
|
797
|
+
let currentIndex = 0;
|
|
798
|
+
const errors = [];
|
|
799
|
+
const workers = Array(Math.min(concurrency, items.length)).fill(null).map(async () => {
|
|
800
|
+
while (currentIndex < items.length) {
|
|
801
|
+
const index = currentIndex++;
|
|
802
|
+
const item = items[index];
|
|
803
|
+
try {
|
|
804
|
+
results[index] = await processor(item);
|
|
805
|
+
} catch (error) {
|
|
806
|
+
errors.push({ index, error });
|
|
807
|
+
results[index] = null;
|
|
808
|
+
}
|
|
809
|
+
completed++;
|
|
810
|
+
if (onProgress) {
|
|
811
|
+
onProgress(completed, items.length);
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
});
|
|
815
|
+
await Promise.all(workers);
|
|
816
|
+
if (errors.length > 0) {
|
|
817
|
+
console.warn(`Processed ${items.length} items with ${errors.length} errors`);
|
|
818
|
+
}
|
|
819
|
+
return results;
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
// src/core/extractor.ts
|
|
823
|
+
async function extractAllUrls(sitemapUrls, config, onProgress) {
|
|
824
|
+
const allUrls = [];
|
|
825
|
+
const allErrors = [];
|
|
826
|
+
let sitemapsProcessed = 0;
|
|
827
|
+
let sitemapsFailed = 0;
|
|
828
|
+
if (config.verbose) {
|
|
829
|
+
console.log(`
|
|
830
|
+
Extracting URLs from ${sitemapUrls.length} sitemap(s)...`);
|
|
831
|
+
}
|
|
832
|
+
const CONCURRENCY = config.parsingConcurrency || 50;
|
|
833
|
+
if (!config.silent && config.verbose) {
|
|
834
|
+
console.log(`Using parsing concurrency: ${CONCURRENCY}`);
|
|
835
|
+
}
|
|
836
|
+
const results = await processInBatches(
|
|
837
|
+
sitemapUrls,
|
|
838
|
+
CONCURRENCY,
|
|
839
|
+
async (sitemapUrl) => {
|
|
840
|
+
try {
|
|
841
|
+
if (config.verbose) {
|
|
842
|
+
console.log(`Extracting URLs from: ${sitemapUrl}`);
|
|
843
|
+
}
|
|
844
|
+
const response = await fetchUrl(sitemapUrl, {
|
|
845
|
+
timeout: 10,
|
|
846
|
+
// Fast timeout for sitemaps
|
|
847
|
+
maxRetries: 0,
|
|
848
|
+
// No retries - fail fast
|
|
849
|
+
disableBrowserFallback: true
|
|
850
|
+
// Don't use browser for bulk parsing
|
|
851
|
+
});
|
|
852
|
+
const parseResult = await parseSitemap(response.content, sitemapUrl);
|
|
853
|
+
const extractedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
854
|
+
parseResult.urls.forEach((url) => {
|
|
855
|
+
url.extractedAt = extractedAt;
|
|
856
|
+
});
|
|
857
|
+
if (config.verbose) {
|
|
858
|
+
console.log(` \u2713 Extracted ${parseResult.urls.length} URLs from ${sitemapUrl}`);
|
|
859
|
+
}
|
|
860
|
+
return {
|
|
861
|
+
success: true,
|
|
862
|
+
urls: parseResult.urls,
|
|
863
|
+
errors: parseResult.errors
|
|
864
|
+
};
|
|
865
|
+
} catch (error) {
|
|
866
|
+
const errorMsg = `Failed to process ${sitemapUrl}: ${error instanceof Error ? error.message : String(error)}`;
|
|
867
|
+
if (config.verbose) {
|
|
868
|
+
console.error(` \u2717 ${errorMsg}`);
|
|
869
|
+
}
|
|
870
|
+
return {
|
|
871
|
+
success: false,
|
|
872
|
+
urls: [],
|
|
873
|
+
errors: [errorMsg]
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
},
|
|
877
|
+
onProgress
|
|
878
|
+
// Pass progress callback to batch processor
|
|
879
|
+
);
|
|
880
|
+
for (const result of results) {
|
|
881
|
+
if (result.success) {
|
|
882
|
+
sitemapsProcessed++;
|
|
883
|
+
allUrls.push(...result.urls);
|
|
884
|
+
} else {
|
|
885
|
+
sitemapsFailed++;
|
|
886
|
+
}
|
|
887
|
+
allErrors.push(...result.errors);
|
|
888
|
+
}
|
|
889
|
+
if (config.verbose) {
|
|
890
|
+
console.log(`
|
|
891
|
+
Extraction complete:`);
|
|
892
|
+
console.log(` - Sitemaps processed: ${sitemapsProcessed}`);
|
|
893
|
+
console.log(` - Sitemaps failed: ${sitemapsFailed}`);
|
|
894
|
+
console.log(` - Total URLs: ${allUrls.length}`);
|
|
895
|
+
console.log(` - Errors: ${allErrors.length}`);
|
|
896
|
+
}
|
|
897
|
+
return {
|
|
898
|
+
allUrls,
|
|
899
|
+
sitemapsProcessed,
|
|
900
|
+
sitemapsFailed,
|
|
901
|
+
totalUrls: allUrls.length,
|
|
902
|
+
errors: allErrors
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
// src/core/consolidator.ts
|
|
907
|
+
function normalizeUrl(url) {
|
|
908
|
+
try {
|
|
909
|
+
const parsed = new URL(url);
|
|
910
|
+
let pathname = parsed.pathname;
|
|
911
|
+
if (pathname.endsWith("/") && pathname !== "/") {
|
|
912
|
+
pathname = pathname.slice(0, -1);
|
|
913
|
+
}
|
|
914
|
+
const params = Array.from(parsed.searchParams.entries()).sort(
|
|
915
|
+
([a], [b]) => a.localeCompare(b)
|
|
916
|
+
);
|
|
917
|
+
const sortedParams = new URLSearchParams(params);
|
|
918
|
+
return `${parsed.protocol}//${parsed.host}${pathname}${sortedParams.toString() ? "?" + sortedParams.toString() : ""}${parsed.hash}`;
|
|
919
|
+
} catch {
|
|
920
|
+
return url;
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
function mergeUrlEntries(entries) {
|
|
924
|
+
if (entries.length === 1) return entries[0];
|
|
925
|
+
const merged = { ...entries[0] };
|
|
926
|
+
const sources = entries.map((e) => e.source);
|
|
927
|
+
merged.source = sources.join(", ");
|
|
928
|
+
const lastmods = entries.map((e) => e.lastmod).filter((lm) => !!lm).map((lm) => new Date(lm).getTime()).sort((a, b) => b - a);
|
|
929
|
+
if (lastmods.length > 0) {
|
|
930
|
+
merged.lastmod = new Date(lastmods[0]).toISOString();
|
|
931
|
+
}
|
|
932
|
+
const priorities = entries.map((e) => e.priority).filter((p) => p !== void 0);
|
|
933
|
+
if (priorities.length > 0) {
|
|
934
|
+
merged.priority = Math.max(...priorities);
|
|
935
|
+
}
|
|
936
|
+
const changefreqs = entries.map((e) => e.changefreq).filter((cf) => !!cf);
|
|
937
|
+
if (changefreqs.length > 0) {
|
|
938
|
+
const counts = /* @__PURE__ */ new Map();
|
|
939
|
+
for (const cf of changefreqs) {
|
|
940
|
+
counts.set(cf, (counts.get(cf) || 0) + 1);
|
|
941
|
+
}
|
|
942
|
+
const sorted = Array.from(counts.entries()).sort((a, b) => b[1] - a[1]);
|
|
943
|
+
merged.changefreq = sorted[0][0];
|
|
944
|
+
}
|
|
945
|
+
const extractedAts = entries.map((e) => e.extractedAt).filter((ea) => !!ea).map((ea) => new Date(ea).getTime()).sort((a, b) => b - a);
|
|
946
|
+
if (extractedAts.length > 0) {
|
|
947
|
+
merged.extractedAt = new Date(extractedAts[0]).toISOString();
|
|
948
|
+
}
|
|
949
|
+
return merged;
|
|
950
|
+
}
|
|
951
|
+
function consolidateUrls(urls, verbose = false) {
|
|
952
|
+
const totalInputUrls = urls.length;
|
|
953
|
+
if (verbose) {
|
|
954
|
+
console.log(`
|
|
955
|
+
Consolidating ${urls.length} URL(s)...`);
|
|
956
|
+
}
|
|
957
|
+
const urlMap = /* @__PURE__ */ new Map();
|
|
958
|
+
for (const entry of urls) {
|
|
959
|
+
const normalized = normalizeUrl(entry.loc);
|
|
960
|
+
if (!urlMap.has(normalized)) {
|
|
961
|
+
urlMap.set(normalized, []);
|
|
962
|
+
}
|
|
963
|
+
urlMap.get(normalized).push(entry);
|
|
964
|
+
}
|
|
965
|
+
const uniqueUrls = [];
|
|
966
|
+
const duplicateGroups = [];
|
|
967
|
+
for (const [normalized, entries] of urlMap.entries()) {
|
|
968
|
+
const merged = mergeUrlEntries(entries);
|
|
969
|
+
uniqueUrls.push(merged);
|
|
970
|
+
if (entries.length > 1) {
|
|
971
|
+
duplicateGroups.push({
|
|
972
|
+
url: normalized,
|
|
973
|
+
count: entries.length,
|
|
974
|
+
sources: entries.map((e) => e.source)
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
if (verbose) {
|
|
979
|
+
console.log(`Consolidation complete:`);
|
|
980
|
+
console.log(` - Input URLs: ${totalInputUrls}`);
|
|
981
|
+
console.log(` - Unique URLs: ${uniqueUrls.length}`);
|
|
982
|
+
console.log(` - Duplicates removed: ${totalInputUrls - uniqueUrls.length}`);
|
|
983
|
+
if (duplicateGroups.length > 0) {
|
|
984
|
+
console.log(`
|
|
985
|
+
Top duplicates:`);
|
|
986
|
+
const top5 = duplicateGroups.sort((a, b) => b.count - a.count).slice(0, 5);
|
|
987
|
+
for (const group of top5) {
|
|
988
|
+
console.log(` - ${group.url} (${group.count} times)`);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
return {
|
|
993
|
+
uniqueUrls,
|
|
994
|
+
totalInputUrls,
|
|
995
|
+
duplicatesRemoved: totalInputUrls - uniqueUrls.length,
|
|
996
|
+
duplicateGroups
|
|
997
|
+
};
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
// src/core/patterns/risk-patterns.ts
|
|
1001
|
+
var RISK_PATTERNS = [
|
|
1002
|
+
// Note: Environment leakage patterns moved to domain-patterns.ts
|
|
1003
|
+
// Note: Admin path patterns moved to admin-patterns.ts
|
|
1004
|
+
// to avoid duplication and improve maintainability
|
|
1005
|
+
// Sensitive Parameter Patterns (HIGH)
|
|
1006
|
+
{
|
|
1007
|
+
name: "Authentication Parameter",
|
|
1008
|
+
category: "sensitive_params",
|
|
1009
|
+
severity: "high",
|
|
1010
|
+
regex: /[?&](token|auth|key|password|secret|apikey|session|credentials)=/i,
|
|
1011
|
+
description: "Query parameter may contain sensitive authentication data"
|
|
1012
|
+
},
|
|
1013
|
+
{
|
|
1014
|
+
name: "Debug Parameter",
|
|
1015
|
+
category: "sensitive_params",
|
|
1016
|
+
severity: "medium",
|
|
1017
|
+
regex: /[?&](debug|trace|verbose|test_mode)=/i,
|
|
1018
|
+
description: "Query parameter may contain debug or diagnostic flag"
|
|
1019
|
+
},
|
|
1020
|
+
// Protocol Inconsistency Patterns (MEDIUM)
|
|
1021
|
+
{
|
|
1022
|
+
name: "HTTP in HTTPS Site",
|
|
1023
|
+
category: "protocol_inconsistency",
|
|
1024
|
+
severity: "medium",
|
|
1025
|
+
regex: /^http:\/\//,
|
|
1026
|
+
description: "HTTP URL in HTTPS sitemap (potential mixed content)"
|
|
1027
|
+
},
|
|
1028
|
+
// Test/Unfinished Content Patterns (MEDIUM)
|
|
1029
|
+
// Focuses on obvious test/placeholder patterns, avoiding false positives with legitimate content
|
|
1030
|
+
{
|
|
1031
|
+
name: "Test Content Path",
|
|
1032
|
+
category: "test_content",
|
|
1033
|
+
severity: "medium",
|
|
1034
|
+
regex: /\/(?:test-|demo-|sample-|temp-|temporary-|placeholder-)|\/(test|demo|sample|temp|temporary|placeholder)(?:\/|$)/i,
|
|
1035
|
+
description: "URL path suggests test, demo, or unfinished content that may not be intended for indexing"
|
|
1036
|
+
}
|
|
1037
|
+
];
|
|
1038
|
+
|
|
1039
|
+
// src/core/patterns/domain-patterns.ts
|
|
1040
|
+
function escapeRegex(str) {
|
|
1041
|
+
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1042
|
+
}
|
|
1043
|
+
function extractRootDomain(hostname) {
|
|
1044
|
+
const parts = hostname.split(".");
|
|
1045
|
+
if (parts.length >= 2) {
|
|
1046
|
+
return parts.slice(-2).join(".");
|
|
1047
|
+
}
|
|
1048
|
+
return hostname;
|
|
1049
|
+
}
|
|
1050
|
+
function createDomainMismatchPattern(baseUrl, options) {
|
|
1051
|
+
const baseDomain = new URL(baseUrl).hostname;
|
|
1052
|
+
const rootDomain = extractRootDomain(baseDomain);
|
|
1053
|
+
if (options?.allowedSubdomains && options.allowedSubdomains.length > 0) {
|
|
1054
|
+
const escapedRoot2 = escapeRegex(rootDomain);
|
|
1055
|
+
const escapedSubdomains = options.allowedSubdomains.map(escapeRegex).join("|");
|
|
1056
|
+
const pattern2 = `^https?://(?!(?:(?:${escapedSubdomains})\\.)?${escapedRoot2}(?:/|$))`;
|
|
1057
|
+
return {
|
|
1058
|
+
name: "Domain Mismatch",
|
|
1059
|
+
category: "domain_mismatch",
|
|
1060
|
+
severity: "high",
|
|
1061
|
+
regex: new RegExp(pattern2),
|
|
1062
|
+
description: `URL does not match expected domain or allowed subdomains`
|
|
1063
|
+
};
|
|
1064
|
+
}
|
|
1065
|
+
const escapedRoot = escapeRegex(rootDomain);
|
|
1066
|
+
const pattern = `^https?://(?!(?:www\\.)?${escapedRoot}(?:/|$))`;
|
|
1067
|
+
return {
|
|
1068
|
+
name: "Domain Mismatch",
|
|
1069
|
+
category: "domain_mismatch",
|
|
1070
|
+
severity: "high",
|
|
1071
|
+
regex: new RegExp(pattern),
|
|
1072
|
+
description: `URL does not match expected domain: ${rootDomain} (including www variant)`
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
1075
|
+
var ENVIRONMENT_PATTERNS = [
|
|
1076
|
+
{
|
|
1077
|
+
name: "Staging Subdomain",
|
|
1078
|
+
category: "environment_leakage",
|
|
1079
|
+
severity: "high",
|
|
1080
|
+
regex: /^https?:\/\/(staging|stg)\./i,
|
|
1081
|
+
description: "URL uses staging subdomain"
|
|
1082
|
+
},
|
|
1083
|
+
{
|
|
1084
|
+
name: "Development Subdomain",
|
|
1085
|
+
category: "environment_leakage",
|
|
1086
|
+
severity: "high",
|
|
1087
|
+
regex: /^https?:\/\/(dev|development)\./i,
|
|
1088
|
+
description: "URL uses development subdomain"
|
|
1089
|
+
},
|
|
1090
|
+
{
|
|
1091
|
+
name: "QA/Test Subdomain",
|
|
1092
|
+
category: "environment_leakage",
|
|
1093
|
+
severity: "high",
|
|
1094
|
+
regex: /^https?:\/\/(qa|test|uat|preprod)\./i,
|
|
1095
|
+
description: "URL uses test environment subdomain"
|
|
1096
|
+
},
|
|
1097
|
+
{
|
|
1098
|
+
name: "Localhost URL",
|
|
1099
|
+
category: "environment_leakage",
|
|
1100
|
+
severity: "high",
|
|
1101
|
+
regex: /^https?:\/\/(localhost|127\.0\.0\.1|0\.0\.0\.0)/,
|
|
1102
|
+
description: "URL points to localhost (development environment)"
|
|
1103
|
+
},
|
|
1104
|
+
{
|
|
1105
|
+
name: "Environment in Path",
|
|
1106
|
+
category: "environment_leakage",
|
|
1107
|
+
severity: "high",
|
|
1108
|
+
regex: /^https?:\/\/[^/]+\/(staging|dev|qa|uat|preprod)\//i,
|
|
1109
|
+
description: "URL path contains environment identifier at root level"
|
|
1110
|
+
}
|
|
1111
|
+
];
|
|
1112
|
+
|
|
1113
|
+
// src/core/patterns/admin-patterns.ts
|
|
1114
|
+
var ADMIN_PATH_PATTERNS = [
|
|
1115
|
+
{
|
|
1116
|
+
name: "Admin Path",
|
|
1117
|
+
category: "admin_paths",
|
|
1118
|
+
severity: "high",
|
|
1119
|
+
regex: /\/(admin|administrator)(?:\/|$|\?)/i,
|
|
1120
|
+
description: "URL contains /admin or /administrator as a path segment"
|
|
1121
|
+
},
|
|
1122
|
+
{
|
|
1123
|
+
name: "Dashboard Path",
|
|
1124
|
+
category: "admin_paths",
|
|
1125
|
+
severity: "high",
|
|
1126
|
+
regex: /\/dashboard(?:\/|$|\?)/i,
|
|
1127
|
+
description: "URL contains /dashboard as a path segment"
|
|
1128
|
+
},
|
|
1129
|
+
{
|
|
1130
|
+
name: "Config Path",
|
|
1131
|
+
category: "admin_paths",
|
|
1132
|
+
severity: "high",
|
|
1133
|
+
regex: /\/(config|configuration)(?:\/|$|\?)/i,
|
|
1134
|
+
description: "URL contains /config or /configuration as a path segment"
|
|
1135
|
+
},
|
|
1136
|
+
{
|
|
1137
|
+
name: "Console Path",
|
|
1138
|
+
category: "admin_paths",
|
|
1139
|
+
severity: "high",
|
|
1140
|
+
regex: /\/console(?:\/|$|\?)/i,
|
|
1141
|
+
description: "URL contains /console as a path segment"
|
|
1142
|
+
},
|
|
1143
|
+
{
|
|
1144
|
+
name: "Control Panel Path",
|
|
1145
|
+
category: "admin_paths",
|
|
1146
|
+
severity: "high",
|
|
1147
|
+
regex: /\/(cpanel|control-panel)(?:\/|$|\?)/i,
|
|
1148
|
+
description: "URL contains control panel as a path segment"
|
|
1149
|
+
}
|
|
1150
|
+
];
|
|
1151
|
+
var INTERNAL_CONTENT_PATTERNS = [
|
|
1152
|
+
{
|
|
1153
|
+
name: "Internal Content Path",
|
|
1154
|
+
category: "internal_content",
|
|
1155
|
+
severity: "medium",
|
|
1156
|
+
regex: /\/internal\b/i,
|
|
1157
|
+
description: "URL contains /internal path segment - may be internal-only content not intended for public indexing"
|
|
1158
|
+
}
|
|
1159
|
+
];
|
|
1160
|
+
var SENSITIVE_PARAM_PATTERNS = [
|
|
1161
|
+
{
|
|
1162
|
+
name: "Authentication Token Parameter",
|
|
1163
|
+
category: "sensitive_params",
|
|
1164
|
+
severity: "high",
|
|
1165
|
+
regex: /[?&](token|auth_token|access_token|api_token)=/i,
|
|
1166
|
+
description: "Query parameter may contain authentication token"
|
|
1167
|
+
},
|
|
1168
|
+
{
|
|
1169
|
+
name: "API Key Parameter",
|
|
1170
|
+
category: "sensitive_params",
|
|
1171
|
+
severity: "high",
|
|
1172
|
+
regex: /[?&](apikey|api_key|key)=/i,
|
|
1173
|
+
description: "Query parameter may contain API key"
|
|
1174
|
+
},
|
|
1175
|
+
{
|
|
1176
|
+
name: "Password Parameter",
|
|
1177
|
+
category: "sensitive_params",
|
|
1178
|
+
severity: "high",
|
|
1179
|
+
regex: /[?&](password|passwd|pwd)=/i,
|
|
1180
|
+
description: "Query parameter may contain password"
|
|
1181
|
+
},
|
|
1182
|
+
{
|
|
1183
|
+
name: "Secret Parameter",
|
|
1184
|
+
category: "sensitive_params",
|
|
1185
|
+
severity: "high",
|
|
1186
|
+
regex: /[?&](secret|client_secret)=/i,
|
|
1187
|
+
description: "Query parameter may contain secret value"
|
|
1188
|
+
},
|
|
1189
|
+
{
|
|
1190
|
+
name: "Session Parameter",
|
|
1191
|
+
category: "sensitive_params",
|
|
1192
|
+
severity: "high",
|
|
1193
|
+
regex: /[?&](session|sessionid|sid)=/i,
|
|
1194
|
+
description: "Query parameter may contain session identifier"
|
|
1195
|
+
},
|
|
1196
|
+
{
|
|
1197
|
+
name: "Credentials Parameter",
|
|
1198
|
+
category: "sensitive_params",
|
|
1199
|
+
severity: "high",
|
|
1200
|
+
regex: /[?&]credentials=/i,
|
|
1201
|
+
description: "Query parameter may contain credentials"
|
|
1202
|
+
},
|
|
1203
|
+
{
|
|
1204
|
+
name: "Debug Parameter",
|
|
1205
|
+
category: "sensitive_params",
|
|
1206
|
+
severity: "medium",
|
|
1207
|
+
regex: /[?&](debug|trace|verbose)=/i,
|
|
1208
|
+
description: "Query parameter contains debug or diagnostic flag"
|
|
1209
|
+
},
|
|
1210
|
+
{
|
|
1211
|
+
name: "Test Mode Parameter",
|
|
1212
|
+
category: "sensitive_params",
|
|
1213
|
+
severity: "medium",
|
|
1214
|
+
regex: /[?&](test_mode|test|testing)=/i,
|
|
1215
|
+
description: "Query parameter indicates test mode"
|
|
1216
|
+
}
|
|
1217
|
+
];
|
|
1218
|
+
|
|
1219
|
+
// src/utils/sanitizer.ts
|
|
1220
|
+
function sanitizeUrl(url) {
|
|
1221
|
+
try {
|
|
1222
|
+
const parsed = new URL(url);
|
|
1223
|
+
const sensitiveParams = [
|
|
1224
|
+
"token",
|
|
1225
|
+
"auth",
|
|
1226
|
+
"auth_token",
|
|
1227
|
+
"access_token",
|
|
1228
|
+
"api_token",
|
|
1229
|
+
"apikey",
|
|
1230
|
+
"api_key",
|
|
1231
|
+
"key",
|
|
1232
|
+
"password",
|
|
1233
|
+
"passwd",
|
|
1234
|
+
"pwd",
|
|
1235
|
+
"secret",
|
|
1236
|
+
"client_secret",
|
|
1237
|
+
"session",
|
|
1238
|
+
"sessionid",
|
|
1239
|
+
"sid",
|
|
1240
|
+
"credentials"
|
|
1241
|
+
];
|
|
1242
|
+
for (const param of sensitiveParams) {
|
|
1243
|
+
if (parsed.searchParams.has(param)) {
|
|
1244
|
+
parsed.searchParams.set(param, "[REDACTED]");
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
return parsed.toString();
|
|
1248
|
+
} catch {
|
|
1249
|
+
return url;
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
// src/core/risk-grouper.ts
|
|
1254
|
+
function generateRecommendation(category, _severity, count) {
|
|
1255
|
+
switch (category) {
|
|
1256
|
+
case "environment_leakage":
|
|
1257
|
+
return {
|
|
1258
|
+
rationale: `Production sitemap contains ${count} URL(s) from non-production environments (staging, dev, QA, test). This indicates configuration errors or environment leakage.`,
|
|
1259
|
+
recommendedAction: "Verify sitemap generation excludes non-production environments. Review deployment configuration and environment filtering rules."
|
|
1260
|
+
};
|
|
1261
|
+
case "admin_paths":
|
|
1262
|
+
return {
|
|
1263
|
+
rationale: `${count} administrative path(s) detected in public sitemap (admin, dashboard, config). These paths may expose privileged access points.`,
|
|
1264
|
+
recommendedAction: "Confirm if admin paths should be publicly indexed. Consider excluding via robots.txt or removing from sitemap. Verify access controls."
|
|
1265
|
+
};
|
|
1266
|
+
case "internal_content":
|
|
1267
|
+
return {
|
|
1268
|
+
rationale: `${count} URL(s) contain "internal" in the path. These may be internal-facing content not intended for public indexing.`,
|
|
1269
|
+
recommendedAction: "Review URLs to determine if they should be publicly accessible. Consider excluding internal content from sitemap or adding noindex meta tags."
|
|
1270
|
+
};
|
|
1271
|
+
case "test_content":
|
|
1272
|
+
return {
|
|
1273
|
+
rationale: `${count} URL(s) contain test/demo/sample identifiers. These may be placeholder or unfinished content not intended for indexing.`,
|
|
1274
|
+
recommendedAction: "Review and remove test content from production sitemaps. Verify content is production-ready before including in sitemap."
|
|
1275
|
+
};
|
|
1276
|
+
case "sensitive_params":
|
|
1277
|
+
return {
|
|
1278
|
+
rationale: `${count} URL(s) contain sensitive query parameters (token, auth, key, password, session). This may expose authentication credentials or debugging flags.`,
|
|
1279
|
+
recommendedAction: "Review why sensitive parameters are in sitemap URLs. Remove authentication tokens from URLs. Consider POST requests for sensitive data."
|
|
1280
|
+
};
|
|
1281
|
+
case "protocol_inconsistency":
|
|
1282
|
+
return {
|
|
1283
|
+
rationale: `${count} URL(s) use HTTP protocol in HTTPS sitemap. This creates mixed content warnings and potential security issues.`,
|
|
1284
|
+
recommendedAction: "Update URLs to use HTTPS consistently. Verify SSL certificate coverage. Check for hardcoded HTTP URLs in content."
|
|
1285
|
+
};
|
|
1286
|
+
case "domain_mismatch":
|
|
1287
|
+
return {
|
|
1288
|
+
rationale: `${count} URL(s) do not match expected base domain. This may indicate external links, CDN URLs, or configuration errors.`,
|
|
1289
|
+
recommendedAction: "Verify if external domains are intentional. Review sitemap generation logic. Confirm CDN or subdomain configuration is correct."
|
|
1290
|
+
};
|
|
1291
|
+
default:
|
|
1292
|
+
return {
|
|
1293
|
+
rationale: `${count} URL(s) flagged in category: ${category}`,
|
|
1294
|
+
recommendedAction: "Review flagged URLs and determine appropriate action."
|
|
1295
|
+
};
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
function groupRiskFindings(findings, maxSampleUrls = 5) {
|
|
1299
|
+
const categoryMap = /* @__PURE__ */ new Map();
|
|
1300
|
+
for (const finding of findings) {
|
|
1301
|
+
if (!categoryMap.has(finding.category)) {
|
|
1302
|
+
categoryMap.set(finding.category, []);
|
|
1303
|
+
}
|
|
1304
|
+
categoryMap.get(finding.category).push(finding);
|
|
1305
|
+
}
|
|
1306
|
+
const groups = [];
|
|
1307
|
+
for (const [category, categoryFindings] of categoryMap.entries()) {
|
|
1308
|
+
const uniqueUrls = Array.from(new Set(categoryFindings.map((f) => f.url)));
|
|
1309
|
+
const severity = categoryFindings.reduce((highest, finding) => {
|
|
1310
|
+
const severityOrder = ["low", "medium", "high"];
|
|
1311
|
+
return severityOrder.indexOf(finding.severity) > severityOrder.indexOf(highest) ? finding.severity : highest;
|
|
1312
|
+
}, "low");
|
|
1313
|
+
const sampleUrls = uniqueUrls.slice(0, maxSampleUrls);
|
|
1314
|
+
const { rationale, recommendedAction } = generateRecommendation(category, severity, uniqueUrls.length);
|
|
1315
|
+
groups.push({
|
|
1316
|
+
category,
|
|
1317
|
+
severity,
|
|
1318
|
+
count: uniqueUrls.length,
|
|
1319
|
+
rationale,
|
|
1320
|
+
sampleUrls,
|
|
1321
|
+
recommendedAction,
|
|
1322
|
+
allUrls: uniqueUrls
|
|
1323
|
+
});
|
|
1324
|
+
}
|
|
1325
|
+
groups.sort((a, b) => {
|
|
1326
|
+
const severityOrder = ["high", "medium", "low"];
|
|
1327
|
+
return severityOrder.indexOf(a.severity) - severityOrder.indexOf(b.severity);
|
|
1328
|
+
});
|
|
1329
|
+
const totalRiskUrls = new Set(findings.map((f) => f.url)).size;
|
|
1330
|
+
const highSeverityCount = groups.filter((g) => g.severity === "high").reduce((sum, g) => sum + g.count, 0);
|
|
1331
|
+
const mediumSeverityCount = groups.filter((g) => g.severity === "medium").reduce((sum, g) => sum + g.count, 0);
|
|
1332
|
+
const lowSeverityCount = groups.filter((g) => g.severity === "low").reduce((sum, g) => sum + g.count, 0);
|
|
1333
|
+
return {
|
|
1334
|
+
groups,
|
|
1335
|
+
totalRiskUrls,
|
|
1336
|
+
highSeverityCount,
|
|
1337
|
+
mediumSeverityCount,
|
|
1338
|
+
lowSeverityCount
|
|
1339
|
+
};
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
// src/core/risk-detector.ts
|
|
1343
|
+
var import_os2 = __toESM(require("os"), 1);
|
|
1344
|
+
function compileAcceptedPatterns(config) {
|
|
1345
|
+
const patterns = [];
|
|
1346
|
+
if (config.acceptedPatterns && config.acceptedPatterns.length > 0) {
|
|
1347
|
+
for (const pattern of config.acceptedPatterns) {
|
|
1348
|
+
try {
|
|
1349
|
+
let regexPattern = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, "[^/]*");
|
|
1350
|
+
if (!regexPattern.endsWith("$") && !regexPattern.includes("(?:")) {
|
|
1351
|
+
regexPattern = regexPattern + "(?:/|$|\\?|#)";
|
|
1352
|
+
}
|
|
1353
|
+
patterns.push(new RegExp(regexPattern, "i"));
|
|
1354
|
+
} catch (error) {
|
|
1355
|
+
if (config.verbose) {
|
|
1356
|
+
console.warn(`Invalid accepted pattern: ${pattern}`);
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
return patterns;
|
|
1362
|
+
}
|
|
1363
|
+
async function detectRisksInBatch(urls, allPatterns, acceptedPatterns, expectedProtocol, verbose) {
|
|
1364
|
+
const findings = [];
|
|
1365
|
+
for (const urlEntry of urls) {
|
|
1366
|
+
const url = urlEntry.loc;
|
|
1367
|
+
let isAccepted = false;
|
|
1368
|
+
for (const acceptedPattern of acceptedPatterns) {
|
|
1369
|
+
if (acceptedPattern.test(url)) {
|
|
1370
|
+
isAccepted = true;
|
|
1371
|
+
break;
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
if (isAccepted) continue;
|
|
1375
|
+
for (const pattern of allPatterns) {
|
|
1376
|
+
if (pattern.category === "protocol_inconsistency") {
|
|
1377
|
+
try {
|
|
1378
|
+
const urlProtocol = new URL(url).protocol;
|
|
1379
|
+
if (expectedProtocol === "https:" && urlProtocol === "http:") {
|
|
1380
|
+
findings.push({
|
|
1381
|
+
url,
|
|
1382
|
+
category: pattern.category,
|
|
1383
|
+
severity: pattern.severity,
|
|
1384
|
+
pattern: pattern.name,
|
|
1385
|
+
rationale: pattern.description,
|
|
1386
|
+
matchedValue: "http://"
|
|
1387
|
+
});
|
|
1388
|
+
}
|
|
1389
|
+
} catch (error) {
|
|
1390
|
+
continue;
|
|
1391
|
+
}
|
|
1392
|
+
} else {
|
|
1393
|
+
try {
|
|
1394
|
+
const match = url.match(pattern.regex);
|
|
1395
|
+
if (match) {
|
|
1396
|
+
findings.push({
|
|
1397
|
+
url: pattern.category === "sensitive_params" ? sanitizeUrl(url) : url,
|
|
1398
|
+
category: pattern.category,
|
|
1399
|
+
severity: pattern.severity,
|
|
1400
|
+
pattern: pattern.name,
|
|
1401
|
+
rationale: pattern.description,
|
|
1402
|
+
matchedValue: match[0]
|
|
1403
|
+
});
|
|
1404
|
+
}
|
|
1405
|
+
} catch (error) {
|
|
1406
|
+
if (verbose) {
|
|
1407
|
+
console.error(`Pattern matching failed for ${pattern.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1408
|
+
}
|
|
1409
|
+
continue;
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
return { findings, urlsProcessed: urls.length };
|
|
1415
|
+
}
|
|
1416
|
+
async function detectRisks(urls, baseUrl, config) {
|
|
1417
|
+
const startTime = Date.now();
|
|
1418
|
+
const domainPattern = createDomainMismatchPattern(baseUrl);
|
|
1419
|
+
const allPatterns = [
|
|
1420
|
+
...RISK_PATTERNS,
|
|
1421
|
+
...ENVIRONMENT_PATTERNS,
|
|
1422
|
+
...ADMIN_PATH_PATTERNS,
|
|
1423
|
+
...SENSITIVE_PARAM_PATTERNS,
|
|
1424
|
+
...INTERNAL_CONTENT_PATTERNS,
|
|
1425
|
+
domainPattern
|
|
1426
|
+
];
|
|
1427
|
+
const acceptedPatterns = compileAcceptedPatterns(config);
|
|
1428
|
+
let expectedProtocol;
|
|
1429
|
+
try {
|
|
1430
|
+
expectedProtocol = new URL(baseUrl).protocol;
|
|
1431
|
+
} catch (error) {
|
|
1432
|
+
if (config.verbose) {
|
|
1433
|
+
console.warn(`Invalid base URL: ${baseUrl}, defaulting to https:`);
|
|
1434
|
+
}
|
|
1435
|
+
expectedProtocol = "https:";
|
|
1436
|
+
}
|
|
1437
|
+
const BATCH_SIZE = config.riskDetectionBatchSize || 1e4;
|
|
1438
|
+
const CONCURRENCY = config.riskDetectionConcurrency || Math.max(2, import_os2.default.cpus().length - 1);
|
|
1439
|
+
const batches = chunkArray(urls, BATCH_SIZE);
|
|
1440
|
+
if (config.verbose) {
|
|
1441
|
+
console.log(`
|
|
1442
|
+
Risk Detection Configuration:`);
|
|
1443
|
+
console.log(` - Total URLs: ${urls.length.toLocaleString()}`);
|
|
1444
|
+
console.log(` - Batch size: ${BATCH_SIZE.toLocaleString()}`);
|
|
1445
|
+
console.log(` - Concurrency: ${CONCURRENCY}`);
|
|
1446
|
+
console.log(` - Total batches: ${batches.length}`);
|
|
1447
|
+
try {
|
|
1448
|
+
console.log(` - Base domain: ${new URL(baseUrl).hostname}`);
|
|
1449
|
+
} catch (error) {
|
|
1450
|
+
console.log(` - Base URL: ${baseUrl}`);
|
|
1451
|
+
}
|
|
1452
|
+
if (acceptedPatterns.length > 0) {
|
|
1453
|
+
console.log(` - Accepted patterns: ${acceptedPatterns.length}`);
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
let completedBatches = 0;
|
|
1457
|
+
const totalBatches = batches.length;
|
|
1458
|
+
const batchStartTime = Date.now();
|
|
1459
|
+
const batchResults = await processInBatches(
|
|
1460
|
+
batches,
|
|
1461
|
+
CONCURRENCY,
|
|
1462
|
+
(batch) => detectRisksInBatch(batch, allPatterns, acceptedPatterns, expectedProtocol, config.verbose),
|
|
1463
|
+
(completed) => {
|
|
1464
|
+
completedBatches = completed;
|
|
1465
|
+
const pct = (completed / totalBatches * 100).toFixed(1);
|
|
1466
|
+
const elapsed = (Date.now() - batchStartTime) / 1e3;
|
|
1467
|
+
const urlsProcessed = completed * BATCH_SIZE;
|
|
1468
|
+
const speed = Math.round(urlsProcessed / elapsed);
|
|
1469
|
+
const remaining = totalBatches - completed;
|
|
1470
|
+
const eta = Math.round(remaining * BATCH_SIZE / speed);
|
|
1471
|
+
process.stdout.write(
|
|
1472
|
+
`\r\x1B[K Analyzing batch ${completed}/${totalBatches} (${pct}%) | ETA: ~${eta}s | ${speed.toLocaleString()} URLs/sec`
|
|
1473
|
+
);
|
|
1474
|
+
}
|
|
1475
|
+
);
|
|
1476
|
+
process.stdout.write("\r\x1B[K");
|
|
1477
|
+
const allFindings = batchResults.flatMap((r) => r.findings);
|
|
1478
|
+
const groupingResult = groupRiskFindings(allFindings);
|
|
1479
|
+
const processingTimeMs = Date.now() - startTime;
|
|
1480
|
+
if (config.verbose) {
|
|
1481
|
+
console.log(`
|
|
1482
|
+
Risk Detection Summary:`);
|
|
1483
|
+
console.log(` - Total URLs analyzed: ${urls.length.toLocaleString()}`);
|
|
1484
|
+
console.log(` - Risk URLs found: ${groupingResult.totalRiskUrls.toLocaleString()}`);
|
|
1485
|
+
console.log(` - HIGH severity: ${groupingResult.highSeverityCount}`);
|
|
1486
|
+
console.log(` - MEDIUM severity: ${groupingResult.mediumSeverityCount}`);
|
|
1487
|
+
console.log(` - LOW severity: ${groupingResult.lowSeverityCount}`);
|
|
1488
|
+
console.log(` - Processing time: ${(processingTimeMs / 1e3).toFixed(1)}s`);
|
|
1489
|
+
if (groupingResult.groups.length > 0) {
|
|
1490
|
+
console.log(`
|
|
1491
|
+
Risk Categories Found:`);
|
|
1492
|
+
for (const group of groupingResult.groups) {
|
|
1493
|
+
console.log(` - ${group.category}: ${group.count} URLs (${group.severity.toUpperCase()})`);
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
return {
|
|
1498
|
+
findings: allFindings,
|
|
1499
|
+
groups: groupingResult.groups,
|
|
1500
|
+
totalUrlsAnalyzed: urls.length,
|
|
1501
|
+
riskUrlCount: groupingResult.totalRiskUrls,
|
|
1502
|
+
cleanUrlCount: urls.length - groupingResult.totalRiskUrls,
|
|
1503
|
+
highSeverityCount: groupingResult.highSeverityCount,
|
|
1504
|
+
mediumSeverityCount: groupingResult.mediumSeverityCount,
|
|
1505
|
+
lowSeverityCount: groupingResult.lowSeverityCount,
|
|
1506
|
+
processingTimeMs
|
|
1507
|
+
};
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
// src/summarizer.ts
|
|
1511
|
+
function summarizeRisks(request) {
|
|
1512
|
+
const severityBreakdown = {
|
|
1513
|
+
high: 0,
|
|
1514
|
+
medium: 0,
|
|
1515
|
+
low: 0
|
|
1516
|
+
};
|
|
1517
|
+
const categoryInsights = request.riskGroups.map((group) => {
|
|
1518
|
+
severityBreakdown[group.severity] += group.count;
|
|
1519
|
+
const urls = group.allUrls || group.sampleUrls;
|
|
1520
|
+
return {
|
|
1521
|
+
category: group.category,
|
|
1522
|
+
count: group.count,
|
|
1523
|
+
severity: group.severity,
|
|
1524
|
+
summary: group.rationale,
|
|
1525
|
+
examples: urls.slice(0, 3),
|
|
1526
|
+
allUrls: urls
|
|
1527
|
+
// Include all URLs for download functionality
|
|
1528
|
+
};
|
|
1529
|
+
});
|
|
1530
|
+
const totalRisks = request.riskGroups.reduce((sum, g) => sum + g.count, 0);
|
|
1531
|
+
const overview = totalRisks > 0 ? `Found ${totalRisks} potentially risky URLs across ${request.riskGroups.length} categories in ${request.totalUrls} total URLs.` : `Analyzed ${request.totalUrls} URLs. No suspicious patterns detected.`;
|
|
1532
|
+
const keyFindings = [];
|
|
1533
|
+
if (severityBreakdown.high > 0) {
|
|
1534
|
+
keyFindings.push(`${severityBreakdown.high} high-severity issues require immediate attention`);
|
|
1535
|
+
}
|
|
1536
|
+
if (severityBreakdown.medium > 0) {
|
|
1537
|
+
keyFindings.push(`${severityBreakdown.medium} medium-severity issues should be reviewed`);
|
|
1538
|
+
}
|
|
1539
|
+
if (severityBreakdown.low > 0) {
|
|
1540
|
+
keyFindings.push(`${severityBreakdown.low} low-severity items flagged for awareness`);
|
|
1541
|
+
}
|
|
1542
|
+
return {
|
|
1543
|
+
overview,
|
|
1544
|
+
keyFindings,
|
|
1545
|
+
categoryInsights,
|
|
1546
|
+
severityBreakdown,
|
|
1547
|
+
recommendations: [],
|
|
1548
|
+
generatedBy: "rule-based analysis",
|
|
1549
|
+
metadata: {
|
|
1550
|
+
tokensUsed: 0,
|
|
1551
|
+
processingTime: request.processingTime || 0,
|
|
1552
|
+
model: "pattern-matching"
|
|
1553
|
+
}
|
|
1554
|
+
};
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
// src/reporters/json-reporter.ts
|
|
1558
|
+
var TOOL_VERSION = true ? "1.0.0-alpha.1" : "dev";
|
|
1559
|
+
function generateJsonReport(summary, discoveryResult, parseResult, riskGroups, config, startTime, options = {}) {
|
|
1560
|
+
const {
|
|
1561
|
+
pretty = true,
|
|
1562
|
+
indent = 2,
|
|
1563
|
+
performanceMetrics
|
|
1564
|
+
} = options;
|
|
1565
|
+
const result = buildAnalysisResult(
|
|
1566
|
+
summary,
|
|
1567
|
+
discoveryResult,
|
|
1568
|
+
parseResult,
|
|
1569
|
+
riskGroups,
|
|
1570
|
+
config,
|
|
1571
|
+
startTime
|
|
1572
|
+
);
|
|
1573
|
+
const jsonOutput = transformToJsonOutput(result, performanceMetrics);
|
|
1574
|
+
if (pretty) {
|
|
1575
|
+
return JSON.stringify(jsonOutput, null, indent);
|
|
1576
|
+
} else {
|
|
1577
|
+
return JSON.stringify(jsonOutput);
|
|
1578
|
+
}
|
|
1579
|
+
}
|
|
1580
|
+
function buildAnalysisResult(summary, discoveryResult, parseResult, riskGroups, config, startTime) {
|
|
1581
|
+
const metadata = buildAnalysisMetadata(
|
|
1582
|
+
config.baseUrl || "unknown",
|
|
1583
|
+
startTime,
|
|
1584
|
+
summary
|
|
1585
|
+
);
|
|
1586
|
+
const suspiciousGroups = riskGroups.map((group) => ({
|
|
1587
|
+
category: group.category,
|
|
1588
|
+
severity: group.severity,
|
|
1589
|
+
count: group.count,
|
|
1590
|
+
pattern: group.category,
|
|
1591
|
+
// Use category as pattern identifier
|
|
1592
|
+
rationale: group.rationale,
|
|
1593
|
+
sampleUrls: group.sampleUrls.slice(0, 5),
|
|
1594
|
+
// Limit to 5 samples
|
|
1595
|
+
recommendedAction: group.recommendedAction
|
|
1596
|
+
}));
|
|
1597
|
+
const summaryStats = {
|
|
1598
|
+
highSeverityCount: summary.severityBreakdown.high,
|
|
1599
|
+
mediumSeverityCount: summary.severityBreakdown.medium,
|
|
1600
|
+
lowSeverityCount: summary.severityBreakdown.low,
|
|
1601
|
+
totalRiskyUrls: riskGroups.reduce((sum, g) => sum + g.count, 0),
|
|
1602
|
+
overallStatus: determineOverallStatus(
|
|
1603
|
+
summary.severityBreakdown,
|
|
1604
|
+
parseResult.errors
|
|
1605
|
+
)
|
|
1606
|
+
};
|
|
1607
|
+
const riskSummary = {
|
|
1608
|
+
overview: summary.overview,
|
|
1609
|
+
keyFindings: summary.keyFindings,
|
|
1610
|
+
recommendations: summary.recommendations
|
|
1611
|
+
};
|
|
1612
|
+
const errors = parseResult.errors.map(transformError);
|
|
1613
|
+
return {
|
|
1614
|
+
analysisMetadata: metadata,
|
|
1615
|
+
sitemapsDiscovered: discoveryResult.sitemaps,
|
|
1616
|
+
totalUrlCount: parseResult.totalCount,
|
|
1617
|
+
urlsAnalyzed: parseResult.totalCount,
|
|
1618
|
+
suspiciousGroups,
|
|
1619
|
+
riskSummary,
|
|
1620
|
+
summary: summaryStats,
|
|
1621
|
+
errors
|
|
1622
|
+
};
|
|
1623
|
+
}
|
|
1624
|
+
function buildAnalysisMetadata(baseUrl, startTime, summary) {
|
|
1625
|
+
return {
|
|
1626
|
+
baseUrl,
|
|
1627
|
+
analysisTimestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1628
|
+
toolVersion: TOOL_VERSION,
|
|
1629
|
+
executionTimeMs: Date.now() - startTime,
|
|
1630
|
+
analysisType: summary.generatedBy
|
|
1631
|
+
};
|
|
1632
|
+
}
|
|
1633
|
+
function determineOverallStatus(severityBreakdown, errors) {
|
|
1634
|
+
if (errors.length > 0) {
|
|
1635
|
+
return "errors";
|
|
1636
|
+
}
|
|
1637
|
+
const totalIssues = severityBreakdown.high + severityBreakdown.medium + severityBreakdown.low;
|
|
1638
|
+
return totalIssues > 0 ? "issues_found" : "clean";
|
|
1639
|
+
}
|
|
1640
|
+
function transformToJsonOutput(result, performanceMetrics) {
|
|
1641
|
+
const output = {
|
|
1642
|
+
analysis_metadata: transformMetadata(result.analysisMetadata),
|
|
1643
|
+
sitemaps_discovered: result.sitemapsDiscovered,
|
|
1644
|
+
total_url_count: result.totalUrlCount,
|
|
1645
|
+
urls_analyzed: result.urlsAnalyzed,
|
|
1646
|
+
suspicious_groups: result.suspiciousGroups.map(transformGroup),
|
|
1647
|
+
risk_summary: transformRiskSummary(result.riskSummary),
|
|
1648
|
+
summary: transformSummary(result.summary),
|
|
1649
|
+
errors: result.errors
|
|
1650
|
+
};
|
|
1651
|
+
if (performanceMetrics) {
|
|
1652
|
+
output.performance_metrics = {
|
|
1653
|
+
total_execution_time_ms: performanceMetrics.totalExecutionTimeMs,
|
|
1654
|
+
phase_timings: performanceMetrics.phaseTimings,
|
|
1655
|
+
throughput: performanceMetrics.throughput,
|
|
1656
|
+
resource_usage: performanceMetrics.resourceUsage
|
|
1657
|
+
};
|
|
1658
|
+
}
|
|
1659
|
+
return output;
|
|
1660
|
+
}
|
|
1661
|
+
function transformMetadata(meta) {
|
|
1662
|
+
return {
|
|
1663
|
+
base_url: meta.baseUrl,
|
|
1664
|
+
analysis_timestamp: meta.analysisTimestamp,
|
|
1665
|
+
tool_version: meta.toolVersion,
|
|
1666
|
+
execution_time_ms: meta.executionTimeMs,
|
|
1667
|
+
analysis_type: meta.analysisType
|
|
1668
|
+
};
|
|
1669
|
+
}
|
|
1670
|
+
function transformGroup(group) {
|
|
1671
|
+
return {
|
|
1672
|
+
category: group.category,
|
|
1673
|
+
severity: group.severity,
|
|
1674
|
+
count: group.count,
|
|
1675
|
+
pattern: group.pattern,
|
|
1676
|
+
rationale: group.rationale,
|
|
1677
|
+
sample_urls: group.sampleUrls,
|
|
1678
|
+
recommended_action: group.recommendedAction
|
|
1679
|
+
};
|
|
1680
|
+
}
|
|
1681
|
+
function transformRiskSummary(summary) {
|
|
1682
|
+
return {
|
|
1683
|
+
overview: summary.overview,
|
|
1684
|
+
key_findings: summary.keyFindings,
|
|
1685
|
+
recommendations: summary.recommendations
|
|
1686
|
+
};
|
|
1687
|
+
}
|
|
1688
|
+
function transformSummary(summary) {
|
|
1689
|
+
return {
|
|
1690
|
+
high_severity_count: summary.highSeverityCount,
|
|
1691
|
+
medium_severity_count: summary.mediumSeverityCount,
|
|
1692
|
+
low_severity_count: summary.lowSeverityCount,
|
|
1693
|
+
total_risky_urls: summary.totalRiskyUrls,
|
|
1694
|
+
overall_status: summary.overallStatus
|
|
1695
|
+
};
|
|
1696
|
+
}
|
|
1697
|
+
function transformError(error) {
|
|
1698
|
+
if ("code" in error) {
|
|
1699
|
+
const customError = error;
|
|
1700
|
+
const errorDetail = {
|
|
1701
|
+
code: customError.code || "UNKNOWN_ERROR",
|
|
1702
|
+
message: error.message
|
|
1703
|
+
};
|
|
1704
|
+
if ("attemptedPaths" in customError) {
|
|
1705
|
+
errorDetail.context = {
|
|
1706
|
+
attempted_paths: customError.attemptedPaths
|
|
1707
|
+
};
|
|
1708
|
+
} else if ("sitemapUrl" in customError && "lineNumber" in customError) {
|
|
1709
|
+
errorDetail.context = {
|
|
1710
|
+
sitemap_url: customError.sitemapUrl,
|
|
1711
|
+
line_number: customError.lineNumber
|
|
1712
|
+
};
|
|
1713
|
+
} else if ("url" in customError) {
|
|
1714
|
+
errorDetail.context = {
|
|
1715
|
+
url: customError.url
|
|
1716
|
+
};
|
|
1717
|
+
}
|
|
1718
|
+
return errorDetail;
|
|
1719
|
+
}
|
|
1720
|
+
return {
|
|
1721
|
+
code: "UNKNOWN_ERROR",
|
|
1722
|
+
message: error.message
|
|
1723
|
+
};
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
// src/reporters/html-reporter.ts
|
|
1727
|
+
var import_fs2 = require("fs");
|
|
1728
|
+
var TOOL_VERSION2 = "1.0.0-alpha.1";
|
|
1729
|
+
function generateHtmlReport(summary, discoveryResult, totalUrls, config, errors, options = {}) {
|
|
1730
|
+
const maxUrls = options.maxUrlsPerGroup ?? 10;
|
|
1731
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
1732
|
+
const riskyUrlCount = summary.categoryInsights.reduce((sum, g) => sum + g.count, 0);
|
|
1733
|
+
const highSeverity = summary.categoryInsights.filter((g) => g.severity === "high");
|
|
1734
|
+
const mediumSeverity = summary.categoryInsights.filter((g) => g.severity === "medium");
|
|
1735
|
+
const lowSeverity = summary.categoryInsights.filter((g) => g.severity === "low");
|
|
1736
|
+
const html = `<!DOCTYPE html>
|
|
20
1737
|
<html lang="en">
|
|
21
1738
|
<head>
|
|
22
1739
|
<meta charset="UTF-8">
|
|
23
1740
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
24
|
-
<title>Sitemap QA Report - ${
|
|
1741
|
+
<title>Sitemap QA Report - ${config.baseUrl}</title>
|
|
25
1742
|
<style>
|
|
26
1743
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
27
1744
|
body {
|
|
@@ -339,93 +2056,88 @@ Risk Categories Found:`);for(let p of l.groups)console.log(` - ${p.category}: $
|
|
|
339
2056
|
<div class="header">
|
|
340
2057
|
<h1>Sitemap Analysis</h1>
|
|
341
2058
|
<div class="meta">
|
|
342
|
-
<div>${
|
|
343
|
-
<div>${new Date(
|
|
2059
|
+
<div>${config.baseUrl}</div>
|
|
2060
|
+
<div>${new Date(timestamp).toLocaleString()}</div>
|
|
344
2061
|
</div>
|
|
345
2062
|
</div>
|
|
346
2063
|
|
|
347
2064
|
<div class="summary">
|
|
348
2065
|
<div class="summary-card">
|
|
349
2066
|
<div class="label">Sitemaps</div>
|
|
350
|
-
<div class="value">${
|
|
2067
|
+
<div class="value">${discoveryResult.sitemaps.length}</div>
|
|
351
2068
|
</div>
|
|
352
2069
|
<div class="summary-card">
|
|
353
2070
|
<div class="label">URLs Analyzed</div>
|
|
354
|
-
<div class="value">${
|
|
2071
|
+
<div class="value">${totalUrls.toLocaleString()}</div>
|
|
355
2072
|
</div>
|
|
356
2073
|
<div class="summary-card">
|
|
357
2074
|
<div class="label">Issues Found</div>
|
|
358
|
-
<div class="value" style="color: ${
|
|
2075
|
+
<div class="value" style="color: ${riskyUrlCount > 0 ? "#dc2626" : "#059669"}">${riskyUrlCount}</div>
|
|
359
2076
|
</div>
|
|
360
2077
|
<div class="summary-card">
|
|
361
2078
|
<div class="label">Scan Time</div>
|
|
362
|
-
<div class="value">${(
|
|
2079
|
+
<div class="value">${(summary.metadata.processingTime / 1e3).toFixed(1)}s</div>
|
|
363
2080
|
</div>
|
|
364
2081
|
</div>
|
|
365
2082
|
|
|
366
2083
|
<div class="content">
|
|
367
|
-
${
|
|
2084
|
+
${errors.length > 0 ? `
|
|
368
2085
|
<div class="errors-section">
|
|
369
|
-
<h3>Parsing Errors & Warnings (${
|
|
2086
|
+
<h3>Parsing Errors & Warnings (${errors.length})</h3>
|
|
370
2087
|
<ul>
|
|
371
|
-
${
|
|
372
|
-
`)}
|
|
2088
|
+
${errors.map((err) => `<li>${err.message}</li>`).join("\n ")}
|
|
373
2089
|
</ul>
|
|
374
2090
|
</div>
|
|
375
|
-
|
|
2091
|
+
` : ""}
|
|
376
2092
|
|
|
377
|
-
${
|
|
2093
|
+
${discoveryResult.sitemaps.length > 0 ? `
|
|
378
2094
|
<div class="sitemaps">
|
|
379
|
-
<h3 class="collapsed" onclick="toggleSection(this)">Sitemaps Discovered (${
|
|
2095
|
+
<h3 class="collapsed" onclick="toggleSection(this)">Sitemaps Discovered (${discoveryResult.sitemaps.length})</h3>
|
|
380
2096
|
<div class="sitemaps-content collapsed">
|
|
381
2097
|
<ul>
|
|
382
|
-
${
|
|
383
|
-
`)}
|
|
2098
|
+
${discoveryResult.sitemaps.map((s) => `<li>\u2022 ${s}</li>`).join("\n ")}
|
|
384
2099
|
</ul>
|
|
385
2100
|
</div>
|
|
386
2101
|
</div>
|
|
387
|
-
|
|
2102
|
+
` : ""}
|
|
388
2103
|
|
|
389
|
-
${
|
|
2104
|
+
${riskyUrlCount === 0 ? `
|
|
390
2105
|
<div class="status-clean">
|
|
391
2106
|
<h2>No Issues Found</h2>
|
|
392
2107
|
<p>All URLs in the sitemap passed validation checks.</p>
|
|
393
2108
|
</div>
|
|
394
|
-
|
|
2109
|
+
` : ""}
|
|
395
2110
|
|
|
396
|
-
${
|
|
2111
|
+
${highSeverity.length > 0 ? `
|
|
397
2112
|
<div class="severity-section">
|
|
398
|
-
<h2 class="severity-high" onclick="toggleSection(this)">High Severity (${
|
|
2113
|
+
<h2 class="severity-high" onclick="toggleSection(this)">High Severity (${highSeverity.reduce((sum, g) => sum + g.count, 0)} URLs)</h2>
|
|
399
2114
|
<div class="severity-content">
|
|
400
|
-
${
|
|
401
|
-
`)}
|
|
2115
|
+
${highSeverity.map((group) => renderRiskGroup(group, maxUrls)).join("\n ")}
|
|
402
2116
|
</div>
|
|
403
2117
|
</div>
|
|
404
|
-
|
|
2118
|
+
` : ""}
|
|
405
2119
|
|
|
406
|
-
${
|
|
2120
|
+
${mediumSeverity.length > 0 ? `
|
|
407
2121
|
<div class="severity-section">
|
|
408
|
-
<h2 class="severity-medium" onclick="toggleSection(this)">Medium Severity (${
|
|
2122
|
+
<h2 class="severity-medium" onclick="toggleSection(this)">Medium Severity (${mediumSeverity.reduce((sum, g) => sum + g.count, 0)} URLs)</h2>
|
|
409
2123
|
<div class="severity-content">
|
|
410
|
-
${
|
|
411
|
-
`)}
|
|
2124
|
+
${mediumSeverity.map((group) => renderRiskGroup(group, maxUrls)).join("\n ")}
|
|
412
2125
|
</div>
|
|
413
2126
|
</div>
|
|
414
|
-
|
|
2127
|
+
` : ""}
|
|
415
2128
|
|
|
416
|
-
${
|
|
2129
|
+
${lowSeverity.length > 0 ? `
|
|
417
2130
|
<div class="severity-section">
|
|
418
|
-
<h2 class="severity-low" onclick="toggleSection(this)">Low Severity (${
|
|
2131
|
+
<h2 class="severity-low" onclick="toggleSection(this)">Low Severity (${lowSeverity.reduce((sum, g) => sum + g.count, 0)} URLs)</h2>
|
|
419
2132
|
<div class="severity-content">
|
|
420
|
-
${
|
|
421
|
-
`)}
|
|
2133
|
+
${lowSeverity.map((group) => renderRiskGroup(group, maxUrls)).join("\n ")}
|
|
422
2134
|
</div>
|
|
423
2135
|
</div>
|
|
424
|
-
|
|
2136
|
+
` : ""}
|
|
425
2137
|
</div>
|
|
426
2138
|
|
|
427
2139
|
<div class="footer">
|
|
428
|
-
Generated by <strong>sitemap-qa</strong> v${
|
|
2140
|
+
Generated by <strong>sitemap-qa</strong> v${TOOL_VERSION2}
|
|
429
2141
|
</div>
|
|
430
2142
|
</div>
|
|
431
2143
|
|
|
@@ -458,27 +2170,354 @@ Risk Categories Found:`);for(let p of l.groups)console.log(` - ${p.category}: $
|
|
|
458
2170
|
}
|
|
459
2171
|
</script>
|
|
460
2172
|
</body>
|
|
461
|
-
</html
|
|
462
|
-
|
|
463
|
-
|
|
2173
|
+
</html>`;
|
|
2174
|
+
return html;
|
|
2175
|
+
}
|
|
2176
|
+
function renderRiskGroup(group, maxUrls) {
|
|
2177
|
+
const categoryTitle = group.category.split("_").map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
|
|
2178
|
+
const urlsToShow = group.examples.slice(0, maxUrls);
|
|
2179
|
+
const remaining = group.count - urlsToShow.length;
|
|
2180
|
+
const categorySlug = group.category.toLowerCase();
|
|
2181
|
+
const allUrlsJson = JSON.stringify(group.allUrls);
|
|
2182
|
+
const encodedUrls = escapeHtml(allUrlsJson);
|
|
2183
|
+
return `<div class="risk-group">
|
|
2184
|
+
<h3>${categoryTitle} <span class="count">${group.count} URLs</span></h3>
|
|
2185
|
+
<div class="impact">${group.summary}</div>
|
|
464
2186
|
<div class="urls">
|
|
465
2187
|
<h4>Sample URLs</h4>
|
|
466
2188
|
<ul>
|
|
467
|
-
${
|
|
468
|
-
`)}
|
|
2189
|
+
${urlsToShow.map((url) => `<li>${escapeHtml(url)}</li>`).join("\n ")}
|
|
469
2190
|
</ul>
|
|
470
|
-
${
|
|
471
|
-
<button class="download-btn" onclick="downloadUrls('${
|
|
2191
|
+
${remaining > 0 ? `<div class="more">... and ${remaining} more</div>` : ""}
|
|
2192
|
+
<button class="download-btn" onclick="downloadUrls('${categorySlug}', '${encodedUrls}')">\u{1F4E5} Download All ${group.count} URLs</button>
|
|
472
2193
|
</div>
|
|
473
|
-
</div
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
2194
|
+
</div>`;
|
|
2195
|
+
}
|
|
2196
|
+
function escapeHtml(text) {
|
|
2197
|
+
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
2198
|
+
}
|
|
2199
|
+
async function writeHtmlReport(summary, discoveryResult, totalUrls, config, outputPath, errors, options = {}) {
|
|
2200
|
+
const htmlContent = generateHtmlReport(summary, discoveryResult, totalUrls, config, errors, options);
|
|
2201
|
+
await import_fs2.promises.writeFile(outputPath, htmlContent, "utf-8");
|
|
2202
|
+
}
|
|
2203
|
+
|
|
2204
|
+
// src/commands/analyze.ts
|
|
2205
|
+
var analyzeCommand = new import_commander.Command("analyze").description("Analyze sitemap for QA issues").argument("<url>", "Base URL to analyze").option("--timeout <seconds>", "HTTP timeout in seconds", "30").option("--no-progress", "Disable progress bar").option("--output <format>", "Output format: html or json", "html").option("--output-dir <path>", "Output directory for reports").option("--output-file <path>", "Custom output filename").option("--accepted-patterns <patterns>", "Comma-separated regex patterns to exclude from risk detection").option("--concurrency <number>", "Number of concurrent workers for risk detection").option("--batch-size <number>", "URLs per batch for risk detection", "10000").option("--parsing-concurrency <number>", "Number of concurrent sitemap parsers", "50").option("--discovery-concurrency <number>", "Number of concurrent sitemap index fetches", "50").option("--silent", "Disable all progress output").option("--benchmark", "Save performance profile").option("--no-color", "Disable ANSI color codes in CLI output").option("--verbose", "Enable verbose logging", false).action(async (url, options) => {
|
|
2206
|
+
let config;
|
|
2207
|
+
try {
|
|
2208
|
+
validateAnalyzeOptions(options);
|
|
2209
|
+
const loadedConfig = await loadConfig({
|
|
2210
|
+
...options,
|
|
2211
|
+
baseUrl: url,
|
|
2212
|
+
outputFormat: options.output,
|
|
2213
|
+
riskDetectionConcurrency: options.concurrency ? parseInt(options.concurrency) : void 0,
|
|
2214
|
+
riskDetectionBatchSize: options.batchSize ? parseInt(options.batchSize) : void 0,
|
|
2215
|
+
parsingConcurrency: options.parsingConcurrency ? parseInt(options.parsingConcurrency) : void 0,
|
|
2216
|
+
discoveryConcurrency: options.discoveryConcurrency ? parseInt(options.discoveryConcurrency) : void 0,
|
|
2217
|
+
silent: options.silent,
|
|
2218
|
+
benchmark: options.benchmark,
|
|
2219
|
+
progressBar: options.progress
|
|
2220
|
+
});
|
|
2221
|
+
config = loadedConfig;
|
|
2222
|
+
console.log(`
|
|
2223
|
+
\u{1F50D} Analyzing ${url}...
|
|
2224
|
+
`);
|
|
2225
|
+
const result = await runAnalysisPipeline(url, config);
|
|
2226
|
+
await import_fs3.promises.mkdir(config.outputDir, { recursive: true });
|
|
2227
|
+
if (options.output === "json") {
|
|
2228
|
+
const jsonReport = generateJsonReport(
|
|
2229
|
+
result.summary,
|
|
2230
|
+
result.discoveryResult,
|
|
2231
|
+
{ totalCount: result.totalUrls, uniqueUrls: [], errors: [] },
|
|
2232
|
+
result.riskGroups,
|
|
2233
|
+
config,
|
|
2234
|
+
result.executionTime,
|
|
2235
|
+
{ pretty: true, indent: 2 }
|
|
2236
|
+
);
|
|
2237
|
+
console.log("\n" + jsonReport);
|
|
2238
|
+
if (options.outputFile) {
|
|
2239
|
+
const jsonFilePath = `${config.outputDir}/${options.outputFile}`;
|
|
2240
|
+
await import_fs3.promises.writeFile(jsonFilePath, jsonReport, "utf-8");
|
|
2241
|
+
console.log(`
|
|
2242
|
+
\u{1F4C4} JSON report saved to: ${import_chalk.default.cyan(jsonFilePath)}`);
|
|
2243
|
+
}
|
|
2244
|
+
} else {
|
|
2245
|
+
showCliSummary(result);
|
|
2246
|
+
const htmlFileName = options.outputFile || `sitemap-qa-report-${Date.now()}.html`;
|
|
2247
|
+
const htmlFilePath = `${config.outputDir}/${htmlFileName}`;
|
|
2248
|
+
await writeHtmlReport(
|
|
2249
|
+
result.summary,
|
|
2250
|
+
result.discoveryResult,
|
|
2251
|
+
result.totalUrls,
|
|
2252
|
+
config,
|
|
2253
|
+
htmlFilePath,
|
|
2254
|
+
result.errors,
|
|
2255
|
+
{ maxUrlsPerGroup: 10 }
|
|
2256
|
+
);
|
|
2257
|
+
console.log(`
|
|
2258
|
+
\u{1F4C4} Full report saved to: ${import_chalk.default.cyan(htmlFilePath)}`);
|
|
2259
|
+
}
|
|
2260
|
+
const exitCode = determineExitCode(result);
|
|
2261
|
+
process.exit(exitCode);
|
|
2262
|
+
} catch (error) {
|
|
2263
|
+
handleAnalysisError(error, config);
|
|
2264
|
+
process.exit(2);
|
|
2265
|
+
}
|
|
2266
|
+
});
|
|
2267
|
+
function validateAnalyzeOptions(options) {
|
|
2268
|
+
const validFormats = ["json", "html"];
|
|
2269
|
+
if (!validFormats.includes(options.output)) {
|
|
2270
|
+
throw new Error(
|
|
2271
|
+
`Invalid output format: ${options.output}. Must be one of: ${validFormats.join(", ")}`
|
|
2272
|
+
);
|
|
2273
|
+
}
|
|
2274
|
+
const timeout = parseInt(options.timeout);
|
|
2275
|
+
if (isNaN(timeout) || timeout <= 0) {
|
|
2276
|
+
throw new Error(`Invalid timeout: ${options.timeout}. Must be a positive number.`);
|
|
2277
|
+
}
|
|
2278
|
+
}
|
|
2279
|
+
function showCliSummary(result) {
|
|
2280
|
+
const riskyUrlCount = result.summary.categoryInsights.reduce((sum, g) => sum + g.count, 0);
|
|
2281
|
+
console.log(import_chalk.default.dim("\u2500".repeat(50)));
|
|
2282
|
+
if (riskyUrlCount === 0) {
|
|
2283
|
+
console.log(import_chalk.default.green("\u2705 No issues found - sitemap looks clean!"));
|
|
2284
|
+
} else {
|
|
2285
|
+
const { high, medium, low } = result.summary.severityBreakdown;
|
|
2286
|
+
const severityParts = [];
|
|
2287
|
+
if (high > 0) severityParts.push(import_chalk.default.red(`High: ${high}`));
|
|
2288
|
+
if (medium > 0) severityParts.push(import_chalk.default.yellow(`Medium: ${medium}`));
|
|
2289
|
+
if (low > 0) severityParts.push(import_chalk.default.blue(`Low: ${low}`));
|
|
2290
|
+
const severitySummary = severityParts.length > 0 ? ` (${severityParts.join(", ")})` : "";
|
|
2291
|
+
console.log(import_chalk.default.yellow(`\u26A0\uFE0F ${riskyUrlCount} risky URLs found${severitySummary}`));
|
|
2292
|
+
}
|
|
2293
|
+
console.log("");
|
|
2294
|
+
}
|
|
2295
|
+
async function runAnalysisPipeline(url, config) {
|
|
2296
|
+
const overallStartTime = Date.now();
|
|
2297
|
+
const phaseTimings = [];
|
|
2298
|
+
const errors = [];
|
|
2299
|
+
const showProgress = !config.silent && config.progressBar !== false && process.stdout.isTTY;
|
|
2300
|
+
let phaseStart = Date.now();
|
|
2301
|
+
const discoverySpinner = showProgress ? (0, import_ora.default)({ text: "Discovering sitemaps...", color: "cyan" }).start() : null;
|
|
2302
|
+
const discoveryResult = await discoverSitemaps(url, config);
|
|
2303
|
+
if (discoverySpinner) {
|
|
2304
|
+
discoverySpinner.stop();
|
|
2305
|
+
}
|
|
2306
|
+
phaseTimings.push({
|
|
2307
|
+
name: "Discovery",
|
|
2308
|
+
startTime: phaseStart,
|
|
2309
|
+
endTime: Date.now(),
|
|
2310
|
+
duration: Date.now() - phaseStart
|
|
2311
|
+
});
|
|
2312
|
+
if (discoveryResult.accessIssues.length > 0) {
|
|
2313
|
+
if (!config.silent) {
|
|
2314
|
+
console.warn(import_chalk.default.yellow(`\u26A0\uFE0F Warning: ${discoveryResult.accessIssues.length} sitemap(s) are access-blocked`));
|
|
2315
|
+
}
|
|
2316
|
+
for (const issue of discoveryResult.accessIssues) {
|
|
2317
|
+
errors.push(new Error(`Access blocked: ${issue.url} (${issue.statusCode})`));
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
if (discoveryResult.sitemaps.length === 0) {
|
|
2321
|
+
throw new Error(`No sitemaps found at ${url}. Tried: /sitemap.xml, /sitemap_index.xml, /robots.txt`);
|
|
2322
|
+
}
|
|
2323
|
+
phaseStart = Date.now();
|
|
2324
|
+
let extractionResult;
|
|
2325
|
+
if (showProgress && discoveryResult.sitemaps.length > 10) {
|
|
2326
|
+
const parseBar = new import_cli_progress.default.SingleBar({
|
|
2327
|
+
format: "{bar} {percentage}% | {value}/{total} | ETA: {eta}s | {speed} sitemaps/sec",
|
|
2328
|
+
barCompleteChar: "\u2588",
|
|
2329
|
+
barIncompleteChar: "\u2591",
|
|
2330
|
+
hideCursor: true
|
|
2331
|
+
});
|
|
2332
|
+
parseBar.start(discoveryResult.sitemaps.length, 0, { speed: "0" });
|
|
2333
|
+
extractionResult = await extractAllUrls(
|
|
2334
|
+
discoveryResult.sitemaps,
|
|
2335
|
+
config,
|
|
2336
|
+
(completed, total) => {
|
|
2337
|
+
const elapsed = (Date.now() - phaseStart) / 1e3;
|
|
2338
|
+
const speed = elapsed > 0 ? (completed / elapsed).toFixed(1) : "0";
|
|
2339
|
+
parseBar.update(completed, { speed });
|
|
2340
|
+
}
|
|
2341
|
+
);
|
|
2342
|
+
parseBar.stop();
|
|
2343
|
+
} else {
|
|
2344
|
+
extractionResult = await extractAllUrls(discoveryResult.sitemaps, config);
|
|
2345
|
+
}
|
|
2346
|
+
phaseTimings.push({
|
|
2347
|
+
name: "Parsing",
|
|
2348
|
+
startTime: phaseStart,
|
|
2349
|
+
endTime: Date.now(),
|
|
2350
|
+
duration: Date.now() - phaseStart
|
|
2351
|
+
});
|
|
2352
|
+
if (extractionResult.errors.length > 0) {
|
|
2353
|
+
for (const err of extractionResult.errors) {
|
|
2354
|
+
if (typeof err === "string") {
|
|
2355
|
+
errors.push(new Error(err));
|
|
2356
|
+
} else {
|
|
2357
|
+
errors.push(err);
|
|
2358
|
+
}
|
|
2359
|
+
}
|
|
2360
|
+
}
|
|
2361
|
+
if (extractionResult.allUrls.length === 0) {
|
|
2362
|
+
throw new Error("No URLs extracted from sitemaps");
|
|
2363
|
+
}
|
|
2364
|
+
phaseStart = Date.now();
|
|
2365
|
+
const consolidatedResult = consolidateUrls(extractionResult.allUrls);
|
|
2366
|
+
phaseTimings.push({
|
|
2367
|
+
name: "Deduplication",
|
|
2368
|
+
startTime: phaseStart,
|
|
2369
|
+
endTime: Date.now(),
|
|
2370
|
+
duration: Date.now() - phaseStart
|
|
2371
|
+
});
|
|
2372
|
+
const duplicatesRemoved = extractionResult.allUrls.length - consolidatedResult.uniqueUrls.length;
|
|
2373
|
+
const duplicatePercentage = duplicatesRemoved / extractionResult.allUrls.length * 100;
|
|
2374
|
+
if (!config.silent) {
|
|
2375
|
+
if (duplicatesRemoved > 100 || duplicatePercentage > 1) {
|
|
2376
|
+
console.log(import_chalk.default.green(`\u2713 Analyzed ${discoveryResult.sitemaps.length} sitemap(s) \u2192 ${extractionResult.allUrls.length.toLocaleString()} URLs (${consolidatedResult.uniqueUrls.length.toLocaleString()} unique)`));
|
|
2377
|
+
} else {
|
|
2378
|
+
console.log(import_chalk.default.green(`\u2713 Analyzed ${discoveryResult.sitemaps.length} sitemap(s) \u2192 ${extractionResult.allUrls.length.toLocaleString()} URLs`));
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
2381
|
+
phaseStart = Date.now();
|
|
2382
|
+
const riskResult = await detectRisks(consolidatedResult.uniqueUrls, url, config);
|
|
2383
|
+
const riskGroups = groupRiskFindings(riskResult.findings);
|
|
2384
|
+
phaseTimings.push({
|
|
2385
|
+
name: "Risk Detection",
|
|
2386
|
+
startTime: phaseStart,
|
|
2387
|
+
endTime: Date.now(),
|
|
2388
|
+
duration: Date.now() - phaseStart
|
|
2389
|
+
});
|
|
2390
|
+
phaseStart = Date.now();
|
|
2391
|
+
const executionTime = Date.now() - overallStartTime;
|
|
2392
|
+
const summary = summarizeRisks({
|
|
2393
|
+
riskGroups: riskGroups.groups,
|
|
2394
|
+
totalUrls: consolidatedResult.uniqueUrls.length,
|
|
2395
|
+
sitemapUrl: url,
|
|
2396
|
+
processingTime: executionTime
|
|
2397
|
+
});
|
|
2398
|
+
phaseTimings.push({
|
|
2399
|
+
name: "Summarization",
|
|
2400
|
+
startTime: phaseStart,
|
|
2401
|
+
endTime: Date.now(),
|
|
2402
|
+
duration: Date.now() - phaseStart
|
|
2403
|
+
});
|
|
2404
|
+
if (!config.silent && config.verbose) {
|
|
2405
|
+
displayPhaseSummary(phaseTimings, executionTime);
|
|
2406
|
+
} else if (!config.silent) {
|
|
2407
|
+
const parsingPhase = phaseTimings.find((p) => p.name === "Parsing");
|
|
2408
|
+
const sitemapsPerSec = parsingPhase ? (discoveryResult.sitemaps.length / (parsingPhase.duration / 1e3)).toFixed(1) : "0";
|
|
2409
|
+
console.log(import_chalk.default.green(`\u2705 Analysis complete (${(executionTime / 1e3).toFixed(1)}s \xB7 ${sitemapsPerSec} sitemaps/sec)
|
|
2410
|
+
`));
|
|
2411
|
+
}
|
|
2412
|
+
if (config.benchmark) {
|
|
2413
|
+
await saveBenchmark(phaseTimings, url, executionTime, discoveryResult.sitemaps.length, consolidatedResult.uniqueUrls.length, config);
|
|
2414
|
+
}
|
|
2415
|
+
return {
|
|
2416
|
+
discoveryResult,
|
|
2417
|
+
totalUrls: consolidatedResult.uniqueUrls.length,
|
|
2418
|
+
riskGroups: riskGroups.groups,
|
|
2419
|
+
summary,
|
|
2420
|
+
errors,
|
|
2421
|
+
executionTime,
|
|
2422
|
+
phaseTimings
|
|
2423
|
+
};
|
|
2424
|
+
}
|
|
2425
|
+
function determineExitCode(result) {
|
|
2426
|
+
const highSeverityCount = result.summary.severityBreakdown.high;
|
|
2427
|
+
if (highSeverityCount > 0) {
|
|
2428
|
+
return 1;
|
|
2429
|
+
}
|
|
2430
|
+
return 0;
|
|
2431
|
+
}
|
|
2432
|
+
function handleAnalysisError(error, config) {
|
|
2433
|
+
console.error("\n\u274C Analysis failed\n");
|
|
2434
|
+
if (error instanceof Error) {
|
|
2435
|
+
console.error(`Error: ${error.message}`);
|
|
2436
|
+
if (config?.verbose && error.stack) {
|
|
2437
|
+
console.error("\nStack trace:");
|
|
2438
|
+
console.error(error.stack);
|
|
2439
|
+
}
|
|
2440
|
+
if (error.message.includes("No sitemaps found")) {
|
|
2441
|
+
console.error("\nSuggestions:");
|
|
2442
|
+
console.error(" \u2022 Verify the base URL is correct");
|
|
2443
|
+
console.error(" \u2022 Check if the site has a sitemap");
|
|
2444
|
+
console.error(" \u2022 Ensure the sitemap is publicly accessible");
|
|
2445
|
+
} else if (error.message.includes("Network") || error.message.includes("timeout")) {
|
|
2446
|
+
console.error("\nSuggestions:");
|
|
2447
|
+
console.error(" \u2022 Check your internet connection");
|
|
2448
|
+
console.error(" \u2022 Verify the URL is accessible");
|
|
2449
|
+
console.error(" \u2022 Try increasing the timeout with --timeout option");
|
|
2450
|
+
}
|
|
2451
|
+
} else {
|
|
2452
|
+
console.error("Unknown error occurred");
|
|
2453
|
+
console.error(String(error));
|
|
2454
|
+
}
|
|
2455
|
+
}
|
|
2456
|
+
function displayPhaseSummary(timings, totalTime) {
|
|
2457
|
+
console.log(import_chalk.default.green(`
|
|
2458
|
+
\u2705 Analysis Complete (Total: ${(totalTime / 1e3).toFixed(1)}s)
|
|
2459
|
+
`));
|
|
2460
|
+
console.log(import_chalk.default.cyan("Phase Breakdown:"));
|
|
2461
|
+
for (const timing of timings) {
|
|
2462
|
+
const seconds = (timing.duration / 1e3).toFixed(1);
|
|
2463
|
+
const percentage = (timing.duration / totalTime * 100).toFixed(1);
|
|
2464
|
+
const bar = "\u2022";
|
|
2465
|
+
console.log(` ${bar} ${timing.name.padEnd(15)}: ${seconds.padStart(5)}s (${percentage.padStart(5)}%)`);
|
|
2466
|
+
}
|
|
2467
|
+
console.log("");
|
|
2468
|
+
}
|
|
2469
|
+
async function saveBenchmark(timings, url, totalTime, sitemapCount, urlCount, config) {
|
|
2470
|
+
const benchmark = {
|
|
2471
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2472
|
+
url,
|
|
2473
|
+
total_duration_ms: totalTime,
|
|
2474
|
+
phases: timings.map((t) => ({
|
|
2475
|
+
name: t.name.toLowerCase(),
|
|
2476
|
+
start_ms: t.startTime,
|
|
2477
|
+
end_ms: t.endTime,
|
|
2478
|
+
duration_ms: t.duration
|
|
2479
|
+
})),
|
|
2480
|
+
metrics: {
|
|
2481
|
+
sitemaps_processed: sitemapCount,
|
|
2482
|
+
urls_analyzed: urlCount,
|
|
2483
|
+
throughput: {
|
|
2484
|
+
urls_per_second: Math.round(urlCount / totalTime * 1e3),
|
|
2485
|
+
sitemaps_per_second: (sitemapCount / totalTime * 1e3).toFixed(2)
|
|
2486
|
+
}
|
|
2487
|
+
},
|
|
2488
|
+
system_info: {
|
|
2489
|
+
cpu_count: import_os3.default.cpus().length,
|
|
2490
|
+
node_version: process.version,
|
|
2491
|
+
platform: process.platform,
|
|
2492
|
+
memory_total_mb: Math.round(import_os3.default.totalmem() / 1024 / 1024)
|
|
2493
|
+
},
|
|
2494
|
+
config: {
|
|
2495
|
+
discovery_concurrency: config.discoveryConcurrency,
|
|
2496
|
+
parsing_concurrency: config.parsingConcurrency,
|
|
2497
|
+
risk_detection_concurrency: config.riskDetectionConcurrency,
|
|
2498
|
+
risk_detection_batch_size: config.riskDetectionBatchSize
|
|
2499
|
+
}
|
|
2500
|
+
};
|
|
2501
|
+
const filename = `performance-profile-${Date.now()}.json`;
|
|
2502
|
+
await import_fs3.promises.writeFile(filename, JSON.stringify(benchmark, null, 2));
|
|
2503
|
+
console.log(import_chalk.default.blue(`\u{1F4CA} Benchmark saved to: ${filename}`));
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2506
|
+
// src/index.ts
|
|
2507
|
+
var program = new import_commander2.Command();
|
|
2508
|
+
program.name("sitemap-qa").version("1.0.0").description("sitemap analysis for QA teams");
|
|
2509
|
+
program.addCommand(analyzeCommand);
|
|
2510
|
+
process.on("unhandledRejection", (reason, promise) => {
|
|
2511
|
+
console.error("Unhandled Rejection at:", promise, "reason:", reason);
|
|
2512
|
+
process.exit(1);
|
|
2513
|
+
});
|
|
2514
|
+
process.on("SIGINT", () => {
|
|
2515
|
+
console.log("\nGracefully shutting down...");
|
|
2516
|
+
process.exit(0);
|
|
2517
|
+
});
|
|
2518
|
+
process.on("SIGTERM", () => {
|
|
2519
|
+
console.log("\nGracefully shutting down...");
|
|
2520
|
+
process.exit(0);
|
|
2521
|
+
});
|
|
2522
|
+
program.parse();
|
|
484
2523
|
//# sourceMappingURL=index.cjs.map
|