@djangocfg/seo 2.1.140 → 2.1.143
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +1329 -1329
- package/dist/cli.mjs.map +1 -1
- package/dist/crawler/index.mjs +1 -1
- package/dist/crawler/index.mjs.map +1 -1
- package/dist/google-console/index.mjs +1 -1
- package/dist/google-console/index.mjs.map +1 -1
- package/dist/index.mjs +1227 -1227
- package/dist/index.mjs.map +1 -1
- package/dist/link-checker/index.mjs +2 -2
- package/dist/link-checker/index.mjs.map +1 -1
- package/dist/reports/index.mjs +184 -184
- package/dist/reports/index.mjs.map +1 -1
- package/dist/routes/index.mjs.map +1 -1
- package/package.json +2 -2
- package/src/analyzer.ts +5 -10
- package/src/cli/commands/audit.ts +11 -8
- package/src/cli/commands/content.ts +6 -9
- package/src/cli/commands/crawl.ts +3 -2
- package/src/cli/commands/inspect.ts +3 -2
- package/src/cli/commands/links.ts +2 -1
- package/src/cli/commands/robots.ts +2 -0
- package/src/cli/commands/routes.ts +7 -3
- package/src/cli/commands/sitemap.ts +2 -0
- package/src/cli/index.ts +7 -3
- package/src/config.ts +2 -2
- package/src/content/link-checker.ts +3 -2
- package/src/content/link-fixer.ts +2 -1
- package/src/content/scanner.ts +1 -0
- package/src/content/sitemap-generator.ts +3 -2
- package/src/crawler/crawler.ts +2 -1
- package/src/crawler/robots-parser.ts +2 -1
- package/src/crawler/sitemap-validator.ts +2 -1
- package/src/google-console/auth.ts +3 -2
- package/src/google-console/client.ts +5 -2
- package/src/link-checker/index.ts +3 -2
- package/src/reports/generator.ts +7 -5
- package/src/reports/split-report.ts +2 -1
- package/src/routes/analyzer.ts +1 -0
- package/src/routes/scanner.ts +1 -1
- package/src/utils/index.ts +1 -1
package/dist/cli.mjs
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { parseArgs } from 'util';
|
|
3
|
-
import consola3 from 'consola';
|
|
4
2
|
import chalk2 from 'chalk';
|
|
3
|
+
import consola6 from 'consola';
|
|
4
|
+
import { parseArgs } from 'util';
|
|
5
5
|
import fs4, { existsSync, readFileSync, readdirSync, rmSync, mkdirSync, writeFileSync, statSync } from 'fs';
|
|
6
6
|
import path, { resolve, join, dirname } from 'path';
|
|
7
|
-
import {
|
|
7
|
+
import { parseHTML, DOMParser } from 'linkedom';
|
|
8
8
|
import pLimit from 'p-limit';
|
|
9
|
+
import robotsParser from 'robots-parser';
|
|
9
10
|
import pRetry from 'p-retry';
|
|
11
|
+
import { searchconsole } from '@googleapis/searchconsole';
|
|
10
12
|
import { JWT } from 'google-auth-library';
|
|
11
|
-
import { parseHTML, DOMParser } from 'linkedom';
|
|
12
|
-
import robotsParser from 'robots-parser';
|
|
13
|
-
import { mkdir, writeFile } from 'fs/promises';
|
|
14
13
|
import * as linkinator from 'linkinator';
|
|
14
|
+
import { mkdir, writeFile } from 'fs/promises';
|
|
15
15
|
|
|
16
16
|
var config = {
|
|
17
17
|
env: {
|
|
@@ -74,26 +74,26 @@ function getSiteUrl(options) {
|
|
|
74
74
|
const url = isProd ? config.env.prod : config.env.dev;
|
|
75
75
|
if (url) {
|
|
76
76
|
const envLabel = isProd ? "production" : "development";
|
|
77
|
-
|
|
77
|
+
consola6.info(`Using ${chalk2.cyan(envLabel)} URL: ${chalk2.bold(url)}`);
|
|
78
78
|
return url;
|
|
79
79
|
}
|
|
80
80
|
const fallbackUrl = process.env.NEXT_PUBLIC_SITE_URL || process.env.SITE_URL || process.env.BASE_URL;
|
|
81
81
|
if (fallbackUrl) {
|
|
82
|
-
|
|
82
|
+
consola6.info(`Using URL from environment: ${fallbackUrl}`);
|
|
83
83
|
return fallbackUrl;
|
|
84
84
|
}
|
|
85
85
|
console.log("");
|
|
86
|
-
|
|
86
|
+
consola6.error("No site URL found!");
|
|
87
87
|
console.log("");
|
|
88
88
|
if (config.env.prod || config.env.dev) {
|
|
89
|
-
|
|
90
|
-
if (config.env.prod)
|
|
91
|
-
if (config.env.dev)
|
|
89
|
+
consola6.info("Available environments:");
|
|
90
|
+
if (config.env.prod) consola6.log(` ${chalk2.green("prod")}: ${config.env.prod}`);
|
|
91
|
+
if (config.env.dev) consola6.log(` ${chalk2.yellow("dev")}: ${config.env.dev}`);
|
|
92
92
|
console.log("");
|
|
93
|
-
|
|
93
|
+
consola6.info(`Use ${chalk2.cyan("--env prod")} or ${chalk2.cyan("--env dev")} to select`);
|
|
94
94
|
} else {
|
|
95
|
-
|
|
96
|
-
|
|
95
|
+
consola6.info("Create .env.production or .env.development with NEXT_PUBLIC_SITE_URL");
|
|
96
|
+
consola6.info("Or use --site https://example.com");
|
|
97
97
|
}
|
|
98
98
|
process.exit(1);
|
|
99
99
|
}
|
|
@@ -104,12 +104,12 @@ function findGoogleServiceAccount(explicitPath) {
|
|
|
104
104
|
if (existsSync(resolved)) {
|
|
105
105
|
return resolved;
|
|
106
106
|
}
|
|
107
|
-
|
|
107
|
+
consola6.warn(`Service account file not found: ${explicitPath}`);
|
|
108
108
|
return void 0;
|
|
109
109
|
}
|
|
110
110
|
const defaultPath = resolve(config.cwd, GSC_KEY_FILENAME);
|
|
111
111
|
if (existsSync(defaultPath)) {
|
|
112
|
-
|
|
112
|
+
consola6.info(`Found Google service account: ${chalk2.cyan(GSC_KEY_FILENAME)}`);
|
|
113
113
|
return defaultPath;
|
|
114
114
|
}
|
|
115
115
|
return void 0;
|
|
@@ -117,511 +117,336 @@ function findGoogleServiceAccount(explicitPath) {
|
|
|
117
117
|
function getGscKeyFilename() {
|
|
118
118
|
return GSC_KEY_FILENAME;
|
|
119
119
|
}
|
|
120
|
-
var
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
scopes: SCOPES
|
|
154
|
-
});
|
|
155
|
-
auth._serviceAccountEmail = credentials.client_email;
|
|
156
|
-
return auth;
|
|
157
|
-
}
|
|
158
|
-
async function verifyAuth(auth, siteUrl) {
|
|
159
|
-
const email = auth._serviceAccountEmail || auth.email;
|
|
160
|
-
try {
|
|
161
|
-
await auth.authorize();
|
|
162
|
-
consola3.success("Google Search Console authentication verified");
|
|
163
|
-
consola3.info(`Service account: ${email}`);
|
|
164
|
-
if (siteUrl) {
|
|
165
|
-
const domain = new URL(siteUrl).hostname;
|
|
166
|
-
const gscUrl = `https://search.google.com/search-console/users?resource_id=sc-domain%3A${domain}`;
|
|
167
|
-
consola3.info(`Ensure this email has Full access in GSC: ${gscUrl}`);
|
|
168
|
-
}
|
|
169
|
-
return true;
|
|
170
|
-
} catch (error) {
|
|
171
|
-
consola3.error("Authentication failed");
|
|
172
|
-
consola3.info(`Service account email: ${email}`);
|
|
173
|
-
consola3.info("Make sure this email is added to GSC with Full access");
|
|
174
|
-
return false;
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
// src/google-console/client.ts
|
|
179
|
-
var GoogleConsoleClient = class {
|
|
180
|
-
auth;
|
|
181
|
-
searchconsole;
|
|
182
|
-
siteUrl;
|
|
183
|
-
gscSiteUrl;
|
|
184
|
-
// Format for GSC API (may be sc-domain:xxx)
|
|
185
|
-
limit = pLimit(2);
|
|
186
|
-
// Max 2 concurrent requests (Cloudflare-friendly)
|
|
187
|
-
requestDelay = 500;
|
|
188
|
-
// Delay between requests in ms
|
|
189
|
-
constructor(config2) {
|
|
190
|
-
this.auth = createAuthClient(config2);
|
|
191
|
-
this.searchconsole = searchconsole({ version: "v1", auth: this.auth });
|
|
192
|
-
this.siteUrl = config2.siteUrl;
|
|
193
|
-
if (config2.gscSiteUrl) {
|
|
194
|
-
this.gscSiteUrl = config2.gscSiteUrl;
|
|
195
|
-
} else {
|
|
196
|
-
const domain = new URL(config2.siteUrl).hostname;
|
|
197
|
-
this.gscSiteUrl = `sc-domain:${domain}`;
|
|
198
|
-
}
|
|
199
|
-
consola3.debug(`GSC site URL: ${this.gscSiteUrl}`);
|
|
200
|
-
}
|
|
201
|
-
/**
|
|
202
|
-
* Delay helper for rate limiting
|
|
203
|
-
*/
|
|
204
|
-
delay(ms) {
|
|
205
|
-
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
206
|
-
}
|
|
207
|
-
/**
|
|
208
|
-
* Verify the client is authenticated
|
|
209
|
-
*/
|
|
210
|
-
async verify() {
|
|
211
|
-
return verifyAuth(this.auth, this.siteUrl);
|
|
120
|
+
var DEFAULT_CONFIG = {
|
|
121
|
+
maxPages: 100,
|
|
122
|
+
maxDepth: 3,
|
|
123
|
+
concurrency: 5,
|
|
124
|
+
timeout: 3e4,
|
|
125
|
+
userAgent: "DjangoCFG-SEO-Crawler/1.0 (+https://djangocfg.com/bot)",
|
|
126
|
+
respectRobotsTxt: true,
|
|
127
|
+
includePatterns: [],
|
|
128
|
+
excludePatterns: [
|
|
129
|
+
"/api/",
|
|
130
|
+
"/admin/",
|
|
131
|
+
"/_next/",
|
|
132
|
+
"/static/",
|
|
133
|
+
".pdf",
|
|
134
|
+
".jpg",
|
|
135
|
+
".png",
|
|
136
|
+
".gif",
|
|
137
|
+
".svg",
|
|
138
|
+
".css",
|
|
139
|
+
".js"
|
|
140
|
+
]
|
|
141
|
+
};
|
|
142
|
+
var SiteCrawler = class {
|
|
143
|
+
config;
|
|
144
|
+
baseUrl;
|
|
145
|
+
visited = /* @__PURE__ */ new Set();
|
|
146
|
+
queue = [];
|
|
147
|
+
results = [];
|
|
148
|
+
limit;
|
|
149
|
+
constructor(siteUrl, config2) {
|
|
150
|
+
this.config = { ...DEFAULT_CONFIG, ...config2 };
|
|
151
|
+
this.baseUrl = new URL(siteUrl);
|
|
152
|
+
this.limit = pLimit(this.config.concurrency);
|
|
212
153
|
}
|
|
213
154
|
/**
|
|
214
|
-
*
|
|
155
|
+
* Start crawling the site
|
|
215
156
|
*/
|
|
216
|
-
async
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
157
|
+
async crawl() {
|
|
158
|
+
consola6.info(`Starting crawl of ${this.baseUrl.origin}`);
|
|
159
|
+
consola6.info(`Config: maxPages=${this.config.maxPages}, maxDepth=${this.config.maxDepth}`);
|
|
160
|
+
this.queue.push({ url: this.baseUrl.href, depth: 0 });
|
|
161
|
+
while (this.queue.length > 0 && this.results.length < this.config.maxPages) {
|
|
162
|
+
const batch = this.queue.splice(0, this.config.concurrency);
|
|
163
|
+
const promises = batch.map(
|
|
164
|
+
({ url, depth }) => this.limit(() => this.crawlPage(url, depth))
|
|
165
|
+
);
|
|
166
|
+
await Promise.all(promises);
|
|
223
167
|
}
|
|
168
|
+
consola6.success(`Crawl complete. Crawled ${this.results.length} pages.`);
|
|
169
|
+
return this.results;
|
|
224
170
|
}
|
|
225
171
|
/**
|
|
226
|
-
*
|
|
172
|
+
* Crawl a single page
|
|
227
173
|
*/
|
|
228
|
-
async
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
174
|
+
async crawlPage(url, depth) {
|
|
175
|
+
const normalizedUrl = this.normalizeUrl(url);
|
|
176
|
+
if (this.visited.has(normalizedUrl)) return;
|
|
177
|
+
if (this.shouldExclude(normalizedUrl)) return;
|
|
178
|
+
this.visited.add(normalizedUrl);
|
|
179
|
+
const startTime = Date.now();
|
|
180
|
+
const result = {
|
|
181
|
+
url: normalizedUrl,
|
|
182
|
+
statusCode: 0,
|
|
183
|
+
links: { internal: [], external: [] },
|
|
184
|
+
images: [],
|
|
185
|
+
loadTime: 0,
|
|
186
|
+
errors: [],
|
|
187
|
+
warnings: [],
|
|
188
|
+
crawledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
189
|
+
};
|
|
190
|
+
try {
|
|
191
|
+
const controller = new AbortController();
|
|
192
|
+
const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
|
|
193
|
+
const response = await fetch(normalizedUrl, {
|
|
194
|
+
headers: {
|
|
195
|
+
"User-Agent": this.config.userAgent,
|
|
196
|
+
Accept: "text/html,application/xhtml+xml"
|
|
244
197
|
},
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
consola3.info(`Inspecting ${urls.length} URLs...`);
|
|
266
|
-
const results = [];
|
|
267
|
-
const errors = [];
|
|
268
|
-
let consecutiveErrors = 0;
|
|
269
|
-
const maxConsecutiveErrors = 3;
|
|
270
|
-
for (const url of urls) {
|
|
271
|
-
try {
|
|
272
|
-
const result = await this.inspectUrl(url);
|
|
273
|
-
results.push(result);
|
|
274
|
-
consecutiveErrors = 0;
|
|
275
|
-
await this.delay(this.requestDelay);
|
|
276
|
-
} catch (error) {
|
|
277
|
-
const err = error;
|
|
278
|
-
errors.push({ url, error: err });
|
|
279
|
-
consecutiveErrors++;
|
|
280
|
-
if (consecutiveErrors >= maxConsecutiveErrors) {
|
|
281
|
-
console.log("");
|
|
282
|
-
consola3.error(`Stopping after ${maxConsecutiveErrors} consecutive failures`);
|
|
283
|
-
this.showRateLimitHelp();
|
|
284
|
-
break;
|
|
198
|
+
signal: controller.signal,
|
|
199
|
+
redirect: "follow"
|
|
200
|
+
});
|
|
201
|
+
result.ttfb = Date.now() - startTime;
|
|
202
|
+
clearTimeout(timeoutId);
|
|
203
|
+
result.statusCode = response.status;
|
|
204
|
+
result.contentType = response.headers.get("content-type") || void 0;
|
|
205
|
+
result.contentLength = Number(response.headers.get("content-length")) || void 0;
|
|
206
|
+
if (response.ok && result.contentType?.includes("text/html")) {
|
|
207
|
+
const html = await response.text();
|
|
208
|
+
this.parseHtml(html, result, normalizedUrl, depth);
|
|
209
|
+
} else if (!response.ok) {
|
|
210
|
+
result.errors.push(`HTTP ${response.status}: ${response.statusText}`);
|
|
211
|
+
}
|
|
212
|
+
} catch (error) {
|
|
213
|
+
if (error instanceof Error) {
|
|
214
|
+
if (error.name === "AbortError") {
|
|
215
|
+
result.errors.push("Request timeout");
|
|
216
|
+
} else {
|
|
217
|
+
result.errors.push(error.message);
|
|
285
218
|
}
|
|
286
219
|
}
|
|
287
220
|
}
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
}
|
|
291
|
-
if (results.length > 0) {
|
|
292
|
-
consola3.success(`Successfully inspected ${results.length}/${urls.length} URLs`);
|
|
293
|
-
} else if (errors.length > 0) {
|
|
294
|
-
consola3.warn("No URLs were successfully inspected");
|
|
295
|
-
}
|
|
296
|
-
return results;
|
|
221
|
+
result.loadTime = Date.now() - startTime;
|
|
222
|
+
this.results.push(result);
|
|
223
|
+
consola6.debug(`Crawled: ${normalizedUrl} (${result.statusCode}) - ${result.loadTime}ms`);
|
|
297
224
|
}
|
|
298
225
|
/**
|
|
299
|
-
*
|
|
226
|
+
* Parse HTML and extract SEO-relevant data
|
|
300
227
|
*/
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
228
|
+
parseHtml(html, result, pageUrl, depth) {
|
|
229
|
+
const { document } = parseHTML(html);
|
|
230
|
+
const titleEl = document.querySelector("title");
|
|
231
|
+
result.title = titleEl?.textContent?.trim() || void 0;
|
|
232
|
+
if (!result.title) {
|
|
233
|
+
result.warnings.push("Missing title tag");
|
|
234
|
+
} else if (result.title.length > 60) {
|
|
235
|
+
result.warnings.push(`Title too long (${result.title.length} chars, recommended: <60)`);
|
|
236
|
+
}
|
|
237
|
+
const metaDesc = document.querySelector('meta[name="description"]');
|
|
238
|
+
result.metaDescription = metaDesc?.getAttribute("content")?.trim() || void 0;
|
|
239
|
+
if (!result.metaDescription) {
|
|
240
|
+
result.warnings.push("Missing meta description");
|
|
241
|
+
} else if (result.metaDescription.length > 160) {
|
|
242
|
+
result.warnings.push(
|
|
243
|
+
`Meta description too long (${result.metaDescription.length} chars, recommended: <160)`
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
const metaRobots = document.querySelector('meta[name="robots"]');
|
|
247
|
+
result.metaRobots = metaRobots?.getAttribute("content")?.trim() || void 0;
|
|
248
|
+
const xRobots = document.querySelector('meta[http-equiv="X-Robots-Tag"]');
|
|
249
|
+
const xRobotsContent = xRobots?.getAttribute("content")?.trim();
|
|
250
|
+
if (xRobotsContent) {
|
|
251
|
+
result.metaRobots = result.metaRobots ? `${result.metaRobots}, ${xRobotsContent}` : xRobotsContent;
|
|
252
|
+
}
|
|
253
|
+
const canonical = document.querySelector('link[rel="canonical"]');
|
|
254
|
+
result.canonicalUrl = canonical?.getAttribute("href")?.trim() || void 0;
|
|
255
|
+
if (!result.canonicalUrl) {
|
|
256
|
+
result.warnings.push("Missing canonical tag");
|
|
257
|
+
}
|
|
258
|
+
result.h1 = Array.from(document.querySelectorAll("h1")).map((el) => el.textContent?.trim() || "");
|
|
259
|
+
result.h2 = Array.from(document.querySelectorAll("h2")).map((el) => el.textContent?.trim() || "");
|
|
260
|
+
if (result.h1.length === 0) {
|
|
261
|
+
result.warnings.push("Missing H1 tag");
|
|
262
|
+
} else if (result.h1.length > 1) {
|
|
263
|
+
result.warnings.push(`Multiple H1 tags (${result.h1.length})`);
|
|
264
|
+
}
|
|
265
|
+
for (const el of document.querySelectorAll("a[href]")) {
|
|
266
|
+
const href = el.getAttribute("href");
|
|
267
|
+
if (!href) continue;
|
|
268
|
+
try {
|
|
269
|
+
const linkUrl = new URL(href, pageUrl);
|
|
270
|
+
if (linkUrl.hostname === this.baseUrl.hostname) {
|
|
271
|
+
const internalUrl = this.normalizeUrl(linkUrl.href);
|
|
272
|
+
result.links.internal.push(internalUrl);
|
|
273
|
+
if (depth < this.config.maxDepth && !this.visited.has(internalUrl)) {
|
|
274
|
+
this.queue.push({ url: internalUrl, depth: depth + 1 });
|
|
275
|
+
}
|
|
276
|
+
} else {
|
|
277
|
+
result.links.external.push(linkUrl.href);
|
|
332
278
|
}
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
|
|
279
|
+
} catch {
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
for (const el of document.querySelectorAll("img")) {
|
|
283
|
+
const src = el.getAttribute("src");
|
|
284
|
+
const alt = el.getAttribute("alt");
|
|
285
|
+
const hasAltAttr = alt !== null;
|
|
286
|
+
if (src) {
|
|
287
|
+
result.images.push({
|
|
288
|
+
src,
|
|
289
|
+
alt: alt ?? void 0,
|
|
290
|
+
hasAlt: hasAltAttr && alt.trim().length > 0
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
const imagesWithoutAlt = result.images.filter((img) => !img.hasAlt);
|
|
295
|
+
if (imagesWithoutAlt.length > 0) {
|
|
296
|
+
result.warnings.push(`${imagesWithoutAlt.length} images without alt text`);
|
|
338
297
|
}
|
|
339
298
|
}
|
|
340
299
|
/**
|
|
341
|
-
*
|
|
300
|
+
* Normalize URL for deduplication
|
|
342
301
|
*/
|
|
343
|
-
|
|
302
|
+
normalizeUrl(url) {
|
|
344
303
|
try {
|
|
345
|
-
const
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
304
|
+
const parsed = new URL(url, this.baseUrl.href);
|
|
305
|
+
parsed.hash = "";
|
|
306
|
+
let pathname = parsed.pathname;
|
|
307
|
+
if (pathname.endsWith("/") && pathname !== "/") {
|
|
308
|
+
pathname = pathname.slice(0, -1);
|
|
309
|
+
}
|
|
310
|
+
parsed.pathname = pathname;
|
|
311
|
+
return parsed.href;
|
|
312
|
+
} catch {
|
|
313
|
+
return url;
|
|
352
314
|
}
|
|
353
315
|
}
|
|
354
316
|
/**
|
|
355
|
-
*
|
|
317
|
+
* Check if URL should be excluded
|
|
356
318
|
*/
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
indexingState: indexStatus.indexingState || "INDEXING_STATE_UNSPECIFIED",
|
|
366
|
-
robotsTxtState: indexStatus.robotsTxtState || "ROBOTS_TXT_STATE_UNSPECIFIED",
|
|
367
|
-
pageFetchState: indexStatus.pageFetchState || "PAGE_FETCH_STATE_UNSPECIFIED",
|
|
368
|
-
lastCrawlTime: indexStatus.lastCrawlTime || void 0,
|
|
369
|
-
crawledAs: indexStatus.crawledAs,
|
|
370
|
-
googleCanonical: indexStatus.googleCanonical || void 0,
|
|
371
|
-
userCanonical: indexStatus.userCanonical || void 0,
|
|
372
|
-
sitemap: indexStatus.sitemap || void 0,
|
|
373
|
-
referringUrls: indexStatus.referringUrls || void 0
|
|
374
|
-
},
|
|
375
|
-
mobileUsabilityResult: result.mobileUsabilityResult ? {
|
|
376
|
-
verdict: result.mobileUsabilityResult.verdict || "VERDICT_UNSPECIFIED",
|
|
377
|
-
issues: result.mobileUsabilityResult.issues?.map((issue) => ({
|
|
378
|
-
issueType: issue.issueType || "UNKNOWN",
|
|
379
|
-
message: issue.message || ""
|
|
380
|
-
}))
|
|
381
|
-
} : void 0,
|
|
382
|
-
richResultsResult: result.richResultsResult ? {
|
|
383
|
-
verdict: result.richResultsResult.verdict || "VERDICT_UNSPECIFIED",
|
|
384
|
-
detectedItems: result.richResultsResult.detectedItems?.map((item) => ({
|
|
385
|
-
richResultType: item.richResultType || "UNKNOWN",
|
|
386
|
-
items: item.items?.map((i) => ({
|
|
387
|
-
name: i.name || "",
|
|
388
|
-
issues: i.issues?.map((issue) => ({
|
|
389
|
-
issueMessage: issue.issueMessage || "",
|
|
390
|
-
severity: issue.severity || "WARNING"
|
|
391
|
-
}))
|
|
392
|
-
}))
|
|
393
|
-
}))
|
|
394
|
-
} : void 0
|
|
395
|
-
};
|
|
319
|
+
shouldExclude(url) {
|
|
320
|
+
if (this.config.includePatterns.length > 0) {
|
|
321
|
+
const included = this.config.includePatterns.some(
|
|
322
|
+
(pattern) => url.includes(pattern)
|
|
323
|
+
);
|
|
324
|
+
if (!included) return true;
|
|
325
|
+
}
|
|
326
|
+
return this.config.excludePatterns.some((pattern) => url.includes(pattern));
|
|
396
327
|
}
|
|
397
328
|
};
|
|
398
|
-
|
|
399
|
-
// src/google-console/analyzer.ts
|
|
400
|
-
function analyzeInspectionResults(results) {
|
|
329
|
+
function analyzeCrawlResults(results) {
|
|
401
330
|
const issues = [];
|
|
402
331
|
for (const result of results) {
|
|
403
|
-
|
|
404
|
-
}
|
|
405
|
-
return issues.sort((a, b) => severityOrder(a.severity) - severityOrder(b.severity));
|
|
406
|
-
}
|
|
407
|
-
function analyzeUrlInspection(result) {
|
|
408
|
-
const issues = [];
|
|
409
|
-
const { indexStatusResult, mobileUsabilityResult, richResultsResult } = result;
|
|
410
|
-
switch (indexStatusResult.coverageState) {
|
|
411
|
-
case "CRAWLED_CURRENTLY_NOT_INDEXED":
|
|
332
|
+
if (result.statusCode >= 400) {
|
|
412
333
|
issues.push({
|
|
413
|
-
id: `
|
|
334
|
+
id: `http-error-${hash(result.url)}`,
|
|
414
335
|
url: result.url,
|
|
415
|
-
category: "
|
|
336
|
+
category: "technical",
|
|
337
|
+
severity: result.statusCode >= 500 ? "critical" : "error",
|
|
338
|
+
title: `HTTP ${result.statusCode} error`,
|
|
339
|
+
description: `Page returns ${result.statusCode} status code.`,
|
|
340
|
+
recommendation: result.statusCode === 404 ? "Either restore the content or set up a redirect." : "Fix the server error and ensure the page is accessible.",
|
|
341
|
+
detectedAt: result.crawledAt,
|
|
342
|
+
metadata: { statusCode: result.statusCode }
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
if (!result.title && result.statusCode === 200) {
|
|
346
|
+
issues.push({
|
|
347
|
+
id: `missing-title-${hash(result.url)}`,
|
|
348
|
+
url: result.url,
|
|
349
|
+
category: "content",
|
|
416
350
|
severity: "error",
|
|
417
|
-
title: "
|
|
418
|
-
description: "
|
|
419
|
-
recommendation: "
|
|
420
|
-
detectedAt:
|
|
421
|
-
metadata: { coverageState: indexStatusResult.coverageState }
|
|
351
|
+
title: "Missing title tag",
|
|
352
|
+
description: "This page does not have a title tag.",
|
|
353
|
+
recommendation: "Add a unique, descriptive title tag (50-60 characters).",
|
|
354
|
+
detectedAt: result.crawledAt
|
|
422
355
|
});
|
|
423
|
-
|
|
424
|
-
|
|
356
|
+
}
|
|
357
|
+
if (!result.metaDescription && result.statusCode === 200) {
|
|
425
358
|
issues.push({
|
|
426
|
-
id: `
|
|
359
|
+
id: `missing-meta-desc-${hash(result.url)}`,
|
|
427
360
|
url: result.url,
|
|
428
|
-
category: "
|
|
361
|
+
category: "content",
|
|
429
362
|
severity: "warning",
|
|
430
|
-
title: "
|
|
431
|
-
description: "
|
|
432
|
-
recommendation: "
|
|
433
|
-
detectedAt:
|
|
434
|
-
metadata: { coverageState: indexStatusResult.coverageState }
|
|
363
|
+
title: "Missing meta description",
|
|
364
|
+
description: "This page does not have a meta description.",
|
|
365
|
+
recommendation: "Add a unique meta description (120-160 characters).",
|
|
366
|
+
detectedAt: result.crawledAt
|
|
435
367
|
});
|
|
436
|
-
|
|
437
|
-
|
|
368
|
+
}
|
|
369
|
+
if (result.h1 && result.h1.length === 0 && result.statusCode === 200) {
|
|
438
370
|
issues.push({
|
|
439
|
-
id: `
|
|
371
|
+
id: `missing-h1-${hash(result.url)}`,
|
|
440
372
|
url: result.url,
|
|
441
|
-
category: "
|
|
373
|
+
category: "content",
|
|
442
374
|
severity: "warning",
|
|
443
|
-
title: "
|
|
444
|
-
description: "This page
|
|
445
|
-
recommendation: "Add a
|
|
446
|
-
detectedAt:
|
|
447
|
-
metadata: {
|
|
448
|
-
coverageState: indexStatusResult.coverageState,
|
|
449
|
-
googleCanonical: indexStatusResult.googleCanonical
|
|
450
|
-
}
|
|
375
|
+
title: "Missing H1 heading",
|
|
376
|
+
description: "This page does not have an H1 heading.",
|
|
377
|
+
recommendation: "Add a single H1 heading that describes the page content.",
|
|
378
|
+
detectedAt: result.crawledAt
|
|
451
379
|
});
|
|
452
|
-
|
|
453
|
-
|
|
380
|
+
}
|
|
381
|
+
if (result.h1 && result.h1.length > 1) {
|
|
454
382
|
issues.push({
|
|
455
|
-
id: `
|
|
383
|
+
id: `multiple-h1-${hash(result.url)}`,
|
|
456
384
|
url: result.url,
|
|
457
|
-
category: "
|
|
385
|
+
category: "content",
|
|
458
386
|
severity: "warning",
|
|
459
|
-
title: "
|
|
460
|
-
description:
|
|
461
|
-
recommendation: "
|
|
462
|
-
detectedAt:
|
|
463
|
-
metadata: {
|
|
464
|
-
coverageState: indexStatusResult.coverageState,
|
|
465
|
-
userCanonical: indexStatusResult.userCanonical,
|
|
466
|
-
googleCanonical: indexStatusResult.googleCanonical
|
|
467
|
-
}
|
|
387
|
+
title: "Multiple H1 headings",
|
|
388
|
+
description: `This page has ${result.h1.length} H1 headings.`,
|
|
389
|
+
recommendation: "Use only one H1 heading per page.",
|
|
390
|
+
detectedAt: result.crawledAt,
|
|
391
|
+
metadata: { h1Count: result.h1.length }
|
|
468
392
|
});
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
case "BLOCKED_BY_META_TAG":
|
|
393
|
+
}
|
|
394
|
+
const imagesWithoutAlt = result.images.filter((img) => !img.hasAlt);
|
|
395
|
+
if (imagesWithoutAlt.length > 0) {
|
|
473
396
|
issues.push({
|
|
474
|
-
id: `
|
|
397
|
+
id: `images-no-alt-${hash(result.url)}`,
|
|
475
398
|
url: result.url,
|
|
476
|
-
category: "
|
|
477
|
-
severity: "
|
|
478
|
-
title: "
|
|
479
|
-
description:
|
|
480
|
-
recommendation: "
|
|
481
|
-
detectedAt:
|
|
482
|
-
metadata: {
|
|
483
|
-
});
|
|
484
|
-
break;
|
|
485
|
-
case "BLOCKED_BY_HTTP_HEADER":
|
|
486
|
-
issues.push({
|
|
487
|
-
id: `blocked-http-header-${hash(result.url)}`,
|
|
488
|
-
url: result.url,
|
|
489
|
-
category: "indexing",
|
|
490
|
-
severity: "error",
|
|
491
|
-
title: "Blocked by X-Robots-Tag header",
|
|
492
|
-
description: "This page has a noindex directive in the X-Robots-Tag HTTP header.",
|
|
493
|
-
recommendation: "Remove the X-Robots-Tag: noindex header if you want this page to be indexed.",
|
|
494
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
495
|
-
metadata: { indexingState: indexStatusResult.indexingState }
|
|
496
|
-
});
|
|
497
|
-
break;
|
|
498
|
-
case "BLOCKED_BY_ROBOTS_TXT":
|
|
499
|
-
issues.push({
|
|
500
|
-
id: `blocked-robots-txt-${hash(result.url)}`,
|
|
501
|
-
url: result.url,
|
|
502
|
-
category: "crawling",
|
|
503
|
-
severity: "error",
|
|
504
|
-
title: "Blocked by robots.txt",
|
|
505
|
-
description: "This page is blocked from crawling by robots.txt rules.",
|
|
506
|
-
recommendation: "Update robots.txt to allow crawling if you want this page to be indexed.",
|
|
507
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
508
|
-
metadata: { indexingState: indexStatusResult.indexingState }
|
|
509
|
-
});
|
|
510
|
-
break;
|
|
511
|
-
}
|
|
512
|
-
switch (indexStatusResult.pageFetchState) {
|
|
513
|
-
case "SOFT_404":
|
|
514
|
-
issues.push({
|
|
515
|
-
id: `soft-404-${hash(result.url)}`,
|
|
516
|
-
url: result.url,
|
|
517
|
-
category: "technical",
|
|
518
|
-
severity: "error",
|
|
519
|
-
title: "Soft 404 error",
|
|
520
|
-
description: "This page returns a 200 status but Google detected it as a 404 page (empty or low-value content).",
|
|
521
|
-
recommendation: "Either return a proper 404 status code or add meaningful content to this page.",
|
|
522
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
523
|
-
metadata: { pageFetchState: indexStatusResult.pageFetchState }
|
|
524
|
-
});
|
|
525
|
-
break;
|
|
526
|
-
case "NOT_FOUND":
|
|
527
|
-
issues.push({
|
|
528
|
-
id: `404-error-${hash(result.url)}`,
|
|
529
|
-
url: result.url,
|
|
530
|
-
category: "technical",
|
|
531
|
-
severity: "error",
|
|
532
|
-
title: "404 Not Found",
|
|
533
|
-
description: "This page returns a 404 error.",
|
|
534
|
-
recommendation: "Either restore the page content or set up a redirect to a relevant page.",
|
|
535
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
536
|
-
metadata: { pageFetchState: indexStatusResult.pageFetchState }
|
|
537
|
-
});
|
|
538
|
-
break;
|
|
539
|
-
case "SERVER_ERROR":
|
|
540
|
-
issues.push({
|
|
541
|
-
id: `server-error-${hash(result.url)}`,
|
|
542
|
-
url: result.url,
|
|
543
|
-
category: "technical",
|
|
544
|
-
severity: "critical",
|
|
545
|
-
title: "Server error (5xx)",
|
|
546
|
-
description: "This page returns a server error when Google tries to crawl it.",
|
|
547
|
-
recommendation: "Fix the server-side error. Check server logs for details.",
|
|
548
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
549
|
-
metadata: { pageFetchState: indexStatusResult.pageFetchState }
|
|
399
|
+
category: "content",
|
|
400
|
+
severity: "info",
|
|
401
|
+
title: "Images without alt text",
|
|
402
|
+
description: `${imagesWithoutAlt.length} images are missing alt text.`,
|
|
403
|
+
recommendation: "Add descriptive alt text to all images for accessibility and SEO.",
|
|
404
|
+
detectedAt: result.crawledAt,
|
|
405
|
+
metadata: { count: imagesWithoutAlt.length }
|
|
550
406
|
});
|
|
551
|
-
|
|
552
|
-
|
|
407
|
+
}
|
|
408
|
+
if (result.loadTime > 3e3) {
|
|
553
409
|
issues.push({
|
|
554
|
-
id: `
|
|
410
|
+
id: `slow-page-${hash(result.url)}`,
|
|
555
411
|
url: result.url,
|
|
556
|
-
category: "
|
|
557
|
-
severity: "error",
|
|
558
|
-
title: "
|
|
559
|
-
description:
|
|
560
|
-
recommendation: "
|
|
561
|
-
detectedAt:
|
|
562
|
-
metadata: {
|
|
412
|
+
category: "performance",
|
|
413
|
+
severity: result.loadTime > 5e3 ? "error" : "warning",
|
|
414
|
+
title: "Slow page load time",
|
|
415
|
+
description: `Page took ${result.loadTime}ms to load.`,
|
|
416
|
+
recommendation: "Optimize page load time. Target under 3 seconds.",
|
|
417
|
+
detectedAt: result.crawledAt,
|
|
418
|
+
metadata: { loadTime: result.loadTime }
|
|
563
419
|
});
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
case "ACCESS_FORBIDDEN":
|
|
420
|
+
}
|
|
421
|
+
if (result.ttfb && result.ttfb > 800) {
|
|
567
422
|
issues.push({
|
|
568
|
-
id: `
|
|
423
|
+
id: `slow-ttfb-${hash(result.url)}`,
|
|
569
424
|
url: result.url,
|
|
570
|
-
category: "
|
|
571
|
-
severity: "error",
|
|
572
|
-
title: "
|
|
573
|
-
description:
|
|
574
|
-
recommendation: "
|
|
575
|
-
detectedAt:
|
|
576
|
-
metadata: {
|
|
425
|
+
category: "performance",
|
|
426
|
+
severity: result.ttfb > 1500 ? "error" : "warning",
|
|
427
|
+
title: "Slow Time to First Byte",
|
|
428
|
+
description: `TTFB is ${result.ttfb}ms. Server responded slowly.`,
|
|
429
|
+
recommendation: "Optimize server response. Target TTFB under 800ms. Consider CDN, caching, or server upgrades.",
|
|
430
|
+
detectedAt: result.crawledAt,
|
|
431
|
+
metadata: { ttfb: result.ttfb }
|
|
577
432
|
});
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
if (mobileUsabilityResult?.verdict === "FAIL" && mobileUsabilityResult.issues) {
|
|
581
|
-
for (const issue of mobileUsabilityResult.issues) {
|
|
433
|
+
}
|
|
434
|
+
if (result.metaRobots?.includes("noindex")) {
|
|
582
435
|
issues.push({
|
|
583
|
-
id: `
|
|
436
|
+
id: `noindex-${hash(result.url)}`,
|
|
584
437
|
url: result.url,
|
|
585
|
-
category: "
|
|
586
|
-
severity: "
|
|
587
|
-
title:
|
|
588
|
-
description:
|
|
589
|
-
recommendation:
|
|
590
|
-
detectedAt:
|
|
591
|
-
metadata: {
|
|
438
|
+
category: "indexing",
|
|
439
|
+
severity: "info",
|
|
440
|
+
title: "Page marked as noindex",
|
|
441
|
+
description: "This page has a noindex directive.",
|
|
442
|
+
recommendation: "Verify this is intentional. Remove noindex if the page should be indexed.",
|
|
443
|
+
detectedAt: result.crawledAt,
|
|
444
|
+
metadata: { metaRobots: result.metaRobots }
|
|
592
445
|
});
|
|
593
446
|
}
|
|
594
447
|
}
|
|
595
|
-
if (richResultsResult?.verdict === "FAIL" && richResultsResult.detectedItems) {
|
|
596
|
-
for (const item of richResultsResult.detectedItems) {
|
|
597
|
-
for (const i of item.items || []) {
|
|
598
|
-
for (const issueDetail of i.issues || []) {
|
|
599
|
-
issues.push({
|
|
600
|
-
id: `rich-result-${item.richResultType}-${hash(result.url)}`,
|
|
601
|
-
url: result.url,
|
|
602
|
-
category: "structured-data",
|
|
603
|
-
severity: issueDetail.severity === "ERROR" ? "error" : "warning",
|
|
604
|
-
title: `${item.richResultType}: ${i.name}`,
|
|
605
|
-
description: issueDetail.issueMessage,
|
|
606
|
-
recommendation: "Fix the structured data markup according to Google guidelines.",
|
|
607
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
608
|
-
metadata: { richResultType: item.richResultType }
|
|
609
|
-
});
|
|
610
|
-
}
|
|
611
|
-
}
|
|
612
|
-
}
|
|
613
|
-
}
|
|
614
448
|
return issues;
|
|
615
449
|
}
|
|
616
|
-
function severityOrder(severity) {
|
|
617
|
-
const order = {
|
|
618
|
-
critical: 0,
|
|
619
|
-
error: 1,
|
|
620
|
-
warning: 2,
|
|
621
|
-
info: 3
|
|
622
|
-
};
|
|
623
|
-
return order[severity];
|
|
624
|
-
}
|
|
625
450
|
function hash(str) {
|
|
626
451
|
let hash5 = 0;
|
|
627
452
|
for (let i = 0; i < str.length; i++) {
|
|
@@ -631,622 +456,784 @@ function hash(str) {
|
|
|
631
456
|
}
|
|
632
457
|
return Math.abs(hash5).toString(36);
|
|
633
458
|
}
|
|
634
|
-
function
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
MOBILE_FRIENDLY_RULE_TAP_TARGETS_TOO_SMALL: "Increase the size of touch targets (buttons, links) to at least 48x48 pixels.",
|
|
643
|
-
MOBILE_FRIENDLY_RULE_TEXT_TOO_SMALL: "Use at least 16px font size for body text."
|
|
459
|
+
async function analyzeRobotsTxt(siteUrl) {
|
|
460
|
+
const robotsUrl = new URL("/robots.txt", siteUrl).href;
|
|
461
|
+
const analysis = {
|
|
462
|
+
exists: false,
|
|
463
|
+
sitemaps: [],
|
|
464
|
+
allowedPaths: [],
|
|
465
|
+
disallowedPaths: [],
|
|
466
|
+
issues: []
|
|
644
467
|
};
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
"/static/",
|
|
660
|
-
".pdf",
|
|
661
|
-
".jpg",
|
|
662
|
-
".png",
|
|
663
|
-
".gif",
|
|
664
|
-
".svg",
|
|
665
|
-
".css",
|
|
666
|
-
".js"
|
|
667
|
-
]
|
|
668
|
-
};
|
|
669
|
-
var SiteCrawler = class {
|
|
670
|
-
config;
|
|
671
|
-
baseUrl;
|
|
672
|
-
visited = /* @__PURE__ */ new Set();
|
|
673
|
-
queue = [];
|
|
674
|
-
results = [];
|
|
675
|
-
limit;
|
|
676
|
-
constructor(siteUrl, config2) {
|
|
677
|
-
this.config = { ...DEFAULT_CONFIG, ...config2 };
|
|
678
|
-
this.baseUrl = new URL(siteUrl);
|
|
679
|
-
this.limit = pLimit(this.config.concurrency);
|
|
680
|
-
}
|
|
681
|
-
/**
|
|
682
|
-
* Start crawling the site
|
|
683
|
-
*/
|
|
684
|
-
async crawl() {
|
|
685
|
-
consola3.info(`Starting crawl of ${this.baseUrl.origin}`);
|
|
686
|
-
consola3.info(`Config: maxPages=${this.config.maxPages}, maxDepth=${this.config.maxDepth}`);
|
|
687
|
-
this.queue.push({ url: this.baseUrl.href, depth: 0 });
|
|
688
|
-
while (this.queue.length > 0 && this.results.length < this.config.maxPages) {
|
|
689
|
-
const batch = this.queue.splice(0, this.config.concurrency);
|
|
690
|
-
const promises = batch.map(
|
|
691
|
-
({ url, depth }) => this.limit(() => this.crawlPage(url, depth))
|
|
692
|
-
);
|
|
693
|
-
await Promise.all(promises);
|
|
468
|
+
try {
|
|
469
|
+
const response = await fetch(robotsUrl);
|
|
470
|
+
if (!response.ok) {
|
|
471
|
+
analysis.issues.push({
|
|
472
|
+
id: "missing-robots-txt",
|
|
473
|
+
url: robotsUrl,
|
|
474
|
+
category: "technical",
|
|
475
|
+
severity: "warning",
|
|
476
|
+
title: "Missing robots.txt",
|
|
477
|
+
description: `No robots.txt file found (HTTP ${response.status}).`,
|
|
478
|
+
recommendation: "Create a robots.txt file to control crawler access.",
|
|
479
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
480
|
+
});
|
|
481
|
+
return analysis;
|
|
694
482
|
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
483
|
+
analysis.exists = true;
|
|
484
|
+
analysis.content = await response.text();
|
|
485
|
+
if (analysis.content.includes("content-signal") || analysis.content.includes("Content-Signal") || analysis.content.includes("ai-input") || analysis.content.includes("ai-train")) {
|
|
486
|
+
analysis.issues.push({
|
|
487
|
+
id: "cloudflare-managed-robots",
|
|
488
|
+
url: robotsUrl,
|
|
489
|
+
category: "technical",
|
|
490
|
+
severity: "warning",
|
|
491
|
+
title: "Cloudflare managed robots.txt detected",
|
|
492
|
+
description: `Your robots.txt is being overwritten by Cloudflare's "Content Signals Policy". Your app/robots.ts file is not being served.`,
|
|
493
|
+
recommendation: 'Disable in Cloudflare Dashboard: Security \u2192 Settings \u2192 "Manage your robots.txt" \u2192 Set to "Off".',
|
|
494
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
495
|
+
metadata: {
|
|
496
|
+
cloudflareFeature: "Managed robots.txt",
|
|
497
|
+
docsUrl: "https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/"
|
|
498
|
+
}
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
const robots = robotsParser(robotsUrl, analysis.content);
|
|
502
|
+
analysis.sitemaps = robots.getSitemaps();
|
|
503
|
+
if (analysis.sitemaps.length === 0) {
|
|
504
|
+
analysis.issues.push({
|
|
505
|
+
id: "no-sitemap-in-robots",
|
|
506
|
+
url: robotsUrl,
|
|
507
|
+
category: "technical",
|
|
508
|
+
severity: "info",
|
|
509
|
+
title: "No sitemap in robots.txt",
|
|
510
|
+
description: "No sitemap URL is declared in robots.txt.",
|
|
511
|
+
recommendation: "Add a Sitemap directive pointing to your XML sitemap.",
|
|
512
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
const lines = analysis.content.split("\n");
|
|
516
|
+
let currentUserAgent = "*";
|
|
517
|
+
for (const line of lines) {
|
|
518
|
+
const trimmed = line.trim().toLowerCase();
|
|
519
|
+
if (trimmed.startsWith("user-agent:")) {
|
|
520
|
+
currentUserAgent = trimmed.replace("user-agent:", "").trim();
|
|
521
|
+
} else if (trimmed.startsWith("disallow:")) {
|
|
522
|
+
const path6 = line.trim().replace(/disallow:/i, "").trim();
|
|
523
|
+
if (path6) {
|
|
524
|
+
analysis.disallowedPaths.push(path6);
|
|
525
|
+
}
|
|
526
|
+
} else if (trimmed.startsWith("allow:")) {
|
|
527
|
+
const path6 = line.trim().replace(/allow:/i, "").trim();
|
|
528
|
+
if (path6) {
|
|
529
|
+
analysis.allowedPaths.push(path6);
|
|
530
|
+
}
|
|
531
|
+
} else if (trimmed.startsWith("crawl-delay:")) {
|
|
532
|
+
const delay = parseInt(trimmed.replace("crawl-delay:", "").trim(), 10);
|
|
533
|
+
if (!isNaN(delay)) {
|
|
534
|
+
analysis.crawlDelay = delay;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
const importantPaths = ["/", "/sitemap.xml"];
|
|
539
|
+
for (const path6 of importantPaths) {
|
|
540
|
+
if (!robots.isAllowed(new URL(path6, siteUrl).href, "Googlebot")) {
|
|
541
|
+
analysis.issues.push({
|
|
542
|
+
id: `blocked-important-path-${path6.replace(/\//g, "-")}`,
|
|
543
|
+
url: siteUrl,
|
|
544
|
+
category: "crawling",
|
|
545
|
+
severity: "error",
|
|
546
|
+
title: `Important path blocked: ${path6}`,
|
|
547
|
+
description: `The path ${path6} is blocked in robots.txt.`,
|
|
548
|
+
recommendation: `Ensure ${path6} is accessible to search engines.`,
|
|
549
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
550
|
+
metadata: { path: path6 }
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
if (analysis.disallowedPaths.includes("/")) {
|
|
555
|
+
analysis.issues.push({
|
|
556
|
+
id: "all-blocked",
|
|
557
|
+
url: robotsUrl,
|
|
558
|
+
category: "crawling",
|
|
559
|
+
severity: "critical",
|
|
560
|
+
title: "Entire site blocked",
|
|
561
|
+
description: "robots.txt blocks access to the entire site (Disallow: /).",
|
|
562
|
+
recommendation: "Remove or modify this rule if you want your site to be indexed.",
|
|
563
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
consola6.debug(`Analyzed robots.txt: ${analysis.disallowedPaths.length} disallow rules`);
|
|
567
|
+
} catch (error) {
|
|
568
|
+
consola6.error("Failed to fetch robots.txt:", error);
|
|
569
|
+
analysis.issues.push({
|
|
570
|
+
id: "robots-txt-error",
|
|
571
|
+
url: robotsUrl,
|
|
572
|
+
category: "technical",
|
|
573
|
+
severity: "warning",
|
|
574
|
+
title: "Failed to fetch robots.txt",
|
|
575
|
+
description: `Error fetching robots.txt: ${error instanceof Error ? error.message : "Unknown error"}`,
|
|
576
|
+
recommendation: "Ensure robots.txt is accessible.",
|
|
577
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
578
|
+
});
|
|
579
|
+
}
|
|
580
|
+
return analysis;
|
|
581
|
+
}
|
|
582
|
+
async function analyzeSitemap(sitemapUrl) {
|
|
583
|
+
const analysis = {
|
|
584
|
+
url: sitemapUrl,
|
|
585
|
+
exists: false,
|
|
586
|
+
type: "unknown",
|
|
587
|
+
urls: [],
|
|
588
|
+
childSitemaps: [],
|
|
589
|
+
issues: []
|
|
590
|
+
};
|
|
591
|
+
try {
|
|
592
|
+
const response = await fetch(sitemapUrl, {
|
|
593
|
+
headers: {
|
|
594
|
+
Accept: "application/xml, text/xml, */*"
|
|
595
|
+
}
|
|
596
|
+
});
|
|
597
|
+
if (!response.ok) {
|
|
598
|
+
analysis.issues.push({
|
|
599
|
+
id: `sitemap-not-found-${hash2(sitemapUrl)}`,
|
|
600
|
+
url: sitemapUrl,
|
|
601
|
+
category: "technical",
|
|
602
|
+
severity: "error",
|
|
603
|
+
title: "Sitemap not accessible",
|
|
604
|
+
description: `Sitemap returned HTTP ${response.status}.`,
|
|
605
|
+
recommendation: "Ensure the sitemap URL is correct and accessible.",
|
|
606
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
607
|
+
metadata: { statusCode: response.status }
|
|
608
|
+
});
|
|
609
|
+
return analysis;
|
|
610
|
+
}
|
|
611
|
+
analysis.exists = true;
|
|
612
|
+
const content = await response.text();
|
|
613
|
+
const contentType = response.headers.get("content-type") || "";
|
|
614
|
+
if (!contentType.includes("xml") && !content.trim().startsWith("<?xml")) {
|
|
615
|
+
analysis.issues.push({
|
|
616
|
+
id: `sitemap-not-xml-${hash2(sitemapUrl)}`,
|
|
617
|
+
url: sitemapUrl,
|
|
618
|
+
category: "technical",
|
|
619
|
+
severity: "warning",
|
|
620
|
+
title: "Sitemap is not XML",
|
|
621
|
+
description: "The sitemap does not have an XML content type.",
|
|
622
|
+
recommendation: "Ensure sitemap is served with Content-Type: application/xml.",
|
|
623
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
624
|
+
metadata: { contentType }
|
|
625
|
+
});
|
|
626
|
+
}
|
|
627
|
+
const parser = new DOMParser();
|
|
628
|
+
const doc = parser.parseFromString(content, "text/xml");
|
|
629
|
+
const sitemapIndex = doc.querySelector("sitemapindex");
|
|
630
|
+
if (sitemapIndex) {
|
|
631
|
+
analysis.type = "sitemap-index";
|
|
632
|
+
for (const sitemap of doc.querySelectorAll("sitemap")) {
|
|
633
|
+
const loc = sitemap.querySelector("loc")?.textContent?.trim();
|
|
634
|
+
if (loc) {
|
|
635
|
+
analysis.childSitemaps.push(loc);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
consola6.debug(`Sitemap index contains ${analysis.childSitemaps.length} sitemaps`);
|
|
639
|
+
} else {
|
|
640
|
+
analysis.type = "sitemap";
|
|
641
|
+
for (const url of doc.querySelectorAll("url")) {
|
|
642
|
+
const loc = url.querySelector("loc")?.textContent?.trim();
|
|
643
|
+
if (loc) {
|
|
644
|
+
analysis.urls.push(loc);
|
|
645
|
+
}
|
|
646
|
+
if (!analysis.lastmod) {
|
|
647
|
+
const lastmod = url.querySelector("lastmod")?.textContent?.trim();
|
|
648
|
+
if (lastmod) {
|
|
649
|
+
analysis.lastmod = lastmod;
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
consola6.debug(`Sitemap contains ${analysis.urls.length} URLs`);
|
|
654
|
+
}
|
|
655
|
+
if (analysis.type === "sitemap" && analysis.urls.length === 0) {
|
|
656
|
+
analysis.issues.push({
|
|
657
|
+
id: `sitemap-empty-${hash2(sitemapUrl)}`,
|
|
658
|
+
url: sitemapUrl,
|
|
659
|
+
category: "technical",
|
|
660
|
+
severity: "warning",
|
|
661
|
+
title: "Sitemap is empty",
|
|
662
|
+
description: "The sitemap contains no URLs.",
|
|
663
|
+
recommendation: "Add URLs to your sitemap or remove it if not needed.",
|
|
664
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
if (analysis.urls.length > 5e4) {
|
|
668
|
+
analysis.issues.push({
|
|
669
|
+
id: `sitemap-too-large-${hash2(sitemapUrl)}`,
|
|
670
|
+
url: sitemapUrl,
|
|
671
|
+
category: "technical",
|
|
672
|
+
severity: "error",
|
|
673
|
+
title: "Sitemap exceeds URL limit",
|
|
674
|
+
description: `Sitemap contains ${analysis.urls.length} URLs. Maximum is 50,000.`,
|
|
675
|
+
recommendation: "Split the sitemap into multiple files using a sitemap index.",
|
|
676
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
677
|
+
metadata: { urlCount: analysis.urls.length }
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
const sizeInMB = new Blob([content]).size / (1024 * 1024);
|
|
681
|
+
if (sizeInMB > 50) {
|
|
682
|
+
analysis.issues.push({
|
|
683
|
+
id: `sitemap-too-large-size-${hash2(sitemapUrl)}`,
|
|
684
|
+
url: sitemapUrl,
|
|
685
|
+
category: "technical",
|
|
686
|
+
severity: "error",
|
|
687
|
+
title: "Sitemap exceeds size limit",
|
|
688
|
+
description: `Sitemap is ${sizeInMB.toFixed(2)}MB. Maximum is 50MB.`,
|
|
689
|
+
recommendation: "Split the sitemap or compress it.",
|
|
690
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
691
|
+
metadata: { sizeMB: sizeInMB }
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
} catch (error) {
|
|
695
|
+
consola6.error("Failed to analyze sitemap:", error);
|
|
696
|
+
analysis.issues.push({
|
|
697
|
+
id: `sitemap-error-${hash2(sitemapUrl)}`,
|
|
698
|
+
url: sitemapUrl,
|
|
699
|
+
category: "technical",
|
|
700
|
+
severity: "error",
|
|
701
|
+
title: "Failed to parse sitemap",
|
|
702
|
+
description: `Error: ${error instanceof Error ? error.message : "Unknown error"}`,
|
|
703
|
+
recommendation: "Check sitemap validity using Google Search Console.",
|
|
704
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
705
|
+
});
|
|
706
|
+
}
|
|
707
|
+
return analysis;
|
|
708
|
+
}
|
|
709
|
+
async function analyzeAllSitemaps(sitemapUrl, maxDepth = 3) {
|
|
710
|
+
const results = [];
|
|
711
|
+
const visited = /* @__PURE__ */ new Set();
|
|
712
|
+
async function analyze(url, depth) {
|
|
713
|
+
if (depth > maxDepth || visited.has(url)) return;
|
|
714
|
+
visited.add(url);
|
|
715
|
+
const analysis = await analyzeSitemap(url);
|
|
716
|
+
results.push(analysis);
|
|
717
|
+
for (const childUrl of analysis.childSitemaps) {
|
|
718
|
+
await analyze(childUrl, depth + 1);
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
await analyze(sitemapUrl, 0);
|
|
722
|
+
return results;
|
|
723
|
+
}
|
|
724
|
+
function hash2(str) {
|
|
725
|
+
let hash5 = 0;
|
|
726
|
+
for (let i = 0; i < str.length; i++) {
|
|
727
|
+
const char = str.charCodeAt(i);
|
|
728
|
+
hash5 = (hash5 << 5) - hash5 + char;
|
|
729
|
+
hash5 = hash5 & hash5;
|
|
730
|
+
}
|
|
731
|
+
return Math.abs(hash5).toString(36);
|
|
732
|
+
}
|
|
733
|
+
var SCOPES = [
|
|
734
|
+
"https://www.googleapis.com/auth/webmasters.readonly",
|
|
735
|
+
"https://www.googleapis.com/auth/webmasters"
|
|
736
|
+
];
|
|
737
|
+
function loadCredentials(config2) {
|
|
738
|
+
if (config2.serviceAccountJson) {
|
|
739
|
+
return config2.serviceAccountJson;
|
|
740
|
+
}
|
|
741
|
+
if (config2.serviceAccountPath) {
|
|
742
|
+
if (!existsSync(config2.serviceAccountPath)) {
|
|
743
|
+
throw new Error(`Service account file not found: ${config2.serviceAccountPath}`);
|
|
744
|
+
}
|
|
745
|
+
const content = readFileSync(config2.serviceAccountPath, "utf-8");
|
|
746
|
+
return JSON.parse(content);
|
|
747
|
+
}
|
|
748
|
+
const envJson = process.env.GOOGLE_SERVICE_ACCOUNT_JSON;
|
|
749
|
+
if (envJson) {
|
|
750
|
+
return JSON.parse(envJson);
|
|
751
|
+
}
|
|
752
|
+
const defaultPath = "./service_account.json";
|
|
753
|
+
if (existsSync(defaultPath)) {
|
|
754
|
+
const content = readFileSync(defaultPath, "utf-8");
|
|
755
|
+
return JSON.parse(content);
|
|
756
|
+
}
|
|
757
|
+
throw new Error(
|
|
758
|
+
"No service account credentials found. Provide serviceAccountPath, serviceAccountJson, or set GOOGLE_SERVICE_ACCOUNT_JSON env variable."
|
|
759
|
+
);
|
|
760
|
+
}
|
|
761
|
+
function createAuthClient(config2) {
|
|
762
|
+
const credentials = loadCredentials(config2);
|
|
763
|
+
const auth = new JWT({
|
|
764
|
+
email: credentials.client_email,
|
|
765
|
+
key: credentials.private_key,
|
|
766
|
+
scopes: SCOPES
|
|
767
|
+
});
|
|
768
|
+
auth._serviceAccountEmail = credentials.client_email;
|
|
769
|
+
return auth;
|
|
770
|
+
}
|
|
771
|
+
async function verifyAuth(auth, siteUrl) {
|
|
772
|
+
const email = auth._serviceAccountEmail || auth.email;
|
|
773
|
+
try {
|
|
774
|
+
await auth.authorize();
|
|
775
|
+
consola6.success("Google Search Console authentication verified");
|
|
776
|
+
consola6.info(`Service account: ${email}`);
|
|
777
|
+
if (siteUrl) {
|
|
778
|
+
const domain = new URL(siteUrl).hostname;
|
|
779
|
+
const gscUrl = `https://search.google.com/search-console/users?resource_id=sc-domain%3A${domain}`;
|
|
780
|
+
consola6.info(`Ensure this email has Full access in GSC: ${gscUrl}`);
|
|
781
|
+
}
|
|
782
|
+
return true;
|
|
783
|
+
} catch (error) {
|
|
784
|
+
consola6.error("Authentication failed");
|
|
785
|
+
consola6.info(`Service account email: ${email}`);
|
|
786
|
+
consola6.info("Make sure this email is added to GSC with Full access");
|
|
787
|
+
return false;
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
// src/google-console/client.ts
|
|
792
|
+
var GoogleConsoleClient = class {
|
|
793
|
+
auth;
|
|
794
|
+
searchconsole;
|
|
795
|
+
siteUrl;
|
|
796
|
+
gscSiteUrl;
|
|
797
|
+
// Format for GSC API (may be sc-domain:xxx)
|
|
798
|
+
limit = pLimit(2);
|
|
799
|
+
// Max 2 concurrent requests (Cloudflare-friendly)
|
|
800
|
+
requestDelay = 500;
|
|
801
|
+
// Delay between requests in ms
|
|
802
|
+
constructor(config2) {
|
|
803
|
+
this.auth = createAuthClient(config2);
|
|
804
|
+
this.searchconsole = searchconsole({ version: "v1", auth: this.auth });
|
|
805
|
+
this.siteUrl = config2.siteUrl;
|
|
806
|
+
if (config2.gscSiteUrl) {
|
|
807
|
+
this.gscSiteUrl = config2.gscSiteUrl;
|
|
808
|
+
} else {
|
|
809
|
+
const domain = new URL(config2.siteUrl).hostname;
|
|
810
|
+
this.gscSiteUrl = `sc-domain:${domain}`;
|
|
811
|
+
}
|
|
812
|
+
consola6.debug(`GSC site URL: ${this.gscSiteUrl}`);
|
|
813
|
+
}
|
|
814
|
+
/**
|
|
815
|
+
* Delay helper for rate limiting
|
|
816
|
+
*/
|
|
817
|
+
delay(ms) {
|
|
818
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
819
|
+
}
|
|
820
|
+
/**
|
|
821
|
+
* Verify the client is authenticated
|
|
822
|
+
*/
|
|
823
|
+
async verify() {
|
|
824
|
+
return verifyAuth(this.auth, this.siteUrl);
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* List all sites in Search Console
|
|
828
|
+
*/
|
|
829
|
+
async listSites() {
|
|
717
830
|
try {
|
|
718
|
-
const
|
|
719
|
-
|
|
720
|
-
const response = await fetch(normalizedUrl, {
|
|
721
|
-
headers: {
|
|
722
|
-
"User-Agent": this.config.userAgent,
|
|
723
|
-
Accept: "text/html,application/xhtml+xml"
|
|
724
|
-
},
|
|
725
|
-
signal: controller.signal,
|
|
726
|
-
redirect: "follow"
|
|
727
|
-
});
|
|
728
|
-
result.ttfb = Date.now() - startTime;
|
|
729
|
-
clearTimeout(timeoutId);
|
|
730
|
-
result.statusCode = response.status;
|
|
731
|
-
result.contentType = response.headers.get("content-type") || void 0;
|
|
732
|
-
result.contentLength = Number(response.headers.get("content-length")) || void 0;
|
|
733
|
-
if (response.ok && result.contentType?.includes("text/html")) {
|
|
734
|
-
const html = await response.text();
|
|
735
|
-
this.parseHtml(html, result, normalizedUrl, depth);
|
|
736
|
-
} else if (!response.ok) {
|
|
737
|
-
result.errors.push(`HTTP ${response.status}: ${response.statusText}`);
|
|
738
|
-
}
|
|
831
|
+
const response = await this.searchconsole.sites.list();
|
|
832
|
+
return response.data.siteEntry?.map((site) => site.siteUrl || "") || [];
|
|
739
833
|
} catch (error) {
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
result.errors.push("Request timeout");
|
|
743
|
-
} else {
|
|
744
|
-
result.errors.push(error.message);
|
|
745
|
-
}
|
|
746
|
-
}
|
|
834
|
+
consola6.error("Failed to list sites:", error);
|
|
835
|
+
throw error;
|
|
747
836
|
}
|
|
748
|
-
result.loadTime = Date.now() - startTime;
|
|
749
|
-
this.results.push(result);
|
|
750
|
-
consola3.debug(`Crawled: ${normalizedUrl} (${result.statusCode}) - ${result.loadTime}ms`);
|
|
751
837
|
}
|
|
752
838
|
/**
|
|
753
|
-
*
|
|
839
|
+
* Inspect a single URL
|
|
754
840
|
*/
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
841
|
+
async inspectUrl(url) {
|
|
842
|
+
return this.limit(async () => {
|
|
843
|
+
return pRetry(
|
|
844
|
+
async () => {
|
|
845
|
+
const response = await this.searchconsole.urlInspection.index.inspect({
|
|
846
|
+
requestBody: {
|
|
847
|
+
inspectionUrl: url,
|
|
848
|
+
siteUrl: this.gscSiteUrl,
|
|
849
|
+
languageCode: "en-US"
|
|
850
|
+
}
|
|
851
|
+
});
|
|
852
|
+
const result = response.data.inspectionResult;
|
|
853
|
+
if (!result?.indexStatusResult) {
|
|
854
|
+
throw new Error(`No inspection result for URL: ${url}`);
|
|
855
|
+
}
|
|
856
|
+
return this.mapInspectionResult(url, result);
|
|
857
|
+
},
|
|
858
|
+
{
|
|
859
|
+
retries: 2,
|
|
860
|
+
minTimeout: 2e3,
|
|
861
|
+
maxTimeout: 1e4,
|
|
862
|
+
factor: 2,
|
|
863
|
+
// Exponential backoff
|
|
864
|
+
onFailedAttempt: (ctx) => {
|
|
865
|
+
if (ctx.retriesLeft === 0) {
|
|
866
|
+
consola6.warn(`Failed: ${url}`);
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
}
|
|
771
870
|
);
|
|
772
|
-
}
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
}
|
|
780
|
-
const
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
result.h1 = Array.from(document.querySelectorAll("h1")).map((el) => el.textContent?.trim() || "");
|
|
786
|
-
result.h2 = Array.from(document.querySelectorAll("h2")).map((el) => el.textContent?.trim() || "");
|
|
787
|
-
if (result.h1.length === 0) {
|
|
788
|
-
result.warnings.push("Missing H1 tag");
|
|
789
|
-
} else if (result.h1.length > 1) {
|
|
790
|
-
result.warnings.push(`Multiple H1 tags (${result.h1.length})`);
|
|
791
|
-
}
|
|
792
|
-
for (const el of document.querySelectorAll("a[href]")) {
|
|
793
|
-
const href = el.getAttribute("href");
|
|
794
|
-
if (!href) continue;
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
/**
|
|
874
|
+
* Inspect multiple URLs in batch
|
|
875
|
+
* Stops early if too many consecutive errors (likely rate limiting)
|
|
876
|
+
*/
|
|
877
|
+
async inspectUrls(urls) {
|
|
878
|
+
consola6.info(`Inspecting ${urls.length} URLs...`);
|
|
879
|
+
const results = [];
|
|
880
|
+
const errors = [];
|
|
881
|
+
let consecutiveErrors = 0;
|
|
882
|
+
const maxConsecutiveErrors = 3;
|
|
883
|
+
for (const url of urls) {
|
|
795
884
|
try {
|
|
796
|
-
const
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
885
|
+
const result = await this.inspectUrl(url);
|
|
886
|
+
results.push(result);
|
|
887
|
+
consecutiveErrors = 0;
|
|
888
|
+
await this.delay(this.requestDelay);
|
|
889
|
+
} catch (error) {
|
|
890
|
+
const err = error;
|
|
891
|
+
errors.push({ url, error: err });
|
|
892
|
+
consecutiveErrors++;
|
|
893
|
+
if (consecutiveErrors >= maxConsecutiveErrors) {
|
|
894
|
+
console.log("");
|
|
895
|
+
consola6.error(`Stopping after ${maxConsecutiveErrors} consecutive failures`);
|
|
896
|
+
this.showRateLimitHelp();
|
|
897
|
+
break;
|
|
805
898
|
}
|
|
806
|
-
} catch {
|
|
807
899
|
}
|
|
808
900
|
}
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
const alt = el.getAttribute("alt");
|
|
812
|
-
const hasAltAttr = alt !== null;
|
|
813
|
-
if (src) {
|
|
814
|
-
result.images.push({
|
|
815
|
-
src,
|
|
816
|
-
alt: alt ?? void 0,
|
|
817
|
-
hasAlt: hasAltAttr && alt.trim().length > 0
|
|
818
|
-
});
|
|
819
|
-
}
|
|
901
|
+
if (errors.length > 0 && consecutiveErrors < maxConsecutiveErrors) {
|
|
902
|
+
consola6.warn(`Failed to inspect ${errors.length} URLs`);
|
|
820
903
|
}
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
904
|
+
if (results.length > 0) {
|
|
905
|
+
consola6.success(`Successfully inspected ${results.length}/${urls.length} URLs`);
|
|
906
|
+
} else if (errors.length > 0) {
|
|
907
|
+
consola6.warn("No URLs were successfully inspected");
|
|
824
908
|
}
|
|
909
|
+
return results;
|
|
825
910
|
}
|
|
826
911
|
/**
|
|
827
|
-
*
|
|
912
|
+
* Show help message for rate limiting issues
|
|
828
913
|
*/
|
|
829
|
-
|
|
914
|
+
showRateLimitHelp() {
|
|
915
|
+
consola6.info("Possible causes:");
|
|
916
|
+
consola6.info(" 1. Google API quota exceeded (2000 requests/day)");
|
|
917
|
+
consola6.info(" 2. Cloudflare blocking Google's crawler");
|
|
918
|
+
consola6.info(" 3. Service account not added to GSC");
|
|
919
|
+
console.log("");
|
|
920
|
+
consola6.info("Solutions:");
|
|
921
|
+
consola6.info(" \u2022 Check GSC access: https://search.google.com/search-console/users");
|
|
922
|
+
console.log("");
|
|
923
|
+
consola6.info(" \u2022 Cloudflare WAF rule to allow Googlebot:");
|
|
924
|
+
consola6.info(" 1. Dashboard \u2192 Security \u2192 WAF \u2192 Custom rules \u2192 Create rule");
|
|
925
|
+
consola6.info(' 2. Name: "Allow Googlebot"');
|
|
926
|
+
consola6.info(' 3. Field: "Known Bots" | Operator: "equals" | Value: "true"');
|
|
927
|
+
consola6.info(' 4. Or click "Edit expression" and paste: (cf.client.bot)');
|
|
928
|
+
consola6.info(" 5. Action: Skip \u2192 check all rules");
|
|
929
|
+
consola6.info(" 6. Deploy");
|
|
930
|
+
consola6.info(" Docs: https://developers.cloudflare.com/waf/custom-rules/use-cases/allow-traffic-from-verified-bots/");
|
|
931
|
+
console.log("");
|
|
932
|
+
}
|
|
933
|
+
/**
|
|
934
|
+
* Get search analytics data
|
|
935
|
+
*/
|
|
936
|
+
async getSearchAnalytics(options) {
|
|
830
937
|
try {
|
|
831
|
-
const
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
return
|
|
938
|
+
const response = await this.searchconsole.searchanalytics.query({
|
|
939
|
+
siteUrl: this.gscSiteUrl,
|
|
940
|
+
requestBody: {
|
|
941
|
+
startDate: options.startDate,
|
|
942
|
+
endDate: options.endDate,
|
|
943
|
+
dimensions: options.dimensions || ["page"],
|
|
944
|
+
rowLimit: options.rowLimit || 1e3
|
|
945
|
+
}
|
|
946
|
+
});
|
|
947
|
+
return response.data.rows || [];
|
|
948
|
+
} catch (error) {
|
|
949
|
+
consola6.error("Failed to get search analytics:", error);
|
|
950
|
+
throw error;
|
|
841
951
|
}
|
|
842
952
|
}
|
|
843
953
|
/**
|
|
844
|
-
*
|
|
954
|
+
* Get list of sitemaps
|
|
845
955
|
*/
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
const
|
|
849
|
-
|
|
850
|
-
);
|
|
851
|
-
|
|
956
|
+
async getSitemaps() {
|
|
957
|
+
try {
|
|
958
|
+
const response = await this.searchconsole.sitemaps.list({
|
|
959
|
+
siteUrl: this.gscSiteUrl
|
|
960
|
+
});
|
|
961
|
+
return response.data.sitemap || [];
|
|
962
|
+
} catch (error) {
|
|
963
|
+
consola6.error("Failed to get sitemaps:", error);
|
|
964
|
+
throw error;
|
|
852
965
|
}
|
|
853
|
-
|
|
966
|
+
}
|
|
967
|
+
/**
|
|
968
|
+
* Map API response to our types
|
|
969
|
+
*/
|
|
970
|
+
mapInspectionResult(url, result) {
|
|
971
|
+
const indexStatus = result.indexStatusResult;
|
|
972
|
+
return {
|
|
973
|
+
url,
|
|
974
|
+
inspectionResultLink: result.inspectionResultLink || void 0,
|
|
975
|
+
indexStatusResult: {
|
|
976
|
+
verdict: indexStatus.verdict || "VERDICT_UNSPECIFIED",
|
|
977
|
+
coverageState: indexStatus.coverageState || "COVERAGE_STATE_UNSPECIFIED",
|
|
978
|
+
indexingState: indexStatus.indexingState || "INDEXING_STATE_UNSPECIFIED",
|
|
979
|
+
robotsTxtState: indexStatus.robotsTxtState || "ROBOTS_TXT_STATE_UNSPECIFIED",
|
|
980
|
+
pageFetchState: indexStatus.pageFetchState || "PAGE_FETCH_STATE_UNSPECIFIED",
|
|
981
|
+
lastCrawlTime: indexStatus.lastCrawlTime || void 0,
|
|
982
|
+
crawledAs: indexStatus.crawledAs,
|
|
983
|
+
googleCanonical: indexStatus.googleCanonical || void 0,
|
|
984
|
+
userCanonical: indexStatus.userCanonical || void 0,
|
|
985
|
+
sitemap: indexStatus.sitemap || void 0,
|
|
986
|
+
referringUrls: indexStatus.referringUrls || void 0
|
|
987
|
+
},
|
|
988
|
+
mobileUsabilityResult: result.mobileUsabilityResult ? {
|
|
989
|
+
verdict: result.mobileUsabilityResult.verdict || "VERDICT_UNSPECIFIED",
|
|
990
|
+
issues: result.mobileUsabilityResult.issues?.map((issue) => ({
|
|
991
|
+
issueType: issue.issueType || "UNKNOWN",
|
|
992
|
+
message: issue.message || ""
|
|
993
|
+
}))
|
|
994
|
+
} : void 0,
|
|
995
|
+
richResultsResult: result.richResultsResult ? {
|
|
996
|
+
verdict: result.richResultsResult.verdict || "VERDICT_UNSPECIFIED",
|
|
997
|
+
detectedItems: result.richResultsResult.detectedItems?.map((item) => ({
|
|
998
|
+
richResultType: item.richResultType || "UNKNOWN",
|
|
999
|
+
items: item.items?.map((i) => ({
|
|
1000
|
+
name: i.name || "",
|
|
1001
|
+
issues: i.issues?.map((issue) => ({
|
|
1002
|
+
issueMessage: issue.issueMessage || "",
|
|
1003
|
+
severity: issue.severity || "WARNING"
|
|
1004
|
+
}))
|
|
1005
|
+
}))
|
|
1006
|
+
}))
|
|
1007
|
+
} : void 0
|
|
1008
|
+
};
|
|
854
1009
|
}
|
|
855
1010
|
};
|
|
856
|
-
|
|
1011
|
+
|
|
1012
|
+
// src/google-console/analyzer.ts
|
|
1013
|
+
function analyzeInspectionResults(results) {
|
|
857
1014
|
const issues = [];
|
|
858
1015
|
for (const result of results) {
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
detectedAt: result.crawledAt,
|
|
869
|
-
metadata: { statusCode: result.statusCode }
|
|
870
|
-
});
|
|
871
|
-
}
|
|
872
|
-
if (!result.title && result.statusCode === 200) {
|
|
1016
|
+
issues.push(...analyzeUrlInspection(result));
|
|
1017
|
+
}
|
|
1018
|
+
return issues.sort((a, b) => severityOrder(a.severity) - severityOrder(b.severity));
|
|
1019
|
+
}
|
|
1020
|
+
function analyzeUrlInspection(result) {
|
|
1021
|
+
const issues = [];
|
|
1022
|
+
const { indexStatusResult, mobileUsabilityResult, richResultsResult } = result;
|
|
1023
|
+
switch (indexStatusResult.coverageState) {
|
|
1024
|
+
case "CRAWLED_CURRENTLY_NOT_INDEXED":
|
|
873
1025
|
issues.push({
|
|
874
|
-
id: `
|
|
1026
|
+
id: `crawled-not-indexed-${hash3(result.url)}`,
|
|
875
1027
|
url: result.url,
|
|
876
|
-
category: "
|
|
1028
|
+
category: "indexing",
|
|
877
1029
|
severity: "error",
|
|
878
|
-
title: "
|
|
879
|
-
description: "
|
|
880
|
-
recommendation: "
|
|
881
|
-
detectedAt:
|
|
882
|
-
|
|
883
|
-
}
|
|
884
|
-
if (!result.metaDescription && result.statusCode === 200) {
|
|
885
|
-
issues.push({
|
|
886
|
-
id: `missing-meta-desc-${hash2(result.url)}`,
|
|
887
|
-
url: result.url,
|
|
888
|
-
category: "content",
|
|
889
|
-
severity: "warning",
|
|
890
|
-
title: "Missing meta description",
|
|
891
|
-
description: "This page does not have a meta description.",
|
|
892
|
-
recommendation: "Add a unique meta description (120-160 characters).",
|
|
893
|
-
detectedAt: result.crawledAt
|
|
894
|
-
});
|
|
895
|
-
}
|
|
896
|
-
if (result.h1 && result.h1.length === 0 && result.statusCode === 200) {
|
|
897
|
-
issues.push({
|
|
898
|
-
id: `missing-h1-${hash2(result.url)}`,
|
|
899
|
-
url: result.url,
|
|
900
|
-
category: "content",
|
|
901
|
-
severity: "warning",
|
|
902
|
-
title: "Missing H1 heading",
|
|
903
|
-
description: "This page does not have an H1 heading.",
|
|
904
|
-
recommendation: "Add a single H1 heading that describes the page content.",
|
|
905
|
-
detectedAt: result.crawledAt
|
|
1030
|
+
title: "Page crawled but not indexed",
|
|
1031
|
+
description: "Google crawled this page but decided not to index it. This often indicates low content quality or duplicate content.",
|
|
1032
|
+
recommendation: "Improve content quality, ensure uniqueness, add more valuable information, and check for duplicate content issues.",
|
|
1033
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1034
|
+
metadata: { coverageState: indexStatusResult.coverageState }
|
|
906
1035
|
});
|
|
907
|
-
|
|
908
|
-
|
|
1036
|
+
break;
|
|
1037
|
+
case "DISCOVERED_CURRENTLY_NOT_INDEXED":
|
|
909
1038
|
issues.push({
|
|
910
|
-
id: `
|
|
1039
|
+
id: `discovered-not-indexed-${hash3(result.url)}`,
|
|
911
1040
|
url: result.url,
|
|
912
|
-
category: "
|
|
1041
|
+
category: "indexing",
|
|
913
1042
|
severity: "warning",
|
|
914
|
-
title: "
|
|
915
|
-
description:
|
|
916
|
-
recommendation: "
|
|
917
|
-
detectedAt:
|
|
918
|
-
metadata: {
|
|
919
|
-
});
|
|
920
|
-
}
|
|
921
|
-
const imagesWithoutAlt = result.images.filter((img) => !img.hasAlt);
|
|
922
|
-
if (imagesWithoutAlt.length > 0) {
|
|
923
|
-
issues.push({
|
|
924
|
-
id: `images-no-alt-${hash2(result.url)}`,
|
|
925
|
-
url: result.url,
|
|
926
|
-
category: "content",
|
|
927
|
-
severity: "info",
|
|
928
|
-
title: "Images without alt text",
|
|
929
|
-
description: `${imagesWithoutAlt.length} images are missing alt text.`,
|
|
930
|
-
recommendation: "Add descriptive alt text to all images for accessibility and SEO.",
|
|
931
|
-
detectedAt: result.crawledAt,
|
|
932
|
-
metadata: { count: imagesWithoutAlt.length }
|
|
933
|
-
});
|
|
934
|
-
}
|
|
935
|
-
if (result.loadTime > 3e3) {
|
|
936
|
-
issues.push({
|
|
937
|
-
id: `slow-page-${hash2(result.url)}`,
|
|
938
|
-
url: result.url,
|
|
939
|
-
category: "performance",
|
|
940
|
-
severity: result.loadTime > 5e3 ? "error" : "warning",
|
|
941
|
-
title: "Slow page load time",
|
|
942
|
-
description: `Page took ${result.loadTime}ms to load.`,
|
|
943
|
-
recommendation: "Optimize page load time. Target under 3 seconds.",
|
|
944
|
-
detectedAt: result.crawledAt,
|
|
945
|
-
metadata: { loadTime: result.loadTime }
|
|
946
|
-
});
|
|
947
|
-
}
|
|
948
|
-
if (result.ttfb && result.ttfb > 800) {
|
|
949
|
-
issues.push({
|
|
950
|
-
id: `slow-ttfb-${hash2(result.url)}`,
|
|
951
|
-
url: result.url,
|
|
952
|
-
category: "performance",
|
|
953
|
-
severity: result.ttfb > 1500 ? "error" : "warning",
|
|
954
|
-
title: "Slow Time to First Byte",
|
|
955
|
-
description: `TTFB is ${result.ttfb}ms. Server responded slowly.`,
|
|
956
|
-
recommendation: "Optimize server response. Target TTFB under 800ms. Consider CDN, caching, or server upgrades.",
|
|
957
|
-
detectedAt: result.crawledAt,
|
|
958
|
-
metadata: { ttfb: result.ttfb }
|
|
1043
|
+
title: "Page discovered but not crawled",
|
|
1044
|
+
description: "Google discovered this URL but has not crawled it yet. This may indicate crawl budget issues or low priority.",
|
|
1045
|
+
recommendation: "Improve internal linking to this page, submit URL through Google Search Console, or add to sitemap.",
|
|
1046
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1047
|
+
metadata: { coverageState: indexStatusResult.coverageState }
|
|
959
1048
|
});
|
|
960
|
-
|
|
961
|
-
|
|
1049
|
+
break;
|
|
1050
|
+
case "DUPLICATE_WITHOUT_USER_SELECTED_CANONICAL":
|
|
962
1051
|
issues.push({
|
|
963
|
-
id: `
|
|
1052
|
+
id: `duplicate-no-canonical-${hash3(result.url)}`,
|
|
964
1053
|
url: result.url,
|
|
965
1054
|
category: "indexing",
|
|
966
|
-
severity: "info",
|
|
967
|
-
title: "Page marked as noindex",
|
|
968
|
-
description: "This page has a noindex directive.",
|
|
969
|
-
recommendation: "Verify this is intentional. Remove noindex if the page should be indexed.",
|
|
970
|
-
detectedAt: result.crawledAt,
|
|
971
|
-
metadata: { metaRobots: result.metaRobots }
|
|
972
|
-
});
|
|
973
|
-
}
|
|
974
|
-
}
|
|
975
|
-
return issues;
|
|
976
|
-
}
|
|
977
|
-
function hash2(str) {
|
|
978
|
-
let hash5 = 0;
|
|
979
|
-
for (let i = 0; i < str.length; i++) {
|
|
980
|
-
const char = str.charCodeAt(i);
|
|
981
|
-
hash5 = (hash5 << 5) - hash5 + char;
|
|
982
|
-
hash5 = hash5 & hash5;
|
|
983
|
-
}
|
|
984
|
-
return Math.abs(hash5).toString(36);
|
|
985
|
-
}
|
|
986
|
-
async function analyzeRobotsTxt(siteUrl) {
|
|
987
|
-
const robotsUrl = new URL("/robots.txt", siteUrl).href;
|
|
988
|
-
const analysis = {
|
|
989
|
-
exists: false,
|
|
990
|
-
sitemaps: [],
|
|
991
|
-
allowedPaths: [],
|
|
992
|
-
disallowedPaths: [],
|
|
993
|
-
issues: []
|
|
994
|
-
};
|
|
995
|
-
try {
|
|
996
|
-
const response = await fetch(robotsUrl);
|
|
997
|
-
if (!response.ok) {
|
|
998
|
-
analysis.issues.push({
|
|
999
|
-
id: "missing-robots-txt",
|
|
1000
|
-
url: robotsUrl,
|
|
1001
|
-
category: "technical",
|
|
1002
|
-
severity: "warning",
|
|
1003
|
-
title: "Missing robots.txt",
|
|
1004
|
-
description: `No robots.txt file found (HTTP ${response.status}).`,
|
|
1005
|
-
recommendation: "Create a robots.txt file to control crawler access.",
|
|
1006
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1007
|
-
});
|
|
1008
|
-
return analysis;
|
|
1009
|
-
}
|
|
1010
|
-
analysis.exists = true;
|
|
1011
|
-
analysis.content = await response.text();
|
|
1012
|
-
if (analysis.content.includes("content-signal") || analysis.content.includes("Content-Signal") || analysis.content.includes("ai-input") || analysis.content.includes("ai-train")) {
|
|
1013
|
-
analysis.issues.push({
|
|
1014
|
-
id: "cloudflare-managed-robots",
|
|
1015
|
-
url: robotsUrl,
|
|
1016
|
-
category: "technical",
|
|
1017
1055
|
severity: "warning",
|
|
1018
|
-
title: "
|
|
1019
|
-
description:
|
|
1020
|
-
recommendation:
|
|
1056
|
+
title: "Duplicate page without canonical",
|
|
1057
|
+
description: "This page is considered a duplicate but no canonical URL has been specified. Google chose a canonical for you.",
|
|
1058
|
+
recommendation: "Add a canonical tag pointing to the preferred version of this page.",
|
|
1021
1059
|
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1022
1060
|
metadata: {
|
|
1023
|
-
|
|
1024
|
-
|
|
1061
|
+
coverageState: indexStatusResult.coverageState,
|
|
1062
|
+
googleCanonical: indexStatusResult.googleCanonical
|
|
1025
1063
|
}
|
|
1026
1064
|
});
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
const lines = analysis.content.split("\n");
|
|
1043
|
-
let currentUserAgent = "*";
|
|
1044
|
-
for (const line of lines) {
|
|
1045
|
-
const trimmed = line.trim().toLowerCase();
|
|
1046
|
-
if (trimmed.startsWith("user-agent:")) {
|
|
1047
|
-
currentUserAgent = trimmed.replace("user-agent:", "").trim();
|
|
1048
|
-
} else if (trimmed.startsWith("disallow:")) {
|
|
1049
|
-
const path6 = line.trim().replace(/disallow:/i, "").trim();
|
|
1050
|
-
if (path6) {
|
|
1051
|
-
analysis.disallowedPaths.push(path6);
|
|
1052
|
-
}
|
|
1053
|
-
} else if (trimmed.startsWith("allow:")) {
|
|
1054
|
-
const path6 = line.trim().replace(/allow:/i, "").trim();
|
|
1055
|
-
if (path6) {
|
|
1056
|
-
analysis.allowedPaths.push(path6);
|
|
1057
|
-
}
|
|
1058
|
-
} else if (trimmed.startsWith("crawl-delay:")) {
|
|
1059
|
-
const delay = parseInt(trimmed.replace("crawl-delay:", "").trim(), 10);
|
|
1060
|
-
if (!isNaN(delay)) {
|
|
1061
|
-
analysis.crawlDelay = delay;
|
|
1065
|
+
break;
|
|
1066
|
+
case "DUPLICATE_GOOGLE_CHOSE_DIFFERENT_CANONICAL":
|
|
1067
|
+
issues.push({
|
|
1068
|
+
id: `canonical-mismatch-${hash3(result.url)}`,
|
|
1069
|
+
url: result.url,
|
|
1070
|
+
category: "indexing",
|
|
1071
|
+
severity: "warning",
|
|
1072
|
+
title: "Google chose different canonical",
|
|
1073
|
+
description: "You specified a canonical URL, but Google chose a different one. This may cause indexing issues.",
|
|
1074
|
+
recommendation: "Review canonical tags and ensure they point to the correct URL. Check for duplicate content.",
|
|
1075
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1076
|
+
metadata: {
|
|
1077
|
+
coverageState: indexStatusResult.coverageState,
|
|
1078
|
+
userCanonical: indexStatusResult.userCanonical,
|
|
1079
|
+
googleCanonical: indexStatusResult.googleCanonical
|
|
1062
1080
|
}
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1081
|
+
});
|
|
1082
|
+
break;
|
|
1083
|
+
}
|
|
1084
|
+
switch (indexStatusResult.indexingState) {
|
|
1085
|
+
case "BLOCKED_BY_META_TAG":
|
|
1086
|
+
issues.push({
|
|
1087
|
+
id: `blocked-meta-noindex-${hash3(result.url)}`,
|
|
1088
|
+
url: result.url,
|
|
1089
|
+
category: "indexing",
|
|
1090
|
+
severity: "error",
|
|
1091
|
+
title: "Blocked by noindex meta tag",
|
|
1092
|
+
description: "This page has a noindex meta tag preventing it from being indexed.",
|
|
1093
|
+
recommendation: "Remove the noindex meta tag if you want this page to be indexed. If intentional, no action needed.",
|
|
1094
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1095
|
+
metadata: { indexingState: indexStatusResult.indexingState }
|
|
1096
|
+
});
|
|
1097
|
+
break;
|
|
1098
|
+
case "BLOCKED_BY_HTTP_HEADER":
|
|
1099
|
+
issues.push({
|
|
1100
|
+
id: `blocked-http-header-${hash3(result.url)}`,
|
|
1101
|
+
url: result.url,
|
|
1102
|
+
category: "indexing",
|
|
1103
|
+
severity: "error",
|
|
1104
|
+
title: "Blocked by X-Robots-Tag header",
|
|
1105
|
+
description: "This page has a noindex directive in the X-Robots-Tag HTTP header.",
|
|
1106
|
+
recommendation: "Remove the X-Robots-Tag: noindex header if you want this page to be indexed.",
|
|
1107
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1108
|
+
metadata: { indexingState: indexStatusResult.indexingState }
|
|
1109
|
+
});
|
|
1110
|
+
break;
|
|
1111
|
+
case "BLOCKED_BY_ROBOTS_TXT":
|
|
1112
|
+
issues.push({
|
|
1113
|
+
id: `blocked-robots-txt-${hash3(result.url)}`,
|
|
1114
|
+
url: result.url,
|
|
1085
1115
|
category: "crawling",
|
|
1086
|
-
severity: "
|
|
1087
|
-
title: "
|
|
1088
|
-
description: "
|
|
1089
|
-
recommendation: "
|
|
1090
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1116
|
+
severity: "error",
|
|
1117
|
+
title: "Blocked by robots.txt",
|
|
1118
|
+
description: "This page is blocked from crawling by robots.txt rules.",
|
|
1119
|
+
recommendation: "Update robots.txt to allow crawling if you want this page to be indexed.",
|
|
1120
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1121
|
+
metadata: { indexingState: indexStatusResult.indexingState }
|
|
1091
1122
|
});
|
|
1092
|
-
|
|
1093
|
-
consola3.debug(`Analyzed robots.txt: ${analysis.disallowedPaths.length} disallow rules`);
|
|
1094
|
-
} catch (error) {
|
|
1095
|
-
consola3.error("Failed to fetch robots.txt:", error);
|
|
1096
|
-
analysis.issues.push({
|
|
1097
|
-
id: "robots-txt-error",
|
|
1098
|
-
url: robotsUrl,
|
|
1099
|
-
category: "technical",
|
|
1100
|
-
severity: "warning",
|
|
1101
|
-
title: "Failed to fetch robots.txt",
|
|
1102
|
-
description: `Error fetching robots.txt: ${error instanceof Error ? error.message : "Unknown error"}`,
|
|
1103
|
-
recommendation: "Ensure robots.txt is accessible.",
|
|
1104
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1105
|
-
});
|
|
1123
|
+
break;
|
|
1106
1124
|
}
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
exists: false,
|
|
1113
|
-
type: "unknown",
|
|
1114
|
-
urls: [],
|
|
1115
|
-
childSitemaps: [],
|
|
1116
|
-
issues: []
|
|
1117
|
-
};
|
|
1118
|
-
try {
|
|
1119
|
-
const response = await fetch(sitemapUrl, {
|
|
1120
|
-
headers: {
|
|
1121
|
-
Accept: "application/xml, text/xml, */*"
|
|
1122
|
-
}
|
|
1123
|
-
});
|
|
1124
|
-
if (!response.ok) {
|
|
1125
|
-
analysis.issues.push({
|
|
1126
|
-
id: `sitemap-not-found-${hash3(sitemapUrl)}`,
|
|
1127
|
-
url: sitemapUrl,
|
|
1125
|
+
switch (indexStatusResult.pageFetchState) {
|
|
1126
|
+
case "SOFT_404":
|
|
1127
|
+
issues.push({
|
|
1128
|
+
id: `soft-404-${hash3(result.url)}`,
|
|
1129
|
+
url: result.url,
|
|
1128
1130
|
category: "technical",
|
|
1129
1131
|
severity: "error",
|
|
1130
|
-
title: "
|
|
1131
|
-
description:
|
|
1132
|
-
recommendation: "
|
|
1132
|
+
title: "Soft 404 error",
|
|
1133
|
+
description: "This page returns a 200 status but Google detected it as a 404 page (empty or low-value content).",
|
|
1134
|
+
recommendation: "Either return a proper 404 status code or add meaningful content to this page.",
|
|
1133
1135
|
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1134
|
-
metadata: {
|
|
1136
|
+
metadata: { pageFetchState: indexStatusResult.pageFetchState }
|
|
1135
1137
|
});
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
if (!contentType.includes("xml") && !content.trim().startsWith("<?xml")) {
|
|
1142
|
-
analysis.issues.push({
|
|
1143
|
-
id: `sitemap-not-xml-${hash3(sitemapUrl)}`,
|
|
1144
|
-
url: sitemapUrl,
|
|
1138
|
+
break;
|
|
1139
|
+
case "NOT_FOUND":
|
|
1140
|
+
issues.push({
|
|
1141
|
+
id: `404-error-${hash3(result.url)}`,
|
|
1142
|
+
url: result.url,
|
|
1145
1143
|
category: "technical",
|
|
1146
|
-
severity: "
|
|
1147
|
-
title: "
|
|
1148
|
-
description: "
|
|
1149
|
-
recommendation: "
|
|
1144
|
+
severity: "error",
|
|
1145
|
+
title: "404 Not Found",
|
|
1146
|
+
description: "This page returns a 404 error.",
|
|
1147
|
+
recommendation: "Either restore the page content or set up a redirect to a relevant page.",
|
|
1150
1148
|
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1151
|
-
metadata: {
|
|
1149
|
+
metadata: { pageFetchState: indexStatusResult.pageFetchState }
|
|
1152
1150
|
});
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
analysis.type = "sitemap-index";
|
|
1159
|
-
for (const sitemap of doc.querySelectorAll("sitemap")) {
|
|
1160
|
-
const loc = sitemap.querySelector("loc")?.textContent?.trim();
|
|
1161
|
-
if (loc) {
|
|
1162
|
-
analysis.childSitemaps.push(loc);
|
|
1163
|
-
}
|
|
1164
|
-
}
|
|
1165
|
-
consola3.debug(`Sitemap index contains ${analysis.childSitemaps.length} sitemaps`);
|
|
1166
|
-
} else {
|
|
1167
|
-
analysis.type = "sitemap";
|
|
1168
|
-
for (const url of doc.querySelectorAll("url")) {
|
|
1169
|
-
const loc = url.querySelector("loc")?.textContent?.trim();
|
|
1170
|
-
if (loc) {
|
|
1171
|
-
analysis.urls.push(loc);
|
|
1172
|
-
}
|
|
1173
|
-
if (!analysis.lastmod) {
|
|
1174
|
-
const lastmod = url.querySelector("lastmod")?.textContent?.trim();
|
|
1175
|
-
if (lastmod) {
|
|
1176
|
-
analysis.lastmod = lastmod;
|
|
1177
|
-
}
|
|
1178
|
-
}
|
|
1179
|
-
}
|
|
1180
|
-
consola3.debug(`Sitemap contains ${analysis.urls.length} URLs`);
|
|
1181
|
-
}
|
|
1182
|
-
if (analysis.type === "sitemap" && analysis.urls.length === 0) {
|
|
1183
|
-
analysis.issues.push({
|
|
1184
|
-
id: `sitemap-empty-${hash3(sitemapUrl)}`,
|
|
1185
|
-
url: sitemapUrl,
|
|
1151
|
+
break;
|
|
1152
|
+
case "SERVER_ERROR":
|
|
1153
|
+
issues.push({
|
|
1154
|
+
id: `server-error-${hash3(result.url)}`,
|
|
1155
|
+
url: result.url,
|
|
1186
1156
|
category: "technical",
|
|
1187
|
-
severity: "
|
|
1188
|
-
title: "
|
|
1189
|
-
description: "
|
|
1190
|
-
recommendation: "
|
|
1191
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1157
|
+
severity: "critical",
|
|
1158
|
+
title: "Server error (5xx)",
|
|
1159
|
+
description: "This page returns a server error when Google tries to crawl it.",
|
|
1160
|
+
recommendation: "Fix the server-side error. Check server logs for details.",
|
|
1161
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1162
|
+
metadata: { pageFetchState: indexStatusResult.pageFetchState }
|
|
1192
1163
|
});
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
id: `
|
|
1197
|
-
url:
|
|
1164
|
+
break;
|
|
1165
|
+
case "REDIRECT_ERROR":
|
|
1166
|
+
issues.push({
|
|
1167
|
+
id: `redirect-error-${hash3(result.url)}`,
|
|
1168
|
+
url: result.url,
|
|
1198
1169
|
category: "technical",
|
|
1199
1170
|
severity: "error",
|
|
1200
|
-
title: "
|
|
1201
|
-
description:
|
|
1202
|
-
recommendation: "
|
|
1171
|
+
title: "Redirect error",
|
|
1172
|
+
description: "There is a redirect issue with this page (redirect loop, too many redirects, or invalid redirect).",
|
|
1173
|
+
recommendation: "Fix the redirect chain. Ensure redirects point to valid, accessible pages.",
|
|
1203
1174
|
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1204
|
-
metadata: {
|
|
1175
|
+
metadata: { pageFetchState: indexStatusResult.pageFetchState }
|
|
1205
1176
|
});
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
id: `
|
|
1211
|
-
url:
|
|
1177
|
+
break;
|
|
1178
|
+
case "ACCESS_DENIED":
|
|
1179
|
+
case "ACCESS_FORBIDDEN":
|
|
1180
|
+
issues.push({
|
|
1181
|
+
id: `access-denied-${hash3(result.url)}`,
|
|
1182
|
+
url: result.url,
|
|
1212
1183
|
category: "technical",
|
|
1213
1184
|
severity: "error",
|
|
1214
|
-
title: "
|
|
1215
|
-
description:
|
|
1216
|
-
recommendation: "
|
|
1185
|
+
title: "Access denied (401/403)",
|
|
1186
|
+
description: "Google cannot access this page due to authentication requirements.",
|
|
1187
|
+
recommendation: "Ensure the page is publicly accessible without authentication for Googlebot.",
|
|
1217
1188
|
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1218
|
-
metadata: {
|
|
1189
|
+
metadata: { pageFetchState: indexStatusResult.pageFetchState }
|
|
1219
1190
|
});
|
|
1220
|
-
|
|
1221
|
-
} catch (error) {
|
|
1222
|
-
consola3.error("Failed to analyze sitemap:", error);
|
|
1223
|
-
analysis.issues.push({
|
|
1224
|
-
id: `sitemap-error-${hash3(sitemapUrl)}`,
|
|
1225
|
-
url: sitemapUrl,
|
|
1226
|
-
category: "technical",
|
|
1227
|
-
severity: "error",
|
|
1228
|
-
title: "Failed to parse sitemap",
|
|
1229
|
-
description: `Error: ${error instanceof Error ? error.message : "Unknown error"}`,
|
|
1230
|
-
recommendation: "Check sitemap validity using Google Search Console.",
|
|
1231
|
-
detectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1232
|
-
});
|
|
1191
|
+
break;
|
|
1233
1192
|
}
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1193
|
+
if (mobileUsabilityResult?.verdict === "FAIL" && mobileUsabilityResult.issues) {
|
|
1194
|
+
for (const issue of mobileUsabilityResult.issues) {
|
|
1195
|
+
issues.push({
|
|
1196
|
+
id: `mobile-${issue.issueType}-${hash3(result.url)}`,
|
|
1197
|
+
url: result.url,
|
|
1198
|
+
category: "mobile",
|
|
1199
|
+
severity: "warning",
|
|
1200
|
+
title: `Mobile usability: ${formatIssueType(issue.issueType)}`,
|
|
1201
|
+
description: issue.message || "Mobile usability issue detected.",
|
|
1202
|
+
recommendation: getMobileRecommendation(issue.issueType),
|
|
1203
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1204
|
+
metadata: { issueType: issue.issueType }
|
|
1205
|
+
});
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
if (richResultsResult?.verdict === "FAIL" && richResultsResult.detectedItems) {
|
|
1209
|
+
for (const item of richResultsResult.detectedItems) {
|
|
1210
|
+
for (const i of item.items || []) {
|
|
1211
|
+
for (const issueDetail of i.issues || []) {
|
|
1212
|
+
issues.push({
|
|
1213
|
+
id: `rich-result-${item.richResultType}-${hash3(result.url)}`,
|
|
1214
|
+
url: result.url,
|
|
1215
|
+
category: "structured-data",
|
|
1216
|
+
severity: issueDetail.severity === "ERROR" ? "error" : "warning",
|
|
1217
|
+
title: `${item.richResultType}: ${i.name}`,
|
|
1218
|
+
description: issueDetail.issueMessage,
|
|
1219
|
+
recommendation: "Fix the structured data markup according to Google guidelines.",
|
|
1220
|
+
detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1221
|
+
metadata: { richResultType: item.richResultType }
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1246
1225
|
}
|
|
1247
1226
|
}
|
|
1248
|
-
|
|
1249
|
-
|
|
1227
|
+
return issues;
|
|
1228
|
+
}
|
|
1229
|
+
function severityOrder(severity) {
|
|
1230
|
+
const order = {
|
|
1231
|
+
critical: 0,
|
|
1232
|
+
error: 1,
|
|
1233
|
+
warning: 2,
|
|
1234
|
+
info: 3
|
|
1235
|
+
};
|
|
1236
|
+
return order[severity];
|
|
1250
1237
|
}
|
|
1251
1238
|
function hash3(str) {
|
|
1252
1239
|
let hash5 = 0;
|
|
@@ -1257,6 +1244,19 @@ function hash3(str) {
|
|
|
1257
1244
|
}
|
|
1258
1245
|
return Math.abs(hash5).toString(36);
|
|
1259
1246
|
}
|
|
1247
|
+
function formatIssueType(type) {
|
|
1248
|
+
return type.replace(/_/g, " ").toLowerCase().replace(/\b\w/g, (c) => c.toUpperCase());
|
|
1249
|
+
}
|
|
1250
|
+
function getMobileRecommendation(issueType) {
|
|
1251
|
+
const recommendations = {
|
|
1252
|
+
MOBILE_FRIENDLY_RULE_USES_INCOMPATIBLE_PLUGINS: "Remove Flash or other incompatible plugins. Use HTML5 alternatives.",
|
|
1253
|
+
MOBILE_FRIENDLY_RULE_CONFIGURE_VIEWPORT: 'Add a viewport meta tag: <meta name="viewport" content="width=device-width, initial-scale=1">',
|
|
1254
|
+
MOBILE_FRIENDLY_RULE_CONTENT_NOT_SIZED_TO_VIEWPORT: "Ensure content width fits the viewport. Use responsive CSS.",
|
|
1255
|
+
MOBILE_FRIENDLY_RULE_TAP_TARGETS_TOO_SMALL: "Increase the size of touch targets (buttons, links) to at least 48x48 pixels.",
|
|
1256
|
+
MOBILE_FRIENDLY_RULE_TEXT_TOO_SMALL: "Use at least 16px font size for body text."
|
|
1257
|
+
};
|
|
1258
|
+
return recommendations[issueType] || "Fix the mobile usability issue according to Google guidelines.";
|
|
1259
|
+
}
|
|
1260
1260
|
var DEFAULT_SKIP_PATTERN = [
|
|
1261
1261
|
"github.com",
|
|
1262
1262
|
"twitter.com",
|
|
@@ -1514,65 +1514,248 @@ function generateMarkdownReport(result) {
|
|
|
1514
1514
|
const lines = [];
|
|
1515
1515
|
lines.push("# Link Check Report");
|
|
1516
1516
|
lines.push("");
|
|
1517
|
-
lines.push(`**URL:** ${result.url}`);
|
|
1518
|
-
lines.push(`**Timestamp:** ${result.timestamp}`);
|
|
1519
|
-
if (result.duration) {
|
|
1520
|
-
lines.push(`**Duration:** ${(result.duration / 1e3).toFixed(2)}s`);
|
|
1521
|
-
}
|
|
1517
|
+
lines.push(`**URL:** ${result.url}`);
|
|
1518
|
+
lines.push(`**Timestamp:** ${result.timestamp}`);
|
|
1519
|
+
if (result.duration) {
|
|
1520
|
+
lines.push(`**Duration:** ${(result.duration / 1e3).toFixed(2)}s`);
|
|
1521
|
+
}
|
|
1522
|
+
lines.push("");
|
|
1523
|
+
lines.push(
|
|
1524
|
+
`**Status:** ${result.success ? "\u2705 All links valid" : "\u274C Broken links found"}`
|
|
1525
|
+
);
|
|
1526
|
+
lines.push(`**Total links:** ${result.total}`);
|
|
1527
|
+
lines.push(`**Broken links:** ${result.broken}`);
|
|
1528
|
+
lines.push("");
|
|
1529
|
+
if (result.errors.length > 0) {
|
|
1530
|
+
lines.push("## Broken Links");
|
|
1531
|
+
lines.push("");
|
|
1532
|
+
lines.push("| Status | URL | Reason |");
|
|
1533
|
+
lines.push("|--------|-----|--------|");
|
|
1534
|
+
for (const { url, status, reason } of result.errors) {
|
|
1535
|
+
lines.push(`| ${status} | ${url} | ${reason || "-"} |`);
|
|
1536
|
+
}
|
|
1537
|
+
lines.push("");
|
|
1538
|
+
}
|
|
1539
|
+
return lines.join("\n");
|
|
1540
|
+
}
|
|
1541
|
+
function generateTextReport(result) {
|
|
1542
|
+
const lines = [];
|
|
1543
|
+
lines.push("Link Check Report");
|
|
1544
|
+
lines.push("=".repeat(50));
|
|
1545
|
+
lines.push(`URL: ${result.url}`);
|
|
1546
|
+
lines.push(`Timestamp: ${result.timestamp}`);
|
|
1547
|
+
if (result.duration) {
|
|
1548
|
+
lines.push(`Duration: ${(result.duration / 1e3).toFixed(2)}s`);
|
|
1549
|
+
}
|
|
1550
|
+
lines.push("");
|
|
1551
|
+
lines.push(
|
|
1552
|
+
`Status: ${result.success ? "\u2705 All links valid" : "\u274C Broken links found"}`
|
|
1553
|
+
);
|
|
1554
|
+
lines.push(`Total links: ${result.total}`);
|
|
1555
|
+
lines.push(`Broken links: ${result.broken}`);
|
|
1556
|
+
lines.push("");
|
|
1557
|
+
if (result.errors.length > 0) {
|
|
1558
|
+
lines.push("Broken Links:");
|
|
1559
|
+
lines.push("-".repeat(50));
|
|
1560
|
+
for (const { url, status, reason } of result.errors) {
|
|
1561
|
+
lines.push(`[${status}] ${url}${reason ? ` (${reason})` : ""}`);
|
|
1562
|
+
}
|
|
1563
|
+
lines.push("");
|
|
1564
|
+
}
|
|
1565
|
+
return lines.join("\n");
|
|
1566
|
+
}
|
|
1567
|
+
function hash4(str) {
|
|
1568
|
+
let h = 0;
|
|
1569
|
+
for (let i = 0; i < str.length; i++) {
|
|
1570
|
+
const char = str.charCodeAt(i);
|
|
1571
|
+
h = (h << 5) - h + char;
|
|
1572
|
+
h = h & h;
|
|
1573
|
+
}
|
|
1574
|
+
return Math.abs(h).toString(36);
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
// src/reports/claude-context.ts
|
|
1578
|
+
function generateClaudeContext(report) {
|
|
1579
|
+
const lines = [];
|
|
1580
|
+
lines.push("# @djangocfg/seo");
|
|
1581
|
+
lines.push("");
|
|
1582
|
+
lines.push("SEO audit toolkit. Generates AI-optimized split reports (max 1000 lines each).");
|
|
1583
|
+
lines.push("");
|
|
1584
|
+
lines.push("## Commands");
|
|
1585
|
+
lines.push("");
|
|
1586
|
+
lines.push("```bash");
|
|
1587
|
+
lines.push("# Audit (HTTP-based, crawls live site)");
|
|
1588
|
+
lines.push("pnpm seo:audit # Full audit (split reports)");
|
|
1589
|
+
lines.push("pnpm seo:audit --env dev # Audit local dev");
|
|
1590
|
+
lines.push("pnpm seo:audit --format all # All formats");
|
|
1591
|
+
lines.push("");
|
|
1592
|
+
lines.push("# Content (file-based, scans MDX/content/)");
|
|
1593
|
+
lines.push("pnpm exec djangocfg-seo content check # Check MDX links");
|
|
1594
|
+
lines.push("pnpm exec djangocfg-seo content fix # Show fixable links");
|
|
1595
|
+
lines.push("pnpm exec djangocfg-seo content fix --fix # Apply fixes");
|
|
1596
|
+
lines.push("pnpm exec djangocfg-seo content sitemap # Generate sitemap.ts");
|
|
1597
|
+
lines.push("```");
|
|
1598
|
+
lines.push("");
|
|
1599
|
+
lines.push("## Options");
|
|
1600
|
+
lines.push("");
|
|
1601
|
+
lines.push("- `--env, -e` - prod (default) or dev");
|
|
1602
|
+
lines.push("- `--site, -s` - Site URL (overrides env)");
|
|
1603
|
+
lines.push("- `--output, -o` - Output directory");
|
|
1604
|
+
lines.push("- `--format, -f` - split (default), json, markdown, ai-summary, all");
|
|
1605
|
+
lines.push("- `--max-pages` - Max pages (default: 100)");
|
|
1606
|
+
lines.push("- `--service-account` - Google service account JSON path");
|
|
1607
|
+
lines.push("- `--content-dir` - Content directory (default: content/)");
|
|
1608
|
+
lines.push("- `--base-path` - Base URL path for docs (default: /docs)");
|
|
1609
|
+
lines.push("");
|
|
1610
|
+
lines.push("## Reports");
|
|
1611
|
+
lines.push("");
|
|
1612
|
+
lines.push("- `seo-*-index.md` - Summary + links to categories");
|
|
1613
|
+
lines.push("- `seo-*-technical.md` - Broken links, sitemap issues");
|
|
1614
|
+
lines.push("- `seo-*-content.md` - H1, meta, title issues");
|
|
1615
|
+
lines.push("- `seo-*-performance.md` - Load time, TTFB issues");
|
|
1616
|
+
lines.push("- `seo-ai-summary-*.md` - Quick overview");
|
|
1617
|
+
lines.push("");
|
|
1618
|
+
lines.push("## Issue Severity");
|
|
1619
|
+
lines.push("");
|
|
1620
|
+
lines.push("- **critical** - Blocks indexing (fix immediately)");
|
|
1621
|
+
lines.push("- **error** - SEO problems (high priority)");
|
|
1622
|
+
lines.push("- **warning** - Recommendations (medium priority)");
|
|
1623
|
+
lines.push("- **info** - Best practices (low priority)");
|
|
1624
|
+
lines.push("");
|
|
1625
|
+
lines.push("## Issue Categories");
|
|
1626
|
+
lines.push("");
|
|
1627
|
+
lines.push("- **technical** - Broken links, sitemap, robots.txt");
|
|
1628
|
+
lines.push("- **content** - Missing H1, meta description, title");
|
|
1629
|
+
lines.push("- **indexing** - Not indexed, crawl errors from GSC");
|
|
1630
|
+
lines.push("- **performance** - Slow load time (>3s), high TTFB (>800ms)");
|
|
1631
|
+
lines.push("");
|
|
1632
|
+
lines.push("## Routes Scanner");
|
|
1633
|
+
lines.push("");
|
|
1634
|
+
lines.push("Scans Next.js App Router `app/` directory. Handles:");
|
|
1635
|
+
lines.push("- Route groups `(group)` - ignored in URL");
|
|
1636
|
+
lines.push("- Dynamic `[slug]` - shown as `:slug`");
|
|
1637
|
+
lines.push("- Catch-all `[...slug]` - shown as `:...slug`");
|
|
1638
|
+
lines.push("- Parallel `@folder` - skipped");
|
|
1639
|
+
lines.push("- Private `_folder` - skipped");
|
|
1640
|
+
lines.push("");
|
|
1641
|
+
lines.push("## SEO Files (Next.js App Router)");
|
|
1642
|
+
lines.push("");
|
|
1643
|
+
lines.push("**Required files in `app/`:**");
|
|
1644
|
+
lines.push("");
|
|
1645
|
+
lines.push("### sitemap.xml/route.ts");
|
|
1646
|
+
lines.push("```typescript");
|
|
1647
|
+
lines.push("import { createSitemapHandler } from '@djangocfg/nextjs/sitemap';");
|
|
1648
|
+
lines.push("");
|
|
1649
|
+
lines.push('export const dynamic = "force-static";');
|
|
1650
|
+
lines.push("export const { GET } = createSitemapHandler({");
|
|
1651
|
+
lines.push(" siteUrl,");
|
|
1652
|
+
lines.push(" staticPages: [");
|
|
1653
|
+
lines.push(' { loc: "/", priority: 1.0, changefreq: "daily" },');
|
|
1654
|
+
lines.push(' { loc: "/about", priority: 0.8 },');
|
|
1655
|
+
lines.push(" ],");
|
|
1656
|
+
lines.push(" dynamicPages: async () => fetchPagesFromAPI(),");
|
|
1657
|
+
lines.push("});");
|
|
1658
|
+
lines.push("```");
|
|
1659
|
+
lines.push("");
|
|
1660
|
+
lines.push("### robots.ts");
|
|
1661
|
+
lines.push("```typescript");
|
|
1662
|
+
lines.push("import type { MetadataRoute } from 'next';");
|
|
1663
|
+
lines.push("");
|
|
1664
|
+
lines.push("export default function robots(): MetadataRoute.Robots {");
|
|
1665
|
+
lines.push(" return {");
|
|
1666
|
+
lines.push(' rules: { userAgent: "*", allow: "/" },');
|
|
1667
|
+
lines.push(" sitemap: `${siteUrl}/sitemap.xml`,");
|
|
1668
|
+
lines.push(" };");
|
|
1669
|
+
lines.push("}");
|
|
1670
|
+
lines.push("```");
|
|
1671
|
+
lines.push("");
|
|
1672
|
+
lines.push("### Cloudflare Override");
|
|
1673
|
+
lines.push("");
|
|
1674
|
+
lines.push('If robots.txt shows "Content-Signal" or "ai-train" \u2014 Cloudflare is overriding your file.');
|
|
1675
|
+
lines.push('**Fix:** Dashboard \u2192 Security \u2192 Settings \u2192 "Manage your robots.txt" \u2192 Set to "Off"');
|
|
1676
|
+
lines.push("");
|
|
1677
|
+
lines.push("### Declarative Routes with SEO");
|
|
1678
|
+
lines.push("");
|
|
1679
|
+
lines.push("Create `app/_routes/` with SEO metadata for sitemap:");
|
|
1680
|
+
lines.push("```typescript");
|
|
1681
|
+
lines.push("import { defineRoute } from '@djangocfg/nextjs/navigation';");
|
|
1682
|
+
lines.push("");
|
|
1683
|
+
lines.push("export const home = defineRoute('/', {");
|
|
1684
|
+
lines.push(" label: 'Home',");
|
|
1685
|
+
lines.push(" protected: false,");
|
|
1686
|
+
lines.push(" priority: 1.0, // Sitemap priority 0.0-1.0");
|
|
1687
|
+
lines.push(" changefreq: 'daily', // always|hourly|daily|weekly|monthly|yearly|never");
|
|
1688
|
+
lines.push(" noindex: false, // Exclude from sitemap");
|
|
1689
|
+
lines.push("});");
|
|
1690
|
+
lines.push("");
|
|
1691
|
+
lines.push("export const staticRoutes = [home, about, contact];");
|
|
1692
|
+
lines.push("```");
|
|
1693
|
+
lines.push("");
|
|
1694
|
+
lines.push("Then in `sitemap.xml/route.ts`:");
|
|
1695
|
+
lines.push("```typescript");
|
|
1696
|
+
lines.push("import { routes } from '@/app/_routes';");
|
|
1697
|
+
lines.push("routes.getAllStaticRoutes().filter(r => !r.metadata.noindex)");
|
|
1698
|
+
lines.push("```");
|
|
1699
|
+
lines.push("");
|
|
1700
|
+
lines.push("## Link Guidelines");
|
|
1701
|
+
lines.push("");
|
|
1702
|
+
lines.push("### Nextra/MDX Projects (content/)");
|
|
1703
|
+
lines.push("");
|
|
1704
|
+
lines.push("For non-index files (e.g., `overview.mdx`):");
|
|
1705
|
+
lines.push("- **Sibling file**: `../sibling` (one level up)");
|
|
1706
|
+
lines.push("- **Other section**: `/docs/full/path` (absolute)");
|
|
1707
|
+
lines.push("- **AVOID**: `./sibling` (browser adds filename to path!)");
|
|
1708
|
+
lines.push("- **AVOID**: `../../deep/path` (hard to maintain)");
|
|
1709
|
+
lines.push("");
|
|
1710
|
+
lines.push("For index files (e.g., `index.mdx`):");
|
|
1711
|
+
lines.push("- **Child file**: `./child` works correctly");
|
|
1712
|
+
lines.push("- **Sibling folder**: `../sibling/` or absolute");
|
|
1713
|
+
lines.push("");
|
|
1714
|
+
lines.push("### Next.js App Router Projects");
|
|
1715
|
+
lines.push("");
|
|
1716
|
+
lines.push("Use declarative routes from `_routes/`:");
|
|
1717
|
+
lines.push("```typescript");
|
|
1718
|
+
lines.push('import { routes } from "@/app/_routes";');
|
|
1719
|
+
lines.push("<Link href={routes.dashboard.machines}>Machines</Link>");
|
|
1720
|
+
lines.push("```");
|
|
1721
|
+
lines.push("");
|
|
1722
|
+
lines.push("Benefits: type-safe, refactor-friendly, centralized.");
|
|
1723
|
+
lines.push("");
|
|
1724
|
+
lines.push("---");
|
|
1725
|
+
lines.push("");
|
|
1726
|
+
lines.push("## Current Audit");
|
|
1727
|
+
lines.push("");
|
|
1728
|
+
lines.push(`Site: ${report.siteUrl}`);
|
|
1729
|
+
lines.push(`Score: ${report.summary.healthScore}/100`);
|
|
1730
|
+
lines.push(`Date: ${report.generatedAt.slice(0, 10)}`);
|
|
1731
|
+
lines.push("");
|
|
1732
|
+
lines.push("### Issues");
|
|
1522
1733
|
lines.push("");
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
);
|
|
1526
|
-
lines.push(
|
|
1527
|
-
lines.push(
|
|
1734
|
+
const { critical = 0, error = 0, warning = 0, info = 0 } = report.summary.issuesBySeverity;
|
|
1735
|
+
if (critical > 0) lines.push(`- Critical: ${critical}`);
|
|
1736
|
+
if (error > 0) lines.push(`- Error: ${error}`);
|
|
1737
|
+
if (warning > 0) lines.push(`- Warning: ${warning}`);
|
|
1738
|
+
if (info > 0) lines.push(`- Info: ${info}`);
|
|
1528
1739
|
lines.push("");
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
}
|
|
1537
|
-
lines.push("");
|
|
1538
|
-
}
|
|
1539
|
-
return lines.join("\n");
|
|
1540
|
-
}
|
|
1541
|
-
function generateTextReport(result) {
|
|
1542
|
-
const lines = [];
|
|
1543
|
-
lines.push("Link Check Report");
|
|
1544
|
-
lines.push("=".repeat(50));
|
|
1545
|
-
lines.push(`URL: ${result.url}`);
|
|
1546
|
-
lines.push(`Timestamp: ${result.timestamp}`);
|
|
1547
|
-
if (result.duration) {
|
|
1548
|
-
lines.push(`Duration: ${(result.duration / 1e3).toFixed(2)}s`);
|
|
1740
|
+
lines.push("### Top Actions");
|
|
1741
|
+
lines.push("");
|
|
1742
|
+
const topRecs = report.recommendations.slice(0, 5);
|
|
1743
|
+
for (let i = 0; i < topRecs.length; i++) {
|
|
1744
|
+
const rec = topRecs[i];
|
|
1745
|
+
if (!rec) continue;
|
|
1746
|
+
lines.push(`${i + 1}. **${rec.title}** (${rec.affectedUrls.length} URLs)`);
|
|
1549
1747
|
}
|
|
1550
1748
|
lines.push("");
|
|
1551
|
-
lines.push(
|
|
1552
|
-
|
|
1553
|
-
);
|
|
1554
|
-
lines.push(`
|
|
1555
|
-
lines.push(`
|
|
1749
|
+
lines.push("### Report Files");
|
|
1750
|
+
lines.push("");
|
|
1751
|
+
lines.push("See split reports in this directory:");
|
|
1752
|
+
lines.push("- `seo-*-index.md` - Start here");
|
|
1753
|
+
lines.push("- `seo-*-technical.md` - Technical issues");
|
|
1754
|
+
lines.push("- `seo-*-content.md` - Content issues");
|
|
1755
|
+
lines.push("- `seo-*-performance.md` - Performance issues");
|
|
1556
1756
|
lines.push("");
|
|
1557
|
-
if (result.errors.length > 0) {
|
|
1558
|
-
lines.push("Broken Links:");
|
|
1559
|
-
lines.push("-".repeat(50));
|
|
1560
|
-
for (const { url, status, reason } of result.errors) {
|
|
1561
|
-
lines.push(`[${status}] ${url}${reason ? ` (${reason})` : ""}`);
|
|
1562
|
-
}
|
|
1563
|
-
lines.push("");
|
|
1564
|
-
}
|
|
1565
1757
|
return lines.join("\n");
|
|
1566
1758
|
}
|
|
1567
|
-
function hash4(str) {
|
|
1568
|
-
let h = 0;
|
|
1569
|
-
for (let i = 0; i < str.length; i++) {
|
|
1570
|
-
const char = str.charCodeAt(i);
|
|
1571
|
-
h = (h << 5) - h + char;
|
|
1572
|
-
h = h & h;
|
|
1573
|
-
}
|
|
1574
|
-
return Math.abs(h).toString(36);
|
|
1575
|
-
}
|
|
1576
1759
|
|
|
1577
1760
|
// src/reports/json-report.ts
|
|
1578
1761
|
function generateJsonReport(siteUrl, data, options = {}) {
|
|
@@ -2058,189 +2241,6 @@ function formatCategory2(category) {
|
|
|
2058
2241
|
return category.split("-").map((s) => s.charAt(0).toUpperCase() + s.slice(1)).join(" ");
|
|
2059
2242
|
}
|
|
2060
2243
|
|
|
2061
|
-
// src/reports/claude-context.ts
|
|
2062
|
-
function generateClaudeContext(report) {
|
|
2063
|
-
const lines = [];
|
|
2064
|
-
lines.push("# @djangocfg/seo");
|
|
2065
|
-
lines.push("");
|
|
2066
|
-
lines.push("SEO audit toolkit. Generates AI-optimized split reports (max 1000 lines each).");
|
|
2067
|
-
lines.push("");
|
|
2068
|
-
lines.push("## Commands");
|
|
2069
|
-
lines.push("");
|
|
2070
|
-
lines.push("```bash");
|
|
2071
|
-
lines.push("# Audit (HTTP-based, crawls live site)");
|
|
2072
|
-
lines.push("pnpm seo:audit # Full audit (split reports)");
|
|
2073
|
-
lines.push("pnpm seo:audit --env dev # Audit local dev");
|
|
2074
|
-
lines.push("pnpm seo:audit --format all # All formats");
|
|
2075
|
-
lines.push("");
|
|
2076
|
-
lines.push("# Content (file-based, scans MDX/content/)");
|
|
2077
|
-
lines.push("pnpm exec djangocfg-seo content check # Check MDX links");
|
|
2078
|
-
lines.push("pnpm exec djangocfg-seo content fix # Show fixable links");
|
|
2079
|
-
lines.push("pnpm exec djangocfg-seo content fix --fix # Apply fixes");
|
|
2080
|
-
lines.push("pnpm exec djangocfg-seo content sitemap # Generate sitemap.ts");
|
|
2081
|
-
lines.push("```");
|
|
2082
|
-
lines.push("");
|
|
2083
|
-
lines.push("## Options");
|
|
2084
|
-
lines.push("");
|
|
2085
|
-
lines.push("- `--env, -e` - prod (default) or dev");
|
|
2086
|
-
lines.push("- `--site, -s` - Site URL (overrides env)");
|
|
2087
|
-
lines.push("- `--output, -o` - Output directory");
|
|
2088
|
-
lines.push("- `--format, -f` - split (default), json, markdown, ai-summary, all");
|
|
2089
|
-
lines.push("- `--max-pages` - Max pages (default: 100)");
|
|
2090
|
-
lines.push("- `--service-account` - Google service account JSON path");
|
|
2091
|
-
lines.push("- `--content-dir` - Content directory (default: content/)");
|
|
2092
|
-
lines.push("- `--base-path` - Base URL path for docs (default: /docs)");
|
|
2093
|
-
lines.push("");
|
|
2094
|
-
lines.push("## Reports");
|
|
2095
|
-
lines.push("");
|
|
2096
|
-
lines.push("- `seo-*-index.md` - Summary + links to categories");
|
|
2097
|
-
lines.push("- `seo-*-technical.md` - Broken links, sitemap issues");
|
|
2098
|
-
lines.push("- `seo-*-content.md` - H1, meta, title issues");
|
|
2099
|
-
lines.push("- `seo-*-performance.md` - Load time, TTFB issues");
|
|
2100
|
-
lines.push("- `seo-ai-summary-*.md` - Quick overview");
|
|
2101
|
-
lines.push("");
|
|
2102
|
-
lines.push("## Issue Severity");
|
|
2103
|
-
lines.push("");
|
|
2104
|
-
lines.push("- **critical** - Blocks indexing (fix immediately)");
|
|
2105
|
-
lines.push("- **error** - SEO problems (high priority)");
|
|
2106
|
-
lines.push("- **warning** - Recommendations (medium priority)");
|
|
2107
|
-
lines.push("- **info** - Best practices (low priority)");
|
|
2108
|
-
lines.push("");
|
|
2109
|
-
lines.push("## Issue Categories");
|
|
2110
|
-
lines.push("");
|
|
2111
|
-
lines.push("- **technical** - Broken links, sitemap, robots.txt");
|
|
2112
|
-
lines.push("- **content** - Missing H1, meta description, title");
|
|
2113
|
-
lines.push("- **indexing** - Not indexed, crawl errors from GSC");
|
|
2114
|
-
lines.push("- **performance** - Slow load time (>3s), high TTFB (>800ms)");
|
|
2115
|
-
lines.push("");
|
|
2116
|
-
lines.push("## Routes Scanner");
|
|
2117
|
-
lines.push("");
|
|
2118
|
-
lines.push("Scans Next.js App Router `app/` directory. Handles:");
|
|
2119
|
-
lines.push("- Route groups `(group)` - ignored in URL");
|
|
2120
|
-
lines.push("- Dynamic `[slug]` - shown as `:slug`");
|
|
2121
|
-
lines.push("- Catch-all `[...slug]` - shown as `:...slug`");
|
|
2122
|
-
lines.push("- Parallel `@folder` - skipped");
|
|
2123
|
-
lines.push("- Private `_folder` - skipped");
|
|
2124
|
-
lines.push("");
|
|
2125
|
-
lines.push("## SEO Files (Next.js App Router)");
|
|
2126
|
-
lines.push("");
|
|
2127
|
-
lines.push("**Required files in `app/`:**");
|
|
2128
|
-
lines.push("");
|
|
2129
|
-
lines.push("### sitemap.xml/route.ts");
|
|
2130
|
-
lines.push("```typescript");
|
|
2131
|
-
lines.push("import { createSitemapHandler } from '@djangocfg/nextjs/sitemap';");
|
|
2132
|
-
lines.push("");
|
|
2133
|
-
lines.push('export const dynamic = "force-static";');
|
|
2134
|
-
lines.push("export const { GET } = createSitemapHandler({");
|
|
2135
|
-
lines.push(" siteUrl,");
|
|
2136
|
-
lines.push(" staticPages: [");
|
|
2137
|
-
lines.push(' { loc: "/", priority: 1.0, changefreq: "daily" },');
|
|
2138
|
-
lines.push(' { loc: "/about", priority: 0.8 },');
|
|
2139
|
-
lines.push(" ],");
|
|
2140
|
-
lines.push(" dynamicPages: async () => fetchPagesFromAPI(),");
|
|
2141
|
-
lines.push("});");
|
|
2142
|
-
lines.push("```");
|
|
2143
|
-
lines.push("");
|
|
2144
|
-
lines.push("### robots.ts");
|
|
2145
|
-
lines.push("```typescript");
|
|
2146
|
-
lines.push("import type { MetadataRoute } from 'next';");
|
|
2147
|
-
lines.push("");
|
|
2148
|
-
lines.push("export default function robots(): MetadataRoute.Robots {");
|
|
2149
|
-
lines.push(" return {");
|
|
2150
|
-
lines.push(' rules: { userAgent: "*", allow: "/" },');
|
|
2151
|
-
lines.push(" sitemap: `${siteUrl}/sitemap.xml`,");
|
|
2152
|
-
lines.push(" };");
|
|
2153
|
-
lines.push("}");
|
|
2154
|
-
lines.push("```");
|
|
2155
|
-
lines.push("");
|
|
2156
|
-
lines.push("### Cloudflare Override");
|
|
2157
|
-
lines.push("");
|
|
2158
|
-
lines.push('If robots.txt shows "Content-Signal" or "ai-train" \u2014 Cloudflare is overriding your file.');
|
|
2159
|
-
lines.push('**Fix:** Dashboard \u2192 Security \u2192 Settings \u2192 "Manage your robots.txt" \u2192 Set to "Off"');
|
|
2160
|
-
lines.push("");
|
|
2161
|
-
lines.push("### Declarative Routes with SEO");
|
|
2162
|
-
lines.push("");
|
|
2163
|
-
lines.push("Create `app/_routes/` with SEO metadata for sitemap:");
|
|
2164
|
-
lines.push("```typescript");
|
|
2165
|
-
lines.push("import { defineRoute } from '@djangocfg/nextjs/navigation';");
|
|
2166
|
-
lines.push("");
|
|
2167
|
-
lines.push("export const home = defineRoute('/', {");
|
|
2168
|
-
lines.push(" label: 'Home',");
|
|
2169
|
-
lines.push(" protected: false,");
|
|
2170
|
-
lines.push(" priority: 1.0, // Sitemap priority 0.0-1.0");
|
|
2171
|
-
lines.push(" changefreq: 'daily', // always|hourly|daily|weekly|monthly|yearly|never");
|
|
2172
|
-
lines.push(" noindex: false, // Exclude from sitemap");
|
|
2173
|
-
lines.push("});");
|
|
2174
|
-
lines.push("");
|
|
2175
|
-
lines.push("export const staticRoutes = [home, about, contact];");
|
|
2176
|
-
lines.push("```");
|
|
2177
|
-
lines.push("");
|
|
2178
|
-
lines.push("Then in `sitemap.xml/route.ts`:");
|
|
2179
|
-
lines.push("```typescript");
|
|
2180
|
-
lines.push("import { routes } from '@/app/_routes';");
|
|
2181
|
-
lines.push("routes.getAllStaticRoutes().filter(r => !r.metadata.noindex)");
|
|
2182
|
-
lines.push("```");
|
|
2183
|
-
lines.push("");
|
|
2184
|
-
lines.push("## Link Guidelines");
|
|
2185
|
-
lines.push("");
|
|
2186
|
-
lines.push("### Nextra/MDX Projects (content/)");
|
|
2187
|
-
lines.push("");
|
|
2188
|
-
lines.push("For non-index files (e.g., `overview.mdx`):");
|
|
2189
|
-
lines.push("- **Sibling file**: `../sibling` (one level up)");
|
|
2190
|
-
lines.push("- **Other section**: `/docs/full/path` (absolute)");
|
|
2191
|
-
lines.push("- **AVOID**: `./sibling` (browser adds filename to path!)");
|
|
2192
|
-
lines.push("- **AVOID**: `../../deep/path` (hard to maintain)");
|
|
2193
|
-
lines.push("");
|
|
2194
|
-
lines.push("For index files (e.g., `index.mdx`):");
|
|
2195
|
-
lines.push("- **Child file**: `./child` works correctly");
|
|
2196
|
-
lines.push("- **Sibling folder**: `../sibling/` or absolute");
|
|
2197
|
-
lines.push("");
|
|
2198
|
-
lines.push("### Next.js App Router Projects");
|
|
2199
|
-
lines.push("");
|
|
2200
|
-
lines.push("Use declarative routes from `_routes/`:");
|
|
2201
|
-
lines.push("```typescript");
|
|
2202
|
-
lines.push('import { routes } from "@/app/_routes";');
|
|
2203
|
-
lines.push("<Link href={routes.dashboard.machines}>Machines</Link>");
|
|
2204
|
-
lines.push("```");
|
|
2205
|
-
lines.push("");
|
|
2206
|
-
lines.push("Benefits: type-safe, refactor-friendly, centralized.");
|
|
2207
|
-
lines.push("");
|
|
2208
|
-
lines.push("---");
|
|
2209
|
-
lines.push("");
|
|
2210
|
-
lines.push("## Current Audit");
|
|
2211
|
-
lines.push("");
|
|
2212
|
-
lines.push(`Site: ${report.siteUrl}`);
|
|
2213
|
-
lines.push(`Score: ${report.summary.healthScore}/100`);
|
|
2214
|
-
lines.push(`Date: ${report.generatedAt.slice(0, 10)}`);
|
|
2215
|
-
lines.push("");
|
|
2216
|
-
lines.push("### Issues");
|
|
2217
|
-
lines.push("");
|
|
2218
|
-
const { critical = 0, error = 0, warning = 0, info = 0 } = report.summary.issuesBySeverity;
|
|
2219
|
-
if (critical > 0) lines.push(`- Critical: ${critical}`);
|
|
2220
|
-
if (error > 0) lines.push(`- Error: ${error}`);
|
|
2221
|
-
if (warning > 0) lines.push(`- Warning: ${warning}`);
|
|
2222
|
-
if (info > 0) lines.push(`- Info: ${info}`);
|
|
2223
|
-
lines.push("");
|
|
2224
|
-
lines.push("### Top Actions");
|
|
2225
|
-
lines.push("");
|
|
2226
|
-
const topRecs = report.recommendations.slice(0, 5);
|
|
2227
|
-
for (let i = 0; i < topRecs.length; i++) {
|
|
2228
|
-
const rec = topRecs[i];
|
|
2229
|
-
if (!rec) continue;
|
|
2230
|
-
lines.push(`${i + 1}. **${rec.title}** (${rec.affectedUrls.length} URLs)`);
|
|
2231
|
-
}
|
|
2232
|
-
lines.push("");
|
|
2233
|
-
lines.push("### Report Files");
|
|
2234
|
-
lines.push("");
|
|
2235
|
-
lines.push("See split reports in this directory:");
|
|
2236
|
-
lines.push("- `seo-*-index.md` - Start here");
|
|
2237
|
-
lines.push("- `seo-*-technical.md` - Technical issues");
|
|
2238
|
-
lines.push("- `seo-*-content.md` - Content issues");
|
|
2239
|
-
lines.push("- `seo-*-performance.md` - Performance issues");
|
|
2240
|
-
lines.push("");
|
|
2241
|
-
return lines.join("\n");
|
|
2242
|
-
}
|
|
2243
|
-
|
|
2244
2244
|
// src/reports/generator.ts
|
|
2245
2245
|
async function generateAndSaveReports(siteUrl, data, options) {
|
|
2246
2246
|
const {
|
|
@@ -2278,7 +2278,7 @@ async function generateAndSaveReports(siteUrl, data, options) {
|
|
|
2278
2278
|
const content = exportJsonReport(report, true);
|
|
2279
2279
|
writeFileSync(filepath, content, "utf-8");
|
|
2280
2280
|
result.files.json = filepath;
|
|
2281
|
-
|
|
2281
|
+
consola6.success(`JSON report saved: ${filepath}`);
|
|
2282
2282
|
}
|
|
2283
2283
|
if (formats.includes("markdown")) {
|
|
2284
2284
|
const filename = `seo-report-${siteName}${ts}.md`;
|
|
@@ -2289,7 +2289,7 @@ async function generateAndSaveReports(siteUrl, data, options) {
|
|
|
2289
2289
|
});
|
|
2290
2290
|
writeFileSync(filepath, content, "utf-8");
|
|
2291
2291
|
result.files.markdown = filepath;
|
|
2292
|
-
|
|
2292
|
+
consola6.success(`Markdown report saved: ${filepath}`);
|
|
2293
2293
|
}
|
|
2294
2294
|
if (formats.includes("ai-summary")) {
|
|
2295
2295
|
const filename = `seo-ai-summary-${siteName}${ts}.md`;
|
|
@@ -2297,7 +2297,7 @@ async function generateAndSaveReports(siteUrl, data, options) {
|
|
|
2297
2297
|
const content = generateAiSummary(report);
|
|
2298
2298
|
writeFileSync(filepath, content, "utf-8");
|
|
2299
2299
|
result.files.aiSummary = filepath;
|
|
2300
|
-
|
|
2300
|
+
consola6.success(`AI summary saved: ${filepath}`);
|
|
2301
2301
|
}
|
|
2302
2302
|
if (formats.includes("split")) {
|
|
2303
2303
|
const splitResult = generateSplitReports(report, {
|
|
@@ -2309,16 +2309,16 @@ async function generateAndSaveReports(siteUrl, data, options) {
|
|
|
2309
2309
|
index: join(outputDir, splitResult.indexFile),
|
|
2310
2310
|
categories: splitResult.categoryFiles.map((f) => join(outputDir, f))
|
|
2311
2311
|
};
|
|
2312
|
-
|
|
2312
|
+
consola6.success(`Split reports saved: ${splitResult.totalFiles} files (index + ${splitResult.categoryFiles.length} categories)`);
|
|
2313
2313
|
}
|
|
2314
2314
|
const claudeContent = generateClaudeContext(report);
|
|
2315
2315
|
const claudeFilepath = join(outputDir, "CLAUDE.md");
|
|
2316
2316
|
writeFileSync(claudeFilepath, claudeContent, "utf-8");
|
|
2317
|
-
|
|
2317
|
+
consola6.success(`AI context saved: ${claudeFilepath}`);
|
|
2318
2318
|
return result;
|
|
2319
2319
|
}
|
|
2320
2320
|
function printReportSummary(report) {
|
|
2321
|
-
|
|
2321
|
+
consola6.box(
|
|
2322
2322
|
`SEO Report: ${report.siteUrl}
|
|
2323
2323
|
Health Score: ${report.summary.healthScore}/100
|
|
2324
2324
|
Total URLs: ${report.summary.totalUrls}
|
|
@@ -2326,18 +2326,18 @@ Indexed: ${report.summary.indexedUrls} | Not Indexed: ${report.summary.notIndexe
|
|
|
2326
2326
|
Issues: ${report.issues.length}`
|
|
2327
2327
|
);
|
|
2328
2328
|
if (report.summary.issuesBySeverity.critical) {
|
|
2329
|
-
|
|
2329
|
+
consola6.error(`Critical issues: ${report.summary.issuesBySeverity.critical}`);
|
|
2330
2330
|
}
|
|
2331
2331
|
if (report.summary.issuesBySeverity.error) {
|
|
2332
|
-
|
|
2332
|
+
consola6.warn(`Errors: ${report.summary.issuesBySeverity.error}`);
|
|
2333
2333
|
}
|
|
2334
2334
|
if (report.summary.issuesBySeverity.warning) {
|
|
2335
|
-
|
|
2335
|
+
consola6.info(`Warnings: ${report.summary.issuesBySeverity.warning}`);
|
|
2336
2336
|
}
|
|
2337
|
-
|
|
2338
|
-
|
|
2337
|
+
consola6.log("");
|
|
2338
|
+
consola6.info("Top recommendations:");
|
|
2339
2339
|
for (const rec of report.recommendations.slice(0, 3)) {
|
|
2340
|
-
|
|
2340
|
+
consola6.log(` ${rec.priority}. ${rec.title} (${rec.affectedUrls.length} URLs)`);
|
|
2341
2341
|
}
|
|
2342
2342
|
}
|
|
2343
2343
|
var PAGE_FILES = ["page.tsx", "page.ts", "page.jsx", "page.js"];
|
|
@@ -2660,7 +2660,7 @@ async function runAudit(options) {
|
|
|
2660
2660
|
const siteUrl = getSiteUrl(options);
|
|
2661
2661
|
const startTime = Date.now();
|
|
2662
2662
|
console.log("");
|
|
2663
|
-
|
|
2663
|
+
consola6.box(`${chalk2.bold("SEO Audit")}
|
|
2664
2664
|
${siteUrl}`);
|
|
2665
2665
|
const serviceAccountPath = findGoogleServiceAccount(options["service-account"]);
|
|
2666
2666
|
const hasGsc = !!serviceAccountPath;
|
|
@@ -2669,7 +2669,7 @@ ${siteUrl}`);
|
|
|
2669
2669
|
if (!serviceAccountPath) {
|
|
2670
2670
|
const keyFile = getGscKeyFilename();
|
|
2671
2671
|
console.log("");
|
|
2672
|
-
|
|
2672
|
+
consola6.info(chalk2.dim("GSC not configured. Save service account as " + chalk2.cyan(keyFile) + " for indexing data."));
|
|
2673
2673
|
}
|
|
2674
2674
|
const allIssues = [];
|
|
2675
2675
|
const allInspections = [];
|
|
@@ -2722,7 +2722,7 @@ ${siteUrl}`);
|
|
|
2722
2722
|
for (const a of analyses) {
|
|
2723
2723
|
issues.push(...a.issues);
|
|
2724
2724
|
totalUrls += a.urls.length;
|
|
2725
|
-
collectedSitemapUrls.push(...a.urls
|
|
2725
|
+
collectedSitemapUrls.push(...a.urls);
|
|
2726
2726
|
}
|
|
2727
2727
|
}
|
|
2728
2728
|
updateProgress("Sitemap", "done");
|
|
@@ -2812,18 +2812,18 @@ ${siteUrl}`);
|
|
|
2812
2812
|
if (errors.length > 0) {
|
|
2813
2813
|
console.log("");
|
|
2814
2814
|
for (const err of errors) {
|
|
2815
|
-
|
|
2815
|
+
consola6.error(err);
|
|
2816
2816
|
}
|
|
2817
2817
|
}
|
|
2818
2818
|
console.log("");
|
|
2819
|
-
|
|
2819
|
+
consola6.log(chalk2.bold("Results:"));
|
|
2820
2820
|
for (const r of results) {
|
|
2821
2821
|
const issueStr = r.issues.length > 0 ? chalk2.yellow(`${r.issues.length} issues`) : chalk2.green("OK");
|
|
2822
2822
|
const metaStr = r.meta ? chalk2.dim(` (${Object.entries(r.meta).map(([k, v]) => `${k}: ${v}`).join(", ")})`) : "";
|
|
2823
|
-
|
|
2823
|
+
consola6.log(` ${r.name}: ${issueStr}${metaStr}`);
|
|
2824
2824
|
}
|
|
2825
2825
|
console.log("");
|
|
2826
|
-
|
|
2826
|
+
consola6.start("Generating reports...");
|
|
2827
2827
|
const formats = parseFormats(options.format);
|
|
2828
2828
|
const { report, files } = await generateAndSaveReports(
|
|
2829
2829
|
siteUrl,
|
|
@@ -2842,36 +2842,36 @@ ${siteUrl}`);
|
|
|
2842
2842
|
console.log("");
|
|
2843
2843
|
printReportSummary(report);
|
|
2844
2844
|
console.log("");
|
|
2845
|
-
|
|
2846
|
-
if (files.json)
|
|
2847
|
-
if (files.markdown)
|
|
2848
|
-
if (files.aiSummary)
|
|
2845
|
+
consola6.info(`Reports saved to: ${chalk2.cyan(options.output)}`);
|
|
2846
|
+
if (files.json) consola6.log(` ${chalk2.dim("\u2192")} ${files.json}`);
|
|
2847
|
+
if (files.markdown) consola6.log(` ${chalk2.dim("\u2192")} ${files.markdown}`);
|
|
2848
|
+
if (files.aiSummary) consola6.log(` ${chalk2.dim("\u2192")} ${files.aiSummary}`);
|
|
2849
2849
|
if (files.split) {
|
|
2850
|
-
|
|
2851
|
-
|
|
2850
|
+
consola6.log(` ${chalk2.dim("\u2192")} ${files.split.index} ${chalk2.dim("(index)")}`);
|
|
2851
|
+
consola6.log(` ${chalk2.dim("\u2192")} ${files.split.categories.length} category files`);
|
|
2852
2852
|
}
|
|
2853
2853
|
console.log("");
|
|
2854
|
-
|
|
2854
|
+
consola6.success(`Audit completed in ${duration}s`);
|
|
2855
2855
|
}
|
|
2856
2856
|
async function runRoutes(options) {
|
|
2857
2857
|
const siteUrl = getSiteUrl(options);
|
|
2858
2858
|
const appDir = options["app-dir"] || findAppDir();
|
|
2859
2859
|
if (!appDir) {
|
|
2860
|
-
|
|
2860
|
+
consola6.error("Could not find app/ directory. Use --app-dir to specify path.");
|
|
2861
2861
|
process.exit(1);
|
|
2862
2862
|
}
|
|
2863
2863
|
console.log("");
|
|
2864
|
-
|
|
2864
|
+
consola6.box(`${chalk2.bold("Routes Scanner")}
|
|
2865
2865
|
${appDir}`);
|
|
2866
|
-
|
|
2866
|
+
consola6.start("Scanning app/ directory...");
|
|
2867
2867
|
const scanResult = scanRoutes({ appDir });
|
|
2868
|
-
|
|
2868
|
+
consola6.success(`Found ${scanResult.routes.length} routes`);
|
|
2869
2869
|
console.log(` \u251C\u2500\u2500 Static: ${scanResult.staticRoutes.length}`);
|
|
2870
2870
|
console.log(` \u251C\u2500\u2500 Dynamic: ${scanResult.dynamicRoutes.length}`);
|
|
2871
2871
|
console.log(` \u2514\u2500\u2500 API: ${scanResult.apiRoutes.length}`);
|
|
2872
2872
|
if (scanResult.staticRoutes.length > 0) {
|
|
2873
2873
|
console.log("");
|
|
2874
|
-
|
|
2874
|
+
consola6.info("Static routes:");
|
|
2875
2875
|
for (const route of scanResult.staticRoutes.slice(0, 20)) {
|
|
2876
2876
|
console.log(` ${chalk2.green("\u2192")} ${route.path}`);
|
|
2877
2877
|
}
|
|
@@ -2881,7 +2881,7 @@ ${appDir}`);
|
|
|
2881
2881
|
}
|
|
2882
2882
|
if (scanResult.dynamicRoutes.length > 0) {
|
|
2883
2883
|
console.log("");
|
|
2884
|
-
|
|
2884
|
+
consola6.info("Dynamic routes:");
|
|
2885
2885
|
for (const route of scanResult.dynamicRoutes.slice(0, 10)) {
|
|
2886
2886
|
const params = route.dynamicSegments.join(", ");
|
|
2887
2887
|
console.log(` ${chalk2.yellow("\u2192")} ${route.path} ${chalk2.dim(`[${params}]`)}`);
|
|
@@ -2892,21 +2892,21 @@ ${appDir}`);
|
|
|
2892
2892
|
}
|
|
2893
2893
|
if (options.check) {
|
|
2894
2894
|
console.log("");
|
|
2895
|
-
|
|
2895
|
+
consola6.start("Loading sitemap...");
|
|
2896
2896
|
try {
|
|
2897
2897
|
const sitemapUrl = new URL("/sitemap.xml", siteUrl).href;
|
|
2898
2898
|
const sitemap = await analyzeSitemap(sitemapUrl);
|
|
2899
|
-
const sitemapUrls = sitemap.urls
|
|
2900
|
-
|
|
2899
|
+
const sitemapUrls = sitemap.urls;
|
|
2900
|
+
consola6.success(`Loaded ${sitemapUrls.length} URLs from sitemap`);
|
|
2901
2901
|
const comparison = compareWithSitemap(scanResult, sitemapUrls, siteUrl);
|
|
2902
2902
|
console.log("");
|
|
2903
|
-
|
|
2903
|
+
consola6.info("Sitemap comparison:");
|
|
2904
2904
|
console.log(` \u251C\u2500\u2500 Matching: ${comparison.matching.length}`);
|
|
2905
2905
|
console.log(` \u251C\u2500\u2500 Missing from sitemap: ${chalk2.yellow(String(comparison.missingFromSitemap.length))}`);
|
|
2906
2906
|
console.log(` \u2514\u2500\u2500 Extra in sitemap: ${comparison.extraInSitemap.length}`);
|
|
2907
2907
|
if (comparison.missingFromSitemap.length > 0) {
|
|
2908
2908
|
console.log("");
|
|
2909
|
-
|
|
2909
|
+
consola6.warn("Routes missing from sitemap:");
|
|
2910
2910
|
for (const route of comparison.missingFromSitemap.slice(0, 10)) {
|
|
2911
2911
|
console.log(` ${chalk2.red("\u2717")} ${route.path}`);
|
|
2912
2912
|
}
|
|
@@ -2915,12 +2915,12 @@ ${appDir}`);
|
|
|
2915
2915
|
}
|
|
2916
2916
|
}
|
|
2917
2917
|
} catch (error) {
|
|
2918
|
-
|
|
2918
|
+
consola6.error(`Failed to load sitemap: ${error.message}`);
|
|
2919
2919
|
}
|
|
2920
2920
|
}
|
|
2921
2921
|
if (options.verify) {
|
|
2922
2922
|
console.log("");
|
|
2923
|
-
|
|
2923
|
+
consola6.start("Verifying routes...");
|
|
2924
2924
|
const verification = await verifyRoutes(scanResult, {
|
|
2925
2925
|
baseUrl: siteUrl,
|
|
2926
2926
|
timeout: parseInt(options.timeout, 10),
|
|
@@ -2929,12 +2929,12 @@ ${appDir}`);
|
|
|
2929
2929
|
});
|
|
2930
2930
|
const accessible = verification.filter((r) => r.isAccessible);
|
|
2931
2931
|
const broken = verification.filter((r) => !r.isAccessible);
|
|
2932
|
-
|
|
2932
|
+
consola6.success(`Verified ${verification.length} routes`);
|
|
2933
2933
|
console.log(` \u251C\u2500\u2500 Accessible: ${chalk2.green(String(accessible.length))}`);
|
|
2934
2934
|
console.log(` \u2514\u2500\u2500 Broken: ${chalk2.red(String(broken.length))}`);
|
|
2935
2935
|
if (broken.length > 0) {
|
|
2936
2936
|
console.log("");
|
|
2937
|
-
|
|
2937
|
+
consola6.error("Broken routes:");
|
|
2938
2938
|
for (const r of broken.slice(0, 10)) {
|
|
2939
2939
|
console.log(` ${chalk2.red("\u2717")} ${r.route.path} \u2192 ${r.statusCode || r.error}`);
|
|
2940
2940
|
}
|
|
@@ -2957,22 +2957,22 @@ function loadUrlsFromFile(filePath) {
|
|
|
2957
2957
|
// src/cli/commands/inspect.ts
|
|
2958
2958
|
async function runInspect(options) {
|
|
2959
2959
|
const siteUrl = getSiteUrl(options);
|
|
2960
|
-
|
|
2960
|
+
consola6.start("Starting URL inspection via Google Search Console");
|
|
2961
2961
|
const client = new GoogleConsoleClient({
|
|
2962
2962
|
siteUrl,
|
|
2963
2963
|
serviceAccountPath: options["service-account"]
|
|
2964
2964
|
});
|
|
2965
2965
|
const isAuth = await client.verify();
|
|
2966
2966
|
if (!isAuth) {
|
|
2967
|
-
|
|
2967
|
+
consola6.error("Failed to authenticate with Google Search Console");
|
|
2968
2968
|
process.exit(1);
|
|
2969
2969
|
}
|
|
2970
2970
|
let urls;
|
|
2971
2971
|
if (options.urls) {
|
|
2972
2972
|
urls = loadUrlsFromFile(options.urls);
|
|
2973
|
-
|
|
2973
|
+
consola6.info(`Loaded ${urls.length} URLs from ${options.urls}`);
|
|
2974
2974
|
} else {
|
|
2975
|
-
|
|
2975
|
+
consola6.info("Fetching URLs from search analytics...");
|
|
2976
2976
|
const today = /* @__PURE__ */ new Date();
|
|
2977
2977
|
const startDate = new Date(today.getTime() - 30 * 24 * 60 * 60 * 1e3);
|
|
2978
2978
|
const rows = await client.getSearchAnalytics({
|
|
@@ -2982,11 +2982,11 @@ async function runInspect(options) {
|
|
|
2982
2982
|
rowLimit: 100
|
|
2983
2983
|
});
|
|
2984
2984
|
urls = rows.map((row) => row.keys?.[0] || "").filter(Boolean);
|
|
2985
|
-
|
|
2985
|
+
consola6.info(`Found ${urls.length} URLs from search analytics`);
|
|
2986
2986
|
}
|
|
2987
2987
|
const results = await client.inspectUrls(urls);
|
|
2988
2988
|
const issues = analyzeInspectionResults(results);
|
|
2989
|
-
|
|
2989
|
+
consola6.info(`Found ${issues.length} issues`);
|
|
2990
2990
|
const formats = parseFormats(options.format);
|
|
2991
2991
|
await generateAndSaveReports(siteUrl, { issues, urlInspections: results }, {
|
|
2992
2992
|
outputDir: options.output,
|
|
@@ -2996,14 +2996,14 @@ async function runInspect(options) {
|
|
|
2996
2996
|
}
|
|
2997
2997
|
async function runCrawl(options) {
|
|
2998
2998
|
const siteUrl = getSiteUrl(options);
|
|
2999
|
-
|
|
2999
|
+
consola6.start(`Starting crawl of ${siteUrl}`);
|
|
3000
3000
|
const crawler = new SiteCrawler(siteUrl, {
|
|
3001
3001
|
maxPages: parseInt(options["max-pages"], 10),
|
|
3002
3002
|
maxDepth: parseInt(options["max-depth"], 10)
|
|
3003
3003
|
});
|
|
3004
3004
|
const crawlResults = await crawler.crawl();
|
|
3005
3005
|
const issues = analyzeCrawlResults(crawlResults);
|
|
3006
|
-
|
|
3006
|
+
consola6.info(`Found ${issues.length} issues from ${crawlResults.length} pages`);
|
|
3007
3007
|
const formats = parseFormats(options.format);
|
|
3008
3008
|
await generateAndSaveReports(siteUrl, { issues, crawlResults }, {
|
|
3009
3009
|
outputDir: options.output,
|
|
@@ -3013,7 +3013,7 @@ async function runCrawl(options) {
|
|
|
3013
3013
|
}
|
|
3014
3014
|
async function runLinks(options) {
|
|
3015
3015
|
const siteUrl = getSiteUrl(options);
|
|
3016
|
-
|
|
3016
|
+
consola6.start(`Checking links on ${siteUrl}`);
|
|
3017
3017
|
const result = await checkLinks({
|
|
3018
3018
|
url: siteUrl,
|
|
3019
3019
|
timeout: parseInt(options.timeout, 10),
|
|
@@ -3021,9 +3021,9 @@ async function runLinks(options) {
|
|
|
3021
3021
|
verbose: true
|
|
3022
3022
|
});
|
|
3023
3023
|
if (result.success) {
|
|
3024
|
-
|
|
3024
|
+
consola6.success(`All ${result.total} links are valid!`);
|
|
3025
3025
|
} else {
|
|
3026
|
-
|
|
3026
|
+
consola6.error(`Found ${result.broken} broken links out of ${result.total}`);
|
|
3027
3027
|
if (options.output !== "./seo-reports" || result.broken > 0) {
|
|
3028
3028
|
const issues = linkResultsToSeoIssues(result);
|
|
3029
3029
|
const formats = parseFormats(options.format);
|
|
@@ -3038,24 +3038,24 @@ async function runLinks(options) {
|
|
|
3038
3038
|
}
|
|
3039
3039
|
async function runRobots(options) {
|
|
3040
3040
|
const siteUrl = getSiteUrl(options);
|
|
3041
|
-
|
|
3041
|
+
consola6.start(`Analyzing robots.txt for ${siteUrl}`);
|
|
3042
3042
|
const analysis = await analyzeRobotsTxt(siteUrl);
|
|
3043
3043
|
if (analysis.exists) {
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3044
|
+
consola6.success("robots.txt found");
|
|
3045
|
+
consola6.info(`Sitemaps: ${analysis.sitemaps.length}`);
|
|
3046
|
+
consola6.info(`Disallow rules: ${analysis.disallowedPaths.length}`);
|
|
3047
|
+
consola6.info(`Allow rules: ${analysis.allowedPaths.length}`);
|
|
3048
3048
|
if (analysis.crawlDelay) {
|
|
3049
|
-
|
|
3049
|
+
consola6.info(`Crawl-delay: ${analysis.crawlDelay}`);
|
|
3050
3050
|
}
|
|
3051
3051
|
if (analysis.issues.length > 0) {
|
|
3052
|
-
|
|
3052
|
+
consola6.warn(`Issues found: ${analysis.issues.length}`);
|
|
3053
3053
|
for (const issue of analysis.issues) {
|
|
3054
|
-
|
|
3054
|
+
consola6.log(` - [${issue.severity}] ${issue.title}`);
|
|
3055
3055
|
}
|
|
3056
3056
|
}
|
|
3057
3057
|
} else {
|
|
3058
|
-
|
|
3058
|
+
consola6.warn("robots.txt not found");
|
|
3059
3059
|
}
|
|
3060
3060
|
}
|
|
3061
3061
|
async function runSitemap(options) {
|
|
@@ -3063,31 +3063,31 @@ async function runSitemap(options) {
|
|
|
3063
3063
|
if (!sitemapUrl.endsWith(".xml")) {
|
|
3064
3064
|
sitemapUrl = new URL("/sitemap.xml", sitemapUrl).href;
|
|
3065
3065
|
}
|
|
3066
|
-
|
|
3066
|
+
consola6.start(`Validating sitemap: ${sitemapUrl}`);
|
|
3067
3067
|
const analyses = await analyzeAllSitemaps(sitemapUrl);
|
|
3068
3068
|
let totalUrls = 0;
|
|
3069
3069
|
let totalIssues = 0;
|
|
3070
3070
|
for (const analysis of analyses) {
|
|
3071
3071
|
if (analysis.exists) {
|
|
3072
|
-
|
|
3073
|
-
|
|
3072
|
+
consola6.success(`${analysis.url}`);
|
|
3073
|
+
consola6.info(` Type: ${analysis.type}`);
|
|
3074
3074
|
if (analysis.type === "sitemap") {
|
|
3075
|
-
|
|
3075
|
+
consola6.info(` URLs: ${analysis.urls.length}`);
|
|
3076
3076
|
totalUrls += analysis.urls.length;
|
|
3077
3077
|
} else {
|
|
3078
|
-
|
|
3078
|
+
consola6.info(` Child sitemaps: ${analysis.childSitemaps.length}`);
|
|
3079
3079
|
}
|
|
3080
3080
|
if (analysis.issues.length > 0) {
|
|
3081
3081
|
totalIssues += analysis.issues.length;
|
|
3082
3082
|
for (const issue of analysis.issues) {
|
|
3083
|
-
|
|
3083
|
+
consola6.warn(` [${issue.severity}] ${issue.title}`);
|
|
3084
3084
|
}
|
|
3085
3085
|
}
|
|
3086
3086
|
} else {
|
|
3087
|
-
|
|
3087
|
+
consola6.error(`${analysis.url} - Not found`);
|
|
3088
3088
|
}
|
|
3089
3089
|
}
|
|
3090
|
-
|
|
3090
|
+
consola6.box(`Total URLs: ${totalUrls}
|
|
3091
3091
|
Total Issues: ${totalIssues}`);
|
|
3092
3092
|
}
|
|
3093
3093
|
function detectProjectType(cwd) {
|
|
@@ -3632,7 +3632,7 @@ ${chalk2.bold("Examples:")}
|
|
|
3632
3632
|
djangocfg-seo content sitemap --output app/_core/sitemap.ts
|
|
3633
3633
|
`;
|
|
3634
3634
|
async function runContent(options) {
|
|
3635
|
-
const subcommand = options._[1];
|
|
3635
|
+
const subcommand = options._?.[1];
|
|
3636
3636
|
if (!subcommand || subcommand === "help") {
|
|
3637
3637
|
console.log(CONTENT_HELP);
|
|
3638
3638
|
return;
|
|
@@ -3640,12 +3640,12 @@ async function runContent(options) {
|
|
|
3640
3640
|
const cwd = process.cwd();
|
|
3641
3641
|
const contentDir = options["content-dir"] ? path.resolve(cwd, options["content-dir"]) : findContentDir(cwd);
|
|
3642
3642
|
if (!contentDir && subcommand !== "sitemap") {
|
|
3643
|
-
|
|
3643
|
+
consola6.error("Could not find content/ directory. Use --content-dir to specify path.");
|
|
3644
3644
|
process.exit(1);
|
|
3645
3645
|
}
|
|
3646
3646
|
const projectType = detectProjectType(cwd);
|
|
3647
3647
|
console.log("");
|
|
3648
|
-
|
|
3648
|
+
consola6.box(`${chalk2.bold("Content Tools")}
|
|
3649
3649
|
Project: ${projectType}
|
|
3650
3650
|
Path: ${contentDir || cwd}`);
|
|
3651
3651
|
switch (subcommand) {
|
|
@@ -3659,23 +3659,23 @@ Path: ${contentDir || cwd}`);
|
|
|
3659
3659
|
await runSitemapGenerate(cwd, options);
|
|
3660
3660
|
break;
|
|
3661
3661
|
default:
|
|
3662
|
-
|
|
3662
|
+
consola6.error(`Unknown subcommand: ${subcommand}`);
|
|
3663
3663
|
console.log(CONTENT_HELP);
|
|
3664
3664
|
process.exit(1);
|
|
3665
3665
|
}
|
|
3666
3666
|
}
|
|
3667
3667
|
async function runCheck(contentDir, options) {
|
|
3668
|
-
|
|
3668
|
+
consola6.start("Checking links in content/ folder...");
|
|
3669
3669
|
const basePath = options["base-path"] || "/docs";
|
|
3670
3670
|
const result = checkContentLinks(contentDir, { basePath });
|
|
3671
3671
|
if (result.success) {
|
|
3672
3672
|
console.log("");
|
|
3673
|
-
|
|
3673
|
+
consola6.success("All links are valid!");
|
|
3674
3674
|
console.log(` Checked ${result.filesChecked} files, ${result.uniqueLinks} unique links.`);
|
|
3675
3675
|
return;
|
|
3676
3676
|
}
|
|
3677
3677
|
console.log("");
|
|
3678
|
-
|
|
3678
|
+
consola6.error(`Found ${result.brokenLinks.length} broken links:`);
|
|
3679
3679
|
console.log("");
|
|
3680
3680
|
const byFile = groupBrokenLinksByFile(result.brokenLinks);
|
|
3681
3681
|
for (const [file, links] of byFile) {
|
|
@@ -3691,11 +3691,11 @@ async function runCheck(contentDir, options) {
|
|
|
3691
3691
|
}
|
|
3692
3692
|
async function runFix(contentDir, options) {
|
|
3693
3693
|
const applyFixes2 = options.fix === true;
|
|
3694
|
-
|
|
3694
|
+
consola6.start(applyFixes2 ? "Fixing links..." : "Checking for absolute links that can be relative...");
|
|
3695
3695
|
const result = fixContentLinks(contentDir, { apply: applyFixes2 });
|
|
3696
3696
|
if (result.totalChanges === 0) {
|
|
3697
3697
|
console.log("");
|
|
3698
|
-
|
|
3698
|
+
consola6.success("No absolute links that can be converted to relative.");
|
|
3699
3699
|
return;
|
|
3700
3700
|
}
|
|
3701
3701
|
console.log("");
|
|
@@ -3709,14 +3709,14 @@ async function runFix(contentDir, options) {
|
|
|
3709
3709
|
console.log("");
|
|
3710
3710
|
}
|
|
3711
3711
|
if (applyFixes2) {
|
|
3712
|
-
|
|
3712
|
+
consola6.success(`Fixed ${result.totalChanges} links in ${result.fileChanges.length} files.`);
|
|
3713
3713
|
} else {
|
|
3714
3714
|
console.log(`${chalk2.yellow("\u{1F4A1}")} Run with --fix to apply changes:`);
|
|
3715
3715
|
console.log(` djangocfg-seo content fix --fix`);
|
|
3716
3716
|
}
|
|
3717
3717
|
}
|
|
3718
3718
|
async function runSitemapGenerate(cwd, options) {
|
|
3719
|
-
|
|
3719
|
+
consola6.start("Generating sitemap...");
|
|
3720
3720
|
const rawOutput = options.output;
|
|
3721
3721
|
const output = rawOutput?.endsWith(".ts") ? rawOutput : "app/_core/sitemap.ts";
|
|
3722
3722
|
const contentDir = options["content-dir"] || "content";
|
|
@@ -3727,7 +3727,7 @@ async function runSitemapGenerate(cwd, options) {
|
|
|
3727
3727
|
});
|
|
3728
3728
|
const counts = countSitemapItems(data);
|
|
3729
3729
|
console.log("");
|
|
3730
|
-
|
|
3730
|
+
consola6.success(`Sitemap generated at ${outputPath}`);
|
|
3731
3731
|
console.log(` \u251C\u2500\u2500 App pages: ${counts.app}`);
|
|
3732
3732
|
console.log(` \u251C\u2500\u2500 Doc pages: ${counts.docs}`);
|
|
3733
3733
|
console.log(` \u2514\u2500\u2500 Total: ${counts.total}`);
|
|
@@ -3852,15 +3852,15 @@ async function main() {
|
|
|
3852
3852
|
await runContent(options);
|
|
3853
3853
|
break;
|
|
3854
3854
|
default:
|
|
3855
|
-
|
|
3855
|
+
consola6.error(`Unknown command: ${command}`);
|
|
3856
3856
|
console.log(HELP);
|
|
3857
3857
|
process.exit(1);
|
|
3858
3858
|
}
|
|
3859
3859
|
} catch (error) {
|
|
3860
|
-
|
|
3860
|
+
consola6.error(error);
|
|
3861
3861
|
process.exit(1);
|
|
3862
3862
|
}
|
|
3863
3863
|
}
|
|
3864
|
-
main().catch(
|
|
3864
|
+
main().catch(consola6.error);
|
|
3865
3865
|
//# sourceMappingURL=cli.mjs.map
|
|
3866
3866
|
//# sourceMappingURL=cli.mjs.map
|