crawlemon 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -5
- package/dist/index.d.ts +5 -0
- package/dist/index.js +2770 -242
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,81 +1,190 @@
|
|
|
1
1
|
// src/index.ts
|
|
2
|
-
import
|
|
3
|
-
import
|
|
2
|
+
import fs6 from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path8 from "node:path";
|
|
4
5
|
import { spawnSync } from "node:child_process";
|
|
5
6
|
|
|
6
7
|
// ../next-adapter/src/parser.ts
|
|
7
|
-
function
|
|
8
|
+
function captured(match) {
|
|
9
|
+
if (!match) return void 0;
|
|
10
|
+
for (let index = 1; index < match.length; index += 1) {
|
|
11
|
+
const value = match[index];
|
|
12
|
+
if (value !== void 0 && value.trim() !== "") return value.trim();
|
|
13
|
+
}
|
|
14
|
+
return void 0;
|
|
15
|
+
}
|
|
16
|
+
function earliest(content, patterns) {
|
|
17
|
+
let best;
|
|
18
|
+
for (const pattern of patterns) {
|
|
19
|
+
const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`;
|
|
20
|
+
const globalPattern = new RegExp(pattern.source, flags);
|
|
21
|
+
for (const match of content.matchAll(globalPattern)) {
|
|
22
|
+
const value = captured(match);
|
|
23
|
+
if (value !== void 0) {
|
|
24
|
+
if (best === void 0 || match.index < best.index) best = { value, index: match.index };
|
|
25
|
+
break;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return best?.value;
|
|
30
|
+
}
|
|
31
|
+
function parseFrontmatter(content) {
|
|
32
|
+
const fence = content.match(/^(?:\uFEFF)?[\s]*?---\s*\n([\s\S]*?)\n---\s*\n/);
|
|
33
|
+
if (!fence) return null;
|
|
34
|
+
const fields = {};
|
|
35
|
+
for (const line of fence[1].split("\n")) {
|
|
36
|
+
const clean = line.replace(/#.*$/, "").trim();
|
|
37
|
+
if (!clean) continue;
|
|
38
|
+
const match = clean.match(/^([A-Za-z][\w-]*)\s*:\s*(.+)$/);
|
|
39
|
+
if (!match) continue;
|
|
40
|
+
const key = match[1].toLowerCase();
|
|
41
|
+
const raw = match[2].trim();
|
|
42
|
+
if (!raw.startsWith('"') && !raw.startsWith("'")) continue;
|
|
43
|
+
const value = raw.replace(/^(['"])([\s\S]*)\1$/, "$2").trim();
|
|
44
|
+
if (!value) continue;
|
|
45
|
+
if (key === "title") fields.title = value;
|
|
46
|
+
else if (key === "description") fields.description = value;
|
|
47
|
+
else if (key === "canonical") fields.canonical = value;
|
|
48
|
+
else if (key === "robots") fields.robots = value;
|
|
49
|
+
}
|
|
50
|
+
return { fields, index: 0 };
|
|
51
|
+
}
|
|
52
|
+
var TITLE_PATTERNS = [
|
|
53
|
+
// HTML <title> block (SvelteKit <svelte:head>, Astro, static HTML, JSX)
|
|
54
|
+
/<title[^>]*>([^<]+)<\/title>/i,
|
|
55
|
+
// Object default (Next.js metadata title: { default: '...', template })
|
|
56
|
+
/\btitle\s*:\s*\{[^{}]*\bdefault\s*:\s*(?:"([^"]+)"|'([^']+)'|`([^`]+)`)/,
|
|
57
|
+
// Literal config (Next.js metadata, Nuxt useHead/definePageMeta, Remix meta)
|
|
58
|
+
/\btitle\s*:\s*(?:"([^"]+)"|'([^']+)'|`([^`]+)`)/
|
|
59
|
+
];
|
|
60
|
+
var DESCRIPTION_PATTERNS = [
|
|
61
|
+
// HTML meta description
|
|
62
|
+
/<meta[^>]*name=["']description["'][^>]*content=["']([^"']+)["']/i,
|
|
63
|
+
/<meta[^>]*content=["']([^"']+)["'][^>]*name=["']description["']/i,
|
|
64
|
+
// meta array item (Nuxt useHead meta, Remix meta function)
|
|
65
|
+
/name\s*:\s*["']description["']\s*,\s*content\s*:\s*(?:"([^"]+)"|'([^']+)'|`([^`]+)`)/,
|
|
66
|
+
// Literal config (Next.js metadata object)
|
|
67
|
+
/\bdescription\s*:\s*(?:"([^"]+)"|'([^']+)'|`([^`]+)`)/
|
|
68
|
+
];
|
|
69
|
+
var CANONICAL_PATTERNS = [
|
|
70
|
+
// HTML <link rel="canonical">
|
|
71
|
+
/<link[^>]*rel=["']canonical["'][^>]*href=(?:"([^"]+)"|'([^']+)')/i,
|
|
72
|
+
/<link[^>]*href=(?:"([^"]+)"|'([^']+)')[^>]*rel=["']canonical["']/i,
|
|
73
|
+
// link array item (Nuxt useHead link, Remix meta)
|
|
74
|
+
/rel\s*:\s*["']canonical["']\s*,\s*href\s*:\s*(?:"([^"]+)"|'([^']+)'|`([^`]+)`)/,
|
|
75
|
+
// Literal config (Next.js metadata alternates.canonical)
|
|
76
|
+
/\bcanonical\s*:\s*(?:"([^"]+)"|'([^']+)'|`([^`]+)`)/
|
|
77
|
+
];
|
|
78
|
+
var ROBOTS_PATTERNS = [
|
|
79
|
+
// HTML meta robots
|
|
80
|
+
/<meta[^>]*name=["']robots["'][^>]*content=["']([^"']+)["']/i,
|
|
81
|
+
// Literal config (Next.js metadata robots string)
|
|
82
|
+
/\brobots\s*:\s*(?:"([^"]+)"|'([^']+)'|`([^`]+)`)/
|
|
83
|
+
];
|
|
84
|
+
function hasConflictingDeclarations(content, framework) {
|
|
85
|
+
if (framework === "nextjs") {
|
|
86
|
+
const mechanisms = [
|
|
87
|
+
/export\s+const\s+metadata\b/.test(content),
|
|
88
|
+
/export\s+(?:async\s+)?function\s+generateMetadata\b/.test(content),
|
|
89
|
+
/<Head\b/.test(content)
|
|
90
|
+
].filter(Boolean).length;
|
|
91
|
+
return mechanisms > 1;
|
|
92
|
+
}
|
|
93
|
+
if (framework === "nuxt") {
|
|
94
|
+
const mechanisms = [
|
|
95
|
+
/\buseHead\s*\(/.test(content),
|
|
96
|
+
/\bdefinePageMeta\s*\(/.test(content),
|
|
97
|
+
/<Head\b/.test(content)
|
|
98
|
+
].filter(Boolean).length;
|
|
99
|
+
return mechanisms > 1;
|
|
100
|
+
}
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
var IMAGE_TAGS = ["img", "Image", "NuxtImg"];
|
|
104
|
+
function lineAt(content, index) {
|
|
105
|
+
return content.slice(0, index).split("\n").length;
|
|
106
|
+
}
|
|
107
|
+
function stripTemplate(text) {
|
|
108
|
+
return text.replace(/<[^>]+>/g, "").replace(/\{\{[\s\S]*?\}\}/g, " ").replace(/\{[^{}]*\}/g, " ").trim();
|
|
109
|
+
}
|
|
110
|
+
function parsePageSource(content, framework = "nextjs") {
|
|
8
111
|
const metadata = {};
|
|
9
112
|
const headings = [];
|
|
10
113
|
const images = [];
|
|
11
114
|
const links = [];
|
|
12
|
-
const
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
if (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
metadata.
|
|
115
|
+
const frontmatter = parseFrontmatter(content);
|
|
116
|
+
const title = earliest(content, TITLE_PATTERNS);
|
|
117
|
+
const description = earliest(content, DESCRIPTION_PATTERNS);
|
|
118
|
+
const canonical = earliest(content, CANONICAL_PATTERNS);
|
|
119
|
+
const robots = earliest(content, ROBOTS_PATTERNS);
|
|
120
|
+
if (title !== void 0) metadata.title = title;
|
|
121
|
+
if (description !== void 0) metadata.description = description;
|
|
122
|
+
if (canonical !== void 0) metadata.canonical = canonical;
|
|
123
|
+
if (robots !== void 0) metadata.robots = robots;
|
|
124
|
+
if (frontmatter) {
|
|
125
|
+
if (metadata.title === void 0 && frontmatter.fields.title !== void 0) metadata.title = frontmatter.fields.title;
|
|
126
|
+
if (metadata.description === void 0 && frontmatter.fields.description !== void 0) {
|
|
127
|
+
metadata.description = frontmatter.fields.description;
|
|
128
|
+
}
|
|
129
|
+
if (metadata.canonical === void 0 && frontmatter.fields.canonical !== void 0) {
|
|
130
|
+
metadata.canonical = frontmatter.fields.canonical;
|
|
131
|
+
}
|
|
132
|
+
if (metadata.robots === void 0 && frontmatter.fields.robots !== void 0) {
|
|
133
|
+
metadata.robots = frontmatter.fields.robots;
|
|
134
|
+
}
|
|
24
135
|
}
|
|
25
|
-
const
|
|
26
|
-
if (
|
|
27
|
-
|
|
136
|
+
const robotsObjectMatch = content.match(/\brobots\s*:\s*\{[^{}]*\b(noindex|index)\b[^{}]*\}/);
|
|
137
|
+
if (metadata.robots === void 0 && robotsObjectMatch) {
|
|
138
|
+
const mode = robotsObjectMatch[1];
|
|
139
|
+
metadata.robots = mode === "noindex" || robotsObjectMatch[0].includes("index: false") ? "noindex" : "";
|
|
28
140
|
}
|
|
29
|
-
const jsonLdMatches = content.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi);
|
|
30
141
|
const jsonLd = [];
|
|
31
|
-
for (const match of
|
|
142
|
+
for (const match of content.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
|
|
32
143
|
try {
|
|
33
|
-
|
|
34
|
-
jsonLd.push(parsed);
|
|
144
|
+
jsonLd.push(JSON.parse(match[1].trim()));
|
|
35
145
|
} catch {
|
|
36
146
|
jsonLd.push({ invalid: true });
|
|
37
147
|
}
|
|
38
148
|
}
|
|
39
|
-
if (jsonLd.length > 0)
|
|
40
|
-
|
|
41
|
-
}
|
|
42
|
-
const declarationCount = [
|
|
43
|
-
/export\s+const\s+metadata\b/.test(content),
|
|
44
|
-
/export\s+(?:async\s+)?function\s+generateMetadata\b/.test(content),
|
|
45
|
-
/<Head\b/.test(content)
|
|
46
|
-
].filter(Boolean).length;
|
|
47
|
-
metadata.hasConflictingDeclarations = declarationCount > 1;
|
|
149
|
+
if (jsonLd.length > 0) metadata.jsonLd = jsonLd;
|
|
150
|
+
metadata.hasConflictingDeclarations = hasConflictingDeclarations(content, framework);
|
|
48
151
|
for (const match of content.matchAll(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi)) {
|
|
49
152
|
headings.push({
|
|
50
153
|
level: parseInt(match[1], 10),
|
|
51
|
-
text: match[2]
|
|
52
|
-
line: lineAt(match.index || 0)
|
|
154
|
+
text: stripTemplate(match[2]),
|
|
155
|
+
line: lineAt(content, match.index || 0)
|
|
53
156
|
});
|
|
54
157
|
}
|
|
55
|
-
for (const
|
|
56
|
-
const
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
158
|
+
for (const tag of IMAGE_TAGS) {
|
|
159
|
+
const re = new RegExp(`<(${tag})\\b([^>]*?)\\/?>`, "gi");
|
|
160
|
+
for (const match of content.matchAll(re)) {
|
|
161
|
+
const attrs = match[2];
|
|
162
|
+
const srcMatch = attrs.match(/\bsrc\s*=\s*["'`]([^"'`]+)["'`]/);
|
|
163
|
+
const altMatch = attrs.match(/\balt\s*=\s*["'`]([^"'`]*?)["'`]/);
|
|
164
|
+
if (!srcMatch && !altMatch) continue;
|
|
165
|
+
images.push({
|
|
166
|
+
src: srcMatch ? srcMatch[1] : "unknown-image",
|
|
167
|
+
alt: altMatch ? altMatch[1] : void 0,
|
|
168
|
+
isNextImage: tag !== "img",
|
|
169
|
+
line: lineAt(content, match.index || 0)
|
|
170
|
+
});
|
|
171
|
+
}
|
|
65
172
|
}
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
const
|
|
173
|
+
const LINK_RE = /<(a|Link|NuxtLink|RouterLink|router-link)\b([^>]*?)(?:href|to)\s*=\s*(?:"([^"]+)"|'([^']+)')([^>]*)>([\s\S]*?)<\/\1>/gi;
|
|
174
|
+
for (const match of content.matchAll(LINK_RE)) {
|
|
175
|
+
const href = (match[3] ?? match[4])?.trim();
|
|
176
|
+
if (href === void 0 || href === "") continue;
|
|
177
|
+
const anchorRaw = stripTemplate(match[6]);
|
|
69
178
|
const isInternal = href.startsWith("/") || !/^(https?:|mailto:|tel:|javascript:)/i.test(href);
|
|
70
179
|
links.push({
|
|
71
180
|
href,
|
|
72
181
|
anchorText: anchorRaw || "(empty anchor)",
|
|
73
|
-
line: lineAt(match.index || 0),
|
|
182
|
+
line: lineAt(content, match.index || 0),
|
|
74
183
|
isInternal
|
|
75
184
|
});
|
|
76
185
|
}
|
|
77
|
-
const cleanBodyText = content.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
78
|
-
const words = cleanBodyText.split(/\s+/).filter((
|
|
186
|
+
const cleanBodyText = content.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\{\{[\s\S]*?\}\}/g, " ").replace(/\{[^{}]*\}/g, " ").replace(/\s+/g, " ").trim();
|
|
187
|
+
const words = cleanBodyText.split(/\s+/).filter((word) => word.length > 2);
|
|
79
188
|
const hasLittleContent = words.length < 20;
|
|
80
189
|
return {
|
|
81
190
|
metadata,
|
|
@@ -88,8 +197,184 @@ function parsePageSource(content) {
|
|
|
88
197
|
}
|
|
89
198
|
|
|
90
199
|
// ../next-adapter/src/scanner.ts
|
|
200
|
+
import fs2 from "node:fs";
|
|
201
|
+
import path2 from "node:path";
|
|
202
|
+
|
|
203
|
+
// ../next-adapter/src/dependency-graph.ts
|
|
91
204
|
import fs from "node:fs";
|
|
92
205
|
import path from "node:path";
|
|
206
|
+
var RouteDependencyGraph = class {
|
|
207
|
+
projectRoot;
|
|
208
|
+
appDir;
|
|
209
|
+
layoutsByDir = /* @__PURE__ */ new Map();
|
|
210
|
+
routes = [];
|
|
211
|
+
constructor(projectRoot, appDir) {
|
|
212
|
+
this.projectRoot = projectRoot;
|
|
213
|
+
this.appDir = appDir;
|
|
214
|
+
}
|
|
215
|
+
indexLayouts() {
|
|
216
|
+
this.layoutsByDir.clear();
|
|
217
|
+
if (!fs.existsSync(this.appDir)) return;
|
|
218
|
+
const walkDir = (currentDir) => {
|
|
219
|
+
const entries = fs.readdirSync(currentDir, { withFileTypes: true });
|
|
220
|
+
for (const entry of entries) {
|
|
221
|
+
const fullPath = path.join(currentDir, entry.name);
|
|
222
|
+
if (entry.isDirectory()) {
|
|
223
|
+
if (!entry.name.startsWith(".") && entry.name !== "node_modules" && entry.name !== "api") {
|
|
224
|
+
walkDir(fullPath);
|
|
225
|
+
}
|
|
226
|
+
} else if (entry.isFile() && /^layout\.(tsx|jsx|js|ts)$/.test(entry.name)) {
|
|
227
|
+
const content = fs.readFileSync(fullPath, "utf8");
|
|
228
|
+
const parsed = parsePageSource(content, "nextjs");
|
|
229
|
+
const relativeDir = path.relative(this.appDir, currentDir);
|
|
230
|
+
let canonicalLine;
|
|
231
|
+
let robotsLine;
|
|
232
|
+
let titleLine;
|
|
233
|
+
const lines = content.split("\n");
|
|
234
|
+
for (let i = 0; i < lines.length; i++) {
|
|
235
|
+
const line = lines[i];
|
|
236
|
+
if (line.includes("canonical")) canonicalLine = i + 1;
|
|
237
|
+
if (line.includes("robots")) robotsLine = i + 1;
|
|
238
|
+
if (line.includes("title")) titleLine = i + 1;
|
|
239
|
+
}
|
|
240
|
+
this.layoutsByDir.set(relativeDir, {
|
|
241
|
+
filePath: fullPath,
|
|
242
|
+
relativeDir,
|
|
243
|
+
metadata: parsed.metadata,
|
|
244
|
+
metadataSourceLocation: {
|
|
245
|
+
canonicalLine,
|
|
246
|
+
robotsLine,
|
|
247
|
+
titleLine
|
|
248
|
+
}
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
};
|
|
253
|
+
walkDir(this.appDir);
|
|
254
|
+
}
|
|
255
|
+
setRoutes(routes) {
|
|
256
|
+
this.routes = routes;
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Resolves merged inherited metadata for a given route based on all parent layouts.
|
|
260
|
+
*/
|
|
261
|
+
getInheritedMetadata(routeFilePath) {
|
|
262
|
+
const routeDir = path.dirname(routeFilePath);
|
|
263
|
+
let relativeDir = path.relative(this.appDir, routeDir);
|
|
264
|
+
const merged = {};
|
|
265
|
+
const chain = [];
|
|
266
|
+
while (true) {
|
|
267
|
+
const layout = this.layoutsByDir.get(relativeDir);
|
|
268
|
+
if (layout) {
|
|
269
|
+
chain.unshift(layout);
|
|
270
|
+
}
|
|
271
|
+
if (relativeDir === "" || relativeDir === ".") break;
|
|
272
|
+
relativeDir = path.dirname(relativeDir);
|
|
273
|
+
if (relativeDir === ".") relativeDir = "";
|
|
274
|
+
}
|
|
275
|
+
for (const layout of chain) {
|
|
276
|
+
Object.assign(merged, layout.metadata);
|
|
277
|
+
}
|
|
278
|
+
return merged;
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Calculates the blast radius (affected routes) when a layout or component file is modified.
|
|
282
|
+
*/
|
|
283
|
+
calculateBlastRadius(modifiedFilePath) {
|
|
284
|
+
const relative = path.relative(this.appDir, modifiedFilePath).replace(/\\/g, "/");
|
|
285
|
+
const layoutDir = path.dirname(relative) === "." ? "" : path.dirname(relative);
|
|
286
|
+
const affected = this.routes.filter((route) => {
|
|
287
|
+
const routeRel = path.relative(this.appDir, route.filePath).replace(/\\/g, "/");
|
|
288
|
+
if (layoutDir === "") return true;
|
|
289
|
+
return routeRel.startsWith(layoutDir + "/");
|
|
290
|
+
});
|
|
291
|
+
const routePattern = layoutDir === "" ? "/*" : `/${layoutDir.replace(/\/\([^)]+\)/g, "")}/*`;
|
|
292
|
+
const hasDynamic = routePattern.includes("[");
|
|
293
|
+
return {
|
|
294
|
+
sourceFile: modifiedFilePath,
|
|
295
|
+
routePattern,
|
|
296
|
+
affectedRoutes: affected.map((r) => r.route),
|
|
297
|
+
confidence: hasDynamic ? "ROUTE_PATTERN" : "KNOWN_ROUTES"
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Returns all indexed layouts.
|
|
302
|
+
*/
|
|
303
|
+
getLayouts() {
|
|
304
|
+
return Array.from(this.layoutsByDir.values());
|
|
305
|
+
}
|
|
306
|
+
/**
|
|
307
|
+
* Returns a unified SEODependencyGraph representation for Next.js app router.
|
|
308
|
+
*/
|
|
309
|
+
toSEODependencyGraph(linkGraph) {
|
|
310
|
+
const nodes = [];
|
|
311
|
+
const edges = [];
|
|
312
|
+
for (const layout of this.layoutsByDir.values()) {
|
|
313
|
+
const layoutId = `layout:${layout.filePath}`;
|
|
314
|
+
nodes.push({
|
|
315
|
+
id: layoutId,
|
|
316
|
+
type: "layout",
|
|
317
|
+
label: layout.filePath,
|
|
318
|
+
path: layout.filePath,
|
|
319
|
+
metadata: layout.metadata
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
for (const route of this.routes) {
|
|
323
|
+
const routeId = `route:${route.route}`;
|
|
324
|
+
nodes.push({
|
|
325
|
+
id: routeId,
|
|
326
|
+
type: "route",
|
|
327
|
+
label: route.route,
|
|
328
|
+
path: route.filePath,
|
|
329
|
+
metadata: route.metadata
|
|
330
|
+
});
|
|
331
|
+
if (route.filePath) {
|
|
332
|
+
const fileId = `file:${route.filePath}`;
|
|
333
|
+
nodes.push({
|
|
334
|
+
id: fileId,
|
|
335
|
+
type: "source_file",
|
|
336
|
+
label: route.filePath,
|
|
337
|
+
path: route.filePath
|
|
338
|
+
});
|
|
339
|
+
edges.push({
|
|
340
|
+
source: fileId,
|
|
341
|
+
target: routeId,
|
|
342
|
+
type: "DEFINES_METADATA",
|
|
343
|
+
detail: "route entrypoint"
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
const routeDir = path.dirname(route.filePath);
|
|
347
|
+
let relativeDir = path.relative(this.appDir, routeDir).replace(/\\/g, "/");
|
|
348
|
+
while (true) {
|
|
349
|
+
const layout = this.layoutsByDir.get(relativeDir);
|
|
350
|
+
if (layout) {
|
|
351
|
+
edges.push({
|
|
352
|
+
source: routeId,
|
|
353
|
+
target: `layout:${layout.filePath}`,
|
|
354
|
+
type: "INHERITS_FROM",
|
|
355
|
+
detail: "layout metadata inheritance"
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
if (relativeDir === "" || relativeDir === ".") break;
|
|
359
|
+
const parent = path.dirname(relativeDir);
|
|
360
|
+
relativeDir = parent === "." ? "" : parent;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
if (linkGraph?.edges) {
|
|
364
|
+
for (const link of linkGraph.edges) {
|
|
365
|
+
edges.push({
|
|
366
|
+
source: `route:${link.source}`,
|
|
367
|
+
target: `route:${link.target}`,
|
|
368
|
+
type: "LINKS_TO",
|
|
369
|
+
detail: link.anchorText
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
return { nodes, edges };
|
|
374
|
+
}
|
|
375
|
+
};
|
|
376
|
+
|
|
377
|
+
// ../next-adapter/src/scanner.ts
|
|
93
378
|
function normalizeRouteGroup(segment) {
|
|
94
379
|
return segment.startsWith("(") && segment.endsWith(")") ? "" : segment;
|
|
95
380
|
}
|
|
@@ -98,33 +383,39 @@ var NextJsAdapter = class {
|
|
|
98
383
|
* Detects if the given directory contains a Next.js application.
|
|
99
384
|
*/
|
|
100
385
|
static async detect(projectRoot) {
|
|
101
|
-
const pkgPath =
|
|
102
|
-
if (
|
|
386
|
+
const pkgPath = path2.join(projectRoot, "package.json");
|
|
387
|
+
if (fs2.existsSync(pkgPath)) {
|
|
103
388
|
try {
|
|
104
|
-
const pkg = JSON.parse(
|
|
389
|
+
const pkg = JSON.parse(fs2.readFileSync(pkgPath, "utf8"));
|
|
105
390
|
const deps = { ...pkg.dependencies || {}, ...pkg.devDependencies || {} };
|
|
106
391
|
if (deps.next) return true;
|
|
107
392
|
} catch {
|
|
108
393
|
}
|
|
109
394
|
}
|
|
110
|
-
return
|
|
395
|
+
return fs2.existsSync(path2.join(projectRoot, "app")) || fs2.existsSync(path2.join(projectRoot, "src", "app")) || fs2.existsSync(path2.join(projectRoot, "pages")) || fs2.existsSync(path2.join(projectRoot, "src", "pages")) || fs2.existsSync(path2.join(projectRoot, "next.config.js")) || fs2.existsSync(path2.join(projectRoot, "next.config.mjs"));
|
|
111
396
|
}
|
|
112
397
|
/**
|
|
113
398
|
* Loads optional seo.config.ts or returns default configuration.
|
|
114
399
|
*/
|
|
115
400
|
static async loadConfig(projectRoot) {
|
|
116
401
|
const configCandidates = [
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
402
|
+
path2.join(projectRoot, "crawlemon.config.ts"),
|
|
403
|
+
path2.join(projectRoot, "crawlemon.config.js"),
|
|
404
|
+
path2.join(projectRoot, "crawlemon.config.mjs"),
|
|
405
|
+
path2.join(projectRoot, "crawlemon.config.json"),
|
|
406
|
+
path2.join(projectRoot, "crawlemon.yml"),
|
|
407
|
+
path2.join(projectRoot, "crawlemon.yaml"),
|
|
408
|
+
path2.join(projectRoot, "seo.config.ts"),
|
|
409
|
+
path2.join(projectRoot, "seo.config.js"),
|
|
410
|
+
path2.join(projectRoot, "seo.config.json")
|
|
120
411
|
];
|
|
121
412
|
for (const candidate of configCandidates) {
|
|
122
|
-
if (
|
|
413
|
+
if (fs2.existsSync(candidate)) {
|
|
123
414
|
try {
|
|
124
415
|
if (candidate.endsWith(".json")) {
|
|
125
|
-
return JSON.parse(
|
|
416
|
+
return JSON.parse(fs2.readFileSync(candidate, "utf8"));
|
|
126
417
|
}
|
|
127
|
-
const content =
|
|
418
|
+
const content = fs2.readFileSync(candidate, "utf8");
|
|
128
419
|
const siteUrlMatch = content.match(/siteUrl\s*:\s*["']([^"']+)["']/);
|
|
129
420
|
const ignoreMatch = content.match(/ignore\s*:\s*\[([\s\S]*?)\]/);
|
|
130
421
|
const ignore = ignoreMatch ? Array.from(ignoreMatch[1].matchAll(/["']([^"']+)["']/g), (match) => match[1]) : void 0;
|
|
@@ -135,6 +426,23 @@ var NextJsAdapter = class {
|
|
|
135
426
|
rules[match[1]] = match[2] === "true";
|
|
136
427
|
}
|
|
137
428
|
}
|
|
429
|
+
const exportMatch = content.match(/export\s+default\s+([\s\S]+?);?$/);
|
|
430
|
+
if (exportMatch) {
|
|
431
|
+
try {
|
|
432
|
+
const evaluated = new Function(`return (${exportMatch[1].trim()})`)();
|
|
433
|
+
if (evaluated && typeof evaluated === "object") {
|
|
434
|
+
return {
|
|
435
|
+
siteUrl: evaluated.siteUrl || siteUrlMatch?.[1],
|
|
436
|
+
ignore: evaluated.ignore || ignore,
|
|
437
|
+
rules: evaluated.rules || rules,
|
|
438
|
+
qualityGate: evaluated.qualityGate,
|
|
439
|
+
contracts: evaluated.contracts,
|
|
440
|
+
opportunityKeywords: evaluated.opportunityKeywords
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
} catch {
|
|
444
|
+
}
|
|
445
|
+
}
|
|
138
446
|
return {
|
|
139
447
|
siteUrl: siteUrlMatch ? siteUrlMatch[1] : void 0,
|
|
140
448
|
ignore,
|
|
@@ -155,64 +463,53 @@ var NextJsAdapter = class {
|
|
|
155
463
|
static async scan(projectRoot) {
|
|
156
464
|
const isNext = await this.detect(projectRoot);
|
|
157
465
|
const config = await this.loadConfig(projectRoot);
|
|
158
|
-
let appDir =
|
|
159
|
-
if (!
|
|
160
|
-
appDir =
|
|
466
|
+
let appDir = path2.join(projectRoot, "app");
|
|
467
|
+
if (!fs2.existsSync(appDir) && fs2.existsSync(path2.join(projectRoot, "src", "app"))) {
|
|
468
|
+
appDir = path2.join(projectRoot, "src", "app");
|
|
161
469
|
}
|
|
162
|
-
let pagesDir =
|
|
163
|
-
if (!
|
|
164
|
-
pagesDir =
|
|
470
|
+
let pagesDir = path2.join(projectRoot, "pages");
|
|
471
|
+
if (!fs2.existsSync(pagesDir) && fs2.existsSync(path2.join(projectRoot, "src", "pages"))) {
|
|
472
|
+
pagesDir = path2.join(projectRoot, "src", "pages");
|
|
165
473
|
}
|
|
166
|
-
const isAppRouter =
|
|
474
|
+
const isAppRouter = fs2.existsSync(appDir);
|
|
167
475
|
const routes = [];
|
|
168
476
|
const redirects = [];
|
|
169
477
|
for (const configName of ["next.config.js", "next.config.mjs", "next.config.ts"]) {
|
|
170
|
-
const configPath =
|
|
171
|
-
if (!
|
|
172
|
-
const content =
|
|
478
|
+
const configPath = path2.join(projectRoot, configName);
|
|
479
|
+
if (!fs2.existsSync(configPath)) continue;
|
|
480
|
+
const content = fs2.readFileSync(configPath, "utf8");
|
|
173
481
|
for (const match of content.matchAll(/source\s*:\s*["'`]([^"'`]+)["'`][\s\S]{0,500}?destination\s*:\s*["'`]([^"'`]+)["'`][\s\S]{0,200}?permanent\s*:\s*(true|false)/g)) {
|
|
174
482
|
redirects.push({ source: match[1], destination: match[2], permanent: match[3] === "true" });
|
|
175
483
|
}
|
|
176
484
|
break;
|
|
177
485
|
}
|
|
178
|
-
|
|
179
|
-
if (
|
|
180
|
-
|
|
181
|
-
path.join(appDir, "layout.tsx"),
|
|
182
|
-
path.join(appDir, "layout.jsx"),
|
|
183
|
-
path.join(appDir, "layout.js")
|
|
184
|
-
];
|
|
185
|
-
for (const layoutFile of layoutFileCandidates) {
|
|
186
|
-
if (fs.existsSync(layoutFile)) {
|
|
187
|
-
const content = fs.readFileSync(layoutFile, "utf8");
|
|
188
|
-
const parsed = parsePageSource(content);
|
|
189
|
-
rootLayoutMetadata = parsed.metadata;
|
|
190
|
-
break;
|
|
191
|
-
}
|
|
192
|
-
}
|
|
486
|
+
const dependencyGraph = isAppRouter ? new RouteDependencyGraph(projectRoot, appDir) : void 0;
|
|
487
|
+
if (dependencyGraph) {
|
|
488
|
+
dependencyGraph.indexLayouts();
|
|
193
489
|
}
|
|
194
490
|
if (isAppRouter) {
|
|
195
491
|
const scanAppDir = (currentDir, relativePath = "") => {
|
|
196
|
-
const entries =
|
|
492
|
+
const entries = fs2.readdirSync(currentDir, { withFileTypes: true });
|
|
197
493
|
for (const entry of entries) {
|
|
198
|
-
const fullPath =
|
|
494
|
+
const fullPath = path2.join(currentDir, entry.name);
|
|
199
495
|
if (entry.isDirectory()) {
|
|
200
496
|
if (entry.name.startsWith(".") || entry.name === "node_modules" || entry.name === "api") {
|
|
201
497
|
continue;
|
|
202
498
|
}
|
|
203
499
|
const normalized = normalizeRouteGroup(entry.name);
|
|
204
|
-
const nextRelative = normalized ?
|
|
500
|
+
const nextRelative = normalized ? path2.join(relativePath, normalized) : relativePath;
|
|
205
501
|
scanAppDir(fullPath, nextRelative);
|
|
206
502
|
} else if (entry.isFile()) {
|
|
207
503
|
if (/^page\.(tsx|jsx|js|ts)$/.test(entry.name)) {
|
|
208
504
|
const routePath = relativePath === "" ? "/" : `/${relativePath.replace(/\\/g, "/")}`;
|
|
209
|
-
const content =
|
|
505
|
+
const content = fs2.readFileSync(fullPath, "utf8");
|
|
210
506
|
const parsed = parsePageSource(content);
|
|
507
|
+
const inheritedMetadata = dependencyGraph ? dependencyGraph.getInheritedMetadata(fullPath) : {};
|
|
211
508
|
routes.push({
|
|
212
509
|
route: routePath,
|
|
213
510
|
filePath: fullPath,
|
|
214
511
|
metadata: {
|
|
215
|
-
...
|
|
512
|
+
...inheritedMetadata,
|
|
216
513
|
...parsed.metadata
|
|
217
514
|
},
|
|
218
515
|
headings: parsed.headings,
|
|
@@ -228,25 +525,25 @@ var NextJsAdapter = class {
|
|
|
228
525
|
};
|
|
229
526
|
scanAppDir(appDir);
|
|
230
527
|
}
|
|
231
|
-
if (
|
|
528
|
+
if (fs2.existsSync(pagesDir)) {
|
|
232
529
|
const scanPagesDir = (currentDir, relativePath = "") => {
|
|
233
|
-
const entries =
|
|
530
|
+
const entries = fs2.readdirSync(currentDir, { withFileTypes: true });
|
|
234
531
|
for (const entry of entries) {
|
|
235
|
-
const fullPath =
|
|
532
|
+
const fullPath = path2.join(currentDir, entry.name);
|
|
236
533
|
if (entry.isDirectory()) {
|
|
237
534
|
if (entry.name.startsWith(".") || entry.name === "node_modules" || entry.name === "api") {
|
|
238
535
|
continue;
|
|
239
536
|
}
|
|
240
|
-
scanPagesDir(fullPath,
|
|
537
|
+
scanPagesDir(fullPath, path2.join(relativePath, entry.name));
|
|
241
538
|
} else if (entry.isFile()) {
|
|
242
539
|
if (/\.(tsx|jsx|js)$/.test(entry.name)) {
|
|
243
540
|
const baseName = entry.name.replace(/\.(tsx|jsx|js)$/, "");
|
|
244
541
|
if (baseName.startsWith("_") || baseName === "api") {
|
|
245
542
|
continue;
|
|
246
543
|
}
|
|
247
|
-
let routePath = `/${
|
|
544
|
+
let routePath = `/${path2.join(relativePath, baseName === "index" ? "" : baseName).replace(/\\/g, "/")}`;
|
|
248
545
|
if (routePath === "//" || routePath === "") routePath = "/";
|
|
249
|
-
const content =
|
|
546
|
+
const content = fs2.readFileSync(fullPath, "utf8");
|
|
250
547
|
const parsed = parsePageSource(content);
|
|
251
548
|
routes.push({
|
|
252
549
|
route: routePath,
|
|
@@ -266,17 +563,17 @@ var NextJsAdapter = class {
|
|
|
266
563
|
scanPagesDir(pagesDir);
|
|
267
564
|
}
|
|
268
565
|
const sitemapCandidates = [
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
566
|
+
path2.join(appDir, "sitemap.ts"),
|
|
567
|
+
path2.join(appDir, "sitemap.js"),
|
|
568
|
+
path2.join(projectRoot, "public", "sitemap.xml")
|
|
272
569
|
];
|
|
273
570
|
let sitemapFound = false;
|
|
274
571
|
let sitemapMalformed = false;
|
|
275
572
|
const sitemapUrls = [];
|
|
276
573
|
for (const candidate of sitemapCandidates) {
|
|
277
|
-
if (
|
|
574
|
+
if (fs2.existsSync(candidate)) {
|
|
278
575
|
sitemapFound = true;
|
|
279
|
-
const content =
|
|
576
|
+
const content = fs2.readFileSync(candidate, "utf8");
|
|
280
577
|
if (candidate.endsWith(".xml") && (!/<urlset\b/i.test(content) || !/<loc>[^<]+<\/loc>/i.test(content))) {
|
|
281
578
|
sitemapMalformed = true;
|
|
282
579
|
}
|
|
@@ -289,14 +586,17 @@ var NextJsAdapter = class {
|
|
|
289
586
|
}
|
|
290
587
|
}
|
|
291
588
|
const robotsCandidates = [
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
589
|
+
path2.join(appDir, "robots.ts"),
|
|
590
|
+
path2.join(appDir, "robots.js"),
|
|
591
|
+
path2.join(projectRoot, "public", "robots.txt")
|
|
295
592
|
];
|
|
296
|
-
const robotsFile = robotsCandidates.find((candidate) =>
|
|
593
|
+
const robotsFile = robotsCandidates.find((candidate) => fs2.existsSync(candidate));
|
|
297
594
|
const robotsFound = Boolean(robotsFile);
|
|
298
|
-
const robotsContent = robotsFile ?
|
|
595
|
+
const robotsContent = robotsFile ? fs2.readFileSync(robotsFile, "utf8") : "";
|
|
299
596
|
routes.sort((a, b) => a.route.localeCompare(b.route));
|
|
597
|
+
if (dependencyGraph) {
|
|
598
|
+
dependencyGraph.setRoutes(routes);
|
|
599
|
+
}
|
|
300
600
|
return {
|
|
301
601
|
isNextJs: isNext,
|
|
302
602
|
isAppRouter,
|
|
@@ -308,7 +608,175 @@ var NextJsAdapter = class {
|
|
|
308
608
|
robotsContent,
|
|
309
609
|
redirects,
|
|
310
610
|
config,
|
|
311
|
-
projectRoot
|
|
611
|
+
projectRoot,
|
|
612
|
+
dependencyGraph
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
};
|
|
616
|
+
|
|
617
|
+
// ../next-adapter/src/framework-adapter.ts
|
|
618
|
+
import fs3 from "node:fs";
|
|
619
|
+
import path3 from "node:path";
|
|
620
|
+
var INHERITED_METADATA_FIELDS = ["title", "description", "canonical", "robots"];
|
|
621
|
+
var LABELS = {
|
|
622
|
+
nextjs: "Next.js",
|
|
623
|
+
nuxt: "Nuxt",
|
|
624
|
+
sveltekit: "SvelteKit",
|
|
625
|
+
astro: "Astro",
|
|
626
|
+
remix: "Remix",
|
|
627
|
+
vite: "Vite",
|
|
628
|
+
static: "Static HTML"
|
|
629
|
+
};
|
|
630
|
+
function dependencies(root) {
|
|
631
|
+
try {
|
|
632
|
+
const pkg = JSON.parse(fs3.readFileSync(path3.join(root, "package.json"), "utf8"));
|
|
633
|
+
return { ...pkg.dependencies || {}, ...pkg.devDependencies || {} };
|
|
634
|
+
} catch {
|
|
635
|
+
return {};
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
function walk(dir, visit) {
|
|
639
|
+
if (!fs3.existsSync(dir)) return;
|
|
640
|
+
for (const entry of fs3.readdirSync(dir, { withFileTypes: true })) {
|
|
641
|
+
if (entry.name.startsWith(".") || ["node_modules", "dist", "build", ".output"].includes(entry.name)) continue;
|
|
642
|
+
const full = path3.join(dir, entry.name);
|
|
643
|
+
if (entry.isDirectory()) walk(full, visit);
|
|
644
|
+
else if (entry.isFile()) visit(full);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
function routeNode(file, route, framework) {
|
|
648
|
+
const parsed = parsePageSource(fs3.readFileSync(file, "utf8"), framework);
|
|
649
|
+
return { route, filePath: file, ...parsed, hasDynamicSegments: /[:[*]/.test(route) };
|
|
650
|
+
}
|
|
651
|
+
function cleanRoute(value) {
|
|
652
|
+
let route = `/${value}`.replace(/\\/g, "/").replace(/\/+/g, "/");
|
|
653
|
+
route = route.replace(/\/(index|\+page)$/i, "").replace(/\/\([^/]+\)/g, "");
|
|
654
|
+
return route === "" ? "/" : route;
|
|
655
|
+
}
|
|
656
|
+
function discoverRoutes(root, framework) {
|
|
657
|
+
const routes = [];
|
|
658
|
+
const add = (base, extensions, convert) => {
|
|
659
|
+
walk(base, (file) => {
|
|
660
|
+
const relative = path3.relative(base, file);
|
|
661
|
+
if (!extensions.test(relative) || /(^|\/)api(\/|\.|$)/.test(relative)) return;
|
|
662
|
+
routes.push(routeNode(file, convert(relative), framework));
|
|
663
|
+
});
|
|
664
|
+
};
|
|
665
|
+
if (framework === "nuxt") {
|
|
666
|
+
const base = fs3.existsSync(path3.join(root, "pages")) ? path3.join(root, "pages") : path3.join(root, "app", "pages");
|
|
667
|
+
add(base, /\.vue$/, (r) => cleanRoute(r.replace(/\.vue$/, "").replace(/\[\.\.\.([^\]]+)\]/g, "*$1").replace(/\[([^\]]+)\]/g, ":$1")));
|
|
668
|
+
} else if (framework === "sveltekit") {
|
|
669
|
+
const base = path3.join(root, "src", "routes");
|
|
670
|
+
add(base, /(^|\/)\+page\.svelte$/, (r) => cleanRoute(r.replace(/\/\+page\.svelte$/, "").replace(/^\+page\.svelte$/, "")));
|
|
671
|
+
} else if (framework === "astro") {
|
|
672
|
+
const base = path3.join(root, "src", "pages");
|
|
673
|
+
add(base, /\.(astro|md|mdx)$/, (r) => cleanRoute(r.replace(/\.(astro|md|mdx)$/, "")));
|
|
674
|
+
} else if (framework === "remix") {
|
|
675
|
+
const base = path3.join(root, "app", "routes");
|
|
676
|
+
add(base, /\.(tsx|jsx|ts|js)$/, (r) => cleanRoute(r.replace(/\.(tsx|jsx|ts|js)$/, "").replace(/\._index$/, "").replace(/^_index$/, "").replace(/\./g, "/").replace(/\$([^/]+)/g, ":$1")));
|
|
677
|
+
} else if (framework === "vite") {
|
|
678
|
+
for (const candidate of ["src/App.tsx", "src/App.jsx", "src/App.vue", "src/App.svelte", "index.html"]) {
|
|
679
|
+
const file = path3.join(root, candidate);
|
|
680
|
+
if (fs3.existsSync(file)) routes.push(routeNode(file, "/", framework));
|
|
681
|
+
}
|
|
682
|
+
} else {
|
|
683
|
+
add(root, /\.html?$/, (r) => cleanRoute(r.replace(/\.html?$/, "")));
|
|
684
|
+
}
|
|
685
|
+
const unique = new Map(routes.map((route) => [route.route, route]));
|
|
686
|
+
return [...unique.values()].sort((a, b) => a.route.localeCompare(b.route));
|
|
687
|
+
}
|
|
688
|
+
function svelteKitLayouts(root) {
|
|
689
|
+
const layoutByDir = /* @__PURE__ */ new Map();
|
|
690
|
+
const base = path3.join(root, "src", "routes");
|
|
691
|
+
const readLayout = (dir) => {
|
|
692
|
+
const layoutFile = path3.join(dir, "+layout.svelte");
|
|
693
|
+
if (fs3.existsSync(layoutFile)) {
|
|
694
|
+
layoutByDir.set(dir, parsePageSource(fs3.readFileSync(layoutFile, "utf8"), "sveltekit").metadata);
|
|
695
|
+
}
|
|
696
|
+
for (const entry of fs3.readdirSync(dir, { withFileTypes: true })) {
|
|
697
|
+
if (entry.isDirectory() && !entry.name.startsWith(".") && entry.name !== "node_modules") {
|
|
698
|
+
readLayout(path3.join(dir, entry.name));
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
};
|
|
702
|
+
if (fs3.existsSync(base)) readLayout(base);
|
|
703
|
+
return layoutByDir;
|
|
704
|
+
}
|
|
705
|
+
function mergeLayoutMetadata(routes, framework, root) {
|
|
706
|
+
if (framework !== "sveltekit") return routes;
|
|
707
|
+
const layoutByDir = svelteKitLayouts(root);
|
|
708
|
+
if (layoutByDir.size === 0) return routes;
|
|
709
|
+
const base = path3.join(root, "src", "routes");
|
|
710
|
+
return routes.map((route) => {
|
|
711
|
+
const merged = {};
|
|
712
|
+
let dir = path3.dirname(route.filePath);
|
|
713
|
+
while (dir === base || dir.startsWith(`${base}${path3.sep}`)) {
|
|
714
|
+
const meta = layoutByDir.get(dir);
|
|
715
|
+
if (meta) {
|
|
716
|
+
for (const field of INHERITED_METADATA_FIELDS) {
|
|
717
|
+
if (merged[field] === void 0 && meta[field] !== void 0) merged[field] = meta[field];
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
if (dir === base) break;
|
|
721
|
+
dir = path3.dirname(dir);
|
|
722
|
+
}
|
|
723
|
+
if (Object.keys(merged).length === 0) return route;
|
|
724
|
+
for (const field of INHERITED_METADATA_FIELDS) {
|
|
725
|
+
if (route.metadata[field] !== void 0) merged[field] = route.metadata[field];
|
|
726
|
+
}
|
|
727
|
+
return { ...route, metadata: { ...route.metadata, ...merged } };
|
|
728
|
+
});
|
|
729
|
+
}
|
|
730
|
+
function crawlFiles(root) {
|
|
731
|
+
const roots = [path3.join(root, "public"), path3.join(root, "static"), root];
|
|
732
|
+
const sitemap = roots.map((dir) => path3.join(dir, "sitemap.xml")).find(fs3.existsSync);
|
|
733
|
+
const robots = roots.map((dir) => path3.join(dir, "robots.txt")).find(fs3.existsSync);
|
|
734
|
+
const sitemapContent = sitemap ? fs3.readFileSync(sitemap, "utf8") : "";
|
|
735
|
+
return {
|
|
736
|
+
sitemapFound: Boolean(sitemap),
|
|
737
|
+
sitemapUrls: [...sitemapContent.matchAll(/<loc>([^<]+)<\/loc>/gi)].map((m) => m[1].trim()),
|
|
738
|
+
sitemapMalformed: Boolean(sitemap && (!/<urlset\b/i.test(sitemapContent) || !/<loc>[^<]+<\/loc>/i.test(sitemapContent))),
|
|
739
|
+
robotsFound: Boolean(robots),
|
|
740
|
+
robotsContent: robots ? fs3.readFileSync(robots, "utf8") : ""
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
var FrameworkAdapter = class {
|
|
744
|
+
static async detect(root) {
|
|
745
|
+
const deps = dependencies(root);
|
|
746
|
+
if (deps.next) return "nextjs";
|
|
747
|
+
if (deps.nuxt) return "nuxt";
|
|
748
|
+
if (deps["@sveltejs/kit"]) return "sveltekit";
|
|
749
|
+
if (deps.astro) return "astro";
|
|
750
|
+
if (deps["@remix-run/react"] || deps["@remix-run/node"]) return "remix";
|
|
751
|
+
if (deps.vite) return "vite";
|
|
752
|
+
if (await NextJsAdapter.detect(root)) return "nextjs";
|
|
753
|
+
if (fs3.existsSync(path3.join(root, "nuxt.config.ts")) || fs3.existsSync(path3.join(root, "nuxt.config.js"))) return "nuxt";
|
|
754
|
+
if (fs3.existsSync(path3.join(root, "svelte.config.js")) && fs3.existsSync(path3.join(root, "src", "routes"))) {
|
|
755
|
+
return "sveltekit";
|
|
756
|
+
}
|
|
757
|
+
if (fs3.existsSync(path3.join(root, "index.html"))) return "static";
|
|
758
|
+
return null;
|
|
759
|
+
}
|
|
760
|
+
static async scan(root) {
|
|
761
|
+
const framework = await this.detect(root);
|
|
762
|
+
if (!framework) throw new Error(`No supported web framework detected in "${root}".`);
|
|
763
|
+
if (framework === "nextjs") {
|
|
764
|
+
const scan = await NextJsAdapter.scan(root);
|
|
765
|
+
return { ...scan, framework, frameworkLabel: LABELS[framework] };
|
|
766
|
+
}
|
|
767
|
+
const routes = mergeLayoutMetadata(discoverRoutes(root, framework), framework, root);
|
|
768
|
+
const config = await NextJsAdapter.loadConfig(root);
|
|
769
|
+
const redirects = [];
|
|
770
|
+
return {
|
|
771
|
+
framework,
|
|
772
|
+
frameworkLabel: LABELS[framework],
|
|
773
|
+
isNextJs: false,
|
|
774
|
+
isAppRouter: false,
|
|
775
|
+
routes,
|
|
776
|
+
redirects,
|
|
777
|
+
config,
|
|
778
|
+
projectRoot: root,
|
|
779
|
+
...crawlFiles(root)
|
|
312
780
|
};
|
|
313
781
|
}
|
|
314
782
|
};
|
|
@@ -717,9 +1185,9 @@ var linksRule = {
|
|
|
717
1185
|
analyze(context) {
|
|
718
1186
|
const findings = [];
|
|
719
1187
|
const validRoutes = new Set(context.routes.map((r) => r.route));
|
|
720
|
-
const
|
|
721
|
-
const normalizedValidRoutes = new Set([...validRoutes].map(
|
|
722
|
-
const redirects = new Map((context.redirects || []).map((redirect) => [
|
|
1188
|
+
const normalizeRoute2 = (r) => r.endsWith("/") && r.length > 1 ? r.slice(0, -1) : r;
|
|
1189
|
+
const normalizedValidRoutes = new Set([...validRoutes].map(normalizeRoute2));
|
|
1190
|
+
const redirects = new Map((context.redirects || []).map((redirect) => [normalizeRoute2(redirect.source), redirect.destination]));
|
|
723
1191
|
for (const route of context.routes) {
|
|
724
1192
|
for (const link of route.links) {
|
|
725
1193
|
if (!link.isInternal) continue;
|
|
@@ -739,7 +1207,7 @@ var linksRule = {
|
|
|
739
1207
|
});
|
|
740
1208
|
continue;
|
|
741
1209
|
}
|
|
742
|
-
const cleanHref =
|
|
1210
|
+
const cleanHref = normalizeRoute2(href.split("?")[0].split("#")[0]);
|
|
743
1211
|
if (cleanHref.startsWith("/")) {
|
|
744
1212
|
const redirectTarget = redirects.get(cleanHref);
|
|
745
1213
|
if (redirectTarget) {
|
|
@@ -797,7 +1265,7 @@ var crawlabilityRule = {
|
|
|
797
1265
|
rule: "crawlability",
|
|
798
1266
|
severity: "warning",
|
|
799
1267
|
category: "technical",
|
|
800
|
-
message: "Missing robots.txt or
|
|
1268
|
+
message: "Missing robots.txt (or a framework robots declaration file).",
|
|
801
1269
|
fixable: true,
|
|
802
1270
|
explanation: "Robots.txt directs search crawler access to public sections and points crawlers to your sitemap."
|
|
803
1271
|
});
|
|
@@ -840,7 +1308,7 @@ var crawlabilityRule = {
|
|
|
840
1308
|
rule: "crawlability",
|
|
841
1309
|
severity: "error",
|
|
842
1310
|
category: "technical",
|
|
843
|
-
message: "Missing sitemap.xml or
|
|
1311
|
+
message: "Missing sitemap.xml (or a framework sitemap declaration file).",
|
|
844
1312
|
fixable: Boolean(context.config.siteUrl),
|
|
845
1313
|
explanation: "An XML sitemap communicates your full URL catalog to Google, ensuring newly published routes are indexed quickly."
|
|
846
1314
|
});
|
|
@@ -1049,7 +1517,7 @@ var metadataConflictsRule = {
|
|
|
1049
1517
|
file: route.filePath,
|
|
1050
1518
|
route: route.route,
|
|
1051
1519
|
fixable: false,
|
|
1052
|
-
explanation: "Mixing
|
|
1520
|
+
explanation: "Mixing multiple metadata declaration mechanisms can produce conflicting search metadata."
|
|
1053
1521
|
}));
|
|
1054
1522
|
}
|
|
1055
1523
|
};
|
|
@@ -1159,7 +1627,7 @@ function calculateScore(findings) {
|
|
|
1159
1627
|
// ../core/src/graph.ts
|
|
1160
1628
|
function buildLinkGraph(routes, redirects = []) {
|
|
1161
1629
|
const nodeSet = new Set(routes.map((r) => r.route));
|
|
1162
|
-
const nodes = Array.from(nodeSet);
|
|
1630
|
+
const nodes = Array.from(nodeSet).sort();
|
|
1163
1631
|
const edges = [];
|
|
1164
1632
|
const incomingCount = {};
|
|
1165
1633
|
const outgoingCount = {};
|
|
@@ -1169,7 +1637,7 @@ function buildLinkGraph(routes, redirects = []) {
|
|
|
1169
1637
|
outgoingCount[node] = 0;
|
|
1170
1638
|
adjacencyList[node] = [];
|
|
1171
1639
|
}
|
|
1172
|
-
const normalize = (
|
|
1640
|
+
const normalize = (path9) => path9.endsWith("/") && path9.length > 1 ? path9.slice(0, -1) : path9;
|
|
1173
1641
|
const redirectMap = new Map(redirects.map((redirect) => [normalize(redirect.source), normalize(redirect.destination)]));
|
|
1174
1642
|
for (const route of routes) {
|
|
1175
1643
|
const source = route.route;
|
|
@@ -1238,13 +1706,13 @@ function buildLinkGraph(routes, redirects = []) {
|
|
|
1238
1706
|
}
|
|
1239
1707
|
return {
|
|
1240
1708
|
nodes,
|
|
1241
|
-
edges,
|
|
1709
|
+
edges: edges.sort((a, b) => `${a.source}\0${a.target}\0${a.anchorText}`.localeCompare(`${b.source}\0${b.target}\0${b.anchorText}`)),
|
|
1242
1710
|
metrics,
|
|
1243
|
-
orphans,
|
|
1244
|
-
deadEnds,
|
|
1245
|
-
brokenEdges,
|
|
1246
|
-
lowConnectivity,
|
|
1247
|
-
highlyLinked
|
|
1711
|
+
orphans: orphans.sort(),
|
|
1712
|
+
deadEnds: deadEnds.sort(),
|
|
1713
|
+
brokenEdges: brokenEdges.sort((a, b) => `${a.source}\0${a.target}`.localeCompare(`${b.source}\0${b.target}`)),
|
|
1714
|
+
lowConnectivity: lowConnectivity.sort(),
|
|
1715
|
+
highlyLinked: highlyLinked.sort()
|
|
1248
1716
|
};
|
|
1249
1717
|
}
|
|
1250
1718
|
|
|
@@ -1435,8 +1903,8 @@ function detectContentOpportunities(routes, providedKeywords = {}) {
|
|
|
1435
1903
|
}
|
|
1436
1904
|
|
|
1437
1905
|
// ../core/src/fixer.ts
|
|
1438
|
-
import
|
|
1439
|
-
import
|
|
1906
|
+
import path4 from "node:path";
|
|
1907
|
+
import fs4 from "node:fs";
|
|
1440
1908
|
function createUnifiedDiff(filename, oldText, newText) {
|
|
1441
1909
|
const oldLines = oldText ? oldText.split("\n") : [];
|
|
1442
1910
|
const newLines = newText ? newText.split("\n") : [];
|
|
@@ -1468,8 +1936,167 @@ function createUnifiedDiff(filename, oldText, newText) {
|
|
|
1468
1936
|
}
|
|
1469
1937
|
return diff;
|
|
1470
1938
|
}
|
|
1939
|
+
function generateRevertSafeFix(options) {
|
|
1940
|
+
const { finding, projectRoot, baseContent, headContent, filePath } = options;
|
|
1941
|
+
if (!baseContent || !headContent) return null;
|
|
1942
|
+
const isCanonical = finding.rule.includes("canonical");
|
|
1943
|
+
const isRobots = finding.rule.includes("robots") || finding.message.includes("noindex");
|
|
1944
|
+
if (isCanonical) {
|
|
1945
|
+
const baseCanonicalMatch = baseContent.match(/canonical\s*:\s*["'`]([^"'`]+)["'`]/) || baseContent.match(/<link\s+rel=["']canonical["']\s+href=["']([^"']+)["']/i);
|
|
1946
|
+
if (baseCanonicalMatch) {
|
|
1947
|
+
const canonicalVal = baseCanonicalMatch[1];
|
|
1948
|
+
if (headContent.includes("export const metadata")) {
|
|
1949
|
+
const updated = headContent.replace(
|
|
1950
|
+
/(export\s+const\s+metadata(?:\s*:\s*Metadata)?\s*=\s*\{)/,
|
|
1951
|
+
`$1
|
|
1952
|
+
alternates: {
|
|
1953
|
+
canonical: '${canonicalVal}',
|
|
1954
|
+
},`
|
|
1955
|
+
);
|
|
1956
|
+
if (updated !== headContent) {
|
|
1957
|
+
return {
|
|
1958
|
+
filePath,
|
|
1959
|
+
originalContent: headContent,
|
|
1960
|
+
newContent: updated,
|
|
1961
|
+
diff: createUnifiedDiff(path4.relative(projectRoot, filePath), headContent, updated),
|
|
1962
|
+
description: `Restored canonical value "${canonicalVal}" from BASE revision.`
|
|
1963
|
+
};
|
|
1964
|
+
}
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1967
|
+
}
|
|
1968
|
+
if (isRobots) {
|
|
1969
|
+
if (headContent.includes("noindex")) {
|
|
1970
|
+
const updated = headContent.replace(/["']noindex["']/g, "'index'").replace(/noindex/g, "index");
|
|
1971
|
+
if (updated !== headContent) {
|
|
1972
|
+
return {
|
|
1973
|
+
filePath,
|
|
1974
|
+
originalContent: headContent,
|
|
1975
|
+
newContent: updated,
|
|
1976
|
+
diff: createUnifiedDiff(path4.relative(projectRoot, filePath), headContent, updated),
|
|
1977
|
+
description: "Restored indexable robots directive present in BASE revision."
|
|
1978
|
+
};
|
|
1979
|
+
}
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
return null;
|
|
1983
|
+
}
|
|
1984
|
+
function addCanonicalDeclaration(framework, content, canonicalUrl) {
|
|
1985
|
+
if (content.includes("canonical")) return null;
|
|
1986
|
+
if (framework === "nextjs") {
|
|
1987
|
+
if (content.includes("export const metadata")) {
|
|
1988
|
+
if (content.includes("alternates")) return null;
|
|
1989
|
+
const updatedContent = content.replace(
|
|
1990
|
+
/(export\s+const\s+metadata(?:\s*:\s*Metadata)?\s*=\s*\{)/,
|
|
1991
|
+
`$1
|
|
1992
|
+
alternates: {
|
|
1993
|
+
canonical: '${canonicalUrl}',
|
|
1994
|
+
},`
|
|
1995
|
+
);
|
|
1996
|
+
return updatedContent !== content ? updatedContent : null;
|
|
1997
|
+
}
|
|
1998
|
+
if (content.includes("export default function")) {
|
|
1999
|
+
return `import type { Metadata } from 'next';
|
|
2000
|
+
|
|
2001
|
+
export const metadata: Metadata = {
|
|
2002
|
+
alternates: {
|
|
2003
|
+
canonical: '${canonicalUrl}',
|
|
2004
|
+
},
|
|
2005
|
+
};
|
|
2006
|
+
|
|
2007
|
+
${content}`;
|
|
2008
|
+
}
|
|
2009
|
+
return null;
|
|
2010
|
+
}
|
|
2011
|
+
if (framework === "sveltekit") {
|
|
2012
|
+
const block = `
|
|
2013
|
+
<link rel="canonical" href="${canonicalUrl}" />
|
|
2014
|
+
`;
|
|
2015
|
+
const close = "</svelte:head>";
|
|
2016
|
+
const closeIndex = content.indexOf(close);
|
|
2017
|
+
if (closeIndex !== -1) {
|
|
2018
|
+
const updatedContent = content.replace(close, `${block}</svelte:head>`);
|
|
2019
|
+
return updatedContent !== content ? updatedContent : null;
|
|
2020
|
+
}
|
|
2021
|
+
const scriptEnd = content.indexOf("</script>");
|
|
2022
|
+
if (scriptEnd !== -1) {
|
|
2023
|
+
const updatedContent = `${content.slice(0, scriptEnd + "</script>".length)}
|
|
2024
|
+
<svelte:head>${block}</svelte:head>${content.slice(scriptEnd + "</script>".length)}`;
|
|
2025
|
+
return updatedContent !== content ? updatedContent : null;
|
|
2026
|
+
}
|
|
2027
|
+
if (/^\s*</.test(content)) {
|
|
2028
|
+
return `<svelte:head>${block}</svelte:head>
|
|
2029
|
+
${content}`;
|
|
2030
|
+
}
|
|
2031
|
+
return null;
|
|
2032
|
+
}
|
|
2033
|
+
if (framework === "astro" || framework === "static" || framework === "vite") {
|
|
2034
|
+
const headMatch = content.match(/<head\b[^>]*>/i);
|
|
2035
|
+
if (headMatch && headMatch.index !== void 0) {
|
|
2036
|
+
const insertAt = headMatch.index + headMatch[0].length;
|
|
2037
|
+
const updatedContent = `${content.slice(0, insertAt)}
|
|
2038
|
+
<link rel="canonical" href="${canonicalUrl}" />${content.slice(insertAt)}`;
|
|
2039
|
+
return updatedContent !== content ? updatedContent : null;
|
|
2040
|
+
}
|
|
2041
|
+
return null;
|
|
2042
|
+
}
|
|
2043
|
+
if (framework === "nuxt") {
|
|
2044
|
+
const anchor = "useHead({";
|
|
2045
|
+
const anchorIndex = content.indexOf(anchor);
|
|
2046
|
+
if (anchorIndex === -1) return null;
|
|
2047
|
+
const updatedContent = content.replace(
|
|
2048
|
+
anchor,
|
|
2049
|
+
`useHead({
|
|
2050
|
+
link: [{ rel: 'canonical', href: '${canonicalUrl}' }],`
|
|
2051
|
+
);
|
|
2052
|
+
return updatedContent !== content ? updatedContent : null;
|
|
2053
|
+
}
|
|
2054
|
+
if (framework === "remix") {
|
|
2055
|
+
const direct = content.match(/=>\s*\[/);
|
|
2056
|
+
if (direct && direct.index !== void 0) {
|
|
2057
|
+
const insertAt = direct.index + direct[0].length;
|
|
2058
|
+
const updatedContent = `${content.slice(0, insertAt)}
|
|
2059
|
+
{ tagName: 'link', rel: 'canonical', href: '${canonicalUrl}' },${content.slice(insertAt)}`;
|
|
2060
|
+
return updatedContent !== content ? updatedContent : null;
|
|
2061
|
+
}
|
|
2062
|
+
const returnStatement = content.match(/\breturn\s+\[/);
|
|
2063
|
+
if (returnStatement && returnStatement.index !== void 0) {
|
|
2064
|
+
const insertAt = returnStatement.index + returnStatement[0].length;
|
|
2065
|
+
const updatedContent = `${content.slice(0, insertAt)}
|
|
2066
|
+
{ tagName: 'link', rel: 'canonical', href: '${canonicalUrl}' },${content.slice(insertAt)}`;
|
|
2067
|
+
return updatedContent !== content ? updatedContent : null;
|
|
2068
|
+
}
|
|
2069
|
+
return null;
|
|
2070
|
+
}
|
|
2071
|
+
return null;
|
|
2072
|
+
}
|
|
2073
|
+
function assetLayout(projectRoot, framework, isAppRouter) {
|
|
2074
|
+
if (framework === "nextjs" && isAppRouter) {
|
|
2075
|
+
const appDirectory = fs4.existsSync(path4.join(projectRoot, "app")) ? path4.join(projectRoot, "app") : path4.join(projectRoot, "src", "app");
|
|
2076
|
+
return { kind: "app-router", directory: appDirectory };
|
|
2077
|
+
}
|
|
2078
|
+
let directory;
|
|
2079
|
+
switch (framework) {
|
|
2080
|
+
case "sveltekit":
|
|
2081
|
+
directory = path4.join(projectRoot, "static");
|
|
2082
|
+
break;
|
|
2083
|
+
case "static":
|
|
2084
|
+
directory = projectRoot;
|
|
2085
|
+
break;
|
|
2086
|
+
default:
|
|
2087
|
+
directory = path4.join(projectRoot, "public");
|
|
2088
|
+
}
|
|
2089
|
+
return { kind: "file", directory };
|
|
2090
|
+
}
|
|
2091
|
+
var ROBOTS_TXT = (siteUrl) => `User-agent: *
|
|
2092
|
+
Allow: /
|
|
2093
|
+
Disallow: /api/
|
|
2094
|
+
Disallow: /admin/
|
|
2095
|
+
${siteUrl ? `
|
|
2096
|
+
Sitemap: ${siteUrl}/sitemap.xml
|
|
2097
|
+
` : ""}`;
|
|
1471
2098
|
async function applySafeFixes(options) {
|
|
1472
|
-
const { projectRoot, routes, findings, config, dryRun = false, isAppRouter = true } = options;
|
|
2099
|
+
const { projectRoot, routes, findings, config, dryRun = false, isAppRouter = true, framework = "nextjs" } = options;
|
|
1473
2100
|
const appliedChanges = [];
|
|
1474
2101
|
const skippedFindings = [];
|
|
1475
2102
|
const scoreBefore = calculateScore(findings);
|
|
@@ -1487,31 +2114,54 @@ async function applySafeFixes(options) {
|
|
|
1487
2114
|
const validRoutes = routes.filter((r) => !r.hasDynamicSegments).map((r) => r.route);
|
|
1488
2115
|
const fixedFindingIds = /* @__PURE__ */ new Set();
|
|
1489
2116
|
const virtualFiles = /* @__PURE__ */ new Map();
|
|
1490
|
-
const appDirectory = fs2.existsSync(path2.join(projectRoot, "app")) ? path2.join(projectRoot, "app") : path2.join(projectRoot, "src", "app");
|
|
1491
2117
|
const safeFile = (candidate) => {
|
|
1492
|
-
const root =
|
|
1493
|
-
const resolved =
|
|
1494
|
-
return resolved === root || resolved.startsWith(`${root}${
|
|
2118
|
+
const root = path4.resolve(projectRoot);
|
|
2119
|
+
const resolved = path4.resolve(candidate);
|
|
2120
|
+
return resolved === root || resolved.startsWith(`${root}${path4.sep}`) ? resolved : null;
|
|
1495
2121
|
};
|
|
1496
2122
|
const readCurrent = (candidate) => {
|
|
1497
2123
|
if (virtualFiles.has(candidate)) return virtualFiles.get(candidate);
|
|
1498
|
-
return
|
|
2124
|
+
return fs4.existsSync(candidate) ? fs4.readFileSync(candidate, "utf8") : "";
|
|
1499
2125
|
};
|
|
1500
2126
|
const recordChange = (change, findingId) => {
|
|
1501
2127
|
appliedChanges.push(change);
|
|
1502
2128
|
fixedFindingIds.add(findingId);
|
|
1503
2129
|
virtualFiles.set(change.filePath, change.newContent);
|
|
1504
2130
|
if (!dryRun) {
|
|
1505
|
-
|
|
1506
|
-
|
|
2131
|
+
fs4.mkdirSync(path4.dirname(change.filePath), { recursive: true });
|
|
2132
|
+
fs4.writeFileSync(change.filePath, change.newContent, "utf8");
|
|
1507
2133
|
}
|
|
1508
2134
|
};
|
|
2135
|
+
const layout = assetLayout(projectRoot, framework, isAppRouter);
|
|
2136
|
+
if (options.baseFiles) {
|
|
2137
|
+
const revertSafeFindings = findings.filter(
|
|
2138
|
+
(f) => f.fixability === "REVERT_SAFE" || f.confidence === "DETERMINISTIC"
|
|
2139
|
+
);
|
|
2140
|
+
for (const f of revertSafeFindings) {
|
|
2141
|
+
const targetFile = f.file ? safeFile(f.file) : null;
|
|
2142
|
+
if (!targetFile) continue;
|
|
2143
|
+
const relPath = path4.relative(projectRoot, targetFile).replace(/\\/g, "/");
|
|
2144
|
+
const baseContent = options.baseFiles.get(relPath) || options.baseFiles.get(targetFile);
|
|
2145
|
+
if (!baseContent) continue;
|
|
2146
|
+
const headContent = readCurrent(targetFile);
|
|
2147
|
+
const fix = generateRevertSafeFix({
|
|
2148
|
+
finding: f,
|
|
2149
|
+
projectRoot,
|
|
2150
|
+
baseContent,
|
|
2151
|
+
headContent,
|
|
2152
|
+
filePath: targetFile
|
|
2153
|
+
});
|
|
2154
|
+
if (fix) {
|
|
2155
|
+
recordChange(fix, f.id);
|
|
2156
|
+
}
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
1509
2159
|
const robotsFinding = findings.find((f) => f.rule === "crawlability" && f.id === "robots-missing");
|
|
1510
2160
|
if (robotsFinding) {
|
|
1511
|
-
if (
|
|
1512
|
-
const robotsFilePath =
|
|
1513
|
-
const relativePath =
|
|
1514
|
-
const oldContent =
|
|
2161
|
+
if (layout.kind === "app-router") {
|
|
2162
|
+
const robotsFilePath = path4.join(layout.directory, "robots.ts");
|
|
2163
|
+
const relativePath = path4.relative(projectRoot, robotsFilePath);
|
|
2164
|
+
const oldContent = fs4.existsSync(robotsFilePath) ? fs4.readFileSync(robotsFilePath, "utf8") : "";
|
|
1515
2165
|
const sitemapLine = siteUrl ? `
|
|
1516
2166
|
sitemap: '${siteUrl}/sitemap.xml',` : "";
|
|
1517
2167
|
const newContent = `import { MetadataRoute } from 'next';
|
|
@@ -1535,22 +2185,16 @@ export default function robots(): MetadataRoute.Robots {
|
|
|
1535
2185
|
description: "Generated app/robots.ts with standard crawler rules and sitemap reference."
|
|
1536
2186
|
}, robotsFinding.id);
|
|
1537
2187
|
} else {
|
|
1538
|
-
const robotsFilePath =
|
|
1539
|
-
const relativePath =
|
|
1540
|
-
const oldContent =
|
|
1541
|
-
const newContent =
|
|
1542
|
-
Allow: /
|
|
1543
|
-
Disallow: /api/
|
|
1544
|
-
Disallow: /admin/
|
|
1545
|
-
${siteUrl ? `
|
|
1546
|
-
Sitemap: ${siteUrl}/sitemap.xml
|
|
1547
|
-
` : ""}`;
|
|
2188
|
+
const robotsFilePath = path4.join(layout.directory, "robots.txt");
|
|
2189
|
+
const relativePath = path4.relative(projectRoot, robotsFilePath);
|
|
2190
|
+
const oldContent = fs4.existsSync(robotsFilePath) ? fs4.readFileSync(robotsFilePath, "utf8") : "";
|
|
2191
|
+
const newContent = ROBOTS_TXT(siteUrl);
|
|
1548
2192
|
recordChange({
|
|
1549
2193
|
filePath: robotsFilePath,
|
|
1550
2194
|
originalContent: oldContent,
|
|
1551
2195
|
newContent,
|
|
1552
2196
|
diff: createUnifiedDiff(relativePath, oldContent, newContent),
|
|
1553
|
-
description:
|
|
2197
|
+
description: `Generated ${relativePath} with standard crawler rules and sitemap pointer.`
|
|
1554
2198
|
}, robotsFinding.id);
|
|
1555
2199
|
}
|
|
1556
2200
|
}
|
|
@@ -1558,10 +2202,10 @@ Sitemap: ${siteUrl}/sitemap.xml
|
|
|
1558
2202
|
(f) => f.rule === "crawlability" && (f.id === "sitemap-missing" || f.id === "sitemap-malformed")
|
|
1559
2203
|
);
|
|
1560
2204
|
if (sitemapFinding && siteUrl) {
|
|
1561
|
-
if (
|
|
1562
|
-
const sitemapFilePath =
|
|
1563
|
-
const relativePath =
|
|
1564
|
-
const oldContent =
|
|
2205
|
+
if (layout.kind === "app-router") {
|
|
2206
|
+
const sitemapFilePath = path4.join(layout.directory, "sitemap.ts");
|
|
2207
|
+
const relativePath = path4.relative(projectRoot, sitemapFilePath);
|
|
2208
|
+
const oldContent = fs4.existsSync(sitemapFilePath) ? fs4.readFileSync(sitemapFilePath, "utf8") : "";
|
|
1565
2209
|
const routeEntries = validRoutes.map(
|
|
1566
2210
|
(r) => ` {
|
|
1567
2211
|
url: '${siteUrl}${r === "/" ? "" : r}',
|
|
@@ -1586,9 +2230,9 @@ ${routeEntries}
|
|
|
1586
2230
|
description: `Generated app/sitemap.ts containing ${validRoutes.length} discovered routes.`
|
|
1587
2231
|
}, sitemapFinding.id);
|
|
1588
2232
|
} else {
|
|
1589
|
-
const sitemapFilePath =
|
|
1590
|
-
const relativePath =
|
|
1591
|
-
const oldContent =
|
|
2233
|
+
const sitemapFilePath = path4.join(layout.directory, "sitemap.xml");
|
|
2234
|
+
const relativePath = path4.relative(projectRoot, sitemapFilePath);
|
|
2235
|
+
const oldContent = fs4.existsSync(sitemapFilePath) ? fs4.readFileSync(sitemapFilePath, "utf8") : "";
|
|
1592
2236
|
const xmlEntries = validRoutes.map(
|
|
1593
2237
|
(r) => ` <url>
|
|
1594
2238
|
<loc>${siteUrl}${r === "/" ? "" : r}</loc>
|
|
@@ -1606,7 +2250,7 @@ ${xmlEntries}
|
|
|
1606
2250
|
originalContent: oldContent,
|
|
1607
2251
|
newContent,
|
|
1608
2252
|
diff: createUnifiedDiff(relativePath, oldContent, newContent),
|
|
1609
|
-
description: `Generated
|
|
2253
|
+
description: `Generated ${relativePath} containing ${validRoutes.length} discovered routes.`
|
|
1610
2254
|
}, sitemapFinding.id);
|
|
1611
2255
|
}
|
|
1612
2256
|
}
|
|
@@ -1614,46 +2258,18 @@ ${xmlEntries}
|
|
|
1614
2258
|
const canonicalFindings = findings.filter((f) => f.rule === "canonical" && f.fixable && f.file);
|
|
1615
2259
|
for (const finding of canonicalFindings) {
|
|
1616
2260
|
const findingFile = finding.file ? safeFile(finding.file) : null;
|
|
1617
|
-
if (!findingFile || !
|
|
2261
|
+
if (!findingFile || !fs4.existsSync(findingFile)) continue;
|
|
1618
2262
|
const fileContent = readCurrent(findingFile);
|
|
1619
2263
|
const route = finding.route || "/";
|
|
1620
2264
|
const canonicalUrl = `${siteUrl}${route === "/" ? "" : route}`;
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
const updatedContent = fileContent.replace(
|
|
1624
|
-
/(export\s+const\s+metadata(?:\s*:\s*Metadata)?\s*=\s*\{)/,
|
|
1625
|
-
`$1
|
|
1626
|
-
alternates: {
|
|
1627
|
-
canonical: '${canonicalUrl}',
|
|
1628
|
-
},`
|
|
1629
|
-
);
|
|
1630
|
-
if (updatedContent !== fileContent) {
|
|
1631
|
-
recordChange({
|
|
1632
|
-
filePath: findingFile,
|
|
1633
|
-
originalContent: fileContent,
|
|
1634
|
-
newContent: updatedContent,
|
|
1635
|
-
diff: createUnifiedDiff(path2.relative(projectRoot, findingFile), fileContent, updatedContent),
|
|
1636
|
-
description: `Added canonical alternate "${canonicalUrl}" to metadata in ${path2.basename(findingFile)}.`
|
|
1637
|
-
}, finding.id);
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
1640
|
-
} else if (fileContent.includes("export default function")) {
|
|
1641
|
-
const metadataBoilerplate = `import type { Metadata } from 'next';
|
|
1642
|
-
|
|
1643
|
-
export const metadata: Metadata = {
|
|
1644
|
-
alternates: {
|
|
1645
|
-
canonical: '${canonicalUrl}',
|
|
1646
|
-
},
|
|
1647
|
-
};
|
|
1648
|
-
|
|
1649
|
-
`;
|
|
1650
|
-
const updatedContent = metadataBoilerplate + fileContent;
|
|
2265
|
+
const updatedContent = addCanonicalDeclaration(framework, fileContent, canonicalUrl);
|
|
2266
|
+
if (updatedContent) {
|
|
1651
2267
|
recordChange({
|
|
1652
2268
|
filePath: findingFile,
|
|
1653
2269
|
originalContent: fileContent,
|
|
1654
2270
|
newContent: updatedContent,
|
|
1655
|
-
diff: createUnifiedDiff(
|
|
1656
|
-
description: `
|
|
2271
|
+
diff: createUnifiedDiff(path4.relative(projectRoot, findingFile), fileContent, updatedContent),
|
|
2272
|
+
description: `Added canonical "${canonicalUrl}" declaration to ${path4.basename(findingFile)}.`
|
|
1657
2273
|
}, finding.id);
|
|
1658
2274
|
}
|
|
1659
2275
|
}
|
|
@@ -1661,7 +2277,7 @@ export const metadata: Metadata = {
|
|
|
1661
2277
|
const fixableLinkFindings = findings.filter((f) => f.rule === "links" && f.fixable && f.file);
|
|
1662
2278
|
for (const lf of fixableLinkFindings) {
|
|
1663
2279
|
const linkFile = lf.file ? safeFile(lf.file) : null;
|
|
1664
|
-
if (!linkFile || !
|
|
2280
|
+
if (!linkFile || !fs4.existsSync(linkFile)) continue;
|
|
1665
2281
|
const fileContent = readCurrent(linkFile);
|
|
1666
2282
|
const match = lf.message.match(/nonexistent route "([^"]+)"/);
|
|
1667
2283
|
if (match) {
|
|
@@ -1672,15 +2288,15 @@ export const metadata: Metadata = {
|
|
|
1672
2288
|
if (normalizedTarget) {
|
|
1673
2289
|
const escapedHref = brokenHref.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1674
2290
|
const updatedContent = fileContent.replace(
|
|
1675
|
-
new RegExp(`href=["']${escapedHref}["']`, "g"),
|
|
1676
|
-
|
|
2291
|
+
new RegExp(`(href|to)=["']${escapedHref}["']`, "g"),
|
|
2292
|
+
`$1="${normalizedTarget}"`
|
|
1677
2293
|
);
|
|
1678
2294
|
if (updatedContent !== fileContent) {
|
|
1679
2295
|
recordChange({
|
|
1680
2296
|
filePath: linkFile,
|
|
1681
2297
|
originalContent: fileContent,
|
|
1682
2298
|
newContent: updatedContent,
|
|
1683
|
-
diff: createUnifiedDiff(
|
|
2299
|
+
diff: createUnifiedDiff(path4.relative(projectRoot, linkFile), fileContent, updatedContent),
|
|
1684
2300
|
description: `Fixed casing of internal link: "${brokenHref}" -> "${normalizedTarget}".`
|
|
1685
2301
|
}, lf.id);
|
|
1686
2302
|
}
|
|
@@ -1703,6 +2319,74 @@ export const metadata: Metadata = {
|
|
|
1703
2319
|
};
|
|
1704
2320
|
}
|
|
1705
2321
|
|
|
2322
|
+
// ../core/src/fingerprint.ts
|
|
2323
|
+
import crypto from "node:crypto";
|
|
2324
|
+
function normalizeFilePath(rawPath) {
|
|
2325
|
+
if (!rawPath) return "";
|
|
2326
|
+
const cleaned = rawPath.replace(/\\/g, "/").trim();
|
|
2327
|
+
const match = cleaned.match(/(?:^|\/)(app\/.*|pages\/.*|src\/.*|public\/.*|[^/]+\.(json|js|ts|mjs|tsx|jsx|html|vue|svelte|astro))$/i);
|
|
2328
|
+
return match ? match[1].toLowerCase() : cleaned.split("/").slice(-2).join("/").toLowerCase();
|
|
2329
|
+
}
|
|
2330
|
+
function computeFindingFingerprint(finding) {
|
|
2331
|
+
const normalizedFile = normalizeFilePath(finding.sourceFile || finding.file || "");
|
|
2332
|
+
const parts = [
|
|
2333
|
+
finding.rule || "unknown-rule",
|
|
2334
|
+
(finding.route || "").trim().toLowerCase(),
|
|
2335
|
+
normalizedFile
|
|
2336
|
+
];
|
|
2337
|
+
const semanticTarget = extractSemanticTarget(finding);
|
|
2338
|
+
if (semanticTarget) {
|
|
2339
|
+
parts.push(semanticTarget.toLowerCase());
|
|
2340
|
+
}
|
|
2341
|
+
const raw = parts.join("::");
|
|
2342
|
+
return crypto.createHash("sha256").update(raw).digest("hex").slice(0, 16);
|
|
2343
|
+
}
|
|
2344
|
+
function extractSemanticTarget(finding) {
|
|
2345
|
+
const id = finding.id || "";
|
|
2346
|
+
if (id.includes("-duplicate-")) {
|
|
2347
|
+
const parts = id.split("-duplicate-");
|
|
2348
|
+
return `${parts[0]}-duplicate`;
|
|
2349
|
+
}
|
|
2350
|
+
if (id.startsWith("img-")) {
|
|
2351
|
+
const segments = id.split("-");
|
|
2352
|
+
return segments.slice(2).join("-");
|
|
2353
|
+
}
|
|
2354
|
+
if (id.startsWith("link-")) {
|
|
2355
|
+
const segments = id.split("-");
|
|
2356
|
+
return segments.slice(2).join("-");
|
|
2357
|
+
}
|
|
2358
|
+
if (id.startsWith("heading-")) {
|
|
2359
|
+
return id.replace(/^heading-/, "");
|
|
2360
|
+
}
|
|
2361
|
+
if (id.startsWith("canonical-")) {
|
|
2362
|
+
return id.replace(/^canonical-/, "");
|
|
2363
|
+
}
|
|
2364
|
+
if (id.startsWith("robots-")) {
|
|
2365
|
+
return id.replace(/^robots-/, "");
|
|
2366
|
+
}
|
|
2367
|
+
if (id.startsWith("title-")) {
|
|
2368
|
+
return id.replace(/^title-/, "");
|
|
2369
|
+
}
|
|
2370
|
+
if (id.startsWith("desc-")) {
|
|
2371
|
+
return id.replace(/^desc-/, "");
|
|
2372
|
+
}
|
|
2373
|
+
const cleanedMessage = (finding.message || "").replace(/\s*\(?(?:\/[a-zA-Z0-9_\-/*]+(?:,\s*)?)+\)?/g, "");
|
|
2374
|
+
return cleanedMessage.trim();
|
|
2375
|
+
}
|
|
2376
|
+
function enrichFinding(finding) {
|
|
2377
|
+
const fingerprint = finding.fingerprint || computeFindingFingerprint(finding);
|
|
2378
|
+
const confidence = finding.confidence || (finding.rule.includes("heuristic") ? "HEURISTIC" : "DETERMINISTIC");
|
|
2379
|
+
const fixability = finding.fixability || (finding.fixable ? "AUTO_FIXABLE" : "NOT_FIXABLE");
|
|
2380
|
+
return {
|
|
2381
|
+
...finding,
|
|
2382
|
+
fingerprint,
|
|
2383
|
+
confidence,
|
|
2384
|
+
fixability,
|
|
2385
|
+
sourceFile: finding.sourceFile || finding.file,
|
|
2386
|
+
sourceLine: finding.sourceLine || finding.line
|
|
2387
|
+
};
|
|
2388
|
+
}
|
|
2389
|
+
|
|
1706
2390
|
// ../core/src/engine.ts
|
|
1707
2391
|
function runSEOAudit(options) {
|
|
1708
2392
|
const {
|
|
@@ -1715,7 +2399,8 @@ function runSEOAudit(options) {
|
|
|
1715
2399
|
sitemapMalformed = false,
|
|
1716
2400
|
redirects = [],
|
|
1717
2401
|
projectRoot = "",
|
|
1718
|
-
isAppRouter = true
|
|
2402
|
+
isAppRouter = true,
|
|
2403
|
+
timestamp = (/* @__PURE__ */ new Date()).toISOString()
|
|
1719
2404
|
} = options;
|
|
1720
2405
|
const ignored = config.ignore || ["/admin/**", "/api/**"];
|
|
1721
2406
|
const matchesIgnore = (route, pattern) => {
|
|
@@ -1740,14 +2425,15 @@ function runSEOAudit(options) {
|
|
|
1740
2425
|
"metadata-description": "requireDescription"
|
|
1741
2426
|
};
|
|
1742
2427
|
const ruleSetting = (ruleId) => config.rules?.[ruleId] ?? (ruleAliases[ruleId] ? config.rules?.[ruleAliases[ruleId]] : void 0);
|
|
1743
|
-
const
|
|
2428
|
+
const rawFindings = allRules.filter((rule) => ruleSetting(rule.id) !== false).flatMap((rule) => {
|
|
1744
2429
|
const results = rule.analyze(ruleContext);
|
|
1745
2430
|
const configured = ruleSetting(rule.id);
|
|
1746
2431
|
if (configured && typeof configured === "object" && configured.severity) {
|
|
1747
2432
|
return results.map((finding) => ({ ...finding, severity: configured.severity }));
|
|
1748
2433
|
}
|
|
1749
2434
|
return results;
|
|
1750
|
-
})
|
|
2435
|
+
});
|
|
2436
|
+
const findings = rawFindings.map(enrichFinding).sort((a, b) => a.id.localeCompare(b.id));
|
|
1751
2437
|
const score = calculateScore(findings);
|
|
1752
2438
|
const linkGraph = buildLinkGraph(analyzedRoutes, redirects);
|
|
1753
2439
|
const recommendations = generateLinkRecommendations(analyzedRoutes);
|
|
@@ -1759,43 +2445,1711 @@ function runSEOAudit(options) {
|
|
|
1759
2445
|
linkGraph,
|
|
1760
2446
|
recommendations,
|
|
1761
2447
|
opportunities,
|
|
1762
|
-
timestamp
|
|
1763
|
-
engineVersion: "0.
|
|
2448
|
+
timestamp,
|
|
2449
|
+
engineVersion: "0.2.0"
|
|
1764
2450
|
};
|
|
1765
2451
|
}
|
|
1766
2452
|
|
|
1767
|
-
// src/
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
red: "\x1B[31m",
|
|
1774
|
-
green: "\x1B[32m",
|
|
1775
|
-
yellow: "\x1B[33m",
|
|
1776
|
-
blue: "\x1B[34m",
|
|
1777
|
-
cyan: "\x1B[36m",
|
|
1778
|
-
bgRed: "\x1B[41m",
|
|
1779
|
-
bgGreen: "\x1B[42m"
|
|
2453
|
+
// ../core/src/quality-gate.ts
|
|
2454
|
+
var DEFAULT_QUALITY_GATE_POLICY = {
|
|
2455
|
+
maxNewCritical: 0,
|
|
2456
|
+
maxNewErrors: 0,
|
|
2457
|
+
maxNewWarnings: 10,
|
|
2458
|
+
blockOnNewOrphanRoutes: true
|
|
1780
2459
|
};
|
|
1781
|
-
function
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
2460
|
+
function evaluateQualityGate(input) {
|
|
2461
|
+
const {
|
|
2462
|
+
newFindings,
|
|
2463
|
+
fixedFindings,
|
|
2464
|
+
baselineDebt,
|
|
2465
|
+
newOrphanRoutes = [],
|
|
2466
|
+
affectedRoutes = [],
|
|
2467
|
+
policy = DEFAULT_QUALITY_GATE_POLICY,
|
|
2468
|
+
baselineTotalIssues,
|
|
2469
|
+
routesResolvedCount = 0
|
|
2470
|
+
} = input;
|
|
2471
|
+
const policyViolations = [];
|
|
2472
|
+
const blockingFindings = newFindings.filter((f) => {
|
|
2473
|
+
const confidence = f.confidence ?? "DETERMINISTIC";
|
|
2474
|
+
return confidence === "DETERMINISTIC" || confidence === "AI_VERIFIED" || confidence === "STRUCTURAL";
|
|
2475
|
+
});
|
|
2476
|
+
const newCritical = blockingFindings.filter(
|
|
2477
|
+
(f) => f.severity === "error" && (f.confidence === "DETERMINISTIC" || f.confidence === "STRUCTURAL")
|
|
2478
|
+
).length;
|
|
2479
|
+
const newErrors = blockingFindings.filter((f) => f.severity === "error").length;
|
|
2480
|
+
const newWarnings = blockingFindings.filter((f) => f.severity === "warning").length;
|
|
2481
|
+
const maxNewCritical = policy.maxNewCritical ?? 0;
|
|
2482
|
+
const maxNewErrors = policy.maxNewErrors ?? 0;
|
|
2483
|
+
const maxNewWarnings = policy.maxNewWarnings ?? 10;
|
|
2484
|
+
if (newCritical > maxNewCritical) {
|
|
2485
|
+
policyViolations.push(`New critical SEO regressions detected (${newCritical} > ${maxNewCritical}).`);
|
|
2486
|
+
}
|
|
2487
|
+
if (newErrors > maxNewErrors) {
|
|
2488
|
+
policyViolations.push(`New deterministic SEO errors introduced (${newErrors} > ${maxNewErrors}).`);
|
|
2489
|
+
}
|
|
2490
|
+
if (policy.blockOnNewOrphanRoutes && newOrphanRoutes.length > 0) {
|
|
2491
|
+
policyViolations.push(`Internal link regression: ${newOrphanRoutes.length} route(s) became newly orphaned (${newOrphanRoutes.join(", ")}).`);
|
|
2492
|
+
}
|
|
2493
|
+
if (newWarnings > maxNewWarnings) {
|
|
2494
|
+
policyViolations.push(`New warnings exceed threshold (${newWarnings} > ${maxNewWarnings}).`);
|
|
2495
|
+
}
|
|
2496
|
+
let status = "PASS";
|
|
2497
|
+
if (policyViolations.length > 0) {
|
|
2498
|
+
status = "FAIL";
|
|
2499
|
+
} else if (newWarnings > 0 || newOrphanRoutes.length > 0 && !policy.blockOnNewOrphanRoutes) {
|
|
2500
|
+
status = "WARN";
|
|
2501
|
+
}
|
|
2502
|
+
const passed = status !== "FAIL";
|
|
2503
|
+
const routesAtRisk = !passed ? affectedRoutes.length : 0;
|
|
2504
|
+
const routesProtected = passed ? routesResolvedCount : 0;
|
|
2505
|
+
return {
|
|
2506
|
+
status,
|
|
2507
|
+
passed,
|
|
2508
|
+
summary: {
|
|
2509
|
+
newCritical,
|
|
2510
|
+
newErrors,
|
|
2511
|
+
newWarnings,
|
|
2512
|
+
fixedFindings: fixedFindings.length,
|
|
2513
|
+
unchangedFindings: baselineDebt.length,
|
|
2514
|
+
affectedRoutesCount: affectedRoutes.length,
|
|
2515
|
+
baselineTotalIssues: baselineTotalIssues ?? baselineDebt.length,
|
|
2516
|
+
routesAtRisk,
|
|
2517
|
+
routesProtected
|
|
2518
|
+
},
|
|
2519
|
+
newRegressions: newFindings,
|
|
2520
|
+
fixedFindings,
|
|
2521
|
+
baselineDebt,
|
|
2522
|
+
policyViolations
|
|
2523
|
+
};
|
|
2524
|
+
}
|
|
2525
|
+
|
|
2526
|
+
// ../core/src/diff-engine.ts
|
|
2527
|
+
import path5 from "node:path";
|
|
2528
|
+
|
|
2529
|
+
// ../core/src/contracts.ts
|
|
2530
|
+
function matchesRoutePattern(route, pattern) {
|
|
2531
|
+
if (pattern === "/**" || pattern === "*") return true;
|
|
2532
|
+
if (pattern === route) return true;
|
|
2533
|
+
const normRoute = route.startsWith("/") ? route : `/${route}`;
|
|
2534
|
+
const normPattern = pattern.startsWith("/") ? pattern : `/${pattern}`;
|
|
2535
|
+
const tokenized = normPattern.replace(/\/\*\*/g, "/___GLOBSTAR___").replace(/\*\*/g, "___GLOBSTAR___").replace(/\*/g, "___SINGLESTAR___");
|
|
2536
|
+
const escaped = tokenized.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
|
|
2537
|
+
const regexStr = "^" + escaped.replace(/\/___GLOBSTAR___/g, "(?:/.*)?").replace(/___GLOBSTAR___/g, ".*").replace(/___SINGLESTAR___/g, "[^/]*") + "$";
|
|
2538
|
+
try {
|
|
2539
|
+
const regex = new RegExp(regexStr);
|
|
2540
|
+
return regex.test(normRoute);
|
|
2541
|
+
} catch {
|
|
2542
|
+
return normRoute.startsWith(pattern.replace(/\*+/g, ""));
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
function evaluateSEOContracts(options) {
|
|
2546
|
+
const { routes, contracts, linkGraph } = options;
|
|
2547
|
+
const findings = [];
|
|
2548
|
+
for (const contract of contracts) {
|
|
2549
|
+
const matchingRoutes = routes.filter((r) => {
|
|
2550
|
+
if (contract.exclude && contract.exclude.some((ex) => matchesRoutePattern(r.route, ex))) {
|
|
2551
|
+
return false;
|
|
2552
|
+
}
|
|
2553
|
+
return contract.routes.some((pattern) => matchesRoutePattern(r.route, pattern));
|
|
2554
|
+
});
|
|
2555
|
+
const req = contract.requirement;
|
|
2556
|
+
for (const r of matchingRoutes) {
|
|
2557
|
+
if (req.indexable === true) {
|
|
2558
|
+
const robots = (r.metadata?.robots || "").toLowerCase();
|
|
2559
|
+
if (robots.includes("noindex")) {
|
|
2560
|
+
findings.push({
|
|
2561
|
+
id: `contract-${contract.name}-indexable-${r.route}`,
|
|
2562
|
+
rule: `contract/${contract.name}`,
|
|
2563
|
+
severity: "error",
|
|
2564
|
+
category: "technical",
|
|
2565
|
+
confidence: "DETERMINISTIC",
|
|
2566
|
+
fixability: "REVERT_SAFE",
|
|
2567
|
+
status: "NEW",
|
|
2568
|
+
message: `SEO Contract violation in "${contract.name}": Route "${r.route}" must remain indexable, but robots contains "noindex".`,
|
|
2569
|
+
file: r.filePath,
|
|
2570
|
+
route: r.route,
|
|
2571
|
+
fixable: true,
|
|
2572
|
+
explanation: `Contract "${contract.name}" mandates that all matching routes remain indexable to search engines.`,
|
|
2573
|
+
evidenceChain: {
|
|
2574
|
+
sourceFile: r.filePath,
|
|
2575
|
+
changeDescription: `robots changed to "${r.metadata?.robots}"`,
|
|
2576
|
+
whyThisHappened: `Route "${r.route}" violated contract "${contract.name}" requirement: indexable=true.`,
|
|
2577
|
+
steps: [
|
|
2578
|
+
{
|
|
2579
|
+
type: "CONTRACT_VIOLATION",
|
|
2580
|
+
from: r.filePath,
|
|
2581
|
+
to: r.route,
|
|
2582
|
+
detail: `robots: "${r.metadata?.robots}" violates indexable requirement`
|
|
2583
|
+
}
|
|
2584
|
+
],
|
|
2585
|
+
affectedRoutesCount: 1,
|
|
2586
|
+
sampleRoutes: [r.route],
|
|
2587
|
+
confidence: "DETERMINISTIC",
|
|
2588
|
+
revertSafeValue: "index, follow",
|
|
2589
|
+
fixRecommendation: `Restore robots declaration to allow indexing.`
|
|
2590
|
+
}
|
|
2591
|
+
});
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
if (req.canonical === "required" && !r.metadata?.canonical) {
|
|
2595
|
+
findings.push({
|
|
2596
|
+
id: `contract-${contract.name}-canonical-${r.route}`,
|
|
2597
|
+
rule: `contract/${contract.name}`,
|
|
2598
|
+
severity: "error",
|
|
2599
|
+
category: "metadata",
|
|
2600
|
+
confidence: "DETERMINISTIC",
|
|
2601
|
+
fixability: "DETERMINISTIC_FIX",
|
|
2602
|
+
status: "NEW",
|
|
2603
|
+
message: `SEO Contract violation in "${contract.name}": Canonical URL is required on "${r.route}".`,
|
|
2604
|
+
file: r.filePath,
|
|
2605
|
+
route: r.route,
|
|
2606
|
+
fixable: true,
|
|
2607
|
+
explanation: `Contract "${contract.name}" requires an explicit canonical tag on all matching routes.`,
|
|
2608
|
+
evidenceChain: {
|
|
2609
|
+
sourceFile: r.filePath,
|
|
2610
|
+
changeDescription: "canonical declaration missing",
|
|
2611
|
+
whyThisHappened: `Route "${r.route}" violated contract "${contract.name}" requirement: canonical=required.`,
|
|
2612
|
+
steps: [
|
|
2613
|
+
{
|
|
2614
|
+
type: "CONTRACT_VIOLATION",
|
|
2615
|
+
from: r.filePath,
|
|
2616
|
+
to: r.route,
|
|
2617
|
+
detail: `canonical missing`
|
|
2618
|
+
}
|
|
2619
|
+
],
|
|
2620
|
+
affectedRoutesCount: 1,
|
|
2621
|
+
sampleRoutes: [r.route],
|
|
2622
|
+
confidence: "DETERMINISTIC",
|
|
2623
|
+
fixRecommendation: `Declare explicit canonical link on "${r.route}".`
|
|
2624
|
+
}
|
|
2625
|
+
});
|
|
2626
|
+
}
|
|
2627
|
+
if (req.internalLinks?.minimumInbound !== void 0 && linkGraph) {
|
|
2628
|
+
const metrics = linkGraph.metrics?.[r.route];
|
|
2629
|
+
const inbound = metrics?.incomingLinks ?? 0;
|
|
2630
|
+
if (inbound < req.internalLinks.minimumInbound) {
|
|
2631
|
+
findings.push({
|
|
2632
|
+
id: `contract-${contract.name}-min-inbound-${r.route}`,
|
|
2633
|
+
rule: `contract/${contract.name}`,
|
|
2634
|
+
severity: "error",
|
|
2635
|
+
category: "links",
|
|
2636
|
+
confidence: "STRUCTURAL",
|
|
2637
|
+
fixability: "SUGGESTED_FIX",
|
|
2638
|
+
status: "NEW",
|
|
2639
|
+
message: `SEO Contract violation in "${contract.name}": Route "${r.route}" has ${inbound} incoming link(s), but contract requires at least ${req.internalLinks.minimumInbound}.`,
|
|
2640
|
+
file: r.filePath,
|
|
2641
|
+
route: r.route,
|
|
2642
|
+
fixable: false,
|
|
2643
|
+
explanation: `Contract "${contract.name}" requires minimum ${req.internalLinks.minimumInbound} inbound internal links to prevent orphan routes.`,
|
|
2644
|
+
evidenceChain: {
|
|
2645
|
+
sourceFile: r.filePath,
|
|
2646
|
+
changeDescription: `incoming links = ${inbound} (required: >= ${req.internalLinks.minimumInbound})`,
|
|
2647
|
+
whyThisHappened: `Route became under-linked or orphaned after code changes.`,
|
|
2648
|
+
steps: [
|
|
2649
|
+
{
|
|
2650
|
+
type: "LINK_EDGE_REMOVAL",
|
|
2651
|
+
from: "Internal Link Graph",
|
|
2652
|
+
to: r.route,
|
|
2653
|
+
detail: `Current inbound: ${inbound}, required: ${req.internalLinks.minimumInbound}`
|
|
2654
|
+
}
|
|
2655
|
+
],
|
|
2656
|
+
affectedRoutesCount: 1,
|
|
2657
|
+
sampleRoutes: [r.route],
|
|
2658
|
+
confidence: "STRUCTURAL",
|
|
2659
|
+
fixRecommendation: `Add internal links pointing to "${r.route}" from navigation or parent routes.`
|
|
2660
|
+
}
|
|
2661
|
+
});
|
|
2662
|
+
}
|
|
2663
|
+
}
|
|
2664
|
+
if (req.schema?.required) {
|
|
2665
|
+
const expectedType = req.schema.type;
|
|
2666
|
+
const jsonLd = r.metadata?.jsonLd || [];
|
|
2667
|
+
const hasMatchingSchema = jsonLd.some((item) => {
|
|
2668
|
+
if (!expectedType) return true;
|
|
2669
|
+
const type = item["@type"] || item.type;
|
|
2670
|
+
return type?.toLowerCase() === expectedType.toLowerCase();
|
|
2671
|
+
});
|
|
2672
|
+
if (!hasMatchingSchema) {
|
|
2673
|
+
findings.push({
|
|
2674
|
+
id: `contract-${contract.name}-schema-${r.route}`,
|
|
2675
|
+
rule: `contract/${contract.name}`,
|
|
2676
|
+
severity: "error",
|
|
2677
|
+
category: "technical",
|
|
2678
|
+
confidence: "DETERMINISTIC",
|
|
2679
|
+
fixability: "SUGGESTED_FIX",
|
|
2680
|
+
status: "NEW",
|
|
2681
|
+
message: `SEO Contract violation in "${contract.name}": Missing required JSON-LD schema${expectedType ? ` "${expectedType}"` : ""} on "${r.route}".`,
|
|
2682
|
+
file: r.filePath,
|
|
2683
|
+
route: r.route,
|
|
2684
|
+
fixable: false,
|
|
2685
|
+
explanation: `Contract "${contract.name}" mandates structured data (${expectedType || "Schema.org"}) on this route pattern.`,
|
|
2686
|
+
evidenceChain: {
|
|
2687
|
+
sourceFile: r.filePath,
|
|
2688
|
+
changeDescription: `Schema ${expectedType || "JSON-LD"} missing`,
|
|
2689
|
+
whyThisHappened: `Route "${r.route}" failed contract requirement: schema.required=true.`,
|
|
2690
|
+
steps: [
|
|
2691
|
+
{
|
|
2692
|
+
type: "CONTRACT_VIOLATION",
|
|
2693
|
+
from: r.filePath,
|
|
2694
|
+
to: r.route,
|
|
2695
|
+
detail: `missing @type: ${expectedType || "any"}`
|
|
2696
|
+
}
|
|
2697
|
+
],
|
|
2698
|
+
affectedRoutesCount: 1,
|
|
2699
|
+
sampleRoutes: [r.route],
|
|
2700
|
+
confidence: "DETERMINISTIC",
|
|
2701
|
+
fixRecommendation: `Add JSON-LD script containing @type: "${expectedType || "WebPage"}".`
|
|
2702
|
+
}
|
|
2703
|
+
});
|
|
2704
|
+
}
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
return findings;
|
|
2709
|
+
}
|
|
2710
|
+
function suggestSEOContracts(routes) {
|
|
2711
|
+
const contracts = [];
|
|
2712
|
+
const sections = /* @__PURE__ */ new Map();
|
|
2713
|
+
for (const r of routes) {
|
|
2714
|
+
const parts = r.route.split("/").filter(Boolean);
|
|
2715
|
+
if (parts.length >= 1) {
|
|
2716
|
+
const section = parts[0];
|
|
2717
|
+
if (!sections.has(section)) sections.set(section, []);
|
|
2718
|
+
sections.get(section).push(r);
|
|
2719
|
+
}
|
|
2720
|
+
}
|
|
2721
|
+
for (const [section, sectionRoutes] of sections) {
|
|
2722
|
+
if (sectionRoutes.length >= 2) {
|
|
2723
|
+
const allIndexable = sectionRoutes.every((r) => !(r.metadata?.robots || "").includes("noindex"));
|
|
2724
|
+
const allCanonical = sectionRoutes.every((r) => Boolean(r.metadata?.canonical));
|
|
2725
|
+
if (allIndexable || allCanonical) {
|
|
2726
|
+
contracts.push({
|
|
2727
|
+
name: `${section}-pages`,
|
|
2728
|
+
routes: [`/${section}/**`],
|
|
2729
|
+
requirement: {
|
|
2730
|
+
indexable: allIndexable ? true : void 0,
|
|
2731
|
+
canonical: allCanonical ? "required" : "optional"
|
|
2732
|
+
}
|
|
2733
|
+
});
|
|
2734
|
+
}
|
|
2735
|
+
}
|
|
2736
|
+
}
|
|
2737
|
+
return contracts;
|
|
2738
|
+
}
|
|
2739
|
+
|
|
2740
|
+
// ../core/src/seo-graph.ts
|
|
2741
|
+
function normalizeRoute(pathStr) {
|
|
2742
|
+
if (pathStr.length > 1 && pathStr.endsWith("/")) {
|
|
2743
|
+
return pathStr.slice(0, -1);
|
|
2744
|
+
}
|
|
2745
|
+
return pathStr;
|
|
2746
|
+
}
|
|
2747
|
+
function buildSEODependencyGraph(routes, options = {}) {
|
|
2748
|
+
const nodeMap = /* @__PURE__ */ new Map();
|
|
2749
|
+
const edges = [];
|
|
2750
|
+
const addNode = (node) => {
|
|
2751
|
+
if (!nodeMap.has(node.id)) {
|
|
2752
|
+
nodeMap.set(node.id, node);
|
|
2753
|
+
}
|
|
2754
|
+
};
|
|
2755
|
+
const addEdge = (edge) => {
|
|
2756
|
+
edges.push(edge);
|
|
2757
|
+
};
|
|
2758
|
+
const layouts = options.layouts || [];
|
|
2759
|
+
for (const layout of layouts) {
|
|
2760
|
+
const layoutId = `layout:${layout.filePath}`;
|
|
2761
|
+
addNode({
|
|
2762
|
+
id: layoutId,
|
|
2763
|
+
type: "layout",
|
|
2764
|
+
label: layout.filePath,
|
|
2765
|
+
path: layout.filePath,
|
|
2766
|
+
metadata: layout.metadata
|
|
2767
|
+
});
|
|
2768
|
+
if (layout.parentDir !== void 0) {
|
|
2769
|
+
const parentLayout = layouts.find((l) => l.relativeDir === layout.parentDir);
|
|
2770
|
+
if (parentLayout) {
|
|
2771
|
+
addEdge({
|
|
2772
|
+
source: layoutId,
|
|
2773
|
+
target: `layout:${parentLayout.filePath}`,
|
|
2774
|
+
type: "INHERITS_FROM",
|
|
2775
|
+
detail: "layout inheritance"
|
|
2776
|
+
});
|
|
2777
|
+
}
|
|
2778
|
+
}
|
|
2779
|
+
}
|
|
2780
|
+
for (const route of routes) {
|
|
2781
|
+
const routeId = `route:${normalizeRoute(route.route)}`;
|
|
2782
|
+
addNode({
|
|
2783
|
+
id: routeId,
|
|
2784
|
+
type: "route",
|
|
2785
|
+
label: route.route,
|
|
2786
|
+
path: route.filePath,
|
|
2787
|
+
metadata: {
|
|
2788
|
+
canonical: route.metadata?.canonical,
|
|
2789
|
+
robots: route.metadata?.robots,
|
|
2790
|
+
title: route.metadata?.title,
|
|
2791
|
+
description: route.metadata?.description
|
|
2792
|
+
}
|
|
2793
|
+
});
|
|
2794
|
+
if (route.filePath) {
|
|
2795
|
+
const fileId = `file:${route.filePath}`;
|
|
2796
|
+
addNode({
|
|
2797
|
+
id: fileId,
|
|
2798
|
+
type: "source_file",
|
|
2799
|
+
label: route.filePath,
|
|
2800
|
+
path: route.filePath
|
|
2801
|
+
});
|
|
2802
|
+
addEdge({
|
|
2803
|
+
source: fileId,
|
|
2804
|
+
target: routeId,
|
|
2805
|
+
type: "DEFINES_METADATA",
|
|
2806
|
+
detail: "page component defines route"
|
|
2807
|
+
});
|
|
2808
|
+
}
|
|
2809
|
+
if (layouts.length > 0) {
|
|
2810
|
+
const matchingLayouts = layouts.filter((l) => {
|
|
2811
|
+
const layoutDir = l.relativeDir.replace(/\\/g, "/");
|
|
2812
|
+
if (layoutDir === "" || layoutDir === ".") return true;
|
|
2813
|
+
const normalizedFilePath = route.filePath.replace(/\\/g, "/");
|
|
2814
|
+
return normalizedFilePath.includes(`/${layoutDir}/`) || normalizedFilePath.includes(`${layoutDir}/`);
|
|
2815
|
+
});
|
|
2816
|
+
for (const layout of matchingLayouts) {
|
|
2817
|
+
addEdge({
|
|
2818
|
+
source: routeId,
|
|
2819
|
+
target: `layout:${layout.filePath}`,
|
|
2820
|
+
type: "INHERITS_FROM",
|
|
2821
|
+
detail: "metadata inheritance"
|
|
2822
|
+
});
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
2825
|
+
if (route.metadata?.canonical) {
|
|
2826
|
+
const cleanCanonical = normalizeRoute(route.metadata.canonical);
|
|
2827
|
+
if (cleanCanonical.startsWith("/")) {
|
|
2828
|
+
const canonicalTargetId = `route:${cleanCanonical}`;
|
|
2829
|
+
addEdge({
|
|
2830
|
+
source: routeId,
|
|
2831
|
+
target: canonicalTargetId,
|
|
2832
|
+
type: "CANONICALIZES_TO",
|
|
2833
|
+
detail: route.metadata.canonical
|
|
2834
|
+
});
|
|
2835
|
+
}
|
|
2836
|
+
}
|
|
2837
|
+
}
|
|
2838
|
+
if (options.linkGraph?.edges) {
|
|
2839
|
+
for (const linkEdge of options.linkGraph.edges) {
|
|
2840
|
+
const sourceId = `route:${normalizeRoute(linkEdge.source)}`;
|
|
2841
|
+
const targetId = `route:${normalizeRoute(linkEdge.target)}`;
|
|
2842
|
+
addEdge({
|
|
2843
|
+
source: sourceId,
|
|
2844
|
+
target: targetId,
|
|
2845
|
+
type: "LINKS_TO",
|
|
2846
|
+
detail: linkEdge.anchorText
|
|
2847
|
+
});
|
|
2848
|
+
}
|
|
2849
|
+
}
|
|
2850
|
+
if (options.redirects) {
|
|
2851
|
+
for (const r of options.redirects) {
|
|
2852
|
+
const sourceId = `route:${normalizeRoute(r.source)}`;
|
|
2853
|
+
const targetId = `route:${normalizeRoute(r.destination)}`;
|
|
2854
|
+
addEdge({
|
|
2855
|
+
source: sourceId,
|
|
2856
|
+
target: targetId,
|
|
2857
|
+
type: "REDIRECTS_TO",
|
|
2858
|
+
detail: r.permanent ? "301 Permanent" : "302 Temporary"
|
|
2859
|
+
});
|
|
2860
|
+
}
|
|
2861
|
+
}
|
|
2862
|
+
if (options.inferredSemanticEdges) {
|
|
2863
|
+
for (const semEdge of options.inferredSemanticEdges) {
|
|
2864
|
+
addEdge(semEdge);
|
|
2865
|
+
}
|
|
2866
|
+
}
|
|
2867
|
+
return {
|
|
2868
|
+
nodes: Array.from(nodeMap.values()).sort((a, b) => a.id.localeCompare(b.id)),
|
|
2869
|
+
edges: edges.sort((a, b) => `${a.source}\0${a.type}\0${a.target}`.localeCompare(`${b.source}\0${b.type}\0${b.target}`))
|
|
2870
|
+
};
|
|
2871
|
+
}
|
|
2872
|
+
function computeGraphDiff(baseGraph, headGraph) {
|
|
2873
|
+
const baseNodeMap = new Map(baseGraph.nodes.map((n) => [n.id, n]));
|
|
2874
|
+
const headNodeMap = new Map(headGraph.nodes.map((n) => [n.id, n]));
|
|
2875
|
+
const addedNodes = [];
|
|
2876
|
+
const removedNodes = [];
|
|
2877
|
+
const mutatedMetadata = [];
|
|
2878
|
+
for (const [id, headNode] of headNodeMap) {
|
|
2879
|
+
if (!baseNodeMap.has(id)) {
|
|
2880
|
+
addedNodes.push(headNode);
|
|
2881
|
+
} else {
|
|
2882
|
+
const baseNode = baseNodeMap.get(id);
|
|
2883
|
+
if (baseNode.metadata || headNode.metadata) {
|
|
2884
|
+
const allKeys = /* @__PURE__ */ new Set([
|
|
2885
|
+
...Object.keys(baseNode.metadata || {}),
|
|
2886
|
+
...Object.keys(headNode.metadata || {})
|
|
2887
|
+
]);
|
|
2888
|
+
for (const key of allKeys) {
|
|
2889
|
+
const oldVal = baseNode.metadata?.[key];
|
|
2890
|
+
const newVal = headNode.metadata?.[key];
|
|
2891
|
+
if (oldVal !== newVal) {
|
|
2892
|
+
mutatedMetadata.push({
|
|
2893
|
+
nodeId: id,
|
|
2894
|
+
property: key,
|
|
2895
|
+
oldValue: oldVal,
|
|
2896
|
+
newValue: newVal
|
|
2897
|
+
});
|
|
2898
|
+
}
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
}
|
|
2903
|
+
for (const [id, baseNode] of baseNodeMap) {
|
|
2904
|
+
if (!headNodeMap.has(id)) {
|
|
2905
|
+
removedNodes.push(baseNode);
|
|
2906
|
+
}
|
|
2907
|
+
}
|
|
2908
|
+
const edgeKey2 = (e) => `${e.source} -[${e.type}]-> ${e.target}`;
|
|
2909
|
+
const baseEdgeMap = new Map(baseGraph.edges.map((e) => [edgeKey2(e), e]));
|
|
2910
|
+
const headEdgeMap = new Map(headGraph.edges.map((e) => [edgeKey2(e), e]));
|
|
2911
|
+
const addedEdges = [];
|
|
2912
|
+
const removedEdges = [];
|
|
2913
|
+
for (const [key, edge] of headEdgeMap) {
|
|
2914
|
+
if (!baseEdgeMap.has(key)) addedEdges.push(edge);
|
|
2915
|
+
}
|
|
2916
|
+
for (const [key, edge] of baseEdgeMap) {
|
|
2917
|
+
if (!headEdgeMap.has(key)) removedEdges.push(edge);
|
|
2918
|
+
}
|
|
2919
|
+
return {
|
|
2920
|
+
addedNodes: addedNodes.sort((a, b) => a.id.localeCompare(b.id)),
|
|
2921
|
+
removedNodes: removedNodes.sort((a, b) => a.id.localeCompare(b.id)),
|
|
2922
|
+
addedEdges: addedEdges.sort((a, b) => edgeKey2(a).localeCompare(edgeKey2(b))),
|
|
2923
|
+
removedEdges: removedEdges.sort((a, b) => edgeKey2(a).localeCompare(edgeKey2(b))),
|
|
2924
|
+
mutatedMetadata: mutatedMetadata.sort((a, b) => `${a.nodeId}\0${a.property}`.localeCompare(`${b.nodeId}\0${b.property}`))
|
|
2925
|
+
};
|
|
2926
|
+
}
|
|
2927
|
+
function traceDownstreamRoutes(graph, sourceFileOrLayout) {
|
|
2928
|
+
const affectedRoutes = /* @__PURE__ */ new Set();
|
|
2929
|
+
const matchingNodeIds = graph.nodes.filter((n) => n.path === sourceFileOrLayout || n.id.includes(sourceFileOrLayout)).map((n) => n.id);
|
|
2930
|
+
if (matchingNodeIds.length === 0) return [];
|
|
2931
|
+
for (const edge of graph.edges) {
|
|
2932
|
+
if ((edge.type === "INHERITS_FROM" || edge.type === "DEFINES_METADATA") && matchingNodeIds.includes(edge.target)) {
|
|
2933
|
+
if (edge.source.startsWith("route:")) {
|
|
2934
|
+
affectedRoutes.add(edge.source.replace(/^route:/, ""));
|
|
2935
|
+
}
|
|
2936
|
+
}
|
|
2937
|
+
if (edge.type === "DEFINES_METADATA" && matchingNodeIds.includes(edge.source)) {
|
|
2938
|
+
if (edge.target.startsWith("route:")) {
|
|
2939
|
+
affectedRoutes.add(edge.target.replace(/^route:/, ""));
|
|
2940
|
+
}
|
|
2941
|
+
}
|
|
2942
|
+
}
|
|
2943
|
+
return Array.from(affectedRoutes).sort();
|
|
2944
|
+
}
|
|
2945
|
+
|
|
2946
|
+
// ../core/src/diff-engine.ts
|
|
2947
|
+
function edgeKey(edge) {
|
|
2948
|
+
return `${edge.source} -> ${edge.target} [${edge.anchorText}]`;
|
|
2949
|
+
}
|
|
2950
|
+
function enrichEvidenceChain(finding, isRegression) {
|
|
2951
|
+
if (finding.evidenceChain) return finding;
|
|
2952
|
+
const isCanonical = finding.rule.includes("canonical");
|
|
2953
|
+
const isRobots = finding.rule.includes("robots") || finding.message.includes("noindex");
|
|
2954
|
+
const isCriticalTechnical = isCanonical || isRobots;
|
|
2955
|
+
const isLayoutSource = Boolean(
|
|
2956
|
+
finding.sourceFile?.includes("layout") || finding.file?.includes("layout") || finding.affectedRoutes && finding.affectedRoutes.length > 1
|
|
2957
|
+
);
|
|
2958
|
+
let confidence = finding.confidence || "DETERMINISTIC";
|
|
2959
|
+
let fixability = finding.fixability || "SUGGESTED_FIX";
|
|
2960
|
+
if (isCriticalTechnical && isRegression) {
|
|
2961
|
+
confidence = isLayoutSource ? "STRUCTURAL" : "DETERMINISTIC";
|
|
2962
|
+
fixability = "REVERT_SAFE";
|
|
2963
|
+
} else if (finding.rule.includes("contract")) {
|
|
2964
|
+
confidence = "DETERMINISTIC";
|
|
2965
|
+
} else if (finding.severity === "info" || finding.category === "content") {
|
|
2966
|
+
confidence = "HEURISTIC";
|
|
2967
|
+
}
|
|
2968
|
+
const steps = [];
|
|
2969
|
+
const source = finding.sourceFile || finding.file || "unknown source";
|
|
2970
|
+
const targetRoute = finding.route || finding.routePattern || "/";
|
|
2971
|
+
if (isLayoutSource) {
|
|
2972
|
+
steps.push({
|
|
2973
|
+
type: "LAYOUT_INHERITANCE",
|
|
2974
|
+
from: source,
|
|
2975
|
+
to: targetRoute,
|
|
2976
|
+
detail: "Metadata inherited by nested routes"
|
|
2977
|
+
});
|
|
2978
|
+
} else {
|
|
2979
|
+
steps.push({
|
|
2980
|
+
type: "DIRECT_MUTATION",
|
|
2981
|
+
from: source,
|
|
2982
|
+
to: targetRoute,
|
|
2983
|
+
detail: finding.message
|
|
2984
|
+
});
|
|
2985
|
+
}
|
|
2986
|
+
const affectedCount = Math.max(1, finding.affectedRoutes?.length || 1);
|
|
2987
|
+
const sampleRoutes = finding.affectedRoutes && finding.affectedRoutes.length > 0 ? finding.affectedRoutes.slice(0, 3) : [targetRoute];
|
|
2988
|
+
const evidenceChain = {
|
|
2989
|
+
sourceFile: source,
|
|
2990
|
+
sourceLine: finding.sourceLine || finding.line,
|
|
2991
|
+
changeDescription: isRegression ? `Regression introduced: ${finding.message}` : finding.message,
|
|
2992
|
+
whyThisHappened: isLayoutSource ? `${source} defines metadata inherited by all nested routes.` : `File ${source} directly configures SEO attributes for ${targetRoute}.`,
|
|
2993
|
+
steps,
|
|
2994
|
+
affectedRoutesCount: affectedCount,
|
|
2995
|
+
sampleRoutes,
|
|
2996
|
+
confidence,
|
|
2997
|
+
revertSafeValue: fixability === "REVERT_SAFE" ? "Restore previous declaration from BASE" : void 0,
|
|
2998
|
+
fixRecommendation: fixability === "REVERT_SAFE" ? "Restore previous declaration from BASE." : "Update file declaration according to best practices."
|
|
2999
|
+
};
|
|
3000
|
+
return {
|
|
3001
|
+
...finding,
|
|
3002
|
+
confidence,
|
|
3003
|
+
fixability,
|
|
3004
|
+
evidenceChain
|
|
3005
|
+
};
|
|
3006
|
+
}
|
|
3007
|
+
function consolidateRegressionsByRootCause(rawNewFindings, baseAudit, headAudit, headSEOGraph, seoGraphDiff) {
|
|
3008
|
+
const consolidated = [];
|
|
3009
|
+
const handledIds = /* @__PURE__ */ new Set();
|
|
3010
|
+
const layoutMutations = seoGraphDiff.mutatedMetadata.filter((m) => m.nodeId.startsWith("layout:"));
|
|
3011
|
+
for (const mut of layoutMutations) {
|
|
3012
|
+
const layoutPath = mut.nodeId.replace(/^layout:/, "");
|
|
3013
|
+
const downstream = traceDownstreamRoutes(headSEOGraph, layoutPath);
|
|
3014
|
+
if (downstream.length === 0) continue;
|
|
3015
|
+
const prop = mut.property;
|
|
3016
|
+
const matching = rawNewFindings.filter((f) => {
|
|
3017
|
+
if (handledIds.has(f.id)) return false;
|
|
3018
|
+
const routeMatch = f.route && downstream.includes(f.route);
|
|
3019
|
+
const propMatch = prop === "canonical" ? f.rule.includes("canonical") : f.rule.includes("robots") || f.message.includes("noindex");
|
|
3020
|
+
return routeMatch && propMatch;
|
|
3021
|
+
});
|
|
3022
|
+
if (matching.length > 0) {
|
|
3023
|
+
for (const m of matching) handledIds.add(m.id);
|
|
3024
|
+
const isCanonical = prop === "canonical";
|
|
3025
|
+
const baseName = path5.basename(layoutPath);
|
|
3026
|
+
consolidated.push({
|
|
3027
|
+
id: `regression-${baseName}-${prop}`,
|
|
3028
|
+
rule: isCanonical ? "metadata/canonical-regression" : "crawlability/robots-regression",
|
|
3029
|
+
severity: "error",
|
|
3030
|
+
category: isCanonical ? "metadata" : "technical",
|
|
3031
|
+
confidence: "STRUCTURAL",
|
|
3032
|
+
fixability: "REVERT_SAFE",
|
|
3033
|
+
status: "NEW",
|
|
3034
|
+
message: isCanonical ? `Canonical declaration removed from ${baseName}` : `Robots directive changed in ${baseName}`,
|
|
3035
|
+
file: layoutPath,
|
|
3036
|
+
sourceFile: layoutPath,
|
|
3037
|
+
rootCause: {
|
|
3038
|
+
file: layoutPath,
|
|
3039
|
+
changeDescription: isCanonical ? "alternates.canonical removed from layout" : `robots directive changed in layout`
|
|
3040
|
+
},
|
|
3041
|
+
affectedRoutes: downstream,
|
|
3042
|
+
fixable: true,
|
|
3043
|
+
explanation: `${layoutPath} defines inherited metadata for ${downstream.length} nested route(s). Modifying it propagated this regression to all child routes.`,
|
|
3044
|
+
evidenceChain: {
|
|
3045
|
+
sourceFile: layoutPath,
|
|
3046
|
+
changeDescription: isCanonical ? "alternates.canonical removed from layout" : `robots directive changed in layout`,
|
|
3047
|
+
whyThisHappened: `${layoutPath} defines metadata inherited by all nested routes.`,
|
|
3048
|
+
steps: [
|
|
3049
|
+
{
|
|
3050
|
+
type: "LAYOUT_INHERITANCE",
|
|
3051
|
+
from: layoutPath,
|
|
3052
|
+
to: downstream.length === 1 ? downstream[0] : `${downstream.length} nested routes`,
|
|
3053
|
+
detail: "metadata inheritance"
|
|
3054
|
+
}
|
|
3055
|
+
],
|
|
3056
|
+
affectedRoutesCount: downstream.length,
|
|
3057
|
+
sampleRoutes: downstream.slice(0, 3),
|
|
3058
|
+
confidence: "STRUCTURAL",
|
|
3059
|
+
revertSafeValue: mut.oldValue ? String(mut.oldValue) : void 0,
|
|
3060
|
+
fixRecommendation: `Restore previous ${prop} declaration from BASE revision in ${baseName}.`
|
|
3061
|
+
}
|
|
3062
|
+
});
|
|
3063
|
+
}
|
|
3064
|
+
}
|
|
3065
|
+
const baseRouteMap = new Map((baseAudit.routes || []).map((r) => [r.route, r]));
|
|
3066
|
+
const unhandled = rawNewFindings.filter((f) => !handledIds.has(f.id));
|
|
3067
|
+
const canonicalRegressions = unhandled.filter((f) => f.rule.includes("canonical"));
|
|
3068
|
+
const robotsRegressions = unhandled.filter((f) => f.rule.includes("robots") || f.message.includes("noindex"));
|
|
3069
|
+
const tryGroupCategory = (groupFindings, signal) => {
|
|
3070
|
+
if (groupFindings.length === 0) return;
|
|
3071
|
+
const byDir = /* @__PURE__ */ new Map();
|
|
3072
|
+
for (const f of groupFindings) {
|
|
3073
|
+
if (!f.route) continue;
|
|
3074
|
+
const parts = f.route.split("/").filter(Boolean);
|
|
3075
|
+
const dir = parts.length > 1 ? `/${parts[0]}` : "/";
|
|
3076
|
+
if (!byDir.has(dir)) byDir.set(dir, []);
|
|
3077
|
+
byDir.get(dir).push(f);
|
|
3078
|
+
}
|
|
3079
|
+
for (const [dir, dirFindings] of byDir) {
|
|
3080
|
+
const hadInBase = dirFindings.every((f) => {
|
|
3081
|
+
const baseR = baseRouteMap.get(f.route);
|
|
3082
|
+
if (signal === "canonical") return Boolean(baseR?.metadata?.canonical);
|
|
3083
|
+
return !(baseR?.metadata?.robots || "").includes("noindex");
|
|
3084
|
+
});
|
|
3085
|
+
if (hadInBase && dirFindings.length > 1) {
|
|
3086
|
+
const layoutNode = headSEOGraph.nodes.find(
|
|
3087
|
+
(n) => n.type === "layout" && (n.path?.includes(dir.replace("/", "")) || n.id.includes(dir.replace("/", "")))
|
|
3088
|
+
);
|
|
3089
|
+
const inferredLayoutFile = layoutNode?.path || `app${dir}/layout.tsx`;
|
|
3090
|
+
const baseName = path5.basename(inferredLayoutFile);
|
|
3091
|
+
const affected = dirFindings.map((f) => f.route).sort();
|
|
3092
|
+
for (const df of dirFindings) handledIds.add(df.id);
|
|
3093
|
+
consolidated.push({
|
|
3094
|
+
id: `regression-${baseName}-${signal}`,
|
|
3095
|
+
rule: signal === "canonical" ? "metadata/canonical-regression" : "crawlability/robots-regression",
|
|
3096
|
+
severity: "error",
|
|
3097
|
+
category: signal === "canonical" ? "metadata" : "technical",
|
|
3098
|
+
confidence: "STRUCTURAL",
|
|
3099
|
+
fixability: "REVERT_SAFE",
|
|
3100
|
+
status: "NEW",
|
|
3101
|
+
message: signal === "canonical" ? `Canonical declaration removed from ${baseName}` : `Robots directive changed in ${baseName}`,
|
|
3102
|
+
file: inferredLayoutFile,
|
|
3103
|
+
sourceFile: inferredLayoutFile,
|
|
3104
|
+
rootCause: {
|
|
3105
|
+
file: inferredLayoutFile,
|
|
3106
|
+
changeDescription: signal === "canonical" ? "alternates.canonical removed from layout" : `robots directive changed in layout`
|
|
3107
|
+
},
|
|
3108
|
+
affectedRoutes: affected,
|
|
3109
|
+
fixable: true,
|
|
3110
|
+
explanation: `${inferredLayoutFile} defines inherited metadata for ${affected.length} nested route(s). Modifying it propagated this regression to all child routes.`,
|
|
3111
|
+
evidenceChain: {
|
|
3112
|
+
sourceFile: inferredLayoutFile,
|
|
3113
|
+
changeDescription: signal === "canonical" ? "alternates.canonical removed from layout" : `robots directive changed in layout`,
|
|
3114
|
+
whyThisHappened: `${inferredLayoutFile} defines metadata inherited by all nested routes.`,
|
|
3115
|
+
steps: [
|
|
3116
|
+
{
|
|
3117
|
+
type: "LAYOUT_INHERITANCE",
|
|
3118
|
+
from: inferredLayoutFile,
|
|
3119
|
+
to: `${affected.length} nested routes`,
|
|
3120
|
+
detail: "metadata inheritance"
|
|
3121
|
+
}
|
|
3122
|
+
],
|
|
3123
|
+
affectedRoutesCount: affected.length,
|
|
3124
|
+
sampleRoutes: affected.slice(0, 3),
|
|
3125
|
+
confidence: "STRUCTURAL",
|
|
3126
|
+
revertSafeValue: "Restore previous declaration from BASE revision",
|
|
3127
|
+
fixRecommendation: `Restore previous ${signal} declaration from BASE revision in ${baseName}.`
|
|
3128
|
+
}
|
|
3129
|
+
});
|
|
3130
|
+
}
|
|
3131
|
+
}
|
|
3132
|
+
};
|
|
3133
|
+
tryGroupCategory(canonicalRegressions, "canonical");
|
|
3134
|
+
tryGroupCategory(robotsRegressions, "robots");
|
|
3135
|
+
for (const f of rawNewFindings) {
|
|
3136
|
+
if (!handledIds.has(f.id)) {
|
|
3137
|
+
consolidated.push(f);
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
return consolidated;
|
|
3141
|
+
}
|
|
3142
|
+
function computeAuditDiff(baseAudit, headAudit, policyOrOptions) {
|
|
3143
|
+
const options = policyOrOptions && ("contracts" in policyOrOptions || "maxNewCritical" in policyOrOptions) ? { policy: policyOrOptions } : policyOrOptions || {};
|
|
3144
|
+
const policy = options.policy;
|
|
3145
|
+
const baseFindings = (baseAudit.findings || []).map(enrichFinding);
|
|
3146
|
+
let headFindings = (headAudit.findings || []).map(enrichFinding);
|
|
3147
|
+
const baselineTotalIssues = baseAudit.findings?.length ?? 0;
|
|
3148
|
+
if (policy?.contracts && headAudit.routes) {
|
|
3149
|
+
const contractsList = Object.entries(policy.contracts).map(([name, c]) => ({
|
|
3150
|
+
name,
|
|
3151
|
+
routes: c.routes,
|
|
3152
|
+
exclude: c.exclude,
|
|
3153
|
+
requirement: c
|
|
3154
|
+
}));
|
|
3155
|
+
const contractFindings = evaluateSEOContracts({
|
|
3156
|
+
routes: headAudit.routes,
|
|
3157
|
+
contracts: contractsList,
|
|
3158
|
+
linkGraph: headAudit.linkGraph
|
|
3159
|
+
}).map(enrichFinding);
|
|
3160
|
+
headFindings = [...headFindings, ...contractFindings];
|
|
3161
|
+
}
|
|
3162
|
+
const baseMap = /* @__PURE__ */ new Map();
|
|
3163
|
+
for (const f of baseFindings) {
|
|
3164
|
+
baseMap.set(f.fingerprint, f);
|
|
3165
|
+
}
|
|
3166
|
+
const headMap = /* @__PURE__ */ new Map();
|
|
3167
|
+
for (const f of headFindings) {
|
|
3168
|
+
headMap.set(f.fingerprint, f);
|
|
3169
|
+
}
|
|
3170
|
+
const rawNewFindings = [];
|
|
3171
|
+
const unchangedFindings = [];
|
|
3172
|
+
for (const f of headFindings) {
|
|
3173
|
+
if (baseMap.has(f.fingerprint)) {
|
|
3174
|
+
unchangedFindings.push({ ...f, status: "UNCHANGED" });
|
|
3175
|
+
} else {
|
|
3176
|
+
const enriched = enrichEvidenceChain({ ...f, status: "NEW" }, true);
|
|
3177
|
+
rawNewFindings.push(enriched);
|
|
3178
|
+
}
|
|
3179
|
+
}
|
|
3180
|
+
const fixedFindings = [];
|
|
3181
|
+
for (const f of baseFindings) {
|
|
3182
|
+
if (!headMap.has(f.fingerprint)) {
|
|
3183
|
+
fixedFindings.push({ ...f, status: "FIXED" });
|
|
3184
|
+
}
|
|
3185
|
+
}
|
|
3186
|
+
const baseEdges = /* @__PURE__ */ new Map();
|
|
3187
|
+
for (const edge of baseAudit.linkGraph?.edges || []) {
|
|
3188
|
+
baseEdges.set(edgeKey(edge), edge);
|
|
3189
|
+
}
|
|
3190
|
+
const headEdges = /* @__PURE__ */ new Map();
|
|
3191
|
+
for (const edge of headAudit.linkGraph?.edges || []) {
|
|
3192
|
+
headEdges.set(edgeKey(edge), edge);
|
|
3193
|
+
}
|
|
3194
|
+
const newEdges = [];
|
|
3195
|
+
for (const [key, edge] of headEdges) {
|
|
3196
|
+
if (!baseEdges.has(key)) newEdges.push(edge);
|
|
3197
|
+
}
|
|
3198
|
+
const removedEdges = [];
|
|
3199
|
+
for (const [key, edge] of baseEdges) {
|
|
3200
|
+
if (!headEdges.has(key)) removedEdges.push(edge);
|
|
3201
|
+
}
|
|
3202
|
+
const baseOrphans = new Set(baseAudit.linkGraph?.orphans || []);
|
|
3203
|
+
const headOrphans = new Set(headAudit.linkGraph?.orphans || []);
|
|
3204
|
+
const newOrphans = [...headOrphans].filter((route) => !baseOrphans.has(route));
|
|
3205
|
+
const resolvedOrphans = [...baseOrphans].filter((route) => !headOrphans.has(route));
|
|
3206
|
+
const linkGraphDiff = {
|
|
3207
|
+
newEdges,
|
|
3208
|
+
removedEdges,
|
|
3209
|
+
newOrphans,
|
|
3210
|
+
resolvedOrphans
|
|
3211
|
+
};
|
|
3212
|
+
const baseSEOGraph = baseAudit.seoDependencyGraph || buildSEODependencyGraph(baseAudit.routes || [], { linkGraph: baseAudit.linkGraph });
|
|
3213
|
+
const headSEOGraph = headAudit.seoDependencyGraph || buildSEODependencyGraph(headAudit.routes || [], {
|
|
3214
|
+
linkGraph: headAudit.linkGraph,
|
|
3215
|
+
inferredSemanticEdges: options.inferredSemanticEdges
|
|
3216
|
+
});
|
|
3217
|
+
const seoGraphDiff = computeGraphDiff(baseSEOGraph, headSEOGraph);
|
|
3218
|
+
const newFindings = consolidateRegressionsByRootCause(
|
|
3219
|
+
rawNewFindings,
|
|
3220
|
+
baseAudit,
|
|
3221
|
+
headAudit,
|
|
3222
|
+
headSEOGraph,
|
|
3223
|
+
seoGraphDiff
|
|
3224
|
+
);
|
|
3225
|
+
const affectedRoutesSet = /* @__PURE__ */ new Set();
|
|
3226
|
+
for (const f of newFindings) {
|
|
3227
|
+
if (f.route) affectedRoutesSet.add(f.route);
|
|
3228
|
+
if (f.affectedRoutes) {
|
|
3229
|
+
for (const r of f.affectedRoutes) affectedRoutesSet.add(r);
|
|
3230
|
+
}
|
|
3231
|
+
}
|
|
3232
|
+
for (const orphan of newOrphans) {
|
|
3233
|
+
affectedRoutesSet.add(orphan);
|
|
3234
|
+
}
|
|
3235
|
+
const affectedRoutes = Array.from(affectedRoutesSet).sort();
|
|
3236
|
+
const qualityGate = evaluateQualityGate({
|
|
3237
|
+
newFindings,
|
|
3238
|
+
fixedFindings,
|
|
3239
|
+
baselineDebt: baseFindings,
|
|
3240
|
+
newOrphanRoutes: newOrphans,
|
|
3241
|
+
affectedRoutes,
|
|
3242
|
+
policy,
|
|
3243
|
+
baselineTotalIssues
|
|
3244
|
+
});
|
|
3245
|
+
const scoreDiff = (headAudit.score?.overall ?? 0) - (baseAudit.score?.overall ?? 0);
|
|
3246
|
+
const routesAtRisk = !qualityGate.passed ? affectedRoutes.length : 0;
|
|
3247
|
+
const routesProtected = qualityGate.passed && fixedFindings.length > 0 ? affectedRoutes.length || fixedFindings.length : 0;
|
|
3248
|
+
const preventionMetrics = {
|
|
3249
|
+
prsAnalyzed: 1,
|
|
3250
|
+
regressionsCaught: newFindings.length,
|
|
3251
|
+
criticalRegressionsPrevented: qualityGate.summary.newCritical,
|
|
3252
|
+
routesAtRisk,
|
|
3253
|
+
routesProtected
|
|
3254
|
+
};
|
|
3255
|
+
return {
|
|
3256
|
+
baseScore: baseAudit.score,
|
|
3257
|
+
headScore: headAudit.score,
|
|
3258
|
+
scoreDiff,
|
|
3259
|
+
newFindings,
|
|
3260
|
+
fixedFindings,
|
|
3261
|
+
unchangedFindings,
|
|
3262
|
+
linkGraphDiff,
|
|
3263
|
+
seoGraphDiff,
|
|
3264
|
+
affectedRoutes,
|
|
3265
|
+
preventionMetrics,
|
|
3266
|
+
qualityGate
|
|
3267
|
+
};
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3270
|
+
// ../security/src/redactor.ts
|
|
3271
|
+
var SECRET_PATTERNS = [
|
|
3272
|
+
/sk-[a-zA-Z0-9_-]{20,}/g,
|
|
3273
|
+
// OpenAI, Anthropic, OpenRouter standard API keys
|
|
3274
|
+
/ghp_[a-zA-Z0-9]{36,}/g,
|
|
3275
|
+
// GitHub Personal Access Token
|
|
3276
|
+
/ghs_[a-zA-Z0-9]{36,}/g,
|
|
3277
|
+
// GitHub Installation Token
|
|
3278
|
+
/github_pat_[a-zA-Z0-9_]{60,}/g,
|
|
3279
|
+
// GitHub Fine-grained PAT
|
|
3280
|
+
/Bearer\s+[a-zA-Z0-9._-]{20,}/gi,
|
|
3281
|
+
// Bearer JWT or token
|
|
3282
|
+
/-----BEGIN [A-Z ]+ PRIVATE KEY-----[\s\S]*?-----END [A-Z ]+ PRIVATE KEY-----/g,
|
|
3283
|
+
// RSA / EC Private Keys
|
|
3284
|
+
/password\s*[:=]\s*["']?([^"' \n\r]+)["']?/gi,
|
|
3285
|
+
// inline passwords
|
|
3286
|
+
/secret\s*[:=]\s*["']?([^"' \n\r]+)["']?/gi
|
|
3287
|
+
// inline secrets
|
|
3288
|
+
];
|
|
3289
|
+
function redactString(text) {
|
|
3290
|
+
if (!text || typeof text !== "string") return text;
|
|
3291
|
+
let redacted = text;
|
|
3292
|
+
for (const pattern of SECRET_PATTERNS) {
|
|
3293
|
+
redacted = redacted.replace(pattern, "[REDACTED_SECRET]");
|
|
3294
|
+
}
|
|
3295
|
+
return redacted;
|
|
3296
|
+
}
|
|
3297
|
+
function redactObject(obj) {
|
|
3298
|
+
if (obj === null || obj === void 0) return obj;
|
|
3299
|
+
if (typeof obj === "string") return redactString(obj);
|
|
3300
|
+
if (typeof obj !== "object") return obj;
|
|
3301
|
+
if (Array.isArray(obj)) {
|
|
3302
|
+
return obj.map((item) => redactObject(item));
|
|
3303
|
+
}
|
|
3304
|
+
const result = {};
|
|
3305
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
3306
|
+
const lowerKey = k.toLowerCase();
|
|
3307
|
+
if (lowerKey.includes("key") || lowerKey.includes("secret") || lowerKey.includes("token") || lowerKey.includes("password") || lowerKey.includes("auth")) {
|
|
3308
|
+
result[k] = "[REDACTED]";
|
|
3309
|
+
} else {
|
|
3310
|
+
result[k] = redactObject(v);
|
|
3311
|
+
}
|
|
3312
|
+
}
|
|
3313
|
+
return result;
|
|
3314
|
+
}
|
|
3315
|
+
var secureLogger = {
|
|
3316
|
+
info: (msg, ...args) => {
|
|
3317
|
+
console.info(redactString(msg), ...args.map(redactObject));
|
|
3318
|
+
},
|
|
3319
|
+
warn: (msg, ...args) => {
|
|
3320
|
+
console.warn(redactString(msg), ...args.map(redactObject));
|
|
3321
|
+
},
|
|
3322
|
+
error: (msg, ...args) => {
|
|
3323
|
+
console.error(redactString(msg), ...args.map(redactObject));
|
|
3324
|
+
}
|
|
3325
|
+
};
|
|
3326
|
+
|
|
3327
|
+
// ../security/src/limiter.ts
|
|
3328
|
+
var RateLimiter = class {
|
|
3329
|
+
entries = /* @__PURE__ */ new Map();
|
|
3330
|
+
maxRequests;
|
|
3331
|
+
windowMs;
|
|
3332
|
+
constructor(maxRequests = 30, windowMs = 6e4) {
|
|
3333
|
+
this.maxRequests = maxRequests;
|
|
3334
|
+
this.windowMs = windowMs;
|
|
3335
|
+
}
|
|
3336
|
+
check(key) {
|
|
3337
|
+
const now = Date.now();
|
|
3338
|
+
const entry = this.entries.get(key) || { timestamps: [] };
|
|
3339
|
+
const validTimestamps = entry.timestamps.filter((ts) => now - ts < this.windowMs);
|
|
3340
|
+
if (validTimestamps.length >= this.maxRequests) {
|
|
3341
|
+
const oldest = validTimestamps[0];
|
|
3342
|
+
const resetMs = Math.max(0, this.windowMs - (now - oldest));
|
|
3343
|
+
return {
|
|
3344
|
+
allowed: false,
|
|
3345
|
+
remaining: 0,
|
|
3346
|
+
resetMs
|
|
3347
|
+
};
|
|
3348
|
+
}
|
|
3349
|
+
validTimestamps.push(now);
|
|
3350
|
+
this.entries.set(key, { timestamps: validTimestamps });
|
|
3351
|
+
return {
|
|
3352
|
+
allowed: true,
|
|
3353
|
+
remaining: this.maxRequests - validTimestamps.length,
|
|
3354
|
+
resetMs: this.windowMs
|
|
3355
|
+
};
|
|
3356
|
+
}
|
|
3357
|
+
reset(key) {
|
|
3358
|
+
this.entries.delete(key);
|
|
3359
|
+
}
|
|
3360
|
+
clear() {
|
|
3361
|
+
this.entries.clear();
|
|
3362
|
+
}
|
|
3363
|
+
};
|
|
3364
|
+
var globalRateLimiter = new RateLimiter(60, 6e4);
|
|
3365
|
+
|
|
3366
|
+
// ../ai/src/provider.ts
|
|
3367
|
+
var CAPABILITY_BUDGETS = {
|
|
3368
|
+
explain_finding: {
|
|
3369
|
+
maxOutputTokens: 400,
|
|
3370
|
+
temperature: 0.2,
|
|
3371
|
+
timeoutMs: 12e3,
|
|
3372
|
+
maxAttempts: 1,
|
|
3373
|
+
maxEstimatedCostUsd: 0.01
|
|
3374
|
+
},
|
|
3375
|
+
detect_intent: {
|
|
3376
|
+
maxOutputTokens: 300,
|
|
3377
|
+
temperature: 0.1,
|
|
3378
|
+
timeoutMs: 1e4,
|
|
3379
|
+
maxAttempts: 1,
|
|
3380
|
+
maxEstimatedCostUsd: 0.01
|
|
3381
|
+
},
|
|
3382
|
+
recommend_contract: {
|
|
3383
|
+
maxOutputTokens: 800,
|
|
3384
|
+
temperature: 0.2,
|
|
3385
|
+
timeoutMs: 2e4,
|
|
3386
|
+
maxAttempts: 1,
|
|
3387
|
+
maxEstimatedCostUsd: 0.02
|
|
3388
|
+
},
|
|
3389
|
+
improve_title: {
|
|
3390
|
+
maxOutputTokens: 250,
|
|
3391
|
+
temperature: 0.3,
|
|
3392
|
+
timeoutMs: 1e4,
|
|
3393
|
+
maxAttempts: 1,
|
|
3394
|
+
maxEstimatedCostUsd: 0.01
|
|
3395
|
+
},
|
|
3396
|
+
generate_description: {
|
|
3397
|
+
maxOutputTokens: 300,
|
|
3398
|
+
temperature: 0.3,
|
|
3399
|
+
timeoutMs: 1e4,
|
|
3400
|
+
maxAttempts: 1,
|
|
3401
|
+
maxEstimatedCostUsd: 0.01
|
|
3402
|
+
},
|
|
3403
|
+
suggest_anchor: {
|
|
3404
|
+
maxOutputTokens: 150,
|
|
3405
|
+
temperature: 0.3,
|
|
3406
|
+
timeoutMs: 8e3,
|
|
3407
|
+
maxAttempts: 1,
|
|
3408
|
+
maxEstimatedCostUsd: 5e-3
|
|
3409
|
+
},
|
|
3410
|
+
generate_brief: {
|
|
3411
|
+
maxOutputTokens: 600,
|
|
3412
|
+
temperature: 0.4,
|
|
3413
|
+
timeoutMs: 15e3,
|
|
3414
|
+
maxAttempts: 1,
|
|
3415
|
+
maxEstimatedCostUsd: 0.02
|
|
3416
|
+
},
|
|
3417
|
+
resolve_semantic: {
|
|
3418
|
+
maxOutputTokens: 1500,
|
|
3419
|
+
temperature: 0.1,
|
|
3420
|
+
timeoutMs: 35e3,
|
|
3421
|
+
maxAttempts: 2,
|
|
3422
|
+
maxEstimatedCostUsd: 0.05
|
|
3423
|
+
},
|
|
3424
|
+
explain_root_cause: {
|
|
3425
|
+
maxOutputTokens: 800,
|
|
3426
|
+
temperature: 0.2,
|
|
3427
|
+
timeoutMs: 2e4,
|
|
3428
|
+
maxAttempts: 1,
|
|
3429
|
+
maxEstimatedCostUsd: 0.03
|
|
3430
|
+
},
|
|
3431
|
+
propose_fix: {
|
|
3432
|
+
maxOutputTokens: 2500,
|
|
3433
|
+
temperature: 0.1,
|
|
3434
|
+
timeoutMs: 45e3,
|
|
3435
|
+
maxAttempts: 1,
|
|
3436
|
+
maxEstimatedCostUsd: 0.1
|
|
3437
|
+
}
|
|
3438
|
+
};
|
|
3439
|
+
var OpenAICompatibleProvider = class {
|
|
3440
|
+
apiKey;
|
|
3441
|
+
baseUrl;
|
|
3442
|
+
model;
|
|
3443
|
+
constructor(options) {
|
|
3444
|
+
this.apiKey = options.apiKey;
|
|
3445
|
+
this.baseUrl = options.baseUrl || "https://api.openai.com/v1";
|
|
3446
|
+
this.model = options.model || "gpt-4o-mini";
|
|
3447
|
+
}
|
|
3448
|
+
async execute(request) {
|
|
3449
|
+
const budget = CAPABILITY_BUDGETS[request.action] || {
|
|
3450
|
+
maxOutputTokens: 1e3,
|
|
3451
|
+
temperature: 0.2,
|
|
3452
|
+
timeoutMs: 2e4,
|
|
3453
|
+
maxAttempts: 1,
|
|
3454
|
+
maxEstimatedCostUsd: 0.02
|
|
3455
|
+
};
|
|
3456
|
+
const isReasoningModel = this.model.includes("deepseek") || this.model.includes("o1") || this.model.includes("o3");
|
|
3457
|
+
const allocatedMaxTokens = isReasoningModel ? Math.max(budget.maxOutputTokens, 3e3) : budget.maxOutputTokens;
|
|
3458
|
+
const prompt = this.buildPrompt(request);
|
|
3459
|
+
const startTime = Date.now();
|
|
3460
|
+
const controller = new AbortController();
|
|
3461
|
+
const timeoutId = setTimeout(() => controller.abort(), budget.timeoutMs);
|
|
3462
|
+
try {
|
|
3463
|
+
const res = await fetch(`${this.baseUrl}/chat/completions`, {
|
|
3464
|
+
method: "POST",
|
|
3465
|
+
headers: {
|
|
3466
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
3467
|
+
"Content-Type": "application/json",
|
|
3468
|
+
"User-Agent": "Crawlemon-AI"
|
|
3469
|
+
},
|
|
3470
|
+
signal: controller.signal,
|
|
3471
|
+
body: JSON.stringify({
|
|
3472
|
+
model: this.model,
|
|
3473
|
+
messages: [
|
|
3474
|
+
{
|
|
3475
|
+
role: "system",
|
|
3476
|
+
content: "You are an expert technical SEO and verification assistant. Keep answers precise, punchy, and developer-friendly. Do not include markdown code block quotes around the primary text suggestion."
|
|
3477
|
+
},
|
|
3478
|
+
{
|
|
3479
|
+
role: "user",
|
|
3480
|
+
content: prompt
|
|
3481
|
+
}
|
|
3482
|
+
],
|
|
3483
|
+
temperature: budget.temperature,
|
|
3484
|
+
max_tokens: allocatedMaxTokens
|
|
3485
|
+
})
|
|
3486
|
+
});
|
|
3487
|
+
clearTimeout(timeoutId);
|
|
3488
|
+
if (!res.ok) {
|
|
3489
|
+
const errText = await res.text();
|
|
3490
|
+
secureLogger.error(`AI provider call failed: ${res.status}`);
|
|
3491
|
+
throw new Error(`AI Provider HTTP ${res.status}: ${redactString(errText)}`);
|
|
3492
|
+
}
|
|
3493
|
+
const latencyMs = Date.now() - startTime;
|
|
3494
|
+
const data = await res.json();
|
|
3495
|
+
const output = data.choices?.[0]?.message?.content?.trim() || "";
|
|
3496
|
+
const inputTokens = data.usage?.prompt_tokens ?? 0;
|
|
3497
|
+
const outputTokens = data.usage?.completion_tokens ?? 0;
|
|
3498
|
+
const estimatedCostUsd = Number((inputTokens * 14e-8 + outputTokens * 28e-8).toFixed(6));
|
|
3499
|
+
const telemetry = {
|
|
3500
|
+
provider: "openai-compatible",
|
|
3501
|
+
model: this.model,
|
|
3502
|
+
latencyMs,
|
|
3503
|
+
inputTokens,
|
|
3504
|
+
outputTokens,
|
|
3505
|
+
estimatedCostUsd
|
|
3506
|
+
};
|
|
3507
|
+
const formatted = this.formatResponse(request, output);
|
|
3508
|
+
return {
|
|
3509
|
+
...formatted,
|
|
3510
|
+
telemetry
|
|
3511
|
+
};
|
|
3512
|
+
} catch (err) {
|
|
3513
|
+
clearTimeout(timeoutId);
|
|
3514
|
+
secureLogger.error("Error executing AI request", err);
|
|
3515
|
+
throw new Error(`Failed to generate AI suggestion: ${err?.message || err}`);
|
|
3516
|
+
}
|
|
3517
|
+
}
|
|
3518
|
+
buildPrompt(request) {
|
|
3519
|
+
const { action, context } = request;
|
|
3520
|
+
switch (action) {
|
|
3521
|
+
case "improve_title":
|
|
3522
|
+
return redactString(
|
|
3523
|
+
`Improve this SEO title for route "${context.route}":
|
|
3524
|
+
Current title: "${context.currentTitle || "Untitled"}"
|
|
3525
|
+
Keywords: ${context.keywords?.join(", ") || "none"}
|
|
3526
|
+
Return a compelling, click-worthy title between 40 and 60 characters.`
|
|
3527
|
+
);
|
|
3528
|
+
case "generate_description":
|
|
3529
|
+
return redactString(
|
|
3530
|
+
`Write a high-CTR meta description for route "${context.route}".
|
|
3531
|
+
Page context / title: "${context.currentTitle || ""}"
|
|
3532
|
+
Keywords: ${context.keywords?.join(", ") || "none"}
|
|
3533
|
+
Length must be strictly between 120 and 155 characters.`
|
|
3534
|
+
);
|
|
3535
|
+
case "explain_finding":
|
|
3536
|
+
return redactString(
|
|
3537
|
+
`Explain this SEO finding to a developer in simple terms and why fixing it matters:
|
|
3538
|
+
Issue: "${context.findingMessage}"
|
|
3539
|
+
Route: "${context.route}"`
|
|
3540
|
+
);
|
|
3541
|
+
case "suggest_anchor":
|
|
3542
|
+
return redactString(
|
|
3543
|
+
`Suggest natural, context-rich anchor text for an internal link pointing to "${context.route}".
|
|
3544
|
+
Topic keywords: ${context.keywords?.join(", ") || ""}`
|
|
3545
|
+
);
|
|
3546
|
+
case "generate_brief":
|
|
3547
|
+
return redactString(
|
|
3548
|
+
`Generate a concise developer content brief for a new page at route "${context.route}".
|
|
3549
|
+
Cluster topic: "${context.clusterTopic || ""}".
|
|
3550
|
+
Include suggested H1, 3 key H2 sections, and target search intent.`
|
|
3551
|
+
);
|
|
3552
|
+
case "resolve_semantic":
|
|
3553
|
+
return redactString(
|
|
3554
|
+
`Analyze this SEO helper code and identify what metadata properties it sets:
|
|
3555
|
+
Helper Name: "${context.helperName || ""}"
|
|
3556
|
+
File: "${context.filePath || ""}"
|
|
3557
|
+
Helper Source:
|
|
3558
|
+
\`\`\`ts
|
|
3559
|
+
${context.helperCode || ""}
|
|
3560
|
+
\`\`\`
|
|
3561
|
+
Page Call Site:
|
|
3562
|
+
\`\`\`ts
|
|
3563
|
+
${context.sourceCode || ""}
|
|
3564
|
+
\`\`\`
|
|
3565
|
+
Respond ONLY with a valid JSON object matching this structure:
|
|
3566
|
+
{
|
|
3567
|
+
"helperName": "${context.helperName || ""}",
|
|
3568
|
+
"confidence": 0.85,
|
|
3569
|
+
"maps": {
|
|
3570
|
+
"canonical": "expression or field",
|
|
3571
|
+
"robots": "expression or field",
|
|
3572
|
+
"title": "expression or field",
|
|
3573
|
+
"description": "expression or field"
|
|
3574
|
+
},
|
|
3575
|
+
"evidenceChain": ["file.ts: helper function", "builds alternates.canonical"]
|
|
3576
|
+
}`
|
|
3577
|
+
);
|
|
3578
|
+
case "explain_root_cause":
|
|
3579
|
+
return redactString(
|
|
3580
|
+
`Explain why this SEO regression happened and how it propagated downstream.
|
|
3581
|
+
Issue: "${context.findingMessage || ""}"
|
|
3582
|
+
File: "${context.filePath || ""}"
|
|
3583
|
+
Affected Routes (${context.regressionContext?.blastRadiusRoutes?.length || 0}): ${context.regressionContext?.blastRadiusRoutes?.slice(0, 5).join(", ") || "none"}
|
|
3584
|
+
Diff context:
|
|
3585
|
+
${context.regressionContext?.diffSnippet || ""}
|
|
3586
|
+
Keep the explanation to 2-3 clear, professional developer-centric sentences.`
|
|
3587
|
+
);
|
|
3588
|
+
case "propose_fix": {
|
|
3589
|
+
const headContent = context.regressionContext?.headFileContent || context.sourceCode || "";
|
|
3590
|
+
const baseContent = context.regressionContext?.baseFileContent || "";
|
|
3591
|
+
return redactString(
|
|
3592
|
+
`Propose a minimal, safe code fix for this SEO regression:
|
|
3593
|
+
File: "${context.filePath}"
|
|
3594
|
+
Regression: "${context.findingMessage}"
|
|
3595
|
+
Current code:
|
|
3596
|
+
\`\`\`ts
|
|
3597
|
+
${headContent}
|
|
3598
|
+
\`\`\`
|
|
3599
|
+
Base (working) code:
|
|
3600
|
+
\`\`\`ts
|
|
3601
|
+
${baseContent}
|
|
3602
|
+
\`\`\`
|
|
3603
|
+
Respond with a valid JSON object:
|
|
3604
|
+
{
|
|
3605
|
+
"filePath": "${context.filePath}",
|
|
3606
|
+
"patch": "unified diff or replacement block",
|
|
3607
|
+
"explanation": "Why this fixes the regression",
|
|
3608
|
+
"replacementCode": "exact full new file content or minimal replacement"
|
|
3609
|
+
}`
|
|
3610
|
+
);
|
|
3611
|
+
}
|
|
3612
|
+
case "detect_intent":
|
|
3613
|
+
return redactString(
|
|
3614
|
+
`Analyze whether this SEO directive change (e.g. noindex or canonical removal) was likely intentional by the developer.
|
|
3615
|
+
Route: "${context.route}"
|
|
3616
|
+
Directive: "${context.findingMessage}"
|
|
3617
|
+
Page Snippet:
|
|
3618
|
+
${context.sourceCode || ""}
|
|
3619
|
+
Respond with a JSON object:
|
|
3620
|
+
{
|
|
3621
|
+
"route": "${context.route}",
|
|
3622
|
+
"likelyIntentional": true/false,
|
|
3623
|
+
"confidence": 0.85,
|
|
3624
|
+
"reasoning": "short reasoning",
|
|
3625
|
+
"suggestedAction": "allow_policy" | "keep_blocking" | "manual_review"
|
|
3626
|
+
}`
|
|
3627
|
+
);
|
|
3628
|
+
case "recommend_contract":
|
|
3629
|
+
return redactString(
|
|
3630
|
+
`Analyze these discovered routes and recommend 1 or 2 strict SEO Contracts for policy enforcement:
|
|
3631
|
+
Routes sample:
|
|
3632
|
+
${(context.candidateRoutes || []).map((r) => `${r.route} (title=${r.metadata?.title ? "yes" : "no"}, canonical=${r.metadata?.canonical ? "yes" : "no"}, robots=${r.metadata?.robots || "index"})`).slice(0, 15).join("\n")}
|
|
3633
|
+
Respond with a JSON object:
|
|
3634
|
+
{
|
|
3635
|
+
"suggestedContracts": [
|
|
3636
|
+
{
|
|
3637
|
+
"name": "contract-name",
|
|
3638
|
+
"routes": ["/pattern/**"],
|
|
3639
|
+
"requirement": { "indexable": true, "canonical": "required" }
|
|
3640
|
+
}
|
|
3641
|
+
],
|
|
3642
|
+
"explanation": "reasoning"
|
|
3643
|
+
}`
|
|
3644
|
+
);
|
|
3645
|
+
default:
|
|
3646
|
+
return "Improve this SEO attribute.";
|
|
3647
|
+
}
|
|
3648
|
+
}
|
|
3649
|
+
formatResponse(request, rawText) {
|
|
3650
|
+
const { action, context } = request;
|
|
3651
|
+
if (action === "improve_title") {
|
|
3652
|
+
const clean = rawText.replace(/^["']|["']$/g, "").trim();
|
|
3653
|
+
return {
|
|
3654
|
+
suggestion: clean,
|
|
3655
|
+
explanation: "Generated based on target route keywords and 50-60 char SERP best practices.",
|
|
3656
|
+
diffPreview: `- title: "${context.currentTitle || ""}"
|
|
3657
|
+
+ title: "${clean}"`
|
|
3658
|
+
};
|
|
3659
|
+
}
|
|
3660
|
+
if (action === "generate_description") {
|
|
3661
|
+
const clean = rawText.replace(/^["']|["']$/g, "").trim();
|
|
3662
|
+
return {
|
|
3663
|
+
suggestion: clean,
|
|
3664
|
+
explanation: `Recommended meta description (${clean.length} chars) optimized for SERP click-through rate.`,
|
|
3665
|
+
diffPreview: `- description: "${context.currentDescription || ""}"
|
|
3666
|
+
+ description: "${clean}"`
|
|
3667
|
+
};
|
|
3668
|
+
}
|
|
3669
|
+
if (action === "resolve_semantic") {
|
|
3670
|
+
try {
|
|
3671
|
+
const jsonMatch = rawText.match(/\{[\s\S]*\}/);
|
|
3672
|
+
const parsed = jsonMatch ? JSON.parse(jsonMatch[0]) : null;
|
|
3673
|
+
if (parsed) {
|
|
3674
|
+
return {
|
|
3675
|
+
suggestion: `Inferred metadata helper ${parsed.helperName}`,
|
|
3676
|
+
explanation: `Identified semantic mapping with confidence ${parsed.confidence || 0.8}`,
|
|
3677
|
+
semanticResolution: {
|
|
3678
|
+
helperName: parsed.helperName || context.helperName || "customHelper",
|
|
3679
|
+
inferredSourceFile: parsed.inferredSourceFile || context.filePath,
|
|
3680
|
+
confidence: parsed.confidence || 0.8,
|
|
3681
|
+
maps: parsed.maps || {},
|
|
3682
|
+
evidenceChain: parsed.evidenceChain || [context.filePath || "unknown"]
|
|
3683
|
+
}
|
|
3684
|
+
};
|
|
3685
|
+
}
|
|
3686
|
+
} catch {
|
|
3687
|
+
}
|
|
3688
|
+
return {
|
|
3689
|
+
suggestion: rawText,
|
|
3690
|
+
explanation: "Raw semantic inference"
|
|
3691
|
+
};
|
|
3692
|
+
}
|
|
3693
|
+
if (action === "explain_root_cause") {
|
|
3694
|
+
return {
|
|
3695
|
+
suggestion: rawText,
|
|
3696
|
+
explanation: "Root cause synthesis based on git diff and dependency graph blast radius."
|
|
3697
|
+
};
|
|
3698
|
+
}
|
|
3699
|
+
if (action === "propose_fix") {
|
|
3700
|
+
try {
|
|
3701
|
+
const jsonMatch = rawText.match(/\{[\s\S]*\}/);
|
|
3702
|
+
const parsed = jsonMatch ? JSON.parse(jsonMatch[0]) : null;
|
|
3703
|
+
if (parsed) {
|
|
3704
|
+
return {
|
|
3705
|
+
suggestion: parsed.explanation || "AI Proposed Fix",
|
|
3706
|
+
explanation: parsed.explanation,
|
|
3707
|
+
diffPreview: parsed.patch,
|
|
3708
|
+
fixProposal: {
|
|
3709
|
+
filePath: parsed.filePath || context.filePath || "",
|
|
3710
|
+
patch: parsed.patch || rawText,
|
|
3711
|
+
explanation: parsed.explanation || "",
|
|
3712
|
+
resolvedFindings: [context.findingMessage || ""]
|
|
3713
|
+
}
|
|
3714
|
+
};
|
|
3715
|
+
}
|
|
3716
|
+
} catch {
|
|
3717
|
+
}
|
|
3718
|
+
return {
|
|
3719
|
+
suggestion: rawText,
|
|
3720
|
+
explanation: "AI Proposed Patch",
|
|
3721
|
+
diffPreview: rawText
|
|
3722
|
+
};
|
|
3723
|
+
}
|
|
3724
|
+
if (action === "detect_intent") {
|
|
3725
|
+
try {
|
|
3726
|
+
const jsonMatch = rawText.match(/\{[\s\S]*\}/);
|
|
3727
|
+
const parsed = jsonMatch ? JSON.parse(jsonMatch[0]) : null;
|
|
3728
|
+
if (parsed) {
|
|
3729
|
+
return {
|
|
3730
|
+
suggestion: parsed.likelyIntentional ? "Likely intentional" : "Likely accidental regression",
|
|
3731
|
+
explanation: parsed.reasoning,
|
|
3732
|
+
intentDetection: parsed
|
|
3733
|
+
};
|
|
3734
|
+
}
|
|
3735
|
+
} catch {
|
|
3736
|
+
}
|
|
3737
|
+
return {
|
|
3738
|
+
suggestion: rawText,
|
|
3739
|
+
explanation: "Intent detection analysis"
|
|
3740
|
+
};
|
|
3741
|
+
}
|
|
3742
|
+
if (action === "recommend_contract") {
|
|
3743
|
+
try {
|
|
3744
|
+
const jsonMatch = rawText.match(/\{[\s\S]*\}/);
|
|
3745
|
+
const parsed = jsonMatch ? JSON.parse(jsonMatch[0]) : null;
|
|
3746
|
+
if (parsed && Array.isArray(parsed.suggestedContracts)) {
|
|
3747
|
+
return {
|
|
3748
|
+
suggestion: `Suggested ${parsed.suggestedContracts.length} SEO contract(s)`,
|
|
3749
|
+
explanation: parsed.explanation,
|
|
3750
|
+
contractRecommendation: parsed
|
|
3751
|
+
};
|
|
3752
|
+
}
|
|
3753
|
+
} catch {
|
|
3754
|
+
}
|
|
3755
|
+
return {
|
|
3756
|
+
suggestion: rawText,
|
|
3757
|
+
explanation: "Contract recommendation analysis"
|
|
3758
|
+
};
|
|
3759
|
+
}
|
|
3760
|
+
return {
|
|
3761
|
+
suggestion: rawText,
|
|
3762
|
+
explanation: "AI generated suggestion. Review and approve before applying."
|
|
3763
|
+
};
|
|
3764
|
+
}
|
|
3765
|
+
};
|
|
3766
|
+
function hasAIKey(options) {
|
|
3767
|
+
if (options?.enabled === false) return false;
|
|
3768
|
+
if (options?.apiKey && options.apiKey.trim().length > 0) return true;
|
|
3769
|
+
const envKey = process.env.DEEPSEEK_API_KEY || process.env.CRAWLEMON_AI_KEY || process.env.OPENAI_API_KEY || process.env.AI_API_KEY;
|
|
3770
|
+
return Boolean(envKey && envKey.trim().length > 0);
|
|
3771
|
+
}
|
|
3772
|
+
function getActiveAIProvider(options) {
|
|
3773
|
+
if (options?.enabled === false) return null;
|
|
3774
|
+
const isDeepSeek = Boolean(process.env.DEEPSEEK_API_KEY && !options?.apiKey && !process.env.CRAWLEMON_AI_KEY && !process.env.OPENAI_API_KEY);
|
|
3775
|
+
const key = options?.apiKey || process.env.DEEPSEEK_API_KEY || process.env.CRAWLEMON_AI_KEY || process.env.OPENAI_API_KEY || process.env.AI_API_KEY;
|
|
3776
|
+
if (!key || key.trim() === "") return null;
|
|
3777
|
+
const defaultBaseUrl = isDeepSeek ? "https://api.deepseek.com" : "https://api.openai.com/v1";
|
|
3778
|
+
const defaultModel = isDeepSeek ? process.env.DEEPSEEK_MODEL || "deepseek-v4-flash" : "gpt-4o-mini";
|
|
3779
|
+
return new OpenAICompatibleProvider({
|
|
3780
|
+
apiKey: key.trim(),
|
|
3781
|
+
baseUrl: options?.baseUrl || process.env.DEEPSEEK_BASE_URL || process.env.CRAWLEMON_AI_BASE_URL || process.env.OPENAI_BASE_URL || process.env.AI_BASE_URL || defaultBaseUrl,
|
|
3782
|
+
model: options?.model || process.env.DEEPSEEK_MODEL || process.env.CRAWLEMON_AI_MODEL || process.env.OPENAI_MODEL || process.env.AI_MODEL || defaultModel
|
|
3783
|
+
});
|
|
3784
|
+
}
|
|
3785
|
+
var SessionKeyStore = class {
|
|
3786
|
+
store = /* @__PURE__ */ new Map();
|
|
3787
|
+
set(sessionId, key, provider = "openai", ttlMs = 15 * 60 * 1e3) {
|
|
3788
|
+
this.store.set(sessionId, {
|
|
3789
|
+
key,
|
|
3790
|
+
provider,
|
|
3791
|
+
expiresAt: Date.now() + ttlMs
|
|
3792
|
+
});
|
|
3793
|
+
}
|
|
3794
|
+
get(sessionId) {
|
|
3795
|
+
const entry = this.store.get(sessionId);
|
|
3796
|
+
if (!entry) return null;
|
|
3797
|
+
if (Date.now() > entry.expiresAt) {
|
|
3798
|
+
this.store.delete(sessionId);
|
|
3799
|
+
return null;
|
|
3800
|
+
}
|
|
3801
|
+
return { key: entry.key, provider: entry.provider };
|
|
3802
|
+
}
|
|
3803
|
+
delete(sessionId) {
|
|
3804
|
+
this.store.delete(sessionId);
|
|
3805
|
+
}
|
|
3806
|
+
};
|
|
3807
|
+
var sessionKeyStore = new SessionKeyStore();
|
|
3808
|
+
|
|
3809
|
+
// ../ai/src/semantic-resolver.ts
|
|
3810
|
+
import fs5 from "node:fs";
|
|
3811
|
+
import path6 from "node:path";
|
|
3812
|
+
var METADATA_KEYS = ["canonical", "robots", "title", "description"];
|
|
3813
|
+
function helperDefinesProperty(helperCode, property) {
|
|
3814
|
+
const escaped = property.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3815
|
+
return new RegExp(`(?:^|[,{;\\s])(?:["']${escaped}["']|${escaped})\\s*:`, "m").test(helperCode);
|
|
3816
|
+
}
|
|
3817
|
+
async function resolveSemanticHelpers(options) {
|
|
3818
|
+
const { projectRoot, routes, aiProvider } = options;
|
|
3819
|
+
if (!aiProvider) return [];
|
|
3820
|
+
const results = [];
|
|
3821
|
+
for (const route of routes) {
|
|
3822
|
+
if (!route.filePath || !fs5.existsSync(route.filePath)) continue;
|
|
3823
|
+
const content = fs5.readFileSync(route.filePath, "utf8");
|
|
3824
|
+
const helperMatch = content.match(
|
|
3825
|
+
/export\s+const\s+metadata\s*=\s*([A-Za-z0-9_]+)\s*\(([\s\S]*?)\)/
|
|
3826
|
+
);
|
|
3827
|
+
if (!helperMatch) continue;
|
|
3828
|
+
const helperName = helperMatch[1];
|
|
3829
|
+
if (["generateMetadata", "fetch", "Promise"].includes(helperName)) continue;
|
|
3830
|
+
const importMatch = content.match(
|
|
3831
|
+
new RegExp(`import\\s*\\{[^}]*\\b${helperName}\\b[^}]*\\}\\s*from\\s*["']([^"']+)["']`)
|
|
3832
|
+
);
|
|
3833
|
+
let helperFilePath;
|
|
3834
|
+
let helperCode;
|
|
3835
|
+
if (importMatch) {
|
|
3836
|
+
const importPath = importMatch[1];
|
|
3837
|
+
const candidates = [
|
|
3838
|
+
path6.resolve(path6.dirname(route.filePath), `${importPath}.ts`),
|
|
3839
|
+
path6.resolve(path6.dirname(route.filePath), `${importPath}.tsx`),
|
|
3840
|
+
path6.resolve(path6.dirname(route.filePath), `${importPath}/index.ts`),
|
|
3841
|
+
path6.resolve(projectRoot, `${importPath.replace(/^@\//, "src/").replace(/^~\//, "")}.ts`),
|
|
3842
|
+
path6.resolve(projectRoot, `${importPath.replace(/^@\//, "src/").replace(/^~\//, "")}.tsx`)
|
|
3843
|
+
];
|
|
3844
|
+
for (const cand of candidates) {
|
|
3845
|
+
if (fs5.existsSync(cand)) {
|
|
3846
|
+
helperFilePath = cand;
|
|
3847
|
+
helperCode = fs5.readFileSync(cand, "utf8");
|
|
3848
|
+
break;
|
|
3849
|
+
}
|
|
3850
|
+
}
|
|
3851
|
+
}
|
|
3852
|
+
try {
|
|
3853
|
+
const res = await aiProvider.execute({
|
|
3854
|
+
action: "resolve_semantic",
|
|
3855
|
+
context: {
|
|
3856
|
+
route: route.route,
|
|
3857
|
+
filePath: route.filePath,
|
|
3858
|
+
helperName,
|
|
3859
|
+
helperCode: helperCode?.slice(0, 1500) || "// Helper source not statically resolved",
|
|
3860
|
+
sourceCode: helperMatch[0]
|
|
3861
|
+
}
|
|
3862
|
+
});
|
|
3863
|
+
const resolution = res.semanticResolution;
|
|
3864
|
+
if (!resolution) continue;
|
|
3865
|
+
const helperVerified = Boolean(helperFilePath && helperCode);
|
|
3866
|
+
const hasMeaningfulProps = Boolean(
|
|
3867
|
+
resolution.maps.canonical || resolution.maps.robots || resolution.maps.title || resolution.maps.description
|
|
3868
|
+
);
|
|
3869
|
+
if (hasMeaningfulProps) {
|
|
3870
|
+
const verifiedProperties = helperCode ? METADATA_KEYS.filter((property) => resolution.maps[property] && helperDefinesProperty(helperCode, property)) : [];
|
|
3871
|
+
const confidence = verifiedProperties.length > 0 ? "AI_VERIFIED" : "AI_ASSISTED";
|
|
3872
|
+
const cleanHelperPath = helperFilePath ? path6.relative(projectRoot, helperFilePath).replace(/\\/g, "/") : helperName;
|
|
3873
|
+
const resolvedMeta = {};
|
|
3874
|
+
if (verifiedProperties.includes("canonical") && !route.metadata?.canonical) {
|
|
3875
|
+
resolvedMeta.canonical = resolution.maps.canonical;
|
|
3876
|
+
}
|
|
3877
|
+
if (verifiedProperties.includes("robots") && !route.metadata?.robots) {
|
|
3878
|
+
resolvedMeta.robots = resolution.maps.robots;
|
|
3879
|
+
}
|
|
3880
|
+
const proofId = `proof-seo-${path6.basename(route.filePath)}-${helperName}`;
|
|
3881
|
+
const evidenceItems = [
|
|
3882
|
+
{
|
|
3883
|
+
id: `ev-callsite-${helperName}`,
|
|
3884
|
+
kind: "CALL_EDGE",
|
|
3885
|
+
source: { file: route.filePath, route: route.route, symbol: helperName },
|
|
3886
|
+
description: `Call site in page file invokes ${helperName}`,
|
|
3887
|
+
value: helperMatch[0]
|
|
3888
|
+
}
|
|
3889
|
+
];
|
|
3890
|
+
if (helperVerified) {
|
|
3891
|
+
evidenceItems.push({
|
|
3892
|
+
id: `ev-def-${helperName}`,
|
|
3893
|
+
kind: "AST_NODE",
|
|
3894
|
+
source: { file: cleanHelperPath, symbol: helperName },
|
|
3895
|
+
description: `Helper source definition verified on filesystem`,
|
|
3896
|
+
value: helperCode?.slice(0, 300)
|
|
3897
|
+
});
|
|
3898
|
+
}
|
|
3899
|
+
const verifiedClaims = [];
|
|
3900
|
+
const rejectedClaims = [];
|
|
3901
|
+
for (const [prop, val] of Object.entries(resolution.maps)) {
|
|
3902
|
+
if (val) {
|
|
3903
|
+
if (helperVerified && verifiedProperties.includes(prop)) {
|
|
3904
|
+
verifiedClaims.push({
|
|
3905
|
+
id: `claim-${prop}-${helperName}`,
|
|
3906
|
+
subject: route.route,
|
|
3907
|
+
predicate: `has${prop.charAt(0).toUpperCase() + prop.slice(1)}`,
|
|
3908
|
+
observedValue: val,
|
|
3909
|
+
evidenceRefs: evidenceItems.map((e) => e.id),
|
|
3910
|
+
source: { file: cleanHelperPath, route: route.route }
|
|
3911
|
+
});
|
|
3912
|
+
} else {
|
|
3913
|
+
rejectedClaims.push({
|
|
3914
|
+
id: `claim-${prop}-${helperName}`,
|
|
3915
|
+
subject: route.route,
|
|
3916
|
+
predicate: `has${prop.charAt(0).toUpperCase() + prop.slice(1)}`,
|
|
3917
|
+
reason: helperVerified ? `Helper source does not deterministically define metadata property "${prop}".` : "Helper source code could not be verified on filesystem."
|
|
3918
|
+
});
|
|
3919
|
+
}
|
|
3920
|
+
}
|
|
3921
|
+
}
|
|
3922
|
+
const proof = {
|
|
3923
|
+
id: proofId,
|
|
3924
|
+
hypothesisId: `hypo-${helperName}`,
|
|
3925
|
+
domain: "seo",
|
|
3926
|
+
discoveredBy: "AI",
|
|
3927
|
+
verifier: verifiedProperties.length > 0 ? "DETERMINISTIC_VERIFIER" : "UNVERIFIED_HEURISTIC",
|
|
3928
|
+
confidence: verifiedProperties.length > 0 ? "AI_VERIFIED" : "AI_ASSISTED",
|
|
3929
|
+
evidence: evidenceItems,
|
|
3930
|
+
verifiedClaims,
|
|
3931
|
+
rejectedClaims,
|
|
3932
|
+
telemetry: res.telemetry
|
|
3933
|
+
};
|
|
3934
|
+
const edge = {
|
|
3935
|
+
source: `file:${route.filePath}`,
|
|
3936
|
+
target: `helper:${cleanHelperPath}`,
|
|
3937
|
+
type: "INFERRED_SEMANTIC_EDGE",
|
|
3938
|
+
detail: `${helperName} maps: ${Object.keys(resolution.maps).filter((k) => resolution.maps[k]).join(", ")}`,
|
|
3939
|
+
confidence,
|
|
3940
|
+
proof,
|
|
3941
|
+
provenance: {
|
|
3942
|
+
discoveredBy: "AI",
|
|
3943
|
+
verifiedBy: verifiedProperties.length > 0 ? "DETERMINISTIC_VERIFIER" : void 0,
|
|
3944
|
+
evidence: [
|
|
3945
|
+
`Call site: ${path6.basename(route.filePath)}`,
|
|
3946
|
+
helperVerified ? `Helper definition: ${cleanHelperPath}` : `Inferred signature from ${helperName}`,
|
|
3947
|
+
...resolution.evidenceChain || []
|
|
3948
|
+
]
|
|
3949
|
+
}
|
|
3950
|
+
};
|
|
3951
|
+
results.push({
|
|
3952
|
+
route: route.route,
|
|
3953
|
+
helperName,
|
|
3954
|
+
sourceFile: route.filePath,
|
|
3955
|
+
resolvedMetadata: resolvedMeta,
|
|
3956
|
+
confidence,
|
|
3957
|
+
evidence: edge.provenance?.evidence || [],
|
|
3958
|
+
edge
|
|
3959
|
+
});
|
|
3960
|
+
}
|
|
3961
|
+
} catch {
|
|
3962
|
+
}
|
|
3963
|
+
}
|
|
3964
|
+
return results;
|
|
3965
|
+
}
|
|
3966
|
+
|
|
3967
|
+
// src/formatter.ts
|
|
3968
|
+
import path7 from "node:path";
|
|
3969
|
+
var colors = {
|
|
3970
|
+
reset: "\x1B[0m",
|
|
3971
|
+
bold: "\x1B[1m",
|
|
3972
|
+
dim: "\x1B[2m",
|
|
3973
|
+
red: "\x1B[31m",
|
|
3974
|
+
green: "\x1B[32m",
|
|
3975
|
+
yellow: "\x1B[33m",
|
|
3976
|
+
blue: "\x1B[34m",
|
|
3977
|
+
cyan: "\x1B[36m",
|
|
3978
|
+
bgRed: "\x1B[41m",
|
|
3979
|
+
bgGreen: "\x1B[42m"
|
|
3980
|
+
};
|
|
3981
|
+
function printQualityGateReport(diff, baseRef, headRef, options = {}) {
|
|
3982
|
+
if (options.json) {
|
|
3983
|
+
console.log(JSON.stringify(diff, null, 2));
|
|
3984
|
+
return;
|
|
3985
|
+
}
|
|
3986
|
+
const { qualityGate, newFindings, fixedFindings, unchangedFindings, affectedRoutes, preventionMetrics } = diff;
|
|
3987
|
+
const projectRoot = options.projectRoot || process.cwd();
|
|
3988
|
+
console.log(`
|
|
3989
|
+
\u{1F34B} ${colors.bold}Crawlemon SEO Gate${colors.reset}
|
|
3990
|
+
`);
|
|
3991
|
+
const gateStatusText = qualityGate.passed ? `${colors.green}${colors.bold}PASSED${colors.reset}` : `${colors.red}${colors.bold}FAILED${colors.reset}`;
|
|
3992
|
+
console.log(`${gateStatusText}
|
|
3993
|
+
`);
|
|
3994
|
+
console.log(`Comparing:
|
|
3995
|
+
${colors.cyan}${baseRef}...${headRef}${colors.reset}
|
|
3996
|
+
`);
|
|
3997
|
+
const summary = qualityGate.summary;
|
|
3998
|
+
const regressionCount = summary.newCritical + summary.newErrors;
|
|
3999
|
+
console.log(
|
|
4000
|
+
`${regressionCount > 0 ? colors.red : colors.green}${regressionCount} new regression(s) across ${affectedRoutes.length} affected route(s)${colors.reset}
|
|
4001
|
+
${summary.newWarnings > 0 ? colors.yellow : colors.dim}${summary.newWarnings} new warning(s)${colors.reset}
|
|
4002
|
+
${colors.green}${summary.fixedFindings} issue(s) fixed${colors.reset}`
|
|
4003
|
+
);
|
|
4004
|
+
if (preventionMetrics) {
|
|
4005
|
+
if (preventionMetrics.routesAtRisk > 0) {
|
|
4006
|
+
console.log(`${colors.yellow}\u26A0\uFE0F ${preventionMetrics.routesAtRisk} route(s) at risk${colors.reset}`);
|
|
4007
|
+
}
|
|
4008
|
+
if (preventionMetrics.routesProtected > 0) {
|
|
4009
|
+
console.log(`${colors.green}${colors.bold}\u{1F6E1}\uFE0F ${preventionMetrics.routesProtected} route(s) protected${colors.reset}`);
|
|
4010
|
+
}
|
|
4011
|
+
}
|
|
4012
|
+
console.log(`
|
|
4013
|
+
${colors.dim}${"\u2501".repeat(45)}${colors.reset}
|
|
4014
|
+
`);
|
|
4015
|
+
if (newFindings.length > 0) {
|
|
4016
|
+
for (const f of newFindings) {
|
|
4017
|
+
const isCritical = f.severity === "error";
|
|
4018
|
+
const titleColor = isCritical ? colors.red : colors.yellow;
|
|
4019
|
+
console.log(`${titleColor}${colors.bold}${f.message}${colors.reset}`);
|
|
4020
|
+
console.log(`${colors.dim}${"\u2501".repeat(45)}${colors.reset}
|
|
4021
|
+
`);
|
|
4022
|
+
const rawFile = f.rootCause?.file || f.sourceFile || f.file || "";
|
|
4023
|
+
const relFile = rawFile ? path7.relative(projectRoot, rawFile).replace(/\\/g, "/") : "";
|
|
4024
|
+
const lineSuffix = f.rootCause?.line || f.sourceLine || f.line ? `:${f.rootCause?.line || f.sourceLine || f.line}` : "";
|
|
4025
|
+
if (f.rootCause) {
|
|
4026
|
+
console.log(`${colors.bold}Root cause${colors.reset}`);
|
|
4027
|
+
console.log(`${colors.cyan}${relFile}${lineSuffix}${colors.reset}`);
|
|
4028
|
+
if (f.rootCause.changeDescription) {
|
|
4029
|
+
console.log(`${colors.dim}${f.rootCause.changeDescription}${colors.reset}`);
|
|
4030
|
+
}
|
|
4031
|
+
console.log();
|
|
4032
|
+
} else if (relFile) {
|
|
4033
|
+
console.log(`${colors.bold}Source${colors.reset}`);
|
|
4034
|
+
console.log(`${colors.cyan}${relFile}${lineSuffix}${colors.reset}
|
|
4035
|
+
`);
|
|
4036
|
+
}
|
|
4037
|
+
if (f.evidenceChain?.whyThisHappened || f.explanation) {
|
|
4038
|
+
console.log(`${colors.bold}Why this happened${colors.reset}`);
|
|
4039
|
+
console.log(`${f.evidenceChain?.whyThisHappened || f.explanation}
|
|
4040
|
+
`);
|
|
4041
|
+
}
|
|
4042
|
+
if (f.evidenceChain?.steps && f.evidenceChain.steps.length > 0) {
|
|
4043
|
+
console.log(`${colors.bold}Dependency${colors.reset}`);
|
|
4044
|
+
const sourceName = path7.basename(relFile || f.evidenceChain.sourceFile);
|
|
4045
|
+
console.log(`${sourceName}`);
|
|
4046
|
+
for (const step of f.evidenceChain.steps) {
|
|
4047
|
+
const detail = step.detail ? ` ${step.detail}` : "";
|
|
4048
|
+
console.log(` ${colors.dim}\u2193${detail}${colors.reset}`);
|
|
4049
|
+
console.log(`${step.to}`);
|
|
4050
|
+
}
|
|
4051
|
+
if (f.affectedRoutes && f.affectedRoutes.length > 1) {
|
|
4052
|
+
console.log(` ${colors.dim}\u2193${colors.reset}`);
|
|
4053
|
+
console.log(`${f.affectedRoutes.length} known routes`);
|
|
4054
|
+
}
|
|
4055
|
+
console.log();
|
|
4056
|
+
}
|
|
4057
|
+
if (f.impact && f.impact.targets.length > 0) {
|
|
4058
|
+
console.log(`${colors.bold}Impact${colors.reset}`);
|
|
4059
|
+
console.log(`${f.impact.totalAffected} target(s) affected (${f.impact.targets[0]?.type || "symbol/entity"})`);
|
|
4060
|
+
if (f.impact.description) {
|
|
4061
|
+
console.log(`${colors.dim}${f.impact.description}${colors.reset}`);
|
|
4062
|
+
}
|
|
4063
|
+
console.log(`
|
|
4064
|
+
${colors.bold}Examples${colors.reset}`);
|
|
4065
|
+
for (const t of f.impact.targets.slice(0, 3)) {
|
|
4066
|
+
const location = t.source?.file ? ` (${t.source.file}${t.source.line ? `:${t.source.line}` : ""})` : "";
|
|
4067
|
+
console.log(`${colors.dim}${t.label}${location}${colors.reset}`);
|
|
4068
|
+
}
|
|
4069
|
+
if (f.impact.targets.length > 3) {
|
|
4070
|
+
console.log(`${colors.dim}...and ${f.impact.targets.length - 3} more target(s)${colors.reset}`);
|
|
4071
|
+
}
|
|
4072
|
+
console.log();
|
|
4073
|
+
} else {
|
|
4074
|
+
const routeList = f.affectedRoutes?.length ? f.affectedRoutes : f.route ? [f.route] : [];
|
|
4075
|
+
if (routeList.length > 0) {
|
|
4076
|
+
console.log(`${colors.bold}Impact${colors.reset}`);
|
|
4077
|
+
console.log(`${routeList.length} known route(s) affected`);
|
|
4078
|
+
if (f.routePattern) {
|
|
4079
|
+
console.log(`Pattern: ${f.routePattern}`);
|
|
4080
|
+
}
|
|
4081
|
+
console.log(`
|
|
4082
|
+
${colors.bold}Examples${colors.reset}`);
|
|
4083
|
+
for (const r of routeList.slice(0, 3)) {
|
|
4084
|
+
console.log(`${colors.dim}${r}${colors.reset}`);
|
|
4085
|
+
}
|
|
4086
|
+
if (routeList.length > 3) {
|
|
4087
|
+
console.log(`${colors.dim}...and ${routeList.length - 3} more routes${colors.reset}`);
|
|
4088
|
+
}
|
|
4089
|
+
console.log();
|
|
4090
|
+
}
|
|
4091
|
+
}
|
|
4092
|
+
console.log(`${colors.bold}Confidence${colors.reset}`);
|
|
4093
|
+
let confBadge = `${colors.bold}${colors.cyan}${f.confidence || "DETERMINISTIC"}${colors.reset}`;
|
|
4094
|
+
if (f.confidence === "AI_VERIFIED") {
|
|
4095
|
+
confBadge = `${colors.bold}${colors.green}\u2713 AI_VERIFIED (Proven by Deterministic Verifier)${colors.reset}`;
|
|
4096
|
+
} else if (f.confidence === "AI_ASSISTED") {
|
|
4097
|
+
confBadge = `${colors.bold}${colors.yellow}\u2139 AI_ASSISTED (Non-blocking hypothesis)${colors.reset}`;
|
|
4098
|
+
} else if (f.confidence === "STRUCTURAL") {
|
|
4099
|
+
confBadge = `${colors.bold}${colors.blue}STRUCTURAL (Inherited / Blast Radius)${colors.reset}`;
|
|
4100
|
+
} else if (f.confidence === "DETERMINISTIC") {
|
|
4101
|
+
confBadge = `${colors.bold}${colors.cyan}DETERMINISTIC (Zero-hallucination AST Proof)${colors.reset}`;
|
|
4102
|
+
}
|
|
4103
|
+
console.log(`${confBadge}
|
|
4104
|
+
`);
|
|
4105
|
+
if (f.fixability === "VERIFIED_AI_FIX") {
|
|
4106
|
+
console.log(`${colors.bold}Fix${colors.reset}`);
|
|
4107
|
+
console.log(`${colors.green}\u2713 Verified AI Fix${colors.reset} (Re-verified in sandbox, 0 regressions)`);
|
|
4108
|
+
console.log(`${f.evidenceChain?.fixRecommendation || "Apply verified patch."}
|
|
4109
|
+
`);
|
|
4110
|
+
} else if (f.fixability === "REVERT_SAFE" || f.evidenceChain?.revertSafeValue) {
|
|
4111
|
+
console.log(`${colors.bold}Fix${colors.reset}`);
|
|
4112
|
+
console.log(`${f.evidenceChain?.fixRecommendation || "Restore previous declaration from BASE."}`);
|
|
4113
|
+
console.log(`${colors.green}[Preview Revert-Safe Fix]${colors.reset}
|
|
4114
|
+
`);
|
|
4115
|
+
} else if (f.fixable) {
|
|
4116
|
+
console.log(`${colors.bold}Fix${colors.reset}`);
|
|
4117
|
+
console.log(`Deterministic fix available via \`npx crawlemon fix\`
|
|
4118
|
+
`);
|
|
4119
|
+
}
|
|
4120
|
+
console.log(`${colors.dim}${"\u2501".repeat(45)}${colors.reset}
|
|
4121
|
+
`);
|
|
4122
|
+
}
|
|
4123
|
+
}
|
|
4124
|
+
if (summary.baselineTotalIssues > 0) {
|
|
4125
|
+
console.log(`${colors.bold}Existing baseline debt${colors.reset}`);
|
|
4126
|
+
console.log(`${summary.baselineTotalIssues} pre-existing finding(s)
|
|
4127
|
+
${colors.dim}Ignored for this gate (Clean-As-You-Code).${colors.reset}
|
|
4128
|
+
`);
|
|
4129
|
+
console.log(`${colors.dim}${"\u2501".repeat(45)}${colors.reset}
|
|
4130
|
+
`);
|
|
4131
|
+
}
|
|
4132
|
+
const gateBadge = qualityGate.passed ? `${colors.bgGreen}${colors.bold} SEO Gate: PASSED ${colors.reset}` : `${colors.bgRed}${colors.bold} SEO Gate: FAILED ${colors.reset}`;
|
|
4133
|
+
console.log(gateBadge + "\n");
|
|
4134
|
+
}
|
|
4135
|
+
function formatScoreBadge(score) {
|
|
4136
|
+
if (score >= 85) return `${colors.green}${colors.bold}${score}/100${colors.reset}`;
|
|
4137
|
+
if (score >= 70) return `${colors.yellow}${colors.bold}${score}/100${colors.reset}`;
|
|
4138
|
+
return `${colors.red}${colors.bold}${score}/100${colors.reset}`;
|
|
4139
|
+
}
|
|
4140
|
+
function printAuditReport(result, projectRoot) {
|
|
4141
|
+
const { score, findings, routes } = result;
|
|
4142
|
+
console.log("\n" + colors.cyan + colors.bold + "\u{1F50D} Crawlemon Audit" + colors.reset + ` (${routes.length} routes scanned)`);
|
|
4143
|
+
console.log(colors.dim + "\u2500".repeat(50) + colors.reset);
|
|
4144
|
+
console.log(`
|
|
4145
|
+
SEO Score: ${formatScoreBadge(score.overall)}`);
|
|
4146
|
+
console.log(` ${colors.dim}Technical: ${score.breakdown.technical.toString().padEnd(4)} Metadata: ${score.breakdown.metadata}`);
|
|
4147
|
+
console.log(` Content: ${score.breakdown.content.toString().padEnd(4)} Internal Links: ${score.breakdown.links}${colors.reset}
|
|
4148
|
+
`);
|
|
4149
|
+
if (findings.length === 0) {
|
|
4150
|
+
console.log(`${colors.green}\u2713 All checks passed! No SEO regressions or issues found.${colors.reset}
|
|
4151
|
+
`);
|
|
4152
|
+
return;
|
|
1799
4153
|
}
|
|
1800
4154
|
const errors = findings.filter((f) => f.severity === "error");
|
|
1801
4155
|
const warnings = findings.filter((f) => f.severity === "warning");
|
|
@@ -1811,7 +4165,7 @@ function printAuditReport(result, projectRoot) {
|
|
|
1811
4165
|
if (f.severity === "warning") icon = colors.yellow + "\u26A0" + colors.reset;
|
|
1812
4166
|
const fixTag = f.fixable ? ` ${colors.green}[fixable]${colors.reset}` : "";
|
|
1813
4167
|
const routeTag = f.route ? ` ${colors.cyan}(${f.route})${colors.reset}` : "";
|
|
1814
|
-
const relativeFile = f.file ?
|
|
4168
|
+
const relativeFile = f.file ? path7.relative(projectRoot, f.file) : "";
|
|
1815
4169
|
const fileTag = relativeFile ? `
|
|
1816
4170
|
${colors.dim}${relativeFile}${f.line ? `:${f.line}` : ""}${colors.reset}` : "";
|
|
1817
4171
|
console.log(` ${icon} ${f.message}${fixTag}${routeTag}${fileTag}`);
|
|
@@ -1903,16 +4257,55 @@ ${colors.bold}Cluster: ${c.topic}${colors.reset} (${colors.dim}${c.pattern}${col
|
|
|
1903
4257
|
}
|
|
1904
4258
|
|
|
1905
4259
|
// src/index.ts
|
|
4260
|
+
function loadLocalEnv(projectRoot) {
|
|
4261
|
+
const dirsToSearch = [projectRoot];
|
|
4262
|
+
let curr = projectRoot;
|
|
4263
|
+
for (let i = 0; i < 4; i++) {
|
|
4264
|
+
const parent = path8.dirname(curr);
|
|
4265
|
+
if (parent && parent !== curr) {
|
|
4266
|
+
dirsToSearch.push(parent);
|
|
4267
|
+
curr = parent;
|
|
4268
|
+
} else {
|
|
4269
|
+
break;
|
|
4270
|
+
}
|
|
4271
|
+
}
|
|
4272
|
+
for (const dir of dirsToSearch) {
|
|
4273
|
+
for (const envFile of [".env", ".env.local"]) {
|
|
4274
|
+
const fullPath = path8.join(dir, envFile);
|
|
4275
|
+
if (fs6.existsSync(fullPath)) {
|
|
4276
|
+
try {
|
|
4277
|
+
const content = fs6.readFileSync(fullPath, "utf8");
|
|
4278
|
+
for (const line of content.split("\n")) {
|
|
4279
|
+
const trimmed = line.trim();
|
|
4280
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
4281
|
+
const eqIdx = trimmed.indexOf("=");
|
|
4282
|
+
if (eqIdx !== -1) {
|
|
4283
|
+
const key = trimmed.slice(0, eqIdx).trim();
|
|
4284
|
+
let val = trimmed.slice(eqIdx + 1).trim();
|
|
4285
|
+
if (val.startsWith('"') && val.endsWith('"') || val.startsWith("'") && val.endsWith("'")) {
|
|
4286
|
+
val = val.slice(1, -1);
|
|
4287
|
+
}
|
|
4288
|
+
if (process.env[key] === void 0) {
|
|
4289
|
+
process.env[key] = val;
|
|
4290
|
+
}
|
|
4291
|
+
}
|
|
4292
|
+
}
|
|
4293
|
+
} catch {
|
|
4294
|
+
}
|
|
4295
|
+
}
|
|
4296
|
+
}
|
|
4297
|
+
}
|
|
4298
|
+
}
|
|
1906
4299
|
function checkSecretRisk(projectRoot) {
|
|
1907
|
-
const envCandidates =
|
|
1908
|
-
const gitignorePath =
|
|
4300
|
+
const envCandidates = fs6.readdirSync(projectRoot).filter((name) => name !== ".env.example" && (name === ".env" || name.startsWith(".env.")));
|
|
4301
|
+
const gitignorePath = path8.join(projectRoot, ".gitignore");
|
|
1909
4302
|
let gitignoreContent = "";
|
|
1910
|
-
if (
|
|
1911
|
-
gitignoreContent =
|
|
4303
|
+
if (fs6.existsSync(gitignorePath)) {
|
|
4304
|
+
gitignoreContent = fs6.readFileSync(gitignorePath, "utf8");
|
|
1912
4305
|
}
|
|
1913
4306
|
for (const env of envCandidates) {
|
|
1914
|
-
const envPath =
|
|
1915
|
-
if (
|
|
4307
|
+
const envPath = path8.join(projectRoot, env);
|
|
4308
|
+
if (fs6.existsSync(envPath)) {
|
|
1916
4309
|
const tracked = spawnSync("git", ["ls-files", "--error-unmatch", "--", env], {
|
|
1917
4310
|
cwd: projectRoot,
|
|
1918
4311
|
stdio: "ignore"
|
|
@@ -1923,24 +4316,65 @@ function checkSecretRisk(projectRoot) {
|
|
|
1923
4316
|
}
|
|
1924
4317
|
}
|
|
1925
4318
|
}
|
|
4319
|
+
async function auditGitRef(projectRoot, ref) {
|
|
4320
|
+
const tmpDir = fs6.mkdtempSync(path8.join(os.tmpdir(), "crawlemon-base-"));
|
|
4321
|
+
try {
|
|
4322
|
+
const gitRootRes = spawnSync("git", ["rev-parse", "--show-toplevel"], { cwd: projectRoot, encoding: "utf8" });
|
|
4323
|
+
const gitRoot = gitRootRes.status === 0 ? gitRootRes.stdout.trim() : projectRoot;
|
|
4324
|
+
const relToGitRoot = path8.relative(gitRoot, projectRoot).replace(/\\/g, "/");
|
|
4325
|
+
const archiveRef = relToGitRoot && relToGitRoot !== "." ? `${ref}:${relToGitRoot}` : ref;
|
|
4326
|
+
const gitArchive = spawnSync("git", ["archive", archiveRef], {
|
|
4327
|
+
cwd: gitRoot,
|
|
4328
|
+
maxBuffer: 64 * 1024 * 1024
|
|
4329
|
+
});
|
|
4330
|
+
if (gitArchive.status !== 0 || !gitArchive.stdout) {
|
|
4331
|
+
throw new Error(`Failed to read git ref "${ref}": ${gitArchive.stderr?.toString() || "ref not found"}`);
|
|
4332
|
+
}
|
|
4333
|
+
const tar = spawnSync("tar", ["-x", "-C", tmpDir], { input: gitArchive.stdout });
|
|
4334
|
+
if (tar.status !== 0) {
|
|
4335
|
+
throw new Error(`Failed to extract git archive for "${ref}".`);
|
|
4336
|
+
}
|
|
4337
|
+
const scanData = await FrameworkAdapter.scan(tmpDir);
|
|
4338
|
+
return runSEOAudit({
|
|
4339
|
+
routes: scanData.routes,
|
|
4340
|
+
config: scanData.config,
|
|
4341
|
+
sitemapFound: scanData.sitemapFound,
|
|
4342
|
+
robotsFound: scanData.robotsFound,
|
|
4343
|
+
robotsContent: scanData.robotsContent,
|
|
4344
|
+
sitemapUrls: scanData.sitemapUrls,
|
|
4345
|
+
sitemapMalformed: scanData.sitemapMalformed,
|
|
4346
|
+
redirects: scanData.redirects,
|
|
4347
|
+
projectRoot: scanData.projectRoot,
|
|
4348
|
+
isAppRouter: scanData.isAppRouter
|
|
4349
|
+
});
|
|
4350
|
+
} finally {
|
|
4351
|
+
fs6.rmSync(tmpDir, { recursive: true, force: true });
|
|
4352
|
+
}
|
|
4353
|
+
}
|
|
1926
4354
|
async function runCli(args) {
|
|
1927
4355
|
const command = args[0] || "audit";
|
|
1928
4356
|
const projectRoot = process.cwd();
|
|
4357
|
+
const isJson = args.includes("--json");
|
|
4358
|
+
const baseFlagIndex = args.indexOf("--base");
|
|
4359
|
+
const baseRefArg = baseFlagIndex !== -1 && args[baseFlagIndex + 1] ? args[baseFlagIndex + 1] : void 0;
|
|
1929
4360
|
if (command === "--help" || command === "help") {
|
|
1930
|
-
console.log(
|
|
4361
|
+
console.log(
|
|
4362
|
+
"Crawlemon \u2014 SEO CI for modern web apps\nThe SEO safety layer between code and production.\n\nCommands:\n diff [--base <ref>] [--ai] [--json] Compare code change vs base ref and evaluate Quality Gate\n audit [--base <ref>] [--ai] [--json] Audit current routes or compare against base ref\n fix [--dry-run] Apply safe deterministic SEO fixes\n contract suggest (AI/Pattern) Propose strict SEO Contracts from codebase\n links Inspect internal link graph and orphans\n opportunities (Labs) Inspect topic expansion clusters\n\nOptions:\n --ai Enable AI semantic resolver & root-cause analysis (auto-fallback if no key)\n"
|
|
4363
|
+
);
|
|
1931
4364
|
return;
|
|
1932
4365
|
}
|
|
1933
4366
|
if (command === "--version" || command === "-v") {
|
|
1934
|
-
console.log("0.
|
|
4367
|
+
console.log("0.2.0");
|
|
1935
4368
|
return;
|
|
1936
4369
|
}
|
|
1937
4370
|
checkSecretRisk(projectRoot);
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
4371
|
+
loadLocalEnv(projectRoot);
|
|
4372
|
+
const framework = await FrameworkAdapter.detect(projectRoot);
|
|
4373
|
+
if (!framework) {
|
|
4374
|
+
console.error(`\x1B[31m\u2716 Error: No supported web framework detected in "${projectRoot}".\x1B[0m`);
|
|
1941
4375
|
process.exit(1);
|
|
1942
4376
|
}
|
|
1943
|
-
const scanData = await
|
|
4377
|
+
const scanData = await FrameworkAdapter.scan(projectRoot);
|
|
1944
4378
|
const auditResult = runSEOAudit({
|
|
1945
4379
|
routes: scanData.routes,
|
|
1946
4380
|
config: scanData.config,
|
|
@@ -1953,7 +4387,70 @@ async function runCli(args) {
|
|
|
1953
4387
|
projectRoot: scanData.projectRoot,
|
|
1954
4388
|
isAppRouter: scanData.isAppRouter
|
|
1955
4389
|
});
|
|
4390
|
+
if (scanData.dependencyGraph) {
|
|
4391
|
+
for (const f of auditResult.findings) {
|
|
4392
|
+
if (f.file && f.file.includes("layout.")) {
|
|
4393
|
+
const impact = scanData.dependencyGraph.calculateBlastRadius(f.file);
|
|
4394
|
+
f.affectedRoutes = impact.affectedRoutes;
|
|
4395
|
+
f.routePattern = impact.routePattern;
|
|
4396
|
+
}
|
|
4397
|
+
}
|
|
4398
|
+
}
|
|
4399
|
+
const useAI = args.includes("--ai");
|
|
4400
|
+
const aiKeyAvailable = hasAIKey();
|
|
4401
|
+
const aiProvider = useAI || aiKeyAvailable ? getActiveAIProvider() : null;
|
|
4402
|
+
if (useAI && !aiKeyAvailable) {
|
|
4403
|
+
console.log(`\x1B[2m\u2139 Note: AI flag specified but no API key configured. Automatically executing in 100% deterministic mode.\x1B[0m
|
|
4404
|
+
`);
|
|
4405
|
+
}
|
|
4406
|
+
let inferredSemanticEdges = [];
|
|
4407
|
+
if (aiProvider) {
|
|
4408
|
+
try {
|
|
4409
|
+
const resolvedEdges = await resolveSemanticHelpers({
|
|
4410
|
+
projectRoot,
|
|
4411
|
+
routes: scanData.routes,
|
|
4412
|
+
aiProvider
|
|
4413
|
+
});
|
|
4414
|
+
inferredSemanticEdges = resolvedEdges.map((r) => r.edge);
|
|
4415
|
+
} catch {
|
|
4416
|
+
}
|
|
4417
|
+
}
|
|
4418
|
+
if (command === "contract" && (args[1] === "suggest" || args[1] === "--suggest")) {
|
|
4419
|
+
const suggestions = suggestSEOContracts(scanData.routes);
|
|
4420
|
+
console.log(`
|
|
4421
|
+
\x1B[1m\u{1F4DC} Suggested SEO Contracts for repository\x1B[0m
|
|
4422
|
+
`);
|
|
4423
|
+
console.log(JSON.stringify({ contracts: Object.fromEntries(suggestions.map((c) => [c.name, { routes: c.routes, ...c.requirement }])) }, null, 2));
|
|
4424
|
+
return;
|
|
4425
|
+
}
|
|
4426
|
+
if (command === "diff" || command === "audit" && baseRefArg) {
|
|
4427
|
+
const baseRef = baseRefArg || "main";
|
|
4428
|
+
try {
|
|
4429
|
+
const baseAudit = await auditGitRef(projectRoot, baseRef);
|
|
4430
|
+
const policy = {
|
|
4431
|
+
...scanData.config.qualityGate || {},
|
|
4432
|
+
contracts: scanData.config.contracts
|
|
4433
|
+
};
|
|
4434
|
+
const diff = computeAuditDiff(baseAudit, auditResult, {
|
|
4435
|
+
policy,
|
|
4436
|
+
aiProvider,
|
|
4437
|
+
inferredSemanticEdges
|
|
4438
|
+
});
|
|
4439
|
+
printQualityGateReport(diff, baseRef, "HEAD", { json: isJson, projectRoot });
|
|
4440
|
+
if (!diff.qualityGate.passed) {
|
|
4441
|
+
process.exit(1);
|
|
4442
|
+
}
|
|
4443
|
+
} catch (err) {
|
|
4444
|
+
console.error(`\x1B[31m\u2716 Quality Gate Error: ${err.message}\x1B[0m`);
|
|
4445
|
+
process.exit(1);
|
|
4446
|
+
}
|
|
4447
|
+
return;
|
|
4448
|
+
}
|
|
1956
4449
|
if (command === "audit") {
|
|
4450
|
+
if (isJson) {
|
|
4451
|
+
console.log(JSON.stringify(auditResult, null, 2));
|
|
4452
|
+
return;
|
|
4453
|
+
}
|
|
1957
4454
|
printAuditReport(auditResult, projectRoot);
|
|
1958
4455
|
const hasErrors = auditResult.findings.some((f) => f.severity === "error");
|
|
1959
4456
|
if (hasErrors && process.env.CI) {
|
|
@@ -1961,13 +4458,43 @@ async function runCli(args) {
|
|
|
1961
4458
|
}
|
|
1962
4459
|
} else if (command === "fix") {
|
|
1963
4460
|
const dryRun = args.includes("--dry-run");
|
|
4461
|
+
let baseFiles;
|
|
4462
|
+
if (baseRefArg) {
|
|
4463
|
+
try {
|
|
4464
|
+
const gitRootRes = spawnSync("git", ["rev-parse", "--show-toplevel"], { cwd: projectRoot, encoding: "utf8" });
|
|
4465
|
+
const gitRoot = gitRootRes.status === 0 ? gitRootRes.stdout.trim() : projectRoot;
|
|
4466
|
+
const relToGitRoot = path8.relative(gitRoot, projectRoot).replace(/\\/g, "/");
|
|
4467
|
+
const archiveRef = relToGitRoot && relToGitRoot !== "." ? `${baseRefArg}:${relToGitRoot}` : baseRefArg;
|
|
4468
|
+
const gitArchive = spawnSync("git", ["archive", archiveRef], { cwd: gitRoot, maxBuffer: 64 * 1024 * 1024 });
|
|
4469
|
+
if (gitArchive.status === 0 && gitArchive.stdout) {
|
|
4470
|
+
const tmpDir = fs6.mkdtempSync(path8.join(os.tmpdir(), "crawlemon-base-fix-"));
|
|
4471
|
+
spawnSync("tar", ["-x", "-C", tmpDir], { input: gitArchive.stdout });
|
|
4472
|
+
baseFiles = /* @__PURE__ */ new Map();
|
|
4473
|
+
const walk2 = (d) => {
|
|
4474
|
+
for (const ent of fs6.readdirSync(d, { withFileTypes: true })) {
|
|
4475
|
+
const full = path8.join(d, ent.name);
|
|
4476
|
+
if (ent.isDirectory()) walk2(full);
|
|
4477
|
+
else if (ent.isFile()) {
|
|
4478
|
+
const rel = path8.relative(tmpDir, full).replace(/\\/g, "/");
|
|
4479
|
+
baseFiles.set(rel, fs6.readFileSync(full, "utf8"));
|
|
4480
|
+
}
|
|
4481
|
+
}
|
|
4482
|
+
};
|
|
4483
|
+
walk2(tmpDir);
|
|
4484
|
+
fs6.rmSync(tmpDir, { recursive: true, force: true });
|
|
4485
|
+
}
|
|
4486
|
+
} catch {
|
|
4487
|
+
}
|
|
4488
|
+
}
|
|
1964
4489
|
const fixResult = await applySafeFixes({
|
|
1965
4490
|
projectRoot,
|
|
1966
4491
|
routes: scanData.routes,
|
|
1967
4492
|
findings: auditResult.findings,
|
|
1968
4493
|
config: scanData.config,
|
|
1969
4494
|
dryRun,
|
|
1970
|
-
isAppRouter: scanData.isAppRouter
|
|
4495
|
+
isAppRouter: scanData.isAppRouter,
|
|
4496
|
+
framework: scanData.framework,
|
|
4497
|
+
baseFiles
|
|
1971
4498
|
});
|
|
1972
4499
|
printFixReport(fixResult);
|
|
1973
4500
|
} else if (command === "links") {
|
|
@@ -1977,10 +4504,11 @@ async function runCli(args) {
|
|
|
1977
4504
|
} else {
|
|
1978
4505
|
console.log(`
|
|
1979
4506
|
Unknown command: "${command}"`);
|
|
1980
|
-
console.log("Available commands: audit, fix [--dry-run], links, opportunities\n");
|
|
4507
|
+
console.log("Available commands: diff, audit, fix [--dry-run], links, opportunities\n");
|
|
1981
4508
|
process.exit(1);
|
|
1982
4509
|
}
|
|
1983
4510
|
}
|
|
1984
4511
|
export {
|
|
4512
|
+
auditGitRef,
|
|
1985
4513
|
runCli
|
|
1986
4514
|
};
|