pagegraph 0.8.4 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -6
- package/dist/audit.d.ts +7 -0
- package/dist/audit.js +33 -7
- package/dist/audit.js.map +1 -1
- package/dist/cli.js +3827 -2198
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +14 -1
- package/dist/config.js.map +1 -1
- package/dist/{checks-6noG_xTS.d.ts → content-DVL2iyGU.d.ts} +274 -126
- package/dist/content-DbyJoRpj.js +1384 -0
- package/dist/content-DbyJoRpj.js.map +1 -0
- package/dist/index.d.ts +19 -8
- package/dist/index.js +31 -555
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/links-BFJc-pJ9.js +0 -318
- package/dist/links-BFJc-pJ9.js.map +0 -1
|
@@ -0,0 +1,1384 @@
|
|
|
1
|
+
//#region src/core/projections.ts
|
|
2
|
+
const escapeXml = (value) => value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
3
|
+
/**
|
|
4
|
+
* A node belongs in the sitemap when it declares a positive sitemap policy, is not
|
|
5
|
+
* a redirect, is not robots-noindexed, and is not a param template (a route whose
|
|
6
|
+
* path still contains a `$` segment — those exist only so their instances inherit).
|
|
7
|
+
*/
|
|
8
|
+
function isSitemapEligible(node) {
|
|
9
|
+
if (node.policy.redirectTo !== void 0) return false;
|
|
10
|
+
if (node.policy.robots?.includes("noindex")) return false;
|
|
11
|
+
if (!node.policy.sitemap) return false;
|
|
12
|
+
if (node.path.includes("$")) return false;
|
|
13
|
+
return true;
|
|
14
|
+
}
|
|
15
|
+
/** Canonical absolute URL for a node under the given origin. */
|
|
16
|
+
function urlForNode(origin, node) {
|
|
17
|
+
return node.path === "/" ? origin : `${origin}${node.path}`;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* The date a node's content last changed, as declared: an instance's
|
|
21
|
+
* `modifiedAt`, else its `publishedAt`; a route's own `modifiedAt`. A node with
|
|
22
|
+
* no date (a docs page without frontmatter dates, an undated route) returns
|
|
23
|
+
* undefined, and the sitemap omits its `<lastmod>`.
|
|
24
|
+
*/
|
|
25
|
+
function lastModified(node) {
|
|
26
|
+
if (node.instance) return node.instance.modifiedAt ?? node.instance.publishedAt;
|
|
27
|
+
return node.policy.modifiedAt;
|
|
28
|
+
}
|
|
29
|
+
/** An ISO 8601 date, or a date-time that states its offset. */
|
|
30
|
+
const ISO_DATE = /^(\d{4})-(\d{2})-(\d{2})(?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2}))?$/;
|
|
31
|
+
/**
|
|
32
|
+
* Parse a declared date to epoch milliseconds, or undefined when it is not an
|
|
33
|
+
* ISO 8601 date (`2026-09-29`) or offset date-time (`2026-09-29T10:00:00Z`) on a
|
|
34
|
+
* real calendar day. `Date.parse` alone would accept "Sep 29" in the host's
|
|
35
|
+
* zone and roll "2026-02-30" into March; either would publish a wrong
|
|
36
|
+
* `<lastmod>`. The `invalid-date` check reports the undefined case, so
|
|
37
|
+
* projections that meet one fail loudly.
|
|
38
|
+
*/
|
|
39
|
+
function parseDeclaredDate(value) {
|
|
40
|
+
const match = ISO_DATE.exec(value);
|
|
41
|
+
if (match === null) return void 0;
|
|
42
|
+
const [year, month, day] = [
|
|
43
|
+
Number(match[1]),
|
|
44
|
+
Number(match[2]),
|
|
45
|
+
Number(match[3])
|
|
46
|
+
];
|
|
47
|
+
const calendar = new Date(Date.UTC(year, month - 1, day));
|
|
48
|
+
if (calendar.getUTCMonth() !== month - 1 || calendar.getUTCDate() !== day) return void 0;
|
|
49
|
+
const time = Date.parse(value);
|
|
50
|
+
return Number.isNaN(time) ? void 0 : time;
|
|
51
|
+
}
|
|
52
|
+
function sitemapLastmod(node) {
|
|
53
|
+
const date = lastModified(node);
|
|
54
|
+
if (date === void 0) return void 0;
|
|
55
|
+
const time = parseDeclaredDate(date);
|
|
56
|
+
if (time === void 0) throw new Error(`"${node.path}" declares an invalid date "${date}"; run \`pagegraph check\`.`);
|
|
57
|
+
return new Date(time).toISOString();
|
|
58
|
+
}
|
|
59
|
+
function renderUrlEntry(url, lastmod, node) {
|
|
60
|
+
const { changeFrequency, priority } = node.policy.sitemap;
|
|
61
|
+
const lines = [` <url>`, ` <loc>${escapeXml(url)}</loc>`];
|
|
62
|
+
if (lastmod) lines.push(` <lastmod>${lastmod}</lastmod>`);
|
|
63
|
+
lines.push(` <changefreq>${changeFrequency}</changefreq>`, ` <priority>${priority.toFixed(1)}</priority>`, ` </url>`);
|
|
64
|
+
return lines.join("\n");
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Render sitemap.xml from the graph. Each entry's `<lastmod>` is its
|
|
68
|
+
* {@link lastModified} date: frontmatter for content instances, the declared
|
|
69
|
+
* `modifiedAt` for routes. Route entries are sorted by path, then instances follow in collection order.
|
|
70
|
+
* `indexable` is intentionally unused — the sitemap body is host-independent;
|
|
71
|
+
* robots.txt is what gates crawling.
|
|
72
|
+
*/
|
|
73
|
+
function renderSitemap(graph, cfg) {
|
|
74
|
+
const nodes = [...graph.nodes.values()];
|
|
75
|
+
const staticNodes = nodes.filter((node) => node.source === "route" && isSitemapEligible(node)).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
76
|
+
const instanceNodes = nodes.filter((node) => node.source !== "route" && isSitemapEligible(node));
|
|
77
|
+
const seen = /* @__PURE__ */ new Set();
|
|
78
|
+
const entries = [];
|
|
79
|
+
for (const node of [...staticNodes, ...instanceNodes]) {
|
|
80
|
+
const url = urlForNode(cfg.origin, node);
|
|
81
|
+
const key = url.toLowerCase().replace(/\/$/, "");
|
|
82
|
+
if (seen.has(key)) continue;
|
|
83
|
+
seen.add(key);
|
|
84
|
+
entries.push(renderUrlEntry(url, sitemapLastmod(node), node));
|
|
85
|
+
}
|
|
86
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${entries.join("\n")}\n</urlset>\n`;
|
|
87
|
+
}
|
|
88
|
+
/** Format a Content-Signal robots.txt directive from the preference list. */
|
|
89
|
+
function contentSignal(value) {
|
|
90
|
+
return `Content-Signal: ${value}`;
|
|
91
|
+
}
|
|
92
|
+
const groupLines = (cfg) => {
|
|
93
|
+
const lines = [];
|
|
94
|
+
if (cfg.contentSignal !== void 0 && cfg.contentSignal !== "") lines.push(contentSignal(cfg.contentSignal));
|
|
95
|
+
for (const directive of cfg.directives ?? []) if (directive !== "") lines.push(directive);
|
|
96
|
+
return lines;
|
|
97
|
+
};
|
|
98
|
+
/**
|
|
99
|
+
* Render robots.txt. A non-indexable host (previews) gets a disallow-all
|
|
100
|
+
* with no Sitemap line; an indexable host disallows exactly the prefixes the caller
|
|
101
|
+
* passes. Pages that declare `robots: noindex` are intentionally NOT added as
|
|
102
|
+
* Disallow entries — a Disallow would stop crawlers reaching the page to read its
|
|
103
|
+
* `noindex, follow` meta, so the graph's per-node robots policy never feeds this
|
|
104
|
+
* list. `graph` is unused — kept for signature parity with the other projections,
|
|
105
|
+
* which callers load the graph once for and pass to each.
|
|
106
|
+
*
|
|
107
|
+
* Origin-wide group directives (`contentSignal`, `directives`) are indexable-host
|
|
108
|
+
* only. {@link RobotsConfig.transform} always runs last so a consumer can override
|
|
109
|
+
* the whole file.
|
|
110
|
+
*/
|
|
111
|
+
function renderRobots(_graph, cfg) {
|
|
112
|
+
const rendered = cfg.indexable ? [
|
|
113
|
+
"User-agent: *",
|
|
114
|
+
...groupLines(cfg),
|
|
115
|
+
"Allow: /",
|
|
116
|
+
...cfg.disallow.map((path) => `Disallow: ${path}`),
|
|
117
|
+
"",
|
|
118
|
+
`Sitemap: ${cfg.origin}/sitemap.xml`,
|
|
119
|
+
`Host: ${cfg.origin}`,
|
|
120
|
+
""
|
|
121
|
+
].join("\n") : [
|
|
122
|
+
"User-agent: *",
|
|
123
|
+
"Disallow: /",
|
|
124
|
+
""
|
|
125
|
+
].join("\n");
|
|
126
|
+
return cfg.transform === void 0 ? rendered : cfg.transform(rendered);
|
|
127
|
+
}
|
|
128
|
+
/** Inspect a single node: its declaration, sitemap eligibility, and edges. */
|
|
129
|
+
function inspectNode(graph, path) {
|
|
130
|
+
const node = graph.nodes.get(path);
|
|
131
|
+
if (!node) return void 0;
|
|
132
|
+
return {
|
|
133
|
+
node,
|
|
134
|
+
inSitemap: isSitemapEligible(node),
|
|
135
|
+
incoming: graph.edges.filter((edge) => edge.to === path),
|
|
136
|
+
outgoing: graph.edges.filter((edge) => edge.from === path)
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
//#endregion
|
|
140
|
+
//#region src/core/freshness.ts
|
|
141
|
+
const DAY_MS = 864e5;
|
|
142
|
+
/**
|
|
143
|
+
* Split the graph's sitemap-eligible pages into stale, fresh, undated, and
|
|
144
|
+
* invalid. An invalid date cannot be aged, but it is listed so a report never
|
|
145
|
+
* reads as complete while it skipped a page.
|
|
146
|
+
*/
|
|
147
|
+
function freshnessReport(graph, policy, now) {
|
|
148
|
+
const stale = [];
|
|
149
|
+
const fresh = [];
|
|
150
|
+
const undated = [];
|
|
151
|
+
const invalid = [];
|
|
152
|
+
for (const node of graph.nodes.values()) {
|
|
153
|
+
if (!isSitemapEligible(node)) continue;
|
|
154
|
+
const declared = lastModified(node);
|
|
155
|
+
if (declared === void 0) {
|
|
156
|
+
undated.push(node.path);
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
const time = parseDeclaredDate(declared);
|
|
160
|
+
if (time === void 0) {
|
|
161
|
+
invalid.push(node.path);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
const entry = {
|
|
165
|
+
path: node.path,
|
|
166
|
+
kind: node.kind,
|
|
167
|
+
source: node.source,
|
|
168
|
+
lastModified: new Date(time).toISOString(),
|
|
169
|
+
ageDays: Math.max(0, Math.floor((now.getTime() - time) / DAY_MS))
|
|
170
|
+
};
|
|
171
|
+
(entry.ageDays > policy.maxAgeDays ? stale : fresh).push(entry);
|
|
172
|
+
}
|
|
173
|
+
stale.sort((a, b) => b.ageDays - a.ageDays || (a.path < b.path ? -1 : 1));
|
|
174
|
+
fresh.sort((a, b) => a.ageDays - b.ageDays || (a.path < b.path ? -1 : 1));
|
|
175
|
+
return {
|
|
176
|
+
asOf: now.toISOString(),
|
|
177
|
+
maxAgeDays: policy.maxAgeDays,
|
|
178
|
+
stale,
|
|
179
|
+
fresh,
|
|
180
|
+
undated,
|
|
181
|
+
invalid
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
//#endregion
|
|
185
|
+
//#region src/core/checks.ts
|
|
186
|
+
/** A positive sitemap policy — the author asked for this page to be indexed. */
|
|
187
|
+
const hasPositiveSitemap = (sitemap) => sitemap !== void 0 && sitemap !== false;
|
|
188
|
+
/** Content and manifest instances carry page-level title/description. */
|
|
189
|
+
const isInstance = (node) => node.source !== "route";
|
|
190
|
+
/** Group instance nodes by a present, non-empty string field for duplicate detection. */
|
|
191
|
+
const groupInstancesBy = (graph, field) => {
|
|
192
|
+
const groups = /* @__PURE__ */ new Map();
|
|
193
|
+
for (const node of graph.nodes.values()) {
|
|
194
|
+
if (!isInstance(node)) continue;
|
|
195
|
+
const value = field(node)?.trim();
|
|
196
|
+
if (!value) continue;
|
|
197
|
+
const bucket = groups.get(value);
|
|
198
|
+
if (bucket) bucket.push(node);
|
|
199
|
+
else groups.set(value, [node]);
|
|
200
|
+
}
|
|
201
|
+
return groups;
|
|
202
|
+
};
|
|
203
|
+
const DESCRIPTION_MAX = 160;
|
|
204
|
+
const DESCRIPTION_MIN = 50;
|
|
205
|
+
/**
|
|
206
|
+
* Every check, in one place. `checkGraph` runs them in order and stamps each
|
|
207
|
+
* finding with its rule name and severity, so the output is grouped by rule and
|
|
208
|
+
* deterministic (nodes iterate in graph insertion order, edges in array order).
|
|
209
|
+
*/
|
|
210
|
+
const CHECK_RULES = [
|
|
211
|
+
{
|
|
212
|
+
name: "path-owner-collision",
|
|
213
|
+
severity: "structural",
|
|
214
|
+
evaluate: (graph) => (graph.collisions ?? []).map((collision) => ({
|
|
215
|
+
path: collision.path,
|
|
216
|
+
message: `Canonical path "${collision.path}" is owned by multiple sources: ${collision.sources.join(", ")}.`,
|
|
217
|
+
fix: "Give every concrete page one canonical path and one graph owner."
|
|
218
|
+
}))
|
|
219
|
+
},
|
|
220
|
+
{
|
|
221
|
+
name: "canonical-path-collision",
|
|
222
|
+
severity: "structural",
|
|
223
|
+
evaluate: (graph) => {
|
|
224
|
+
const groups = /* @__PURE__ */ new Map();
|
|
225
|
+
for (const path of graph.nodes.keys()) {
|
|
226
|
+
const canonical = path.toLowerCase().replace(/\/$/, "") || "/";
|
|
227
|
+
const paths = groups.get(canonical);
|
|
228
|
+
if (paths) paths.push(path);
|
|
229
|
+
else groups.set(canonical, [path]);
|
|
230
|
+
}
|
|
231
|
+
return [...groups.values()].filter((paths) => paths.length > 1).flatMap((paths) => paths.map((path) => ({
|
|
232
|
+
path,
|
|
233
|
+
message: `Canonical path collides with ${paths.filter((candidate) => candidate !== path).join(", ")}.`,
|
|
234
|
+
fix: "Use one lowercase, trailing-slash-normalized canonical path."
|
|
235
|
+
})));
|
|
236
|
+
}
|
|
237
|
+
},
|
|
238
|
+
{
|
|
239
|
+
name: "self-edge",
|
|
240
|
+
severity: "structural",
|
|
241
|
+
evaluate: (graph) => graph.edges.filter((edge) => edge.from === edge.to).map((edge) => ({
|
|
242
|
+
path: edge.from,
|
|
243
|
+
message: `${edge.type} edge points back to its own source node.`,
|
|
244
|
+
fix: "Remove the self-reference from the canonical manifest or route declaration."
|
|
245
|
+
}))
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
name: "duplicate-edge",
|
|
249
|
+
severity: "structural",
|
|
250
|
+
evaluate: (graph) => {
|
|
251
|
+
const seen = /* @__PURE__ */ new Set();
|
|
252
|
+
const duplicates = [];
|
|
253
|
+
for (const edge of graph.edges) {
|
|
254
|
+
const key = `${edge.from}\u0000${edge.to}\u0000${edge.type}`;
|
|
255
|
+
if (seen.has(key)) duplicates.push({
|
|
256
|
+
path: edge.from,
|
|
257
|
+
message: `Duplicate ${edge.type} edge from "${edge.from}" to "${edge.to}".`,
|
|
258
|
+
fix: "Declare each graph relationship exactly once."
|
|
259
|
+
});
|
|
260
|
+
else seen.add(key);
|
|
261
|
+
}
|
|
262
|
+
return duplicates;
|
|
263
|
+
}
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
name: "dead-edge",
|
|
267
|
+
severity: "structural",
|
|
268
|
+
evaluate: (graph) => graph.edges.filter((edge) => edge.type !== "crumb-parent" && !graph.nodes.has(edge.to)).map((edge) => ({
|
|
269
|
+
path: edge.from,
|
|
270
|
+
message: `${edge.type} edge from "${edge.from}" points at "${edge.to}", which is not a node in the graph.`,
|
|
271
|
+
fix: `Update the ${edge.type === "redirect" ? "redirectTo" : "related"} target on the "${edge.from}" route, or restore "${edge.to}".`
|
|
272
|
+
}))
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
name: "instance-missing-title",
|
|
276
|
+
severity: "structural",
|
|
277
|
+
evaluate: (graph) => [...graph.nodes.values()].filter((node) => isInstance(node) && !node.instance?.title.trim()).map((node) => ({
|
|
278
|
+
path: node.path,
|
|
279
|
+
message: `Content page "${node.path}" has no title.`,
|
|
280
|
+
fix: "Add a `title` to the page frontmatter."
|
|
281
|
+
}))
|
|
282
|
+
},
|
|
283
|
+
{
|
|
284
|
+
name: "sitemap-noindex-contradiction",
|
|
285
|
+
severity: "structural",
|
|
286
|
+
evaluate: (graph) => [...graph.nodes.values()].filter((node) => hasPositiveSitemap(node.policy.sitemap) && node.policy.robots?.toLowerCase().includes("noindex")).map((node) => ({
|
|
287
|
+
path: node.path,
|
|
288
|
+
message: `"${node.path}" declares a sitemap policy but its robots value is "${node.policy.robots}".`,
|
|
289
|
+
fix: "Drop the sitemap policy (or set `sitemap: false`) on a noindex page, or remove the noindex robots value."
|
|
290
|
+
}))
|
|
291
|
+
},
|
|
292
|
+
{
|
|
293
|
+
name: "robots-not-lowercase",
|
|
294
|
+
severity: "structural",
|
|
295
|
+
evaluate: (graph) => [...graph.nodes.values()].filter((node) => node.policy.robots !== void 0 && node.policy.robots !== node.policy.robots.toLowerCase()).map((node) => ({
|
|
296
|
+
path: node.path,
|
|
297
|
+
message: `"${node.path}" declares robots "${node.policy.robots}" — robots values must be lowercase.`,
|
|
298
|
+
fix: "Lowercase the robots declaration (e.g. \"noindex, follow\")."
|
|
299
|
+
}))
|
|
300
|
+
},
|
|
301
|
+
{
|
|
302
|
+
name: "related-target-missing-link",
|
|
303
|
+
severity: "structural",
|
|
304
|
+
evaluate: (graph) => {
|
|
305
|
+
const linklessTargets = /* @__PURE__ */ new Set();
|
|
306
|
+
for (const edge of graph.edges) {
|
|
307
|
+
if (edge.type !== "related") continue;
|
|
308
|
+
const target = graph.nodes.get(edge.to);
|
|
309
|
+
if (target && target.source === "route" && target.policy.link === void 0) linklessTargets.add(edge.to);
|
|
310
|
+
}
|
|
311
|
+
return [...linklessTargets].map((path) => ({
|
|
312
|
+
path,
|
|
313
|
+
message: `Route "${path}" is a related-link target but declares no link metadata; its card would render empty.`,
|
|
314
|
+
fix: `Add \`link: { title, description }\` to the "${path}" route's staticData.seo.`
|
|
315
|
+
}));
|
|
316
|
+
}
|
|
317
|
+
},
|
|
318
|
+
{
|
|
319
|
+
name: "redirect-in-sitemap",
|
|
320
|
+
severity: "structural",
|
|
321
|
+
evaluate: (graph) => [...graph.nodes.values()].filter((node) => node.policy.redirectTo !== void 0 && hasPositiveSitemap(node.policy.sitemap)).map((node) => ({
|
|
322
|
+
path: node.path,
|
|
323
|
+
message: `Redirect "${node.path}" (→ "${node.policy.redirectTo}") also declares a sitemap policy.`,
|
|
324
|
+
fix: "Remove the sitemap policy from the redirect route; only its target belongs in the sitemap."
|
|
325
|
+
}))
|
|
326
|
+
},
|
|
327
|
+
{
|
|
328
|
+
name: "invalid-date",
|
|
329
|
+
severity: "structural",
|
|
330
|
+
evaluate: (graph) => [...graph.nodes.values()].flatMap((node) => {
|
|
331
|
+
return (node.instance ? [["publishedAt", node.instance.publishedAt], ["modifiedAt", node.instance.modifiedAt]] : [["modifiedAt", node.policy.modifiedAt]]).flatMap(([field, value]) => value !== void 0 && parseDeclaredDate(value) === void 0 ? [{
|
|
332
|
+
path: node.path,
|
|
333
|
+
message: `${field} "${value}" is not an ISO 8601 date on a real calendar day.`,
|
|
334
|
+
fix: `Write ${field} as an ISO 8601 date, e.g. "2026-09-29".`
|
|
335
|
+
}] : []);
|
|
336
|
+
})
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
name: "duplicate-title",
|
|
340
|
+
severity: "editorial",
|
|
341
|
+
evaluate: (graph) => [...groupInstancesBy(graph, (node) => node.instance?.title).entries()].filter(([, nodes]) => nodes.length > 1).flatMap(([title, nodes]) => nodes.map((node) => ({
|
|
342
|
+
path: node.path,
|
|
343
|
+
message: `Title "${title}" is shared by ${nodes.length} pages.`,
|
|
344
|
+
fix: "Give each page a distinct title."
|
|
345
|
+
})))
|
|
346
|
+
},
|
|
347
|
+
{
|
|
348
|
+
name: "duplicate-description",
|
|
349
|
+
severity: "editorial",
|
|
350
|
+
evaluate: (graph) => [...groupInstancesBy(graph, (node) => node.instance?.description).entries()].filter(([, nodes]) => nodes.length > 1).flatMap(([, nodes]) => nodes.map((node) => ({
|
|
351
|
+
path: node.path,
|
|
352
|
+
message: `Description is shared by ${nodes.length} pages.`,
|
|
353
|
+
fix: "Write a distinct meta description for each page."
|
|
354
|
+
})))
|
|
355
|
+
},
|
|
356
|
+
{
|
|
357
|
+
name: "description-length",
|
|
358
|
+
severity: "editorial",
|
|
359
|
+
evaluate: (graph) => [...graph.nodes.values()].flatMap((node) => {
|
|
360
|
+
if (!isInstance(node)) return [];
|
|
361
|
+
const description = node.instance?.description?.trim();
|
|
362
|
+
if (!description) return [];
|
|
363
|
+
if (description.length > DESCRIPTION_MAX) return [{
|
|
364
|
+
path: node.path,
|
|
365
|
+
message: `Description is ${description.length} chars (max ${DESCRIPTION_MAX}); it will be truncated in results.`,
|
|
366
|
+
fix: `Trim the description to ${DESCRIPTION_MAX} characters or fewer.`
|
|
367
|
+
}];
|
|
368
|
+
if (description.length < DESCRIPTION_MIN) return [{
|
|
369
|
+
path: node.path,
|
|
370
|
+
message: `Description is ${description.length} chars (min ${DESCRIPTION_MIN}); it reads as thin.`,
|
|
371
|
+
fix: `Expand the description to at least ${DESCRIPTION_MIN} characters.`
|
|
372
|
+
}];
|
|
373
|
+
return [];
|
|
374
|
+
})
|
|
375
|
+
}
|
|
376
|
+
];
|
|
377
|
+
/**
|
|
378
|
+
* Run every static rule against the graph, then any caller-supplied
|
|
379
|
+
* {@link CoverageRule}s and freshness policy, and return the flat list of
|
|
380
|
+
* violations. With no options the result is exactly the static rule set.
|
|
381
|
+
*/
|
|
382
|
+
function checkGraph(graph, options = {}) {
|
|
383
|
+
const violations = CHECK_RULES.flatMap((rule) => rule.evaluate(graph).map((raw) => ({
|
|
384
|
+
severity: rule.severity,
|
|
385
|
+
rule: rule.name,
|
|
386
|
+
path: raw.path,
|
|
387
|
+
message: raw.message,
|
|
388
|
+
fix: raw.fix
|
|
389
|
+
})));
|
|
390
|
+
const coverage = options.coverage;
|
|
391
|
+
if (coverage !== void 0 && coverage.length > 0) violations.push(...checkCoverage(graph, coverage));
|
|
392
|
+
if (options.freshness !== void 0) violations.push(...checkFreshness(graph, options.freshness, options.now));
|
|
393
|
+
return violations;
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* One editorial `stale-page` violation per sitemap-eligible page whose last
|
|
397
|
+
* declared change is older than the policy allows. Undated pages are not
|
|
398
|
+
* violations: see {@link freshnessReport} for the full picture.
|
|
399
|
+
*/
|
|
400
|
+
function checkFreshness(graph, policy, now) {
|
|
401
|
+
return freshnessReport(graph, policy, now).stale.map((entry) => ({
|
|
402
|
+
severity: "editorial",
|
|
403
|
+
rule: "stale-page",
|
|
404
|
+
path: entry.path,
|
|
405
|
+
message: `Last changed ${entry.lastModified.slice(0, 10)}, ${entry.ageDays} days ago (limit ${policy.maxAgeDays}).`,
|
|
406
|
+
fix: "Review the page; when its content changes, update its modifiedAt."
|
|
407
|
+
}));
|
|
408
|
+
}
|
|
409
|
+
/** Escape a glob for `RegExp`, then expand `**`, `*`, and `?` to path-aware forms. */
|
|
410
|
+
const globToRegExp = (glob) => {
|
|
411
|
+
const source = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*\*/g, "\0").replace(/\*/g, "[^/]*").replace(/\?/g, "[^/]").replace(/\u0000/g, ".*");
|
|
412
|
+
return new RegExp(`^${source}$`);
|
|
413
|
+
};
|
|
414
|
+
/**
|
|
415
|
+
* Enforce contextual-link coverage rules over an explicit surface.
|
|
416
|
+
*
|
|
417
|
+
* A rule that matches no sitemap-eligible page is itself a violation: a typo or
|
|
418
|
+
* a rule aimed at a noindex page would otherwise pass silently forever.
|
|
419
|
+
*/
|
|
420
|
+
const checkCoverageRules = (surface, rules) => {
|
|
421
|
+
const violations = [];
|
|
422
|
+
for (const rule of rules) {
|
|
423
|
+
const matcher = globToRegExp(rule.path);
|
|
424
|
+
const matched = surface.paths.filter((path) => matcher.test(path));
|
|
425
|
+
const eligible = surface.eligiblePaths.filter((path) => matcher.test(path));
|
|
426
|
+
if (eligible.length === 0) {
|
|
427
|
+
violations.push({
|
|
428
|
+
severity: "structural",
|
|
429
|
+
rule: "coverage-rule-unmatched",
|
|
430
|
+
message: matched.length === 0 ? `Coverage rule "${rule.path}" matches no page in the graph.` : `Coverage rule "${rule.path}" matches only pages that are not sitemap-eligible.`,
|
|
431
|
+
fix: "Point the rule at a sitemap-eligible path, or drop it."
|
|
432
|
+
});
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
for (const path of eligible) {
|
|
436
|
+
const inbound = surface.inbound(path);
|
|
437
|
+
if (inbound >= rule.minInbound) continue;
|
|
438
|
+
violations.push({
|
|
439
|
+
severity: "structural",
|
|
440
|
+
rule: "inbound-link-coverage",
|
|
441
|
+
path,
|
|
442
|
+
message: `"${path}" has ${inbound} incoming contextual link(s); the coverage rule requires ${rule.minInbound}.`,
|
|
443
|
+
fix: `Add ${rule.minInbound - inbound} contextual link(s) pointing at "${path}".`
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
return violations;
|
|
448
|
+
};
|
|
449
|
+
/** The sitemap-eligible path set of a declared graph, in graph insertion order. */
|
|
450
|
+
const eligiblePathsOf = (graph) => [...graph.nodes.values()].filter(isSitemapEligible).map((node) => node.path);
|
|
451
|
+
/**
|
|
452
|
+
* Enforce contextual-link coverage against the *declared* graph: only `related`
|
|
453
|
+
* edges count — breadcrumb ancestry is navigation, not context.
|
|
454
|
+
*/
|
|
455
|
+
function checkCoverage(graph, rules) {
|
|
456
|
+
return checkCoverageRules({
|
|
457
|
+
paths: [...graph.nodes.keys()],
|
|
458
|
+
eligiblePaths: eligiblePathsOf(graph),
|
|
459
|
+
inbound: (path) => graph.edges.filter((edge) => edge.type === "related" && edge.to === path).length
|
|
460
|
+
}, rules);
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* Enforce contextual-link coverage against the *rendered* graph: only same-origin
|
|
464
|
+
* body-region anchors count. Nav, header, and footer anchors are site chrome, so
|
|
465
|
+
* a page linked solely from them has no contextual inbound link — the same
|
|
466
|
+
* distinction the declared pass draws by counting `related` edges alone.
|
|
467
|
+
*
|
|
468
|
+
* The target universe is the declared graph's sitemap-eligible pages: the globs
|
|
469
|
+
* in `seo.config.ts` are written against declared paths, and the crawl is what
|
|
470
|
+
* supplies the served edges. Pass the crawl's `internalEdges` (each edge carries
|
|
471
|
+
* its `region`).
|
|
472
|
+
*/
|
|
473
|
+
function checkRenderedCoverage(graph, renderedEdges, rules) {
|
|
474
|
+
const inbound = /* @__PURE__ */ new Map();
|
|
475
|
+
for (const edge of renderedEdges) {
|
|
476
|
+
if (edge.region !== "body") continue;
|
|
477
|
+
inbound.set(edge.to, (inbound.get(edge.to) ?? 0) + 1);
|
|
478
|
+
}
|
|
479
|
+
return checkCoverageRules({
|
|
480
|
+
paths: [...graph.nodes.keys()],
|
|
481
|
+
eligiblePaths: eligiblePathsOf(graph),
|
|
482
|
+
inbound: (path) => inbound.get(path) ?? 0
|
|
483
|
+
}, rules);
|
|
484
|
+
}
|
|
485
|
+
/** Structural violations fail `pagegraph check`; editorial-only stays green. */
|
|
486
|
+
const hasStructuralViolations = (violations) => violations.some((violation) => violation.severity === "structural");
|
|
487
|
+
//#endregion
|
|
488
|
+
//#region src/core/inspect-html.ts
|
|
489
|
+
const ENTITIES = {
|
|
490
|
+
"&": "&",
|
|
491
|
+
"<": "<",
|
|
492
|
+
">": ">",
|
|
493
|
+
""": "\"",
|
|
494
|
+
"'": "'",
|
|
495
|
+
"'": "'"
|
|
496
|
+
};
|
|
497
|
+
const decodeEntities$2 = (value) => value.replace(/&(?:amp|lt|gt|quot|#39|apos);/g, (match) => ENTITIES[match] ?? match);
|
|
498
|
+
/** Pull double/single-quoted attributes off a single tag string. */
|
|
499
|
+
const parseAttrs = (tag) => {
|
|
500
|
+
const attrs = {};
|
|
501
|
+
const re = /([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
|
|
502
|
+
let match;
|
|
503
|
+
while ((match = re.exec(tag)) !== null) attrs[match[1].toLowerCase()] = decodeEntities$2(match[2] ?? match[3] ?? "");
|
|
504
|
+
return attrs;
|
|
505
|
+
};
|
|
506
|
+
/** Normalize a JSON-LD `@type` (string or array) to a single readable label. */
|
|
507
|
+
const typeName = (value) => {
|
|
508
|
+
if (typeof value === "string") return value;
|
|
509
|
+
if (Array.isArray(value)) return value.filter((v) => typeof v === "string").join(", ") || "unknown";
|
|
510
|
+
return "unknown";
|
|
511
|
+
};
|
|
512
|
+
const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
513
|
+
/** Flatten a parsed JSON-LD payload (single object, array, or `@graph`) to items. */
|
|
514
|
+
const collectItems = (parsed) => {
|
|
515
|
+
if (Array.isArray(parsed)) return parsed.filter(isObject);
|
|
516
|
+
if (isObject(parsed)) {
|
|
517
|
+
if (Array.isArray(parsed["@graph"])) return parsed["@graph"].filter(isObject);
|
|
518
|
+
return [parsed];
|
|
519
|
+
}
|
|
520
|
+
return [];
|
|
521
|
+
};
|
|
522
|
+
const countQuestions = (mainEntity) => {
|
|
523
|
+
if (!Array.isArray(mainEntity)) return 0;
|
|
524
|
+
return mainEntity.filter((entry) => isObject(entry) && typeName(entry["@type"]) === "Question").length;
|
|
525
|
+
};
|
|
526
|
+
const validateItemListElements = (value) => {
|
|
527
|
+
if (!Array.isArray(value) || value.length < 1) return ["ItemList needs at least one `itemListElement` entry."];
|
|
528
|
+
const errors = [];
|
|
529
|
+
value.forEach((entry, index) => {
|
|
530
|
+
if (!isObject(entry) || typeName(entry["@type"]) !== "ListItem") {
|
|
531
|
+
errors.push(`ItemList entry ${index + 1} is not a ListItem.`);
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
if (entry["position"] !== index + 1) errors.push(`ItemList entry ${index + 1} has an invalid position.`);
|
|
535
|
+
if (!entry["name"] || !entry["url"]) errors.push(`ItemList entry ${index + 1} needs a name and URL.`);
|
|
536
|
+
});
|
|
537
|
+
return errors;
|
|
538
|
+
};
|
|
539
|
+
/** Minimal per-type validation — enough to catch an empty or malformed block. */
|
|
540
|
+
const validateItem = (item) => {
|
|
541
|
+
const type = typeName(item["@type"]);
|
|
542
|
+
const errors = [];
|
|
543
|
+
if (type === "Article" || type === "NewsArticle" || type === "BlogPosting") {
|
|
544
|
+
if (!item["headline"]) errors.push("Article is missing `headline`.");
|
|
545
|
+
if (!item["datePublished"]) errors.push("Article is missing `datePublished`.");
|
|
546
|
+
} else if (type === "FAQPage") {
|
|
547
|
+
if (countQuestions(item["mainEntity"]) < 1) errors.push("FAQPage needs at least one Question in `mainEntity`.");
|
|
548
|
+
} else if (type === "BreadcrumbList") {
|
|
549
|
+
const items = item["itemListElement"];
|
|
550
|
+
if (!Array.isArray(items) || items.length < 2) errors.push("BreadcrumbList needs at least two `itemListElement` entries.");
|
|
551
|
+
} else if (type === "ItemList") {
|
|
552
|
+
errors.push(...validateItemListElements(item["itemListElement"]));
|
|
553
|
+
const items = item["itemListElement"];
|
|
554
|
+
if (Array.isArray(items) && item["numberOfItems"] !== items.length) errors.push("ItemList `numberOfItems` does not match its entries.");
|
|
555
|
+
}
|
|
556
|
+
return {
|
|
557
|
+
type,
|
|
558
|
+
valid: errors.length === 0,
|
|
559
|
+
errors
|
|
560
|
+
};
|
|
561
|
+
};
|
|
562
|
+
const validateLdJson = (raw) => {
|
|
563
|
+
let parsed;
|
|
564
|
+
try {
|
|
565
|
+
parsed = JSON.parse(raw);
|
|
566
|
+
} catch (cause) {
|
|
567
|
+
return [{
|
|
568
|
+
type: "unparseable",
|
|
569
|
+
valid: false,
|
|
570
|
+
errors: [`JSON parse error: ${cause instanceof Error ? cause.message : String(cause)}`]
|
|
571
|
+
}];
|
|
572
|
+
}
|
|
573
|
+
const items = collectItems(parsed);
|
|
574
|
+
if (items.length === 0) return [{
|
|
575
|
+
type: "unknown",
|
|
576
|
+
valid: false,
|
|
577
|
+
errors: ["No JSON-LD object found in block."]
|
|
578
|
+
}];
|
|
579
|
+
return items.map(validateItem);
|
|
580
|
+
};
|
|
581
|
+
/** Parse a rendered HTML document's `<head>` into a report. Pure. */
|
|
582
|
+
const inspectHtml = (url, status, html) => {
|
|
583
|
+
const headMatch = html.match(/<head[^>]*>([\s\S]*?)<\/head>/i);
|
|
584
|
+
const head = headMatch ? headMatch[1] : html;
|
|
585
|
+
const titleMatch = head.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
|
586
|
+
const title = titleMatch ? decodeEntities$2(titleMatch[1].trim()) : void 0;
|
|
587
|
+
const og = {};
|
|
588
|
+
const twitter = {};
|
|
589
|
+
let description;
|
|
590
|
+
let robots;
|
|
591
|
+
for (const tag of head.match(/<meta\b[^>]*>/gi) ?? []) {
|
|
592
|
+
const attrs = parseAttrs(tag);
|
|
593
|
+
const content = attrs["content"];
|
|
594
|
+
if (content === void 0) continue;
|
|
595
|
+
const property = attrs["property"];
|
|
596
|
+
const name = attrs["name"];
|
|
597
|
+
if (property?.startsWith("og:")) og[property] = content;
|
|
598
|
+
else if (name?.startsWith("twitter:")) twitter[name] = content;
|
|
599
|
+
else if (name === "description") description = content;
|
|
600
|
+
else if (name === "robots") robots = content;
|
|
601
|
+
}
|
|
602
|
+
let canonical;
|
|
603
|
+
for (const tag of head.match(/<link\b[^>]*>/gi) ?? []) {
|
|
604
|
+
const attrs = parseAttrs(tag);
|
|
605
|
+
if (attrs["rel"] === "canonical") canonical = attrs["href"];
|
|
606
|
+
}
|
|
607
|
+
const jsonLd = [];
|
|
608
|
+
const scriptRe = /<script\b[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
|
|
609
|
+
let scriptMatch;
|
|
610
|
+
while ((scriptMatch = scriptRe.exec(head)) !== null) jsonLd.push(...validateLdJson(scriptMatch[1].trim()));
|
|
611
|
+
const issues = [];
|
|
612
|
+
if (status >= 400) issues.push(`Fetch returned HTTP ${status}.`);
|
|
613
|
+
if (!title) issues.push("Missing <title>.");
|
|
614
|
+
if (!description) issues.push("Missing meta description.");
|
|
615
|
+
if (!canonical) issues.push("Missing canonical link.");
|
|
616
|
+
for (const block of jsonLd) if (!block.valid) issues.push(...block.errors.map((error) => `JSON-LD (${block.type}): ${error}`));
|
|
617
|
+
return {
|
|
618
|
+
url,
|
|
619
|
+
status,
|
|
620
|
+
title,
|
|
621
|
+
description,
|
|
622
|
+
canonical,
|
|
623
|
+
robots,
|
|
624
|
+
og,
|
|
625
|
+
twitter,
|
|
626
|
+
jsonLd,
|
|
627
|
+
issues
|
|
628
|
+
};
|
|
629
|
+
};
|
|
630
|
+
/** A non-empty `issues` list fails `pagegraph inspect --live` (exit 1). */
|
|
631
|
+
const hasBlockingIssues = (report) => report.issues.length > 0;
|
|
632
|
+
//#endregion
|
|
633
|
+
//#region src/core/links.ts
|
|
634
|
+
const TOKEN = /<(\/?)(nav|footer|header)\b[^>]*>|<a\b([^>]*)>([\s\S]*?)<\/a>/gi;
|
|
635
|
+
const HREF = /(?:^|\s)href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/i;
|
|
636
|
+
const BASE_TAG = /<base\b[^>]*>/gi;
|
|
637
|
+
const NON_NAVIGABLE = /^(?:#|mailto:|tel:|javascript:|data:)/i;
|
|
638
|
+
const stripTags = (value) => value.replace(/<[^>]*>/g, " ");
|
|
639
|
+
const decodeEntities$1 = (value) => value.replace(/ /gi, " ").replace(/&/gi, "&").replace(/</gi, "<").replace(/>/gi, ">").replace(/"/gi, "\"").replace(/'|'/gi, "'");
|
|
640
|
+
const anchorText = (value) => decodeEntities$1(stripTags(value)).replace(/\s+/g, " ").trim();
|
|
641
|
+
/** Normalize a same-origin URL to the graph's path key (no query/hash; "/" preserved). */
|
|
642
|
+
const normalizePath = (url) => {
|
|
643
|
+
const path = url.pathname.replace(/\/+$/, "");
|
|
644
|
+
return path === "" ? "/" : path;
|
|
645
|
+
};
|
|
646
|
+
const hrefValue = (attributes) => {
|
|
647
|
+
const match = HREF.exec(attributes);
|
|
648
|
+
if (match === null) return void 0;
|
|
649
|
+
return match[1] ?? match[2] ?? match[3];
|
|
650
|
+
};
|
|
651
|
+
/**
|
|
652
|
+
* The document base URL. Browsers use the first `<base>` element that has an
|
|
653
|
+
* `href` attribute and ignore every later one, so a `<base>` carrying only
|
|
654
|
+
* `target` is skipped, but the first `<base href>` wins even when its value is
|
|
655
|
+
* empty (it then resolves to the document URL) or invalid (the base is ignored,
|
|
656
|
+
* resolving to the document URL). With no `<base href>` at all, links resolve
|
|
657
|
+
* against the response URL.
|
|
658
|
+
*/
|
|
659
|
+
const documentBase = (html, responseUrl) => {
|
|
660
|
+
BASE_TAG.lastIndex = 0;
|
|
661
|
+
let tag;
|
|
662
|
+
while ((tag = BASE_TAG.exec(html)) !== null) {
|
|
663
|
+
const href = hrefValue(tag[0]);
|
|
664
|
+
if (href === void 0) continue;
|
|
665
|
+
if (href.length === 0) return responseUrl;
|
|
666
|
+
try {
|
|
667
|
+
return new URL(href, responseUrl);
|
|
668
|
+
} catch {
|
|
669
|
+
return responseUrl;
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
return responseUrl;
|
|
673
|
+
};
|
|
674
|
+
/**
|
|
675
|
+
* Extract every anchor in document order, tagging each with the region tag it
|
|
676
|
+
* sits inside. Anchor text keeps only its letters and a coarse entity decode;
|
|
677
|
+
* this is a classifier input, not a renderer.
|
|
678
|
+
*/
|
|
679
|
+
const extractAnchors = (html, baseUrl) => {
|
|
680
|
+
const base = new URL(baseUrl);
|
|
681
|
+
const resolutionBase = documentBase(html, base);
|
|
682
|
+
const regions = [];
|
|
683
|
+
const anchors = [];
|
|
684
|
+
TOKEN.lastIndex = 0;
|
|
685
|
+
let match;
|
|
686
|
+
while ((match = TOKEN.exec(html)) !== null) {
|
|
687
|
+
const closing = match[1];
|
|
688
|
+
const regionTag = match[2];
|
|
689
|
+
if (regionTag !== void 0) {
|
|
690
|
+
const region = regionTag.toLowerCase();
|
|
691
|
+
if (closing === "/") {
|
|
692
|
+
const index = regions.lastIndexOf(region);
|
|
693
|
+
if (index !== -1) regions.splice(index, 1);
|
|
694
|
+
} else regions.push(region);
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
const attributes = match[3];
|
|
698
|
+
const raw = hrefValue(attributes);
|
|
699
|
+
if (raw === void 0 || raw.length === 0 || NON_NAVIGABLE.test(raw)) continue;
|
|
700
|
+
let resolved;
|
|
701
|
+
try {
|
|
702
|
+
resolved = new URL(raw, resolutionBase);
|
|
703
|
+
} catch {
|
|
704
|
+
continue;
|
|
705
|
+
}
|
|
706
|
+
if (resolved.protocol !== "http:" && resolved.protocol !== "https:") continue;
|
|
707
|
+
const region = regions.length === 0 ? "body" : regions[regions.length - 1];
|
|
708
|
+
anchors.push({
|
|
709
|
+
href: resolved.href,
|
|
710
|
+
text: anchorText(match[4] ?? ""),
|
|
711
|
+
region,
|
|
712
|
+
internal: resolved.origin === base.origin
|
|
713
|
+
});
|
|
714
|
+
}
|
|
715
|
+
return anchors;
|
|
716
|
+
};
|
|
717
|
+
const breadthFirstDepth = (edges, root) => {
|
|
718
|
+
const adjacency = /* @__PURE__ */ new Map();
|
|
719
|
+
for (const edge of edges) {
|
|
720
|
+
const targets = adjacency.get(edge.from) ?? /* @__PURE__ */ new Set();
|
|
721
|
+
targets.add(edge.to);
|
|
722
|
+
adjacency.set(edge.from, targets);
|
|
723
|
+
}
|
|
724
|
+
const depth = /* @__PURE__ */ new Map([[root, 0]]);
|
|
725
|
+
const queue = [root];
|
|
726
|
+
while (queue.length > 0) {
|
|
727
|
+
const current = queue.shift();
|
|
728
|
+
const currentDepth = depth.get(current);
|
|
729
|
+
for (const next of adjacency.get(current) ?? []) {
|
|
730
|
+
if (depth.has(next)) continue;
|
|
731
|
+
depth.set(next, currentDepth + 1);
|
|
732
|
+
queue.push(next);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
return depth;
|
|
736
|
+
};
|
|
737
|
+
/**
|
|
738
|
+
* Assemble a {@link RenderedGraph} from page/edge sets. Edges are already
|
|
739
|
+
* same-origin and path-keyed by the callers, so this is the shared, pure tail of
|
|
740
|
+
* {@link buildRenderedGraph} and {@link renderedGraphFromEdges}.
|
|
741
|
+
*/
|
|
742
|
+
const graphFromEdges = (nodes, edges, root) => {
|
|
743
|
+
const rootKey = root.replace(/\/+$/, "") || "/";
|
|
744
|
+
const internalEdges = edges;
|
|
745
|
+
const contextualEdges = edges.filter((edge) => edge.region === "body");
|
|
746
|
+
const incoming = new Set(internalEdges.map((edge) => edge.to));
|
|
747
|
+
const orphans = [...nodes].filter((path) => path !== rootKey && !incoming.has(path)).sort();
|
|
748
|
+
const depthByPath = breadthFirstDepth(internalEdges, rootKey);
|
|
749
|
+
let maxDepth = null;
|
|
750
|
+
for (const value of depthByPath.values()) if (maxDepth === null || value > maxDepth) maxDepth = value;
|
|
751
|
+
return {
|
|
752
|
+
nodes: [...nodes],
|
|
753
|
+
edges,
|
|
754
|
+
internalEdges,
|
|
755
|
+
contextualEdges,
|
|
756
|
+
orphans,
|
|
757
|
+
depthByPath,
|
|
758
|
+
maxDepth
|
|
759
|
+
};
|
|
760
|
+
};
|
|
761
|
+
/**
|
|
762
|
+
* Build the rendered graph from crawled pages. `origin` fixes the graph's
|
|
763
|
+
* same-origin boundary — pages and anchors on another origin are dropped, not
|
|
764
|
+
* silently folded in — and `root` is the normalized path the depth BFS starts
|
|
765
|
+
* from (the seed's final path when the homepage redirected). Every page path is
|
|
766
|
+
* keyed the way the declared graph keys its routes.
|
|
767
|
+
*/
|
|
768
|
+
const buildRenderedGraph = (origin, pages, root = "/") => {
|
|
769
|
+
const graphOrigin = new URL(origin).origin;
|
|
770
|
+
const edges = [];
|
|
771
|
+
const nodes = /* @__PURE__ */ new Set();
|
|
772
|
+
for (const page of pages) {
|
|
773
|
+
let pageUrl;
|
|
774
|
+
try {
|
|
775
|
+
pageUrl = new URL(page.url);
|
|
776
|
+
} catch {
|
|
777
|
+
continue;
|
|
778
|
+
}
|
|
779
|
+
if (pageUrl.origin !== graphOrigin) continue;
|
|
780
|
+
const path = normalizePath(pageUrl);
|
|
781
|
+
nodes.add(path);
|
|
782
|
+
for (const anchor of page.anchors) {
|
|
783
|
+
let targetUrl;
|
|
784
|
+
try {
|
|
785
|
+
targetUrl = new URL(anchor.href);
|
|
786
|
+
} catch {
|
|
787
|
+
continue;
|
|
788
|
+
}
|
|
789
|
+
if (targetUrl.origin !== graphOrigin) continue;
|
|
790
|
+
const target = normalizePath(targetUrl);
|
|
791
|
+
if (target === path) continue;
|
|
792
|
+
edges.push({
|
|
793
|
+
from: path,
|
|
794
|
+
to: target,
|
|
795
|
+
region: anchor.region
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
return graphFromEdges(nodes, edges, root);
|
|
800
|
+
};
|
|
801
|
+
/** A URL or an already-normalized graph path key both collapse to the path key. */
|
|
802
|
+
const pathKeyOf = (value) => {
|
|
803
|
+
try {
|
|
804
|
+
return normalizePath(new URL(value));
|
|
805
|
+
} catch {
|
|
806
|
+
return (value.split(/[?#]/, 1)[0] ?? "").replace(/\/+$/, "") || "/";
|
|
807
|
+
}
|
|
808
|
+
};
|
|
809
|
+
/**
|
|
810
|
+
* Rebuild a {@link RenderedGraph} from a rendered-edge set alone — the artifact
|
|
811
|
+
* `pagegraph links verify --emit-rendered` writes. Path keys are normalized the
|
|
812
|
+
* same way as a crawl, and `nodes` carries the rendered page paths the artifact
|
|
813
|
+
* recorded, so an edge-less page (a redirect target with no internal links) stays
|
|
814
|
+
* a node and its orphan/depth report survives replay. Without `nodes` the node set
|
|
815
|
+
* falls back to the edge endpoints.
|
|
816
|
+
*/
|
|
817
|
+
const renderedGraphFromEdges = (edges, root = "/", nodes = []) => {
|
|
818
|
+
const nodeSet = new Set(nodes.map(pathKeyOf));
|
|
819
|
+
const kept = [];
|
|
820
|
+
for (const edge of edges) {
|
|
821
|
+
const from = pathKeyOf(edge.from);
|
|
822
|
+
const to = pathKeyOf(edge.to);
|
|
823
|
+
nodeSet.add(from);
|
|
824
|
+
nodeSet.add(to);
|
|
825
|
+
kept.push({
|
|
826
|
+
from,
|
|
827
|
+
to,
|
|
828
|
+
region: edge.region
|
|
829
|
+
});
|
|
830
|
+
}
|
|
831
|
+
return graphFromEdges(nodeSet, kept, root);
|
|
832
|
+
};
|
|
833
|
+
const edgeKey = (edge) => `${edge.from}\u0000${edge.to}`;
|
|
834
|
+
const normalizeEdge = (edge) => ({
|
|
835
|
+
from: edge.from === "" ? "/" : edge.from.replace(/\/+$/, "") || "/",
|
|
836
|
+
to: edge.to === "" ? "/" : edge.to.replace(/\/+$/, "") || "/"
|
|
837
|
+
});
|
|
838
|
+
/**
|
|
839
|
+
* Diff rendered contextual edges against declared `related`/`crumb` edges. Both
|
|
840
|
+
* sides are path-keyed; the diff is directional and reports each direction
|
|
841
|
+
* separately because they call for different fixes.
|
|
842
|
+
*/
|
|
843
|
+
const diffLinkGraph = (declared, rendered) => {
|
|
844
|
+
const declaredNormalized = declared.map(normalizeEdge);
|
|
845
|
+
const declaredKeys = new Set(declaredNormalized.map(edgeKey));
|
|
846
|
+
const renderedKeys = new Set(rendered.contextualEdges.map(edgeKey));
|
|
847
|
+
return {
|
|
848
|
+
declaredNotRendered: declaredNormalized.filter((edge) => !renderedKeys.has(edgeKey(edge))),
|
|
849
|
+
renderedNotDeclared: rendered.internalEdges.filter((edge) => edge.region === "body").map((edge) => normalizeEdge(edge)).filter((edge) => !declaredKeys.has(edgeKey(edge))),
|
|
850
|
+
declaredCount: declaredNormalized.length,
|
|
851
|
+
renderedContextualCount: rendered.contextualEdges.length
|
|
852
|
+
};
|
|
853
|
+
};
|
|
854
|
+
/** The rendered-edge artifact schema this package writes and accepts. */
|
|
855
|
+
const RENDERED_EDGE_ARTIFACT_SCHEMA_VERSION = 1;
|
|
856
|
+
const ARTIFACT_KIND = "links-rendered";
|
|
857
|
+
const REGIONS = /* @__PURE__ */ new Set([
|
|
858
|
+
"nav",
|
|
859
|
+
"footer",
|
|
860
|
+
"header",
|
|
861
|
+
"body"
|
|
862
|
+
]);
|
|
863
|
+
const decodeArtifactEdges = (input) => {
|
|
864
|
+
if (!Array.isArray(input)) throw new Error("`edges` must be an array of { from, to, region } edges");
|
|
865
|
+
return input.map((item) => {
|
|
866
|
+
if (item === null || typeof item !== "object") throw new Error("each artifact edge must be an object with string `from` and `to`");
|
|
867
|
+
const { from, to, region } = item;
|
|
868
|
+
if (typeof from !== "string" || typeof to !== "string") throw new Error("each artifact edge must have string `from` and `to`");
|
|
869
|
+
if (region !== void 0 && !REGIONS.has(region)) throw new Error(`artifact edge region must be one of nav, footer, header, body (received ${JSON.stringify(region)})`);
|
|
870
|
+
return {
|
|
871
|
+
from,
|
|
872
|
+
to,
|
|
873
|
+
region: region ?? "body"
|
|
874
|
+
};
|
|
875
|
+
});
|
|
876
|
+
};
|
|
877
|
+
const decodeArtifactNodes = (input) => {
|
|
878
|
+
if (input === void 0) return [];
|
|
879
|
+
if (!Array.isArray(input) || !input.every((path) => typeof path === "string")) throw new Error("rendered-edge artifact `nodes` must be an array of paths");
|
|
880
|
+
return input;
|
|
881
|
+
};
|
|
882
|
+
const decodeArtifactFailures = (input) => {
|
|
883
|
+
if (input === void 0) return [];
|
|
884
|
+
if (!Array.isArray(input)) throw new Error("rendered-edge artifact `crawl.failures` must be an array");
|
|
885
|
+
return input.map((item) => {
|
|
886
|
+
if (item === null || typeof item !== "object") throw new Error("each artifact failure must be an object with string `url` and `error`");
|
|
887
|
+
const { url, error } = item;
|
|
888
|
+
if (typeof url !== "string" || typeof error !== "string") throw new Error("each artifact failure must have string `url` and `error`");
|
|
889
|
+
return {
|
|
890
|
+
url,
|
|
891
|
+
error
|
|
892
|
+
};
|
|
893
|
+
});
|
|
894
|
+
};
|
|
895
|
+
const requiredAbsoluteUrl = (value, field) => {
|
|
896
|
+
if (typeof value !== "string" || value === "") throw new Error(`rendered-edge artifact must carry a non-empty string \`${field}\``);
|
|
897
|
+
try {
|
|
898
|
+
new URL(value);
|
|
899
|
+
} catch {
|
|
900
|
+
throw new Error(`rendered-edge artifact \`${field}\` must be an absolute URL (received ${JSON.stringify(value)})`);
|
|
901
|
+
}
|
|
902
|
+
return value;
|
|
903
|
+
};
|
|
904
|
+
const requiredInteger = (value, field, minimum) => {
|
|
905
|
+
if (!Number.isSafeInteger(value) || value < minimum) throw new Error(`rendered-edge artifact \`${field}\` must be an integer >= ${minimum} (received ${JSON.stringify(value)})`);
|
|
906
|
+
return value;
|
|
907
|
+
};
|
|
908
|
+
/**
|
|
909
|
+
* Decode a rendered-edge artifact from user input. Throws on any other shape —
|
|
910
|
+
* this is user input, not a provider payload. `root`, `bodyTruncated`,
|
|
911
|
+
* `truncatedPages`, `failures`, and `nodes` are optional so a hand-written edge
|
|
912
|
+
* dump that only carries provenance can still be replayed.
|
|
913
|
+
*/
|
|
914
|
+
const decodeRenderedEdgeArtifact = (input) => {
|
|
915
|
+
if (input === null || typeof input !== "object") throw new Error("rendered-edge artifact must be an object");
|
|
916
|
+
const value = input;
|
|
917
|
+
if (value["kind"] !== ARTIFACT_KIND) throw new Error(`rendered-edge artifact \`kind\` must be "${ARTIFACT_KIND}" (received ${JSON.stringify(value["kind"])})`);
|
|
918
|
+
if (value["schemaVersion"] !== 1) throw new Error(`unsupported rendered-edge artifact schemaVersion: ${JSON.stringify(value["schemaVersion"])}`);
|
|
919
|
+
const origin = requiredAbsoluteUrl(value["origin"], "origin");
|
|
920
|
+
const seed = requiredAbsoluteUrl(value["seed"], "seed");
|
|
921
|
+
const crawl = value["crawl"];
|
|
922
|
+
if (crawl === null || typeof crawl !== "object") throw new Error("rendered-edge artifact must carry a `crawl` object");
|
|
923
|
+
const provenance = crawl;
|
|
924
|
+
const truncatedPages = provenance["truncatedPages"];
|
|
925
|
+
if (truncatedPages !== void 0 && (!Array.isArray(truncatedPages) || !truncatedPages.every((url) => typeof url === "string"))) throw new Error("rendered-edge artifact `truncatedPages` must be an array of URLs");
|
|
926
|
+
const bodyTruncated = provenance["bodyTruncated"];
|
|
927
|
+
if (bodyTruncated !== void 0 && typeof bodyTruncated !== "boolean") throw new Error("rendered-edge artifact `bodyTruncated` must be a boolean");
|
|
928
|
+
if (typeof provenance["truncated"] !== "boolean") throw new Error("rendered-edge artifact `crawl.truncated` must be a boolean");
|
|
929
|
+
const skipped = provenance["skipped"];
|
|
930
|
+
if (skipped !== void 0 && (!Array.isArray(skipped) || !skipped.every((url) => typeof url === "string"))) throw new Error("rendered-edge artifact `crawl.skipped` must be an array of URLs");
|
|
931
|
+
const nonHtml = provenance["nonHtml"];
|
|
932
|
+
if (nonHtml !== void 0 && (!Array.isArray(nonHtml) || !nonHtml.every((item) => item !== null && typeof item === "object" && typeof item.url === "string" && typeof item.finalUrl === "string" && typeof item.contentType === "string"))) throw new Error("rendered-edge artifact `crawl.nonHtml` must be an array of { url, finalUrl, contentType }");
|
|
933
|
+
if (Array.isArray(nonHtml)) for (const item of nonHtml) {
|
|
934
|
+
requiredAbsoluteUrl(item.url, "crawl.nonHtml.url");
|
|
935
|
+
requiredAbsoluteUrl(item.finalUrl, "crawl.nonHtml.finalUrl");
|
|
936
|
+
}
|
|
937
|
+
for (const field of ["sitemapTruncated", "sitemapUsed"]) if (provenance[field] !== void 0 && typeof provenance[field] !== "boolean") throw new Error(`rendered-edge artifact \`crawl.${field}\` must be a boolean`);
|
|
938
|
+
return {
|
|
939
|
+
kind: ARTIFACT_KIND,
|
|
940
|
+
schemaVersion: 1,
|
|
941
|
+
origin,
|
|
942
|
+
seed,
|
|
943
|
+
crawl: {
|
|
944
|
+
pages: requiredInteger(provenance["pages"], "crawl.pages", 0),
|
|
945
|
+
root: typeof provenance["root"] === "string" ? provenance["root"] : pathKeyOf(seed),
|
|
946
|
+
limit: requiredInteger(provenance["limit"], "crawl.limit", 1),
|
|
947
|
+
truncated: provenance["truncated"],
|
|
948
|
+
bodyTruncated: bodyTruncated === true,
|
|
949
|
+
truncatedPages: truncatedPages ?? [],
|
|
950
|
+
failures: decodeArtifactFailures(provenance["failures"]),
|
|
951
|
+
...nonHtml === void 0 ? {} : { nonHtml },
|
|
952
|
+
...provenance["discoveryFailures"] === void 0 ? {} : { discoveryFailures: decodeArtifactFailures(provenance["discoveryFailures"]) },
|
|
953
|
+
...skipped === void 0 ? {} : { skipped },
|
|
954
|
+
...provenance["sitemapTruncated"] === void 0 ? {} : { sitemapTruncated: provenance["sitemapTruncated"] },
|
|
955
|
+
...provenance["sitemapUsed"] === void 0 ? {} : { sitemapUsed: provenance["sitemapUsed"] }
|
|
956
|
+
},
|
|
957
|
+
nodes: decodeArtifactNodes(value["nodes"]),
|
|
958
|
+
edges: decodeArtifactEdges(value["edges"])
|
|
959
|
+
};
|
|
960
|
+
};
|
|
961
|
+
//#endregion
|
|
962
|
+
//#region src/core/content.ts
|
|
963
|
+
const NAMED_ENTITIES = {
|
|
964
|
+
amp: "&",
|
|
965
|
+
lt: "<",
|
|
966
|
+
gt: ">",
|
|
967
|
+
quot: "\"",
|
|
968
|
+
apos: "'",
|
|
969
|
+
nbsp: " ",
|
|
970
|
+
ndash: "–",
|
|
971
|
+
mdash: "—",
|
|
972
|
+
lsquo: "‘",
|
|
973
|
+
rsquo: "’",
|
|
974
|
+
ldquo: "“",
|
|
975
|
+
rdquo: "”",
|
|
976
|
+
hellip: "…",
|
|
977
|
+
middot: "·",
|
|
978
|
+
bull: "•",
|
|
979
|
+
copy: "©",
|
|
980
|
+
reg: "®",
|
|
981
|
+
trade: "™",
|
|
982
|
+
times: "×",
|
|
983
|
+
euro: "€",
|
|
984
|
+
pound: "£"
|
|
985
|
+
};
|
|
986
|
+
const decodeEntities = (value) => value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (entity, code) => {
|
|
987
|
+
if (code.startsWith("#")) {
|
|
988
|
+
const point = code[1] === "x" || code[1] === "X" ? Number.parseInt(code.slice(2), 16) : Number.parseInt(code.slice(1), 10);
|
|
989
|
+
return Number.isInteger(point) && point > 0 && point <= 1114111 ? String.fromCodePoint(point) : entity;
|
|
990
|
+
}
|
|
991
|
+
return NAMED_ENTITIES[code.toLowerCase()] ?? entity;
|
|
992
|
+
});
|
|
993
|
+
/** Elements whose contents a reader never sees as page text. */
|
|
994
|
+
const INVISIBLE = /<(script|style|template|noscript|svg)\b[\s\S]*?<\/\1\s*>/gi;
|
|
995
|
+
const COMMENT = /<!--[\s\S]*?-->/g;
|
|
996
|
+
const withoutInvisible = (html) => html.replace(COMMENT, " ").replace(INVISIBLE, " ");
|
|
997
|
+
const textOf = (html) => decodeEntities(html.replace(/<[^>]*>/g, " ")).replace(/\s+/g, " ").trim();
|
|
998
|
+
const attributeOf = (tag, name) => {
|
|
999
|
+
const match = new RegExp(`(?:^|\\s)${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i").exec(tag);
|
|
1000
|
+
const value = match?.[1] ?? match?.[2] ?? match?.[3];
|
|
1001
|
+
return value === void 0 ? void 0 : decodeEntities(value);
|
|
1002
|
+
};
|
|
1003
|
+
const bodyOf = (html) => {
|
|
1004
|
+
const body = /<body\b[^>]*>([\s\S]*)<\/body>/i.exec(html);
|
|
1005
|
+
if (body) return body[1];
|
|
1006
|
+
return html.replace(/<head\b[^>]*>[\s\S]*?<\/head>/i, " ");
|
|
1007
|
+
};
|
|
1008
|
+
/**
|
|
1009
|
+
* The main content region: `<main>` when the page has one, otherwise the body
|
|
1010
|
+
* without its `<nav>` and `<footer>` blocks. Site chrome repeats on every page,
|
|
1011
|
+
* so counting its headings or words would judge the layout, not the page.
|
|
1012
|
+
*/
|
|
1013
|
+
const mainRegionOf = (body) => {
|
|
1014
|
+
const main = /<main\b[^>]*>([\s\S]*?)<\/main\s*>/i.exec(body);
|
|
1015
|
+
if (main) return main[1];
|
|
1016
|
+
return body.replace(/<(nav|footer)\b[\s\S]*?<\/\1\s*>/gi, " ");
|
|
1017
|
+
};
|
|
1018
|
+
const HEADING = /<h([1-6])\b([^>]*)>([\s\S]*?)<\/h\1\s*>/gi;
|
|
1019
|
+
/**
|
|
1020
|
+
* A heading's accessible text: its content, else its `aria-label`, else the
|
|
1021
|
+
* `alt` of images inside it (a logo H1).
|
|
1022
|
+
*/
|
|
1023
|
+
const headingText = (attributes, inner) => {
|
|
1024
|
+
const text = textOf(inner);
|
|
1025
|
+
if (text !== "") return text;
|
|
1026
|
+
const label = attributeOf(attributes, "aria-label")?.trim();
|
|
1027
|
+
if (label) return label;
|
|
1028
|
+
return [...inner.matchAll(/<img\b[^>]*>/gi)].map((image) => attributeOf(image[0], "alt")?.trim() ?? "").filter(Boolean).join(" ");
|
|
1029
|
+
};
|
|
1030
|
+
const headingsOf = (html) => [...html.matchAll(HEADING)].map((match) => ({
|
|
1031
|
+
level: Number(match[1]),
|
|
1032
|
+
text: headingText(match[2], match[3])
|
|
1033
|
+
}));
|
|
1034
|
+
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1035
|
+
/** Every object with an `@type`, at any depth — offers nest in products, questions in FAQ pages. */
|
|
1036
|
+
const collectTyped = (value, into) => {
|
|
1037
|
+
if (Array.isArray(value)) {
|
|
1038
|
+
for (const entry of value) collectTyped(entry, into);
|
|
1039
|
+
return;
|
|
1040
|
+
}
|
|
1041
|
+
if (!isRecord(value)) return;
|
|
1042
|
+
if (value["@type"] !== void 0) into.push(value);
|
|
1043
|
+
for (const child of Object.values(value)) collectTyped(child, into);
|
|
1044
|
+
};
|
|
1045
|
+
const JSON_LD = /<script\b[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script\s*>/gi;
|
|
1046
|
+
/** Parse every JSON-LD block. A block that does not parse is `inspectHtml`'s finding, not ours. */
|
|
1047
|
+
const jsonLdOf = (html) => {
|
|
1048
|
+
const nodes = [];
|
|
1049
|
+
for (const match of html.matchAll(JSON_LD)) {
|
|
1050
|
+
let parsed;
|
|
1051
|
+
try {
|
|
1052
|
+
parsed = JSON.parse(match[1].trim());
|
|
1053
|
+
} catch {
|
|
1054
|
+
continue;
|
|
1055
|
+
}
|
|
1056
|
+
collectTyped(parsed, nodes);
|
|
1057
|
+
}
|
|
1058
|
+
return nodes;
|
|
1059
|
+
};
|
|
1060
|
+
/**
|
|
1061
|
+
* Text normalized for "does this appear on the page" comparisons: Unicode-folded,
|
|
1062
|
+
* lowercased, and reduced to runs of letters and digits. Punctuation, quotes,
|
|
1063
|
+
* dashes, and markup spacing differ freely between JSON-LD and rendered copy
|
|
1064
|
+
* without changing what a reader sees.
|
|
1065
|
+
*/
|
|
1066
|
+
const matchText = (value) => value.normalize("NFKC").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean).join(" ");
|
|
1067
|
+
const countWords = (text) => {
|
|
1068
|
+
const normalized = matchText(text);
|
|
1069
|
+
return normalized === "" ? 0 : normalized.split(" ").length;
|
|
1070
|
+
};
|
|
1071
|
+
/** Reduce a rendered HTML document to the fields the content rules read. Pure. */
|
|
1072
|
+
const extractPageContent = (html) => {
|
|
1073
|
+
const head = inspectHtml("", 200, html);
|
|
1074
|
+
const body = withoutInvisible(bodyOf(html));
|
|
1075
|
+
const main = mainRegionOf(body);
|
|
1076
|
+
return {
|
|
1077
|
+
title: head.title,
|
|
1078
|
+
description: head.description,
|
|
1079
|
+
robots: head.robots,
|
|
1080
|
+
canonical: head.canonical,
|
|
1081
|
+
h1: headingsOf(body).filter((heading) => heading.level === 1).map((heading) => heading.text),
|
|
1082
|
+
outline: headingsOf(main),
|
|
1083
|
+
wordCount: countWords(textOf(main)),
|
|
1084
|
+
text: textOf(body),
|
|
1085
|
+
jsonLd: jsonLdOf(html)
|
|
1086
|
+
};
|
|
1087
|
+
};
|
|
1088
|
+
const typesOf = (node) => {
|
|
1089
|
+
const type = node["@type"];
|
|
1090
|
+
if (typeof type === "string") return [type];
|
|
1091
|
+
if (Array.isArray(type)) return type.filter((entry) => typeof entry === "string");
|
|
1092
|
+
return [];
|
|
1093
|
+
};
|
|
1094
|
+
const hasType = (node, types) => typesOf(node).some((type) => types.has(type));
|
|
1095
|
+
const stringField = (node, field) => {
|
|
1096
|
+
const value = node[field];
|
|
1097
|
+
return typeof value === "string" && value.trim() !== "" ? value : void 0;
|
|
1098
|
+
};
|
|
1099
|
+
/** JSON-LD text may carry the HTML Google allows in answers; compare its text. */
|
|
1100
|
+
const jsonLdText = (value) => textOf(value);
|
|
1101
|
+
const isVisible = (visible, candidate) => {
|
|
1102
|
+
const needle = matchText(jsonLdText(candidate));
|
|
1103
|
+
return needle === "" || ` ${visible} `.includes(` ${needle} `);
|
|
1104
|
+
};
|
|
1105
|
+
const clip = (value, max = 80) => value.length > max ? `${value.slice(0, max - 1)}…` : value;
|
|
1106
|
+
const STOPWORDS = new Set("a an and are as at be but by can for from get has have how in into is it its more not of on or our per than that the their this to use via vs was what when who why will with you your".split(" "));
|
|
1107
|
+
/**
|
|
1108
|
+
* Terms that carry meaning, reduced to a crude stem: stopwords drop, a plural
|
|
1109
|
+
* `s` folds, and long words compare by their first six letters, so
|
|
1110
|
+
* "Authentication" and "Authenticate" or "templates" and "template" agree. The
|
|
1111
|
+
* rule only asks whether any term is shared, so a lenient stem errs toward
|
|
1112
|
+
* silence, never toward a false warning.
|
|
1113
|
+
*/
|
|
1114
|
+
const significantTerms = (value) => new Set(matchText(value).split(" ").filter((term) => term !== "" && !STOPWORDS.has(term) && (term.length > 2 || /\d/.test(term))).map((term) => term.length > 3 && term.endsWith("s") ? term.slice(0, -1) : term).map((term) => term.slice(0, 6)));
|
|
1115
|
+
/** schema.org `WebPage` and the subtypes whose `name`/`description` describe the page itself. */
|
|
1116
|
+
const WEB_PAGE_TYPES = /* @__PURE__ */ new Set([
|
|
1117
|
+
"WebPage",
|
|
1118
|
+
"AboutPage",
|
|
1119
|
+
"CheckoutPage",
|
|
1120
|
+
"CollectionPage",
|
|
1121
|
+
"ContactPage",
|
|
1122
|
+
"FAQPage",
|
|
1123
|
+
"ItemPage",
|
|
1124
|
+
"MedicalWebPage",
|
|
1125
|
+
"ProfilePage",
|
|
1126
|
+
"QAPage",
|
|
1127
|
+
"RealEstateListing",
|
|
1128
|
+
"SearchResultsPage"
|
|
1129
|
+
]);
|
|
1130
|
+
const OFFER_TYPES = /* @__PURE__ */ new Set(["Offer", "AggregateOffer"]);
|
|
1131
|
+
const FAQ_TYPES = /* @__PURE__ */ new Set(["FAQPage"]);
|
|
1132
|
+
const asArray = (value) => value === void 0 ? [] : Array.isArray(value) ? value : [value];
|
|
1133
|
+
/** Parse a schema.org price (`19`, `"19.00"`, `"1,000"`); undefined when it is not a number. */
|
|
1134
|
+
const parsePrice = (value) => {
|
|
1135
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : void 0;
|
|
1136
|
+
if (typeof value !== "string") return void 0;
|
|
1137
|
+
const cleaned = value.replace(/[^\d.]/g, "");
|
|
1138
|
+
if (cleaned === "" || !/^\d*\.?\d+$/.test(cleaned)) return void 0;
|
|
1139
|
+
return Number(cleaned);
|
|
1140
|
+
};
|
|
1141
|
+
const pricesOf = (offer) => [
|
|
1142
|
+
offer["price"],
|
|
1143
|
+
offer["lowPrice"],
|
|
1144
|
+
offer["highPrice"],
|
|
1145
|
+
...asArray(offer["priceSpecification"]).flatMap((spec) => isRecord(spec) ? [
|
|
1146
|
+
spec["price"],
|
|
1147
|
+
spec["minPrice"],
|
|
1148
|
+
spec["maxPrice"]
|
|
1149
|
+
] : [])
|
|
1150
|
+
].flatMap((value) => {
|
|
1151
|
+
const price = parsePrice(value);
|
|
1152
|
+
return price === void 0 ? [] : [price];
|
|
1153
|
+
});
|
|
1154
|
+
/**
|
|
1155
|
+
* Whether a price appears as a number in the visible text: `19` matches `$19`,
|
|
1156
|
+
* `19.00`, and `19/mo`; `1000` matches `1,000`; `0.5` matches `0.50`. A price of
|
|
1157
|
+
* zero also matches the word "free".
|
|
1158
|
+
*/
|
|
1159
|
+
const priceIsVisible = (price, text) => {
|
|
1160
|
+
if (price === 0 && /\bfree\b/i.test(text)) return true;
|
|
1161
|
+
const [whole, fraction = ""] = String(price).split(".");
|
|
1162
|
+
const grouped = whole.replace(/\B(?=(\d{3})+(?!\d))/g, "[,.\\u00a0 ]?");
|
|
1163
|
+
const decimals = fraction === "" ? "(?:[.,]0+)?" : `[.,]${fraction}0*`;
|
|
1164
|
+
return new RegExp(`(?<![\\d.,])${grouped}${decimals}(?![\\d]|[.,]\\d)`).test(text);
|
|
1165
|
+
};
|
|
1166
|
+
/**
|
|
1167
|
+
* Whether a JSON-LD `name` agrees with the rendered `<title>`. A title usually
|
|
1168
|
+
* wraps the page name in a site-name affix ("Pricing | Example"), so either
|
|
1169
|
+
* containing the other counts as agreement.
|
|
1170
|
+
*/
|
|
1171
|
+
const namesAgree = (a, b) => {
|
|
1172
|
+
const left = matchText(a);
|
|
1173
|
+
const right = matchText(b);
|
|
1174
|
+
if (left === "" || right === "") return true;
|
|
1175
|
+
return ` ${left} `.includes(` ${right} `) || ` ${right} `.includes(` ${left} `);
|
|
1176
|
+
};
|
|
1177
|
+
/**
|
|
1178
|
+
* Every content rule, in one place. Heading-count and structured-data rules that
|
|
1179
|
+
* Google treats as eligibility or policy problems are structural; outline and
|
|
1180
|
+
* wording smells are editorial.
|
|
1181
|
+
*/
|
|
1182
|
+
const CONTENT_RULES = [
|
|
1183
|
+
{
|
|
1184
|
+
name: "missing-h1",
|
|
1185
|
+
severity: "structural",
|
|
1186
|
+
evaluate: (content) => content.h1.length === 0 ? [{
|
|
1187
|
+
message: "Page has no H1.",
|
|
1188
|
+
fix: "Render exactly one <h1> that names the page's topic."
|
|
1189
|
+
}] : []
|
|
1190
|
+
},
|
|
1191
|
+
{
|
|
1192
|
+
name: "multiple-h1",
|
|
1193
|
+
severity: "structural",
|
|
1194
|
+
evaluate: (content) => content.h1.length > 1 ? [{
|
|
1195
|
+
message: `Page has ${content.h1.length} H1s: ${content.h1.map((text) => `"${clip(text, 40)}"`).join(", ")}.`,
|
|
1196
|
+
fix: "Keep one <h1> for the page topic; demote the others to <h2>."
|
|
1197
|
+
}] : []
|
|
1198
|
+
},
|
|
1199
|
+
{
|
|
1200
|
+
name: "empty-h1",
|
|
1201
|
+
severity: "structural",
|
|
1202
|
+
evaluate: (content) => content.h1.some((text) => text === "") ? [{
|
|
1203
|
+
message: "An H1 has no text, aria-label, or image alt.",
|
|
1204
|
+
fix: "Give the <h1> visible text that names the page's topic."
|
|
1205
|
+
}] : []
|
|
1206
|
+
},
|
|
1207
|
+
{
|
|
1208
|
+
name: "heading-level-skip",
|
|
1209
|
+
severity: "editorial",
|
|
1210
|
+
evaluate: (content) => {
|
|
1211
|
+
const skips = [];
|
|
1212
|
+
let previous = 1;
|
|
1213
|
+
for (const heading of content.outline) {
|
|
1214
|
+
if (heading.level > previous + 1) skips.push(`h${previous} → h${heading.level} "${clip(heading.text, 40)}"`);
|
|
1215
|
+
previous = heading.level;
|
|
1216
|
+
}
|
|
1217
|
+
return skips.length === 0 ? [] : [{
|
|
1218
|
+
message: `Heading levels skip ${skips.length} time(s): ${skips.slice(0, 3).join("; ")}${skips.length > 3 ? "; …" : ""}.`,
|
|
1219
|
+
fix: "Nest headings one level at a time; style a heading with CSS, not by choosing a deeper level."
|
|
1220
|
+
}];
|
|
1221
|
+
}
|
|
1222
|
+
},
|
|
1223
|
+
{
|
|
1224
|
+
name: "h1-title-mismatch",
|
|
1225
|
+
severity: "editorial",
|
|
1226
|
+
evaluate: (content) => {
|
|
1227
|
+
const [h1] = content.h1;
|
|
1228
|
+
if (content.h1.length !== 1 || !h1 || !content.title) return [];
|
|
1229
|
+
const title = significantTerms(content.title);
|
|
1230
|
+
return [...significantTerms(h1)].some((term) => title.has(term)) ? [] : [{
|
|
1231
|
+
message: `H1 "${clip(h1, 60)}" shares no significant term with the title "${clip(content.title, 60)}".`,
|
|
1232
|
+
fix: "Make the H1 and <title> describe the same topic; a searcher who clicks the title should recognize the page."
|
|
1233
|
+
}];
|
|
1234
|
+
}
|
|
1235
|
+
},
|
|
1236
|
+
{
|
|
1237
|
+
name: "faq-not-visible",
|
|
1238
|
+
severity: "structural",
|
|
1239
|
+
evaluate: (content) => {
|
|
1240
|
+
const visible = matchText(content.text);
|
|
1241
|
+
return content.jsonLd.filter((node) => hasType(node, FAQ_TYPES)).flatMap((faq) => asArray(faq["mainEntity"])).filter(isRecord).flatMap((question) => {
|
|
1242
|
+
const name = stringField(question, "name");
|
|
1243
|
+
const answers = asArray(question["acceptedAnswer"]).filter(isRecord).flatMap((answer) => {
|
|
1244
|
+
const text = stringField(answer, "text");
|
|
1245
|
+
return text === void 0 ? [] : [text];
|
|
1246
|
+
});
|
|
1247
|
+
const missing = [...name !== void 0 && !isVisible(visible, name) ? ["question"] : [], ...answers.some((answer) => !isVisible(visible, answer)) ? ["answer"] : []];
|
|
1248
|
+
return missing.length === 0 ? [] : [{
|
|
1249
|
+
message: `FAQPage ${missing.join(" and ")} for "${clip(name ?? "(unnamed)", 70)}" is not in the rendered page text.`,
|
|
1250
|
+
fix: "Render every FAQ question and answer in the server HTML, or drop the entry from the FAQPage JSON-LD."
|
|
1251
|
+
}];
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1254
|
+
},
|
|
1255
|
+
{
|
|
1256
|
+
name: "offer-price-not-visible",
|
|
1257
|
+
severity: "structural",
|
|
1258
|
+
evaluate: (content) => content.jsonLd.filter((node) => hasType(node, OFFER_TYPES)).flatMap((offer) => {
|
|
1259
|
+
const hidden = pricesOf(offer).filter((price) => !priceIsVisible(price, content.text));
|
|
1260
|
+
if (hidden.length === 0) return [];
|
|
1261
|
+
const currency = stringField(offer, "priceCurrency");
|
|
1262
|
+
return [{
|
|
1263
|
+
message: `${stringField(offer, "name") ?? typesOf(offer).join(", ")} price ${hidden.map((price) => `${price}${currency ? ` ${currency}` : ""}`).join(", ")} does not appear in the rendered page text.`,
|
|
1264
|
+
fix: "Show the offer's price in the server-rendered copy, or correct the JSON-LD price."
|
|
1265
|
+
}];
|
|
1266
|
+
})
|
|
1267
|
+
},
|
|
1268
|
+
{
|
|
1269
|
+
name: "webpage-head-mismatch",
|
|
1270
|
+
severity: "editorial",
|
|
1271
|
+
evaluate: (content) => content.jsonLd.filter((node) => hasType(node, WEB_PAGE_TYPES)).flatMap((page) => {
|
|
1272
|
+
const findings = [];
|
|
1273
|
+
const name = stringField(page, "name");
|
|
1274
|
+
if (name !== void 0 && content.title && !namesAgree(name, content.title)) findings.push({
|
|
1275
|
+
message: `${typesOf(page).join(", ")} name "${clip(name, 60)}" does not match the title "${clip(content.title, 60)}".`,
|
|
1276
|
+
fix: "Derive the JSON-LD name from the same value as the <title>."
|
|
1277
|
+
});
|
|
1278
|
+
const description = stringField(page, "description");
|
|
1279
|
+
if (description !== void 0 && content.description && matchText(description) !== matchText(content.description)) findings.push({
|
|
1280
|
+
message: `${typesOf(page).join(", ")} description differs from the meta description.`,
|
|
1281
|
+
fix: "Derive the JSON-LD description from the same value as the meta description."
|
|
1282
|
+
});
|
|
1283
|
+
return findings;
|
|
1284
|
+
})
|
|
1285
|
+
},
|
|
1286
|
+
{
|
|
1287
|
+
name: "thin-content",
|
|
1288
|
+
severity: "editorial",
|
|
1289
|
+
evaluate: (content, options) => options.minWords !== void 0 && content.wordCount < options.minWords ? [{
|
|
1290
|
+
message: `Main content has ${content.wordCount} words (min ${options.minWords}).`,
|
|
1291
|
+
fix: "Answer the page's search intent in more depth, or merge it into a stronger page."
|
|
1292
|
+
}] : []
|
|
1293
|
+
}
|
|
1294
|
+
];
|
|
1295
|
+
/** Run every content rule against one rendered page. Graph-independent; pure. */
|
|
1296
|
+
const checkPageContent = (content, options = {}) => CONTENT_RULES.flatMap((rule) => rule.evaluate(content, options).map((raw) => ({
|
|
1297
|
+
rule: rule.name,
|
|
1298
|
+
severity: rule.severity,
|
|
1299
|
+
message: raw.message,
|
|
1300
|
+
fix: raw.fix
|
|
1301
|
+
})));
|
|
1302
|
+
const declaresNoindex = (node) => node.policy.robots?.toLowerCase().includes("noindex") === true;
|
|
1303
|
+
/**
|
|
1304
|
+
* The declared pages a rendered check fetches: every sitemap-eligible page,
|
|
1305
|
+
* plus concrete pages that declare `noindex` so the rendered robots meta can be
|
|
1306
|
+
* held to that declaration. Param templates and redirects render no page of
|
|
1307
|
+
* their own.
|
|
1308
|
+
*/
|
|
1309
|
+
const renderedCheckTargets = (graph) => [...graph.nodes.values()].filter((node) => isSitemapEligible(node) || declaresNoindex(node) && node.policy.redirectTo === void 0 && !node.path.includes("$")).map((node) => node.path);
|
|
1310
|
+
const PATH_BASE = "https://pagegraph.invalid";
|
|
1311
|
+
/**
|
|
1312
|
+
* A path as a URL serializes it (`/café` → `/caf%C3%A9`), trailing slash
|
|
1313
|
+
* dropped. Declared paths, response URLs, and canonicals are compared in this
|
|
1314
|
+
* one form, or a page with non-ASCII segments would read as redirected.
|
|
1315
|
+
*/
|
|
1316
|
+
const urlPath = (value) => {
|
|
1317
|
+
try {
|
|
1318
|
+
return normalizePath(new URL(value, PATH_BASE));
|
|
1319
|
+
} catch {
|
|
1320
|
+
return;
|
|
1321
|
+
}
|
|
1322
|
+
};
|
|
1323
|
+
/**
|
|
1324
|
+
* Judge rendered documents against the declared graph. Every indexable page must
|
|
1325
|
+
* render, stay on its path, keep a self-referencing canonical and indexable
|
|
1326
|
+
* robots meta, and pass {@link checkPageContent}; a page that declares `noindex`
|
|
1327
|
+
* must render a `noindex` robots meta.
|
|
1328
|
+
*
|
|
1329
|
+
* Robots are read from the `<meta name="robots">` tag only. An `X-Robots-Tag`
|
|
1330
|
+
* header is usually host policy (preview hosts noindex everything), so holding
|
|
1331
|
+
* pages to it would fail every local run for a reason no declaration controls.
|
|
1332
|
+
*/
|
|
1333
|
+
function checkRenderedContent(graph, documents, policy = {}) {
|
|
1334
|
+
const violations = [];
|
|
1335
|
+
const push = (rule, severity, path, message, fix) => {
|
|
1336
|
+
violations.push({
|
|
1337
|
+
rule,
|
|
1338
|
+
severity,
|
|
1339
|
+
path,
|
|
1340
|
+
message,
|
|
1341
|
+
fix
|
|
1342
|
+
});
|
|
1343
|
+
};
|
|
1344
|
+
const wordRules = (policy.minWords ?? []).map((rule) => ({
|
|
1345
|
+
rule,
|
|
1346
|
+
matcher: globToRegExp(rule.path)
|
|
1347
|
+
}));
|
|
1348
|
+
const eligible = [...graph.nodes.values()].filter(isSitemapEligible).map((node) => node.path);
|
|
1349
|
+
for (const { rule, matcher } of wordRules) if (!eligible.some((path) => matcher.test(path))) violations.push({
|
|
1350
|
+
severity: "structural",
|
|
1351
|
+
rule: "content-rule-unmatched",
|
|
1352
|
+
message: `Word-count rule "${rule.path}" matches no sitemap-eligible page.`,
|
|
1353
|
+
fix: "Point the rule at a sitemap-eligible path, or drop it."
|
|
1354
|
+
});
|
|
1355
|
+
for (const document of documents) {
|
|
1356
|
+
const node = graph.nodes.get(document.path);
|
|
1357
|
+
if (node === void 0) continue;
|
|
1358
|
+
const declaredPath = urlPath(document.path);
|
|
1359
|
+
if (!document.ok) {
|
|
1360
|
+
push("rendered-page-unavailable", "structural", document.path, `Could not read the rendered page: ${document.error}.`, "Serve the declared page with a 2xx HTML response.");
|
|
1361
|
+
continue;
|
|
1362
|
+
}
|
|
1363
|
+
if (urlPath(document.finalPath) !== declaredPath) {
|
|
1364
|
+
push("rendered-page-unavailable", "structural", document.path, `Rendered page redirects to "${document.finalPath}".`, "Declare the redirect with `redirectTo`, or serve the page at its declared path.");
|
|
1365
|
+
continue;
|
|
1366
|
+
}
|
|
1367
|
+
const content = extractPageContent(document.html);
|
|
1368
|
+
const renderedNoindex = content.robots?.toLowerCase().includes("noindex") === true;
|
|
1369
|
+
if (!isSitemapEligible(node)) {
|
|
1370
|
+
if (!renderedNoindex) push("rendered-robots-mismatch", "structural", document.path, `Declared robots "${node.policy.robots}" but the rendered page has ${content.robots ? `robots "${content.robots}"` : "no robots meta"}.`, "Render the declared robots value in the page's <meta name=\"robots\">.");
|
|
1371
|
+
continue;
|
|
1372
|
+
}
|
|
1373
|
+
if (renderedNoindex) push("rendered-robots-mismatch", "structural", document.path, `Page is in the sitemap but renders robots "${content.robots}".`, "Drop the noindex from the rendered head, or declare the page noindex and remove its sitemap policy.");
|
|
1374
|
+
if ((content.canonical === void 0 ? void 0 : urlPath(content.canonical)) !== declaredPath) push("rendered-canonical-mismatch", "structural", document.path, content.canonical === void 0 ? "Rendered page has no canonical link." : `Rendered canonical "${content.canonical}" does not point at "${document.path}".`, "Render a self-referencing canonical for every indexable page.");
|
|
1375
|
+
const floors = wordRules.filter(({ matcher }) => matcher.test(document.path));
|
|
1376
|
+
const minWords = floors.length === 0 ? void 0 : Math.max(...floors.map(({ rule }) => rule.minWords));
|
|
1377
|
+
for (const finding of checkPageContent(content, { minWords })) push(finding.rule, finding.severity, document.path, finding.message, finding.fix);
|
|
1378
|
+
}
|
|
1379
|
+
return violations;
|
|
1380
|
+
}
|
|
1381
|
+
//#endregion
|
|
1382
|
+
export { lastModified as C, isSitemapEligible as S, renderSitemap as T, checkRenderedCoverage as _, RENDERED_EDGE_ARTIFACT_SCHEMA_VERSION as a, contentSignal as b, diffLinkGraph as c, renderedGraphFromEdges as d, hasBlockingIssues as f, checkGraph as g, checkFreshness as h, renderedCheckTargets as i, extractAnchors as l, checkCoverage as m, checkRenderedContent as n, buildRenderedGraph as o, inspectHtml as p, extractPageContent as r, decodeRenderedEdgeArtifact as s, checkPageContent as t, normalizePath as u, hasStructuralViolations as v, renderRobots as w, inspectNode as x, freshnessReport as y };
|
|
1383
|
+
|
|
1384
|
+
//# sourceMappingURL=content-DbyJoRpj.js.map
|