pagesight 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/docs/changes.md +118 -0
- package/docs/cloudflare.md +31 -0
- package/docs/crawl.md +64 -0
- package/docs/investigation.md +85 -0
- package/docs/monitoring.md +39 -0
- package/docs/rendering.md +102 -0
- package/docs/seo-agent-workflow.md +153 -0
- package/docs/usage.md +22 -19
- package/package.json +5 -2
- package/src/api/change-record.ts +31 -0
- package/src/api/cloudflare.ts +201 -0
- package/src/api/compare-snapshots.ts +1 -1
- package/src/api/crawl.ts +53 -0
- package/src/api/evaluate-change.ts +189 -0
- package/src/api/execute.ts +29 -0
- package/src/api/followup-changes.ts +264 -0
- package/src/api/http-url.ts +13 -0
- package/src/api/investigation.ts +344 -0
- package/src/api/schema.ts +103 -16
- package/src/api/technical-changes.ts +124 -0
- package/src/api/verify-render.ts +134 -0
- package/src/cli.ts +143 -43
- package/src/followup-manifest.ts +42 -0
- package/src/followup-text.ts +45 -0
- package/src/investigation-text.ts +32 -0
- package/src/providers/cloudflare.ts +46 -0
- package/src/tools/observe.ts +1 -1
- package/src/web/fetch.ts +2 -1
- package/src/web/render-browser.ts +190 -0
- package/src/web/render-dom.ts +72 -0
- package/src/web/render-network.ts +122 -0
- package/src/web/site-graph.ts +443 -0
package/src/api/schema.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { httpUrl } from "./http-url.js";
|
|
2
|
+
export { httpUrl } from "./http-url.js";
|
|
3
|
+
import { changeRecordSchema } from "./change-record.js";
|
|
1
4
|
import { uiFindingsSchema } from "./ui-findings.js";
|
|
2
5
|
import { z } from "zod";
|
|
3
6
|
import { snapshotEvidenceSchema } from "./evidence-schema.js";
|
|
@@ -100,17 +103,6 @@ export const gaRealtimeRequestSchema = z
|
|
|
100
103
|
.strict();
|
|
101
104
|
export type GaRealtimeRequest = z.infer<typeof gaRealtimeRequestSchema>;
|
|
102
105
|
|
|
103
|
-
export const httpUrl = z
|
|
104
|
-
.string()
|
|
105
|
-
.url()
|
|
106
|
-
.refine(
|
|
107
|
-
(v) =>
|
|
108
|
-
URL.canParse(v) &&
|
|
109
|
-
["http:", "https:"].includes(new URL(v).protocol) &&
|
|
110
|
-
!new URL(v).username &&
|
|
111
|
-
!new URL(v).password,
|
|
112
|
-
"Use an HTTP(S) URL without credentials",
|
|
113
|
-
);
|
|
114
106
|
export const configSchema = z
|
|
115
107
|
.object({
|
|
116
108
|
site: httpUrl,
|
|
@@ -154,13 +146,97 @@ export const configSchema = z
|
|
|
154
146
|
"Select at least one page or provider",
|
|
155
147
|
);
|
|
156
148
|
export type SiteConfig = z.infer<typeof configSchema>;
|
|
157
|
-
const assessedSnapshotSchema = snapshotEvidenceSchema.superRefine((snapshot, ctx) => {
|
|
149
|
+
export const assessedSnapshotSchema = snapshotEvidenceSchema.superRefine((snapshot, ctx) => {
|
|
158
150
|
const config = configSchema.safeParse(snapshot.pages[0].response.context.config);
|
|
159
151
|
if (!config.success)
|
|
160
152
|
for (const issue of config.error.issues)
|
|
161
153
|
ctx.addIssue({ ...issue, path: ["pages", 0, "response", "context", "config", ...issue.path] });
|
|
162
154
|
});
|
|
155
|
+
const renderRequestSchema = z
|
|
156
|
+
.object({
|
|
157
|
+
operation: z.literal("page.verify"),
|
|
158
|
+
url: httpUrl,
|
|
159
|
+
navigation: z
|
|
160
|
+
.object({ fromUrl: httpUrl, linkSelector: z.string().min(1).max(500) })
|
|
161
|
+
.strict()
|
|
162
|
+
.optional(),
|
|
163
|
+
settleMs: z.number().int().min(0).max(5000).default(1000),
|
|
164
|
+
timeoutMs: z.number().int().min(1000).max(60000).default(20000),
|
|
165
|
+
viewport: z
|
|
166
|
+
.object({ width: z.number().int().min(320).max(2560), height: z.number().int().min(240).max(2160) })
|
|
167
|
+
.strict()
|
|
168
|
+
.default({ width: 1280, height: 800 }),
|
|
169
|
+
})
|
|
170
|
+
.strict();
|
|
163
171
|
const operationVariants = z.discriminatedUnion("operation", [
|
|
172
|
+
renderRequestSchema,
|
|
173
|
+
z
|
|
174
|
+
.object({
|
|
175
|
+
operation: z.literal("change.followup"),
|
|
176
|
+
experiments: z
|
|
177
|
+
.array(z.unknown())
|
|
178
|
+
.min(1)
|
|
179
|
+
.max(50)
|
|
180
|
+
.describe(
|
|
181
|
+
"Entries: {label, record: change record, baseline: configured snapshot, current?: configured snapshot}. Each entry is validated independently; use embedded objects, not filesystem paths.",
|
|
182
|
+
),
|
|
183
|
+
asOf: z.string().datetime().optional(),
|
|
184
|
+
lagDays: z.number().int().min(1).max(30).default(3),
|
|
185
|
+
})
|
|
186
|
+
.strict(),
|
|
187
|
+
|
|
188
|
+
z
|
|
189
|
+
.object({
|
|
190
|
+
operation: z.literal("cloudflare.audit"),
|
|
191
|
+
zone: z.string().regex(/^[a-f0-9]{32}$/),
|
|
192
|
+
hostname: z
|
|
193
|
+
.string()
|
|
194
|
+
.min(1)
|
|
195
|
+
.max(253)
|
|
196
|
+
.regex(/^[a-zA-Z0-9.-]+$/),
|
|
197
|
+
startTime: z.string().datetime(),
|
|
198
|
+
endTime: z.string().datetime(),
|
|
199
|
+
limit: z.number().int().min(1).max(100).default(50),
|
|
200
|
+
})
|
|
201
|
+
.strict(),
|
|
202
|
+
z
|
|
203
|
+
.object({
|
|
204
|
+
operation: z.literal("change.evaluate"),
|
|
205
|
+
record: changeRecordSchema,
|
|
206
|
+
baseline: assessedSnapshotSchema,
|
|
207
|
+
current: assessedSnapshotSchema.optional(),
|
|
208
|
+
maxRows: z.number().int().min(1).max(100).default(20),
|
|
209
|
+
})
|
|
210
|
+
.strict(),
|
|
211
|
+
z
|
|
212
|
+
.object({
|
|
213
|
+
operation: z.literal("crawl"),
|
|
214
|
+
config: configSchema,
|
|
215
|
+
maxPages: z.number().int().min(1).max(100).default(20),
|
|
216
|
+
maxDepth: z.number().int().min(0).max(10).default(3),
|
|
217
|
+
maxLinks: z.number().int().min(1).max(1000).default(500),
|
|
218
|
+
includeQuery: z.boolean().default(false),
|
|
219
|
+
inspectLimit: z.number().int().min(0).max(10).default(3),
|
|
220
|
+
})
|
|
221
|
+
.strict(),
|
|
222
|
+
z
|
|
223
|
+
.object({
|
|
224
|
+
operation: z.literal("technical.compare"),
|
|
225
|
+
baseline: assessedSnapshotSchema.optional(),
|
|
226
|
+
current: assessedSnapshotSchema,
|
|
227
|
+
})
|
|
228
|
+
.strict(),
|
|
229
|
+
z
|
|
230
|
+
.object({
|
|
231
|
+
operation: z.literal("investigate"),
|
|
232
|
+
config: configSchema,
|
|
233
|
+
url: httpUrl,
|
|
234
|
+
startDate: dateSchema,
|
|
235
|
+
endDate: dateSchema,
|
|
236
|
+
maxPages: maxPagesSchema.default(1),
|
|
237
|
+
maxRows: z.number().int().min(1).max(100).default(28),
|
|
238
|
+
})
|
|
239
|
+
.strict(),
|
|
164
240
|
z
|
|
165
241
|
.object({
|
|
166
242
|
operation: z.literal("opportunities"),
|
|
@@ -264,8 +340,19 @@ const operationVariants = z.discriminatedUnion("operation", [
|
|
|
264
340
|
})
|
|
265
341
|
.strict(),
|
|
266
342
|
]);
|
|
267
|
-
export const operationSchema = operationVariants
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
343
|
+
export const operationSchema = operationVariants
|
|
344
|
+
.refine(
|
|
345
|
+
(op) =>
|
|
346
|
+
op.operation !== "cloudflare.audit" ||
|
|
347
|
+
(Date.parse(op.startTime) < Date.parse(op.endTime) &&
|
|
348
|
+
Date.parse(op.endTime) - Date.parse(op.startTime) <= 86400000 &&
|
|
349
|
+
Date.parse(op.endTime) <= Date.now()),
|
|
350
|
+
"Cloudflare requires a past UTC interval of at most 24 hours",
|
|
351
|
+
)
|
|
352
|
+
.refine(
|
|
353
|
+
(op) =>
|
|
354
|
+
(op.operation !== "snapshot" && op.operation !== "investigate") ||
|
|
355
|
+
(op.startDate <= op.endDate && op.endDate <= pacificDate()),
|
|
356
|
+
"Invalid or future reporting interval",
|
|
357
|
+
);
|
|
271
358
|
export type Operation = z.input<typeof operationSchema>;
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { capture } from "./evidence.js";
|
|
3
|
+
import type { ImportedSnapshot } from "./evidence-schema.js";
|
|
4
|
+
import { configSchema } from "./schema.js";
|
|
5
|
+
import { canonical } from "./report-table.js";
|
|
6
|
+
const pageSchema = z.object({
|
|
7
|
+
url: z.string().url(),
|
|
8
|
+
finalUrl: z.string().url(),
|
|
9
|
+
status: z.number().int(),
|
|
10
|
+
canonical: z.string().nullable(),
|
|
11
|
+
robots: z.array(z.string()),
|
|
12
|
+
xRobotsTag: z.string().nullable(),
|
|
13
|
+
title: z.string().nullable(),
|
|
14
|
+
description: z.string().nullable(),
|
|
15
|
+
sha256: z.string(),
|
|
16
|
+
redirects: z.array(z.unknown()),
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
export function technicalChanges(baseline: ImportedSnapshot | undefined, current: ImportedSnapshot) {
|
|
20
|
+
const hash = (v: unknown) => Bun.CryptoHasher.hash("sha256", canonical(v), "hex");
|
|
21
|
+
const before = baseline?.pages[0].response;
|
|
22
|
+
const after = current.pages[0].response;
|
|
23
|
+
const sameScope = Boolean(
|
|
24
|
+
baseline &&
|
|
25
|
+
baseline.target === current.target &&
|
|
26
|
+
canonical(before?.context.config) === canonical(after.context.config),
|
|
27
|
+
);
|
|
28
|
+
return capture(
|
|
29
|
+
"pagesight",
|
|
30
|
+
"technical.compare",
|
|
31
|
+
current.target,
|
|
32
|
+
{ baselineSha256: baseline ? hash(baseline) : null, currentSha256: hash(current) },
|
|
33
|
+
async () => {
|
|
34
|
+
const alerts: Array<{ kind: string; source: string; message: string; before?: unknown; after?: unknown }> = [];
|
|
35
|
+
for (const url of configSchema.parse(after.context.config).pages) {
|
|
36
|
+
if (!after.observations.some((o) => o.provider === "web" && o.operation === "page" && o.target === url))
|
|
37
|
+
alerts.push({
|
|
38
|
+
kind: "unknown",
|
|
39
|
+
source: `page:${url}`,
|
|
40
|
+
message: "Configured page observation is missing; no regression inferred",
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
for (const observation of after.observations) {
|
|
44
|
+
if (observation.status === "error" || observation.error) {
|
|
45
|
+
alerts.push({
|
|
46
|
+
kind: "availability",
|
|
47
|
+
source: observation.name,
|
|
48
|
+
message: `Observation unavailable/incomplete: ${observation.error?.code ?? observation.status}`,
|
|
49
|
+
});
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (observation.provider !== "web" || observation.operation !== "page") continue;
|
|
53
|
+
const parsed = pageSchema.safeParse(observation.pages[0]?.response);
|
|
54
|
+
const request = observation.pages[0]?.request as { url?: string } | undefined;
|
|
55
|
+
if (!parsed.success || request?.url !== parsed.data.url || observation.target !== parsed.data.url) {
|
|
56
|
+
alerts.push({
|
|
57
|
+
kind: "unknown",
|
|
58
|
+
source: observation.name,
|
|
59
|
+
message: "Page response is unavailable or not bound to the requested URL",
|
|
60
|
+
});
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (parsed.data.status >= 400)
|
|
64
|
+
alerts.push({
|
|
65
|
+
kind: "http-error",
|
|
66
|
+
source: observation.name,
|
|
67
|
+
message: `HTTP ${parsed.data.status} observed`,
|
|
68
|
+
after: parsed.data.status,
|
|
69
|
+
});
|
|
70
|
+
if (!sameScope) continue;
|
|
71
|
+
const previous = before?.observations.find(
|
|
72
|
+
(o) => o.provider === "web" && o.operation === "page" && o.target === observation.target,
|
|
73
|
+
);
|
|
74
|
+
const prior = pageSchema.safeParse(previous?.pages[0]?.response);
|
|
75
|
+
const priorRequest = previous?.pages[0]?.request as { url?: string } | undefined;
|
|
76
|
+
if (
|
|
77
|
+
!prior.success ||
|
|
78
|
+
priorRequest?.url !== parsed.data.url ||
|
|
79
|
+
prior.data.url !== parsed.data.url ||
|
|
80
|
+
previous?.status === "error" ||
|
|
81
|
+
previous?.error
|
|
82
|
+
) {
|
|
83
|
+
alerts.push({
|
|
84
|
+
kind: "unknown",
|
|
85
|
+
source: observation.name,
|
|
86
|
+
message: "No usable matching prior page; no regression inferred",
|
|
87
|
+
});
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
for (const field of [
|
|
91
|
+
"status",
|
|
92
|
+
"finalUrl",
|
|
93
|
+
"canonical",
|
|
94
|
+
"robots",
|
|
95
|
+
"xRobotsTag",
|
|
96
|
+
"redirects",
|
|
97
|
+
"title",
|
|
98
|
+
"description",
|
|
99
|
+
"sha256",
|
|
100
|
+
] as const) {
|
|
101
|
+
if (canonical(prior.data[field]) !== canonical(parsed.data[field]))
|
|
102
|
+
alerts.push({
|
|
103
|
+
kind: ["title", "description", "sha256"].includes(field) ? "content-change" : "technical-change",
|
|
104
|
+
source: observation.name,
|
|
105
|
+
message: `Observed ${field} changed; verify against intended route policy`,
|
|
106
|
+
before: prior.data[field],
|
|
107
|
+
after: parsed.data[field],
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
comparisonVersion: 1,
|
|
113
|
+
baselineState: !baseline ? "first-run" : sameScope ? "compatible" : "scope-reset",
|
|
114
|
+
alerts,
|
|
115
|
+
limitations: [
|
|
116
|
+
"Exact common requested URLs and equal configuration only. Added or removed URLs reset the baseline.",
|
|
117
|
+
"Changes are observations, not automatically defects; noindex and canonical policy may be intentional.",
|
|
118
|
+
"Daily reporting windows overlap. No traffic deltas, disappearance claims, ranking impact or cross-provider comparisons are calculated.",
|
|
119
|
+
"HTML only; rendered content and Google's stored indexing state require separate checks.",
|
|
120
|
+
],
|
|
121
|
+
};
|
|
122
|
+
},
|
|
123
|
+
);
|
|
124
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { startRenderNetwork } from "../web/render-network.js";
|
|
2
|
+
import { capture } from "./evidence.js";
|
|
3
|
+
import { fetchText } from "../web/fetch.js";
|
|
4
|
+
import { launchRenderer, parseServer, captureRendered, type RenderOptions } from "../web/render-browser.js";
|
|
5
|
+
import type { SeoDom } from "../web/render-dom.js";
|
|
6
|
+
import { RequestError } from "../shared/http.js";
|
|
7
|
+
|
|
8
|
+
function compare(a: SeoDom | undefined, b: SeoDom | undefined) {
|
|
9
|
+
if (!a || !b || a.url !== b.url || a.truncated || b.truncated)
|
|
10
|
+
return { status: "unavailable", reason: "Both complete captures must identify the same exact final URL." };
|
|
11
|
+
const fields = ["baseHref", "titles", "descriptions", "canonicals", "robots", "h1", "jsonLd", "links"] as const;
|
|
12
|
+
const differences = fields
|
|
13
|
+
.filter((field) => JSON.stringify(a[field]) !== JSON.stringify(b[field]))
|
|
14
|
+
.map((field) => ({ field, before: a[field], after: b[field] }));
|
|
15
|
+
return {
|
|
16
|
+
status: differences.length ? "different" : "equal",
|
|
17
|
+
scope: "Extracted DOM fields only; HTTP response evidence is compared separately.",
|
|
18
|
+
differences,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export async function verifyRender(options: RenderOptions) {
|
|
22
|
+
return capture("pagesight", "page.verify", options.url, options, async () => {
|
|
23
|
+
const network = await startRenderNetwork(options.url);
|
|
24
|
+
try {
|
|
25
|
+
const browser = await launchRenderer(network.proxy);
|
|
26
|
+
try {
|
|
27
|
+
const server = await capture("web", "server_html", options.url, { url: options.url }, async () => {
|
|
28
|
+
const { body, finalUrl, response, redirects } = await fetchText(
|
|
29
|
+
options.url,
|
|
30
|
+
2_000_000,
|
|
31
|
+
new URL(options.url).origin,
|
|
32
|
+
network.proxy,
|
|
33
|
+
);
|
|
34
|
+
if (!response.headers.get("content-type")?.includes("text/html"))
|
|
35
|
+
throw new RequestError("Server response is not HTML.", response.status, "not_html");
|
|
36
|
+
return {
|
|
37
|
+
finalUrl,
|
|
38
|
+
status: response.status,
|
|
39
|
+
redirects,
|
|
40
|
+
xRobotsTag: response.headers.get("x-robots-tag"),
|
|
41
|
+
contentType: response.headers.get("content-type"),
|
|
42
|
+
sha256: Bun.CryptoHasher.hash("sha256", body, "hex"),
|
|
43
|
+
bytes: Buffer.byteLength(body),
|
|
44
|
+
dom: await parseServer(browser, body, finalUrl),
|
|
45
|
+
};
|
|
46
|
+
});
|
|
47
|
+
const direct = await capture("chromium", "direct", options.url, options, () =>
|
|
48
|
+
captureRendered(browser, options, false),
|
|
49
|
+
);
|
|
50
|
+
const navigation = options.navigation
|
|
51
|
+
? await capture("chromium", "internal_navigation", options.url, options, () =>
|
|
52
|
+
captureRendered(browser, options, true),
|
|
53
|
+
)
|
|
54
|
+
: null;
|
|
55
|
+
const getDom = (e: typeof server | null) => (e?.pages[0]?.response as { dom?: SeoDom } | undefined)?.dom;
|
|
56
|
+
return {
|
|
57
|
+
verificationVersion: 1,
|
|
58
|
+
browser: { engine: "chromium", version: browser.version(), viewport: options.viewport },
|
|
59
|
+
capturePolicy: {
|
|
60
|
+
waitUntil: "domcontentloaded",
|
|
61
|
+
settleMs: options.settleMs,
|
|
62
|
+
timeoutMsPerBrowserPath: options.timeoutMs,
|
|
63
|
+
anonymousFreshContextPerPath: true,
|
|
64
|
+
maxRequestsPerPath: 500,
|
|
65
|
+
allowedMethods: ["GET", "HEAD"],
|
|
66
|
+
serviceWorkers: "blocked",
|
|
67
|
+
webSockets: "blocked",
|
|
68
|
+
topLevelOrigin: new URL(options.url).origin,
|
|
69
|
+
},
|
|
70
|
+
network: {
|
|
71
|
+
blockedRequests: network.blockedRequests,
|
|
72
|
+
policy:
|
|
73
|
+
"Public addresses only; an explicitly requested literal private IP permits that exact host and port.",
|
|
74
|
+
},
|
|
75
|
+
observations: { server, direct, navigation },
|
|
76
|
+
documentComparison: (() => {
|
|
77
|
+
const a = server.pages[0]?.response as { status: number; xRobotsTag: string | null } | undefined;
|
|
78
|
+
const b = (
|
|
79
|
+
direct.pages[0]?.response as
|
|
80
|
+
| { targetDocumentResponse?: { status: number; xRobotsTag: string | null } }
|
|
81
|
+
| undefined
|
|
82
|
+
)?.targetDocumentResponse;
|
|
83
|
+
if (!a || !b)
|
|
84
|
+
return {
|
|
85
|
+
status: "unavailable",
|
|
86
|
+
reason: "Both server and target browser document responses are required.",
|
|
87
|
+
};
|
|
88
|
+
const fields = ["status", "xRobotsTag"] as const;
|
|
89
|
+
return {
|
|
90
|
+
status: fields.every((f) => a[f] === b[f]) ? "equal" : "different",
|
|
91
|
+
fields: fields.map((field) => ({ field, server: a[field], browser: b[field] })),
|
|
92
|
+
};
|
|
93
|
+
})(),
|
|
94
|
+
comparisons: {
|
|
95
|
+
serverToDirect: compare(getDom(server), getDom(direct)),
|
|
96
|
+
directToNavigation: navigation ? compare(getDom(direct), getDom(navigation)) : { status: "not_requested" },
|
|
97
|
+
},
|
|
98
|
+
limitations: [
|
|
99
|
+
"Bounded anonymous Chromium observation, not Google rendering or indexing evidence.",
|
|
100
|
+
"A fixed settle interval does not prove hydration or network completion. Captures occur at different times and can see different deployments or personalized content.",
|
|
101
|
+
"Differences are observations, not automatic SEO defects. Equal extracted fields do not prove visible layout, complete content or functional correctness.",
|
|
102
|
+
"Only GET/HEAD requests run; service workers and cross-origin top-level navigation are blocked. This may alter application behavior; ordinary page requests can still reach analytics.",
|
|
103
|
+
"Server HTML is fetched independently with Pagesight's user agent, then parsed inertly in Chromium. A client route may have no target document response; source-document HTTP status is not target status.",
|
|
104
|
+
"No authenticated profile, storage state, browser actions other than one validated anchor click, or arbitrary evaluation scripts are accepted.",
|
|
105
|
+
],
|
|
106
|
+
};
|
|
107
|
+
} finally {
|
|
108
|
+
await browser.close();
|
|
109
|
+
}
|
|
110
|
+
} finally {
|
|
111
|
+
await network.close();
|
|
112
|
+
}
|
|
113
|
+
}).then((result) => {
|
|
114
|
+
const data = result.pages[0]?.response as
|
|
115
|
+
| {
|
|
116
|
+
observations: Record<
|
|
117
|
+
string,
|
|
118
|
+
{ status: string; pages: Array<{ response: { dom?: SeoDom; documentsTruncated?: boolean } }> } | null
|
|
119
|
+
>;
|
|
120
|
+
comparisons: Record<string, { status: string }>;
|
|
121
|
+
}
|
|
122
|
+
| undefined;
|
|
123
|
+
if (
|
|
124
|
+
data &&
|
|
125
|
+
(Object.values(data.observations).some(
|
|
126
|
+
(e) =>
|
|
127
|
+
e && (e.status !== "ok" || e.pages[0]?.response.dom?.truncated || e.pages[0]?.response.documentsTruncated),
|
|
128
|
+
) ||
|
|
129
|
+
Object.values(data.comparisons).some((c) => c.status === "unavailable"))
|
|
130
|
+
)
|
|
131
|
+
result.status = "partial";
|
|
132
|
+
return result;
|
|
133
|
+
});
|
|
134
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { loadFollowupManifest } from "./followup-manifest.js";
|
|
2
|
+
import { renderFollowup } from "./followup-text.js";
|
|
3
|
+
import { renderInvestigation } from "./investigation-text.js";
|
|
1
4
|
import { renderAssessment } from "./assessment-text.js";
|
|
2
5
|
import { renderOpportunities } from "./opportunities-text.js";
|
|
3
6
|
import { parseArgs } from "node:util";
|
|
@@ -10,6 +13,11 @@ export const help = `Pagesight — read-only site evidence
|
|
|
10
13
|
pagesight Show CLI help
|
|
11
14
|
pagesight mcp Start MCP explicitly
|
|
12
15
|
pagesight discover --url https://example.com/ [--providers gsc,ga]
|
|
16
|
+
pagesight cloudflare audit --zone ZONE_ID --hostname example.com --start UTC_TIME --end UTC_TIME [--limit 50]
|
|
17
|
+
|
|
18
|
+
pagesight change followup --manifest experiments.json [--as-of UTC_TIME] [--lag-days 3] [--format text]
|
|
19
|
+
pagesight change evaluate --record change.json --baseline before.json [--current after.json] [--max-rows 20]
|
|
20
|
+
pagesight technical compare --current current.json [--baseline previous.json]
|
|
13
21
|
pagesight doctor --config seo.config.json
|
|
14
22
|
pagesight gsc sites
|
|
15
23
|
pagesight gsc sitemaps --site sc-domain:example.com
|
|
@@ -29,11 +37,14 @@ pagesight ga accounts
|
|
|
29
37
|
pagesight ga property --property 123456
|
|
30
38
|
pagesight ga key-events --property 123456
|
|
31
39
|
pagesight ga report --property 123456 --request report.json [--max-pages 4]
|
|
40
|
+
pagesight render --url URL [--from-url URL --link-selector CSS] [--settle-ms 1000] [--timeout-ms 20000]
|
|
32
41
|
pagesight page --url https://example.com/
|
|
33
42
|
pagesight speed psi --url https://example.com/ [--strategy mobile]
|
|
34
43
|
pagesight speed crux --url https://example.com/ [--origin] [--form-factor PHONE]
|
|
35
44
|
pagesight speed history --url https://example.com/ [--origin]
|
|
36
45
|
pagesight snapshot --config seo.config.json [--start YYYY-MM-DD --end YYYY-MM-DD]
|
|
46
|
+
pagesight crawl --config seo.config.json [--max-pages 20] [--max-depth 3] [--inspect-limit 3]
|
|
47
|
+
pagesight investigate --config seo.config.json --url https://example.com/page [--start YYYY-MM-DD --end YYYY-MM-DD] [--format text]
|
|
37
48
|
pagesight assess --snapshot saved.json [--format text] [--max-rows 10]
|
|
38
49
|
pagesight opportunities --snapshot saved.json [--min-impressions 20] [--max-clicks 2] [--max-rows 10] [--format text]
|
|
39
50
|
pagesight compare --baseline before.json --current after.json [--max-rows 100]
|
|
@@ -41,7 +52,7 @@ pagesight evidence import --request findings.json
|
|
|
41
52
|
pagesight api --request operation.json
|
|
42
53
|
pagesight serve [--port 6095] Local HTTP API; requires PAGESIGHT_API_TOKEN
|
|
43
54
|
|
|
44
|
-
Data commands emit JSON by default; assess --format text
|
|
55
|
+
Data commands emit JSON by default; assess, opportunities, investigate and change followup accept --format text for readable summaries. --json is accepted for clarity.
|
|
45
56
|
--out FILE saves the same evidence locally. Exit: 0 success, 1 provider failure,
|
|
46
57
|
2 invalid input, 3 partial evidence. Snapshot defaults to 28 days ending Pacific
|
|
47
58
|
today minus 3 days; max-pages defaults to 4 (ad hoc reports: 1; maximum: 20).
|
|
@@ -49,6 +60,48 @@ Credentials: GSC_* env, PAGESIGHT_GA_CREDENTIALS or GOOGLE_APPLICATION_CREDENTIA
|
|
|
49
60
|
Requests, property metadata, provider limits and errors are preserved in evidence.
|
|
50
61
|
`;
|
|
51
62
|
|
|
63
|
+
// Every CLI command and the flags it accepts; surface tests fail when one has no case.
|
|
64
|
+
export const commands: Record<string, string[]> = {
|
|
65
|
+
render: ["url", "from-url", "link-selector", "settle-ms", "timeout-ms"],
|
|
66
|
+
"cloudflare.audit": ["zone", "hostname", "start", "end", "limit"],
|
|
67
|
+
"change.followup": ["manifest", "as-of", "lag-days", "format"],
|
|
68
|
+
"change.evaluate": ["record", "baseline", "current", "max-rows"],
|
|
69
|
+
"technical.compare": ["baseline", "current"],
|
|
70
|
+
"evidence.import": ["request"],
|
|
71
|
+
crawl: ["config", "max-pages", "max-depth", "max-links", "inspect-limit", "include-query"],
|
|
72
|
+
investigate: ["config", "url", "start", "end", "max-pages", "max-rows", "format"],
|
|
73
|
+
assess: ["snapshot", "max-rows", "format"],
|
|
74
|
+
opportunities: ["snapshot", "max-rows", "format", "min-impressions", "max-clicks"],
|
|
75
|
+
discover: ["url", "providers"],
|
|
76
|
+
compare: ["baseline", "current", "max-rows"],
|
|
77
|
+
"bing.crawl-stats": ["site"],
|
|
78
|
+
"bing.crawl-issues": ["site"],
|
|
79
|
+
"bing.url-info": ["site", "url"],
|
|
80
|
+
"bing.link-counts": ["site", "max-pages"],
|
|
81
|
+
"bing.url-links": ["site", "url", "max-pages"],
|
|
82
|
+
"bing.sites": [],
|
|
83
|
+
"bing.queries": ["site"],
|
|
84
|
+
"bing.pages": ["site"],
|
|
85
|
+
"bing.traffic": ["site"],
|
|
86
|
+
"gsc.sites": [],
|
|
87
|
+
"gsc.sitemaps": ["site"],
|
|
88
|
+
"gsc.inspect": ["site", "url"],
|
|
89
|
+
"gsc.report": ["site", "request", "max-pages"],
|
|
90
|
+
"ga.realtime": ["property", "request"],
|
|
91
|
+
"ga.accounts": [],
|
|
92
|
+
"ga.property": ["property"],
|
|
93
|
+
"ga.key-events": ["property"],
|
|
94
|
+
"ga.report": ["property", "request", "max-pages"],
|
|
95
|
+
page: ["url"],
|
|
96
|
+
"speed.psi": ["url", "strategy"],
|
|
97
|
+
"speed.crux": ["url", "origin", "form-factor"],
|
|
98
|
+
"speed.history": ["url", "origin", "form-factor"],
|
|
99
|
+
doctor: ["config"],
|
|
100
|
+
snapshot: ["config", "start", "end", "max-pages"],
|
|
101
|
+
api: ["request"],
|
|
102
|
+
serve: ["port"],
|
|
103
|
+
};
|
|
104
|
+
|
|
52
105
|
async function jsonFile(path: string | undefined, label: string): Promise<unknown> {
|
|
53
106
|
if (!path) throw new RequestError(`Missing --${label}`, null, "invalid_input");
|
|
54
107
|
try {
|
|
@@ -65,6 +118,17 @@ export async function runCli(args: string[]): Promise<number> {
|
|
|
65
118
|
allowPositionals: true,
|
|
66
119
|
strict: true,
|
|
67
120
|
options: {
|
|
121
|
+
zone: { type: "string" },
|
|
122
|
+
hostname: { type: "string" },
|
|
123
|
+
"from-url": { type: "string" },
|
|
124
|
+
"link-selector": { type: "string" },
|
|
125
|
+
"settle-ms": { type: "string" },
|
|
126
|
+
"timeout-ms": { type: "string" },
|
|
127
|
+
limit: { type: "string" },
|
|
128
|
+
record: { type: "string" },
|
|
129
|
+
manifest: { type: "string" },
|
|
130
|
+
"as-of": { type: "string" },
|
|
131
|
+
"lag-days": { type: "string" },
|
|
68
132
|
port: { type: "string" },
|
|
69
133
|
help: { type: "boolean", short: "h" },
|
|
70
134
|
json: { type: "boolean" },
|
|
@@ -88,6 +152,10 @@ export async function runCli(args: string[]): Promise<number> {
|
|
|
88
152
|
"max-rows": { type: "string" },
|
|
89
153
|
"min-impressions": { type: "string" },
|
|
90
154
|
"max-clicks": { type: "string" },
|
|
155
|
+
"max-depth": { type: "string" },
|
|
156
|
+
"max-links": { type: "string" },
|
|
157
|
+
"inspect-limit": { type: "string" },
|
|
158
|
+
"include-query": { type: "boolean" },
|
|
91
159
|
},
|
|
92
160
|
});
|
|
93
161
|
if (args.length === 0 || values.help || positionals[0] === "help") {
|
|
@@ -95,45 +163,15 @@ export async function runCli(args: string[]): Promise<number> {
|
|
|
95
163
|
return 0;
|
|
96
164
|
}
|
|
97
165
|
const [family, action] = positionals;
|
|
98
|
-
if (
|
|
166
|
+
if (
|
|
167
|
+
positionals.length >
|
|
168
|
+
(["gsc", "ga", "bing", "speed", "evidence", "cloudflare", "change", "technical"].includes(family) ? 2 : 1)
|
|
169
|
+
)
|
|
99
170
|
throw new RequestError("Too many command arguments", null, "invalid_input");
|
|
100
|
-
const operation = ["gsc", "ga", "bing", "speed", "evidence"].includes(family)
|
|
171
|
+
const operation = ["gsc", "ga", "bing", "speed", "evidence", "cloudflare", "change", "technical"].includes(family)
|
|
101
172
|
? `${family}.${action ?? ""}`
|
|
102
173
|
: family;
|
|
103
|
-
const
|
|
104
|
-
"evidence.import": ["request"],
|
|
105
|
-
assess: ["snapshot", "max-rows", "format"],
|
|
106
|
-
opportunities: ["snapshot", "max-rows", "format", "min-impressions", "max-clicks"],
|
|
107
|
-
discover: ["url", "providers"],
|
|
108
|
-
compare: ["baseline", "current", "max-rows"],
|
|
109
|
-
"bing.crawl-stats": ["site"],
|
|
110
|
-
"bing.crawl-issues": ["site"],
|
|
111
|
-
"bing.url-info": ["site", "url"],
|
|
112
|
-
"bing.link-counts": ["site", "max-pages"],
|
|
113
|
-
"bing.url-links": ["site", "url", "max-pages"],
|
|
114
|
-
"bing.sites": [],
|
|
115
|
-
"bing.queries": ["site"],
|
|
116
|
-
"bing.pages": ["site"],
|
|
117
|
-
"bing.traffic": ["site"],
|
|
118
|
-
"gsc.sites": [],
|
|
119
|
-
"gsc.sitemaps": ["site"],
|
|
120
|
-
"gsc.inspect": ["site", "url"],
|
|
121
|
-
"gsc.report": ["site", "request", "max-pages"],
|
|
122
|
-
"ga.realtime": ["property", "request"],
|
|
123
|
-
"ga.accounts": [],
|
|
124
|
-
"ga.property": ["property"],
|
|
125
|
-
"ga.key-events": ["property"],
|
|
126
|
-
"ga.report": ["property", "request", "max-pages"],
|
|
127
|
-
page: ["url"],
|
|
128
|
-
"speed.psi": ["url", "strategy"],
|
|
129
|
-
"speed.crux": ["url", "origin", "form-factor"],
|
|
130
|
-
"speed.history": ["url", "origin", "form-factor"],
|
|
131
|
-
doctor: ["config"],
|
|
132
|
-
snapshot: ["config", "start", "end", "max-pages"],
|
|
133
|
-
api: ["request"],
|
|
134
|
-
serve: ["port"],
|
|
135
|
-
};
|
|
136
|
-
const allowed = [...(flags[operation] ?? []), ...(operation === "serve" ? [] : ["out", "json"])];
|
|
174
|
+
const allowed = [...(commands[operation] ?? []), ...(operation === "serve" ? [] : ["out", "json"])];
|
|
137
175
|
for (const flag of Object.keys(values))
|
|
138
176
|
if (!allowed.includes(flag))
|
|
139
177
|
throw new RequestError(`--${flag} is not supported for ${operation}`, null, "invalid_input");
|
|
@@ -152,8 +190,60 @@ export async function runCli(args: string[]): Promise<number> {
|
|
|
152
190
|
throw new RequestError("--json and --format text conflict", null, "invalid_input");
|
|
153
191
|
let input: unknown;
|
|
154
192
|
if (operation === "api") input = await jsonFile(values.request, "request");
|
|
155
|
-
else if (operation === "
|
|
193
|
+
else if (operation === "render") {
|
|
194
|
+
if (Boolean(values["from-url"]) !== Boolean(values["link-selector"]))
|
|
195
|
+
throw new RequestError("Supply both --from-url and --link-selector", null, "invalid_input");
|
|
196
|
+
input = {
|
|
197
|
+
operation: "page.verify",
|
|
198
|
+
url: values.url,
|
|
199
|
+
...(values["from-url"]
|
|
200
|
+
? { navigation: { fromUrl: values["from-url"], linkSelector: values["link-selector"] } }
|
|
201
|
+
: {}),
|
|
202
|
+
settleMs: Number(values["settle-ms"] ?? 1000),
|
|
203
|
+
timeoutMs: Number(values["timeout-ms"] ?? 20000),
|
|
204
|
+
};
|
|
205
|
+
} else if (operation === "evidence.import")
|
|
156
206
|
input = { operation, document: await jsonFile(values.request, "request") };
|
|
207
|
+
else if (operation === "cloudflare.audit")
|
|
208
|
+
input = {
|
|
209
|
+
operation,
|
|
210
|
+
zone: values.zone,
|
|
211
|
+
hostname: values.hostname,
|
|
212
|
+
startTime: values.start,
|
|
213
|
+
endTime: values.end,
|
|
214
|
+
limit: Number(values.limit ?? 50),
|
|
215
|
+
};
|
|
216
|
+
else if (operation === "change.followup")
|
|
217
|
+
input = {
|
|
218
|
+
operation,
|
|
219
|
+
experiments: await loadFollowupManifest(values.manifest),
|
|
220
|
+
asOf: values["as-of"],
|
|
221
|
+
lagDays: Number(values["lag-days"] ?? 3),
|
|
222
|
+
};
|
|
223
|
+
else if (operation === "change.evaluate")
|
|
224
|
+
input = {
|
|
225
|
+
operation,
|
|
226
|
+
record: await jsonFile(values.record, "record"),
|
|
227
|
+
baseline: await jsonFile(values.baseline, "baseline"),
|
|
228
|
+
...(values.current ? { current: await jsonFile(values.current, "current") } : {}),
|
|
229
|
+
maxRows: Number(values["max-rows"] ?? 20),
|
|
230
|
+
};
|
|
231
|
+
else if (operation === "crawl")
|
|
232
|
+
input = {
|
|
233
|
+
operation,
|
|
234
|
+
config: await jsonFile(values.config, "config"),
|
|
235
|
+
maxPages: Number(values["max-pages"] ?? 20),
|
|
236
|
+
maxDepth: Number(values["max-depth"] ?? 3),
|
|
237
|
+
maxLinks: Number(values["max-links"] ?? 500),
|
|
238
|
+
inspectLimit: Number(values["inspect-limit"] ?? 3),
|
|
239
|
+
includeQuery: values["include-query"] ?? false,
|
|
240
|
+
};
|
|
241
|
+
else if (operation === "technical.compare")
|
|
242
|
+
input = {
|
|
243
|
+
operation,
|
|
244
|
+
current: await jsonFile(values.current, "current"),
|
|
245
|
+
...(values.baseline ? { baseline: await jsonFile(values.baseline, "baseline") } : {}),
|
|
246
|
+
};
|
|
157
247
|
else if (operation === "opportunities")
|
|
158
248
|
input = {
|
|
159
249
|
operation,
|
|
@@ -175,14 +265,20 @@ export async function runCli(args: string[]): Promise<number> {
|
|
|
175
265
|
current: await jsonFile(values.current, "current"),
|
|
176
266
|
maxRows: Number(values["max-rows"] ?? 100),
|
|
177
267
|
};
|
|
178
|
-
else if (operation === "snapshot" || operation === "doctor") {
|
|
268
|
+
else if (operation === "snapshot" || operation === "doctor" || operation === "investigate") {
|
|
179
269
|
const config = await jsonFile(values.config, "config");
|
|
180
270
|
if (operation === "doctor") input = { operation, config };
|
|
181
271
|
else {
|
|
182
272
|
if (Boolean(values.start) !== Boolean(values.end))
|
|
183
273
|
throw new RequestError("Supply both --start and --end, or neither", null, "invalid_input");
|
|
184
274
|
const dates = values.start ? { startDate: values.start, endDate: values.end } : defaultDates();
|
|
185
|
-
input = {
|
|
275
|
+
input = {
|
|
276
|
+
operation,
|
|
277
|
+
config,
|
|
278
|
+
...dates,
|
|
279
|
+
maxPages: Number(values["max-pages"] ?? (operation === "investigate" ? 1 : 4)),
|
|
280
|
+
...(operation === "investigate" ? { url: values.url, maxRows: Number(values["max-rows"] ?? 28) } : {}),
|
|
281
|
+
};
|
|
186
282
|
}
|
|
187
283
|
} else {
|
|
188
284
|
input = {
|
|
@@ -201,9 +297,13 @@ export async function runCli(args: string[]): Promise<number> {
|
|
|
201
297
|
const result = await execute(input);
|
|
202
298
|
const output =
|
|
203
299
|
values.format === "text"
|
|
204
|
-
? operation === "
|
|
205
|
-
?
|
|
206
|
-
:
|
|
300
|
+
? operation === "change.followup"
|
|
301
|
+
? renderFollowup(result)
|
|
302
|
+
: operation === "investigate"
|
|
303
|
+
? renderInvestigation(result)
|
|
304
|
+
: operation === "opportunities"
|
|
305
|
+
? renderOpportunities(result)
|
|
306
|
+
: renderAssessment(result)
|
|
207
307
|
: `${JSON.stringify(result, null, 2)}\n`;
|
|
208
308
|
if (values.out) await Bun.write(values.out, output);
|
|
209
309
|
process.stdout.write(output);
|