pagesight 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/docs/changes.md +118 -0
- package/docs/cloudflare.md +31 -0
- package/docs/crawl.md +64 -0
- package/docs/investigation.md +85 -0
- package/docs/monitoring.md +39 -0
- package/docs/rendering.md +102 -0
- package/docs/seo-agent-workflow.md +153 -0
- package/docs/usage.md +22 -19
- package/package.json +5 -2
- package/src/api/change-record.ts +31 -0
- package/src/api/cloudflare.ts +201 -0
- package/src/api/compare-snapshots.ts +1 -1
- package/src/api/crawl.ts +53 -0
- package/src/api/evaluate-change.ts +189 -0
- package/src/api/execute.ts +29 -0
- package/src/api/followup-changes.ts +264 -0
- package/src/api/http-url.ts +13 -0
- package/src/api/investigation.ts +344 -0
- package/src/api/schema.ts +103 -16
- package/src/api/technical-changes.ts +124 -0
- package/src/api/verify-render.ts +134 -0
- package/src/cli.ts +143 -43
- package/src/followup-manifest.ts +42 -0
- package/src/followup-text.ts +45 -0
- package/src/investigation-text.ts +32 -0
- package/src/providers/cloudflare.ts +46 -0
- package/src/tools/observe.ts +1 -1
- package/src/web/fetch.ts +2 -1
- package/src/web/render-browser.ts +190 -0
- package/src/web/render-dom.ts +72 -0
- package/src/web/render-network.ts +122 -0
- package/src/web/site-graph.ts +443 -0
package/docs/usage.md
CHANGED
|
@@ -26,25 +26,27 @@ GA evidence identifies the credential source variable, credential type, and serv
|
|
|
26
26
|
account email when present. It never includes the token or private key. Aggregate
|
|
27
27
|
results include a concise per-observation `summary` alongside full observations.
|
|
28
28
|
|
|
29
|
-
| Operation | Required inputs
|
|
30
|
-
| -------------------------------------------- |
|
|
31
|
-
| `discover` | `url`; optional `providers` (`["gsc", "ga"]` by default)
|
|
32
|
-
| `bing.sites` | None
|
|
33
|
-
| `bing.queries`, `bing.pages`, `bing.traffic` | `site`
|
|
34
|
-
| `gsc.sites` | None
|
|
35
|
-
| `gsc.sitemaps` | `site`
|
|
36
|
-
| `gsc.inspect` | `site`, `url`
|
|
37
|
-
| `gsc.report` | `site`, `request`; optional `maxPages`
|
|
38
|
-
| `ga.accounts` | None
|
|
39
|
-
| `ga.property`, `ga.key-events` | `property`
|
|
40
|
-
| `ga.realtime` | `property`, `request`; moving window, no offset
|
|
41
|
-
| `ga.report` | `property`, `request`; optional `maxPages`
|
|
42
|
-
| `page` | `url`
|
|
43
|
-
| `speed.psi` | `url`; optional `strategy` (`mobile` or `desktop`)
|
|
44
|
-
| `speed.crux`, `speed.history` | `url`; optional `origin: true`, `formFactor`
|
|
45
|
-
| `doctor` | `config`
|
|
46
|
-
| `
|
|
47
|
-
| `
|
|
29
|
+
| Operation | Required inputs |
|
|
30
|
+
| -------------------------------------------- | -------------------------------------------------------------------------- |
|
|
31
|
+
| `discover` | `url`; optional `providers` (`["gsc", "ga"]` by default) |
|
|
32
|
+
| `bing.sites` | None |
|
|
33
|
+
| `bing.queries`, `bing.pages`, `bing.traffic` | `site` |
|
|
34
|
+
| `gsc.sites` | None |
|
|
35
|
+
| `gsc.sitemaps` | `site` |
|
|
36
|
+
| `gsc.inspect` | `site`, `url` |
|
|
37
|
+
| `gsc.report` | `site`, `request`; optional `maxPages` |
|
|
38
|
+
| `ga.accounts` | None |
|
|
39
|
+
| `ga.property`, `ga.key-events` | `property` |
|
|
40
|
+
| `ga.realtime` | `property`, `request`; moving window, no offset |
|
|
41
|
+
| `ga.report` | `property`, `request`; optional `maxPages` |
|
|
42
|
+
| `page` | `url` |
|
|
43
|
+
| `speed.psi` | `url`; optional `strategy` (`mobile` or `desktop`) |
|
|
44
|
+
| `speed.crux`, `speed.history` | `url`; optional `origin: true`, `formFactor` |
|
|
45
|
+
| `doctor` | `config` |
|
|
46
|
+
| `crawl` | `config`; optional crawl bounds and `inspectLimit` (see [crawl](crawl.md)) |
|
|
47
|
+
| `investigate` | `config`, `url`, `startDate`, `endDate`; optional `maxPages`, `maxRows` |
|
|
48
|
+
| `assess` | `snapshot`; optional `maxRows` (default 10, max 100) |
|
|
49
|
+
| `snapshot` | `config`, `startDate`, `endDate`; optional `maxPages` |
|
|
48
50
|
|
|
49
51
|
`operationSchema` and `configSchema` are exported for typed validation. The MCP
|
|
50
52
|
`observe` input uses the same schema. `page` observes fetched HTML, status,
|
|
@@ -54,6 +56,7 @@ link, social-meta and contrast checks.
|
|
|
54
56
|
|
|
55
57
|
See [measurement and verification](measurement.md) for saved-snapshot assessments,
|
|
56
58
|
Realtime requests, freshness limits and repeatable browser checks.
|
|
59
|
+
See [investigating one URL](investigation.md) for fresh exact-page search, organic and technical evidence.
|
|
57
60
|
|
|
58
61
|
## CLI
|
|
59
62
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pagesight",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.20.0",
|
|
4
4
|
"description": "See your site the way search engines and AI see it.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai-crawlers",
|
|
@@ -38,10 +38,13 @@
|
|
|
38
38
|
"scripts": {
|
|
39
39
|
"start": "bun run src/index.ts",
|
|
40
40
|
"typecheck": "tsc -p tsconfig.json",
|
|
41
|
-
"test": "bun test __tests__"
|
|
41
|
+
"test": "bun test __tests__ && bun test browser-tests"
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
44
|
"@modelcontextprotocol/sdk": "^1.12.1",
|
|
45
|
+
"entities": "^8.1.0",
|
|
46
|
+
"ipaddr.js": "^2.5.0",
|
|
47
|
+
"playwright": "1.63.0",
|
|
45
48
|
"saxes": "^6.0.0",
|
|
46
49
|
"zod": "^3.24.4"
|
|
47
50
|
},
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { httpUrl } from "./http-url.js";
|
|
3
|
+
|
|
4
|
+
const contextChange = z.object({ at: z.string().datetime(), description: z.string().min(1).max(2000) }).strict();
|
|
5
|
+
export const changeRecordSchema = z
|
|
6
|
+
.object({
|
|
7
|
+
schemaVersion: z.literal(1),
|
|
8
|
+
id: z.string().min(1).max(200),
|
|
9
|
+
site: httpUrl,
|
|
10
|
+
affectedUrls: z.array(httpUrl).min(1).max(100),
|
|
11
|
+
description: z.string().min(1).max(4000),
|
|
12
|
+
hypothesis: z.string().min(1).max(4000),
|
|
13
|
+
deployedAt: z.string().datetime(),
|
|
14
|
+
expectedSignal: z.string().min(1).max(2000),
|
|
15
|
+
measurementChanges: z.array(contextChange).max(100),
|
|
16
|
+
overlappingChanges: z.array(contextChange).max(100),
|
|
17
|
+
})
|
|
18
|
+
.strict()
|
|
19
|
+
.superRefine((record, ctx) => {
|
|
20
|
+
for (const [index, url] of record.affectedUrls.entries()) {
|
|
21
|
+
if (new URL(url).origin !== new URL(record.site).origin)
|
|
22
|
+
ctx.addIssue({
|
|
23
|
+
code: "custom",
|
|
24
|
+
path: ["affectedUrls", index],
|
|
25
|
+
message: "Affected URL must share the site's origin",
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
if (Date.parse(record.deployedAt) > Date.now())
|
|
29
|
+
ctx.addIssue({ code: "custom", path: ["deployedAt"], message: "Deployment must not be in the future" });
|
|
30
|
+
});
|
|
31
|
+
export type ChangeRecord = z.infer<typeof changeRecordSchema>;
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { cloudflareGraphql, type CloudflareQuery } from "../providers/cloudflare.js";
|
|
3
|
+
import { aggregate, capture, type Evidence } from "./evidence.js";
|
|
4
|
+
|
|
5
|
+
const responseSchema = z
|
|
6
|
+
.object({
|
|
7
|
+
data: z
|
|
8
|
+
.object({ viewer: z.object({ zones: z.array(z.object({ zoneTag: z.string() }).passthrough()) }).nullable() })
|
|
9
|
+
.nullable()
|
|
10
|
+
.optional(),
|
|
11
|
+
errors: z.array(z.unknown()).nullable().optional(),
|
|
12
|
+
})
|
|
13
|
+
.passthrough();
|
|
14
|
+
type Input = { zone: string; hostname: string; startTime: string; endTime: string; limit: number };
|
|
15
|
+
// AIDEV-NOTE: Fixed selections exclude IPs, query strings and credentials. New fields change the privacy contract.
|
|
16
|
+
export function cloudflareRequests(input: Input): Array<{ name: string; dataset: string; request: CloudflareQuery }> {
|
|
17
|
+
const zoneVar = { zone: input.zone };
|
|
18
|
+
const filters = {
|
|
19
|
+
datetime_geq: input.startTime,
|
|
20
|
+
datetime_lt: input.endTime,
|
|
21
|
+
clientRequestHTTPHost: input.hostname.toLowerCase(),
|
|
22
|
+
};
|
|
23
|
+
return [
|
|
24
|
+
{
|
|
25
|
+
name: "settings",
|
|
26
|
+
dataset: "settings",
|
|
27
|
+
request: {
|
|
28
|
+
query: `query Settings($zone: string) { viewer { zones(filter: {zoneTag: $zone}) { zoneTag settings { httpRequestsAdaptiveGroups { enabled maxDuration maxPageSize notOlderThan availableFields } firewallEventsAdaptive { enabled maxDuration maxPageSize notOlderThan availableFields } } } } }`,
|
|
29
|
+
variables: zoneVar,
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
name: "http",
|
|
34
|
+
dataset: "httpRequestsAdaptiveGroups",
|
|
35
|
+
request: {
|
|
36
|
+
query: `query Http($zone: string, $filter: ZoneHttpRequestsAdaptiveGroupsFilter_InputObject, $limit: uint64) { viewer { zones(filter: {zoneTag: $zone}) { zoneTag httpRequestsAdaptiveGroups(filter: $filter, limit: $limit, orderBy: [count_DESC]) { count avg { sampleInterval } dimensions { clientRequestHTTPHost clientRequestPath edgeResponseStatus originResponseStatus cacheStatus userAgent verifiedBotCategory } } } } }`,
|
|
37
|
+
variables: { ...zoneVar, filter: filters, limit: input.limit },
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
name: "security",
|
|
42
|
+
dataset: "firewallEventsAdaptive",
|
|
43
|
+
request: {
|
|
44
|
+
query: `query Security($zone: string, $filter: FirewallEventsAdaptiveFilter_InputObject, $limit: uint64) { viewer { zones(filter: {zoneTag: $zone}) { zoneTag firewallEventsAdaptive(filter: $filter, limit: $limit, orderBy: [datetime_DESC]) { datetime action source clientRequestHTTPHost clientRequestPath edgeResponseStatus userAgent verifiedBotCategory sampleInterval } } } }`,
|
|
45
|
+
variables: { ...zoneVar, filter: filters, limit: input.limit },
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export async function cloudflareAudit(input: Input, query = cloudflareGraphql): Promise<Evidence> {
|
|
52
|
+
const observations: Evidence[] = [];
|
|
53
|
+
for (const planned of cloudflareRequests(input)) {
|
|
54
|
+
const observation = await capture("cloudflare", planned.name, input.zone, planned.request, () =>
|
|
55
|
+
query(planned.request),
|
|
56
|
+
);
|
|
57
|
+
observation.name = `cloudflare.${planned.name}`;
|
|
58
|
+
const raw = observation.pages[0]?.response;
|
|
59
|
+
if (raw !== undefined) {
|
|
60
|
+
const parsed = responseSchema.safeParse(raw);
|
|
61
|
+
const zones = parsed.success ? parsed.data.data?.viewer?.zones : undefined;
|
|
62
|
+
const zone = zones?.length === 1 && zones[0].zoneTag === input.zone ? zones[0] : undefined;
|
|
63
|
+
const usable = Boolean(zone && Object.hasOwn(zone, planned.dataset) && zone[planned.dataset] !== null);
|
|
64
|
+
if (!parsed.success || !usable || parsed.data.errors?.length) {
|
|
65
|
+
observation.status = usable ? "partial" : "error";
|
|
66
|
+
observation.error = {
|
|
67
|
+
code: parsed.success ? "graphql_error" : "invalid_response",
|
|
68
|
+
message: "Cloudflare returned unavailable or incomplete dataset evidence; inspect retained GraphQL response",
|
|
69
|
+
httpStatus: 200,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
observation.credential = { source: "CLOUDFLARE_API_TOKEN", type: "api_token", clientEmail: null };
|
|
74
|
+
observation.warnings.push(
|
|
75
|
+
"Adaptive datasets can contain estimated/sampled values. Retain sampleInterval; do not multiply already estimated counts again.",
|
|
76
|
+
"Dataset availability, retained history and maximum query duration depend on zone settings. Missing or denied data is unknown, not zero.",
|
|
77
|
+
);
|
|
78
|
+
if (planned.name !== "settings")
|
|
79
|
+
observation.warnings.push(
|
|
80
|
+
`At most ${input.limit} ${planned.name === "http" ? "groups ordered by count descending" : "security events ordered newest first"} are retained; this is not exhaustive traffic or crawler coverage.`,
|
|
81
|
+
);
|
|
82
|
+
observations.push(observation);
|
|
83
|
+
}
|
|
84
|
+
const diagnostics: Array<{
|
|
85
|
+
dataset: string;
|
|
86
|
+
observedRows: number | null;
|
|
87
|
+
possiblyTruncated: boolean;
|
|
88
|
+
effectiveRowLimit: number;
|
|
89
|
+
unusableRows: number;
|
|
90
|
+
collectedAt: string;
|
|
91
|
+
errorPaths: unknown[];
|
|
92
|
+
}> = [];
|
|
93
|
+
const unknowns: string[] = [];
|
|
94
|
+
const settingsSchema = z.object({
|
|
95
|
+
enabled: z.boolean(),
|
|
96
|
+
maxDuration: z.number().nonnegative(),
|
|
97
|
+
maxPageSize: z.number().int().positive().optional(),
|
|
98
|
+
notOlderThan: z.number().nonnegative(),
|
|
99
|
+
availableFields: z.array(z.string()),
|
|
100
|
+
});
|
|
101
|
+
const settingsObservation = observations[0];
|
|
102
|
+
const settingsRaw = responseSchema.safeParse(settingsObservation.pages[0]?.response);
|
|
103
|
+
const settingsZone = settingsRaw.success
|
|
104
|
+
? settingsRaw.data.data?.viewer?.zones?.find((z) => z.zoneTag === input.zone)
|
|
105
|
+
: undefined;
|
|
106
|
+
const settings = settingsZone?.settings as Record<string, unknown> | undefined;
|
|
107
|
+
for (let index = 1; index < observations.length; index++) {
|
|
108
|
+
const observation = observations[index];
|
|
109
|
+
const dataset = index === 1 ? "httpRequestsAdaptiveGroups" : "firewallEventsAdaptive";
|
|
110
|
+
const parsed = responseSchema.safeParse(observation.pages[0]?.response);
|
|
111
|
+
const zone = parsed.success ? parsed.data.data?.viewer?.zones?.find((z) => z.zoneTag === input.zone) : undefined;
|
|
112
|
+
const rows = Array.isArray(zone?.[dataset]) ? (zone[dataset] as unknown[]) : null;
|
|
113
|
+
const unusableRows =
|
|
114
|
+
rows?.filter((row) => {
|
|
115
|
+
if (!row || typeof row !== "object") return true;
|
|
116
|
+
const value = row as Record<string, unknown>;
|
|
117
|
+
const dimensions = value.dimensions as Record<string, unknown> | undefined;
|
|
118
|
+
const hostname = index === 1 ? dimensions?.clientRequestHTTPHost : value.clientRequestHTTPHost;
|
|
119
|
+
return typeof hostname !== "string" || hostname.toLowerCase() !== input.hostname.toLowerCase();
|
|
120
|
+
}).length ?? 0;
|
|
121
|
+
const declared = settingsObservation.status === "ok" ? settingsSchema.safeParse(settings?.[dataset]) : null;
|
|
122
|
+
if (!declared?.success)
|
|
123
|
+
unknowns.push(`${dataset}: settings unavailable or incomplete; retention/availability are unknown.`);
|
|
124
|
+
else {
|
|
125
|
+
if (!declared.data.enabled) unknowns.push(`${dataset}: dataset is currently disabled for this zone.`);
|
|
126
|
+
if (Date.parse(input.startTime) < Date.parse(settingsObservation.finishedAt) - declared.data.notOlderThan * 1000)
|
|
127
|
+
unknowns.push(`${dataset}: requested start is outside the currently declared retention window.`);
|
|
128
|
+
if (Date.parse(input.endTime) - Date.parse(input.startTime) > declared.data.maxDuration * 1000)
|
|
129
|
+
unknowns.push(`${dataset}: interval exceeds currently declared maximum duration.`);
|
|
130
|
+
}
|
|
131
|
+
const providerRowLimit = declared?.success ? declared.data.maxPageSize : undefined;
|
|
132
|
+
const effectiveRowLimit = Math.min(input.limit, providerRowLimit ?? input.limit);
|
|
133
|
+
if (providerRowLimit === undefined)
|
|
134
|
+
unknowns.push(`${dataset}: provider maximum page size unavailable; provider truncation is unknown.`);
|
|
135
|
+
if (rows && rows.length >= effectiveRowLimit)
|
|
136
|
+
unknowns.push(`${dataset}: retained row limit reached; additional rows may exist.`);
|
|
137
|
+
if (rows?.length === 0)
|
|
138
|
+
unknowns.push(`${dataset}: empty retained set, not proof of zero traffic or security actions.`);
|
|
139
|
+
if (unusableRows) {
|
|
140
|
+
unknowns.push(
|
|
141
|
+
`${dataset}: ${unusableRows} rows do not match the requested hostname/shape; raw data retained without association.`,
|
|
142
|
+
);
|
|
143
|
+
if (observation.status === "ok") observation.status = "partial";
|
|
144
|
+
}
|
|
145
|
+
if (!rows && observation.status === "ok") {
|
|
146
|
+
observation.status = "error";
|
|
147
|
+
observation.error = {
|
|
148
|
+
code: "invalid_response",
|
|
149
|
+
message: "Cloudflare dataset is not a row array",
|
|
150
|
+
httpStatus: 200,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
const issues = unknowns.filter((u) => u.startsWith(`${dataset}:`));
|
|
154
|
+
observation.warnings.push(...issues);
|
|
155
|
+
diagnostics.push({
|
|
156
|
+
dataset,
|
|
157
|
+
observedRows: rows?.length ?? null,
|
|
158
|
+
possiblyTruncated: Boolean(rows && rows.length >= effectiveRowLimit),
|
|
159
|
+
effectiveRowLimit,
|
|
160
|
+
unusableRows,
|
|
161
|
+
collectedAt: observation.finishedAt,
|
|
162
|
+
errorPaths: parsed.success
|
|
163
|
+
? (parsed.data.errors ?? [])
|
|
164
|
+
.slice(0, 3)
|
|
165
|
+
.map((e) => (e && typeof e === "object" ? ("path" in e ? e.path : null) : null))
|
|
166
|
+
: [],
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
const result = aggregate("cloudflare.audit", input.zone, input, observations);
|
|
170
|
+
Object.assign(result.pages[0].response as object, {
|
|
171
|
+
scope: {
|
|
172
|
+
zone: input.zone,
|
|
173
|
+
hostname: input.hostname,
|
|
174
|
+
startInclusive: input.startTime,
|
|
175
|
+
endExclusive: input.endTime,
|
|
176
|
+
timezone: "UTC",
|
|
177
|
+
limit: input.limit,
|
|
178
|
+
ordering: { http: "count_DESC", security: "datetime_DESC" },
|
|
179
|
+
querySignature: Bun.CryptoHasher.hash(
|
|
180
|
+
"sha256",
|
|
181
|
+
JSON.stringify(cloudflareRequests(input).map((p) => p.request.query)),
|
|
182
|
+
"hex",
|
|
183
|
+
),
|
|
184
|
+
settingsCollectedAt: settingsObservation.finishedAt,
|
|
185
|
+
settingsMeaning: "Current zone settings, not proof of settings during the historical reporting window",
|
|
186
|
+
},
|
|
187
|
+
diagnostics,
|
|
188
|
+
unknowns,
|
|
189
|
+
limitations: [
|
|
190
|
+
"No path joins to GSC/GA: Cloudflare paths omit query strings. Cloudflare is not supported by Pagesight report comparisons or conversion funnels.",
|
|
191
|
+
"HTTP traffic is not verified search-engine crawling. User-Agent can be spoofed; verifiedBotCategory is Cloudflare's classification, not proof of Google indexing or a particular crawler identity.",
|
|
192
|
+
"Security events describe observed actions, not all requests or all challenges. Do not reconcile sampled event counts with HTTP groups as a funnel.",
|
|
193
|
+
"Edge status and origin status differ; cached/edge-handled requests may have no origin response. Status zero does not mean successful origin access.",
|
|
194
|
+
"No client IP, request query string, cookies or authentication headers are requested. Paths and user agents may still be sensitive: keep reports private.",
|
|
195
|
+
"Only read-only GraphQL analytics queries execute; no Cloudflare security, cache, DNS or crawler settings are modified.",
|
|
196
|
+
],
|
|
197
|
+
});
|
|
198
|
+
result.credential = { source: "CLOUDFLARE_API_TOKEN", type: "api_token", clientEmail: null };
|
|
199
|
+
if (result.status === "ok" && unknowns.length) result.status = "partial";
|
|
200
|
+
return result;
|
|
201
|
+
}
|
|
@@ -23,7 +23,7 @@ function metric(baseline: number | string, current: number | string) {
|
|
|
23
23
|
|
|
24
24
|
type SnapshotContext = ImportedSnapshot["pages"][0]["response"]["context"];
|
|
25
25
|
|
|
26
|
-
function compareObservation(
|
|
26
|
+
export function compareObservation(
|
|
27
27
|
baseline: Evidence | undefined,
|
|
28
28
|
current: Evidence | undefined,
|
|
29
29
|
maxRows: number,
|
package/src/api/crawl.ts
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { aggregate, capture, type Evidence } from "./evidence.js";
|
|
2
|
+
import { crawlSite } from "../web/site-graph.js";
|
|
3
|
+
import type { SiteConfig } from "./schema.js";
|
|
4
|
+
import type { Executor } from "./execute.js";
|
|
5
|
+
|
|
6
|
+
type Input = {
|
|
7
|
+
config: SiteConfig;
|
|
8
|
+
maxPages: number;
|
|
9
|
+
maxDepth: number;
|
|
10
|
+
maxLinks: number;
|
|
11
|
+
includeQuery: boolean;
|
|
12
|
+
inspectLimit: number;
|
|
13
|
+
};
|
|
14
|
+
export async function crawl(input: Input, run: Executor): Promise<Evidence> {
|
|
15
|
+
const { config } = input;
|
|
16
|
+
const request = {
|
|
17
|
+
site: config.site,
|
|
18
|
+
seeds: config.pages,
|
|
19
|
+
sitemap: config.sitemap,
|
|
20
|
+
maxPages: input.maxPages,
|
|
21
|
+
maxDepth: input.maxDepth,
|
|
22
|
+
maxLinks: input.maxLinks,
|
|
23
|
+
includeQuery: input.includeQuery,
|
|
24
|
+
};
|
|
25
|
+
const graph = await capture("web", "site-graph", config.site, request, () => crawlSite(request));
|
|
26
|
+
graph.name = "crawl.graph";
|
|
27
|
+
const data = graph.pages[0]?.response as Awaited<ReturnType<typeof crawlSite>> | undefined;
|
|
28
|
+
if (data && !data.complete) graph.status = "partial";
|
|
29
|
+
const observations = [graph];
|
|
30
|
+
const urls =
|
|
31
|
+
data?.pages
|
|
32
|
+
.filter((p) => p.status === 200 && p.contentType?.toLowerCase().includes("text/html"))
|
|
33
|
+
.slice(0, input.inspectLimit)
|
|
34
|
+
.map((p) => p.url) ?? [];
|
|
35
|
+
if (config.gscSite)
|
|
36
|
+
for (const url of urls)
|
|
37
|
+
observations.push({
|
|
38
|
+
...(await run({ operation: "gsc.inspect", site: config.gscSite, url })),
|
|
39
|
+
name: `gsc.inspect:${url}`,
|
|
40
|
+
});
|
|
41
|
+
const result = aggregate("crawl", config.site, input, observations);
|
|
42
|
+
Object.assign(result.pages[0].response as object, {
|
|
43
|
+
context: config.context,
|
|
44
|
+
indexingSample: {
|
|
45
|
+
selected: config.gscSite ? urls : [],
|
|
46
|
+
limit: input.inspectLimit,
|
|
47
|
+
available: Boolean(config.gscSite),
|
|
48
|
+
meaning:
|
|
49
|
+
"Stored Google state for selected fetched HTML pages only; not a site indexing count. Missing inspection is unknown.",
|
|
50
|
+
},
|
|
51
|
+
});
|
|
52
|
+
return result;
|
|
53
|
+
}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import type { ChangeRecord } from "./change-record.js";
|
|
2
|
+
import { compareObservation } from "./compare-snapshots.js";
|
|
3
|
+
import { capture, type Evidence } from "./evidence.js";
|
|
4
|
+
import type { ImportedSnapshot } from "./evidence-schema.js";
|
|
5
|
+
import { canonical, normalizeReport, requireSame, Incompatible } from "./report-table.js";
|
|
6
|
+
import { configSchema, gaRequestSchema, gscRequestSchema } from "./schema.js";
|
|
7
|
+
import { snapshotOperations, observationName } from "./snapshot.js";
|
|
8
|
+
import { normalizeGaProperty } from "../providers/ga.js";
|
|
9
|
+
import { RequestError } from "../shared/http.js";
|
|
10
|
+
|
|
11
|
+
export function localDay(at: string, timeZone: string): string {
|
|
12
|
+
const parts = new Intl.DateTimeFormat("en-US", {
|
|
13
|
+
timeZone,
|
|
14
|
+
year: "numeric",
|
|
15
|
+
month: "2-digit",
|
|
16
|
+
day: "2-digit",
|
|
17
|
+
}).formatToParts(new Date(at));
|
|
18
|
+
const value = (name: string) => parts.find((p) => p.type === name)!.value;
|
|
19
|
+
return `${value("year")}-${value("month")}-${value("day")}`;
|
|
20
|
+
}
|
|
21
|
+
function scope(provider: string, request: unknown) {
|
|
22
|
+
if (provider === "ga") {
|
|
23
|
+
const { offset: _offset, limit: _limit, returnPropertyQuota: _quota, ...rest } = gaRequestSchema.parse(request);
|
|
24
|
+
return rest;
|
|
25
|
+
}
|
|
26
|
+
const { startRow: _offset, rowLimit: _limit, ...rest } = gscRequestSchema.parse(request);
|
|
27
|
+
return rest;
|
|
28
|
+
}
|
|
29
|
+
export function boundReport(snapshot: ImportedSnapshot, name: string) {
|
|
30
|
+
const { context, observations } = snapshot.pages[0].response;
|
|
31
|
+
const config = configSchema.parse(context.config);
|
|
32
|
+
const expected = snapshotOperations(config, context.requestedDates.startDate, context.requestedDates.endDate, 1).find(
|
|
33
|
+
(op) => observationName(op) === name,
|
|
34
|
+
);
|
|
35
|
+
const observation = observations.find((o) => o.name === name);
|
|
36
|
+
if (!expected || !["ga.report", "gsc.report"].includes(expected.operation))
|
|
37
|
+
throw new Incompatible("Not a configured snapshot report.");
|
|
38
|
+
if (!observation || observation.status === "error" || !observation.pages.length)
|
|
39
|
+
throw new Incompatible("Report unavailable; absence is not zero.");
|
|
40
|
+
const provider = expected.operation === "ga.report" ? "ga" : "gsc";
|
|
41
|
+
const target =
|
|
42
|
+
expected.operation === "ga.report"
|
|
43
|
+
? normalizeGaProperty(expected.property)
|
|
44
|
+
: expected.operation === "gsc.report"
|
|
45
|
+
? expected.site
|
|
46
|
+
: "";
|
|
47
|
+
if (observation.provider !== provider || observation.operation !== "report" || observation.target !== target)
|
|
48
|
+
throw new Incompatible("Report identity does not match configured provider/property.");
|
|
49
|
+
if (expected.operation !== "ga.report" && expected.operation !== "gsc.report")
|
|
50
|
+
throw new Incompatible("Unsupported report.");
|
|
51
|
+
for (const page of observation.pages)
|
|
52
|
+
requireSame(
|
|
53
|
+
scope(provider, page.request),
|
|
54
|
+
scope(provider, expected.request),
|
|
55
|
+
"Report request differs from configured scope.",
|
|
56
|
+
);
|
|
57
|
+
return { observation, report: normalizeReport(observation, context) };
|
|
58
|
+
}
|
|
59
|
+
export const fingerprint = (value: unknown) => Bun.CryptoHasher.hash("sha256", canonical(value), "hex");
|
|
60
|
+
|
|
61
|
+
export async function evaluateChange(
|
|
62
|
+
record: ChangeRecord,
|
|
63
|
+
baseline: ImportedSnapshot,
|
|
64
|
+
current: ImportedSnapshot | undefined,
|
|
65
|
+
maxRows: number,
|
|
66
|
+
): Promise<Evidence> {
|
|
67
|
+
if (baseline.target !== record.site || (current && current.target !== record.site))
|
|
68
|
+
throw new RequestError("Change record and snapshots must identify the same exact site", null, "invalid_input");
|
|
69
|
+
const before = baseline.pages[0].response;
|
|
70
|
+
const after = current?.pages[0].response;
|
|
71
|
+
const names = [
|
|
72
|
+
...new Set(
|
|
73
|
+
[...before.observations, ...(after?.observations ?? [])]
|
|
74
|
+
.filter((o) => o.operation === "report" && ["gsc", "ga"].includes(o.provider))
|
|
75
|
+
.map((o) => o.name),
|
|
76
|
+
),
|
|
77
|
+
].sort();
|
|
78
|
+
const result = await capture(
|
|
79
|
+
"pagesight",
|
|
80
|
+
"change.evaluate",
|
|
81
|
+
record.site,
|
|
82
|
+
{
|
|
83
|
+
recordSha256: fingerprint(record),
|
|
84
|
+
baselineSha256: fingerprint(baseline),
|
|
85
|
+
currentSha256: current ? fingerprint(current) : null,
|
|
86
|
+
maxRows,
|
|
87
|
+
},
|
|
88
|
+
async () => {
|
|
89
|
+
const observations = names.map((name) => {
|
|
90
|
+
try {
|
|
91
|
+
const a = boundReport(baseline, name);
|
|
92
|
+
const timezone = (a.report.semantics as { timezone: string }).timezone;
|
|
93
|
+
const deploymentDay = localDay(record.deployedAt, timezone);
|
|
94
|
+
if (a.report.end >= deploymentDay)
|
|
95
|
+
throw new Incompatible("Baseline must end before the deployment day in this report's timezone.");
|
|
96
|
+
if (!current || !after)
|
|
97
|
+
return {
|
|
98
|
+
name,
|
|
99
|
+
status: "pending",
|
|
100
|
+
reason: "After snapshot not supplied; no outcome is available.",
|
|
101
|
+
timezone,
|
|
102
|
+
deploymentDay,
|
|
103
|
+
};
|
|
104
|
+
const b = boundReport(current, name);
|
|
105
|
+
if (b.report.start <= deploymentDay)
|
|
106
|
+
throw new Incompatible("After period must start after the deployment day in this report's timezone.");
|
|
107
|
+
const identity = (snapshot: ImportedSnapshot) => {
|
|
108
|
+
const config = configSchema.parse(snapshot.pages[0].response.context.config);
|
|
109
|
+
return {
|
|
110
|
+
site: config.site,
|
|
111
|
+
gscSite: config.gscSite,
|
|
112
|
+
gaProperty: config.gaProperty ? normalizeGaProperty(config.gaProperty) : null,
|
|
113
|
+
productionHostname: config.productionHostname,
|
|
114
|
+
};
|
|
115
|
+
};
|
|
116
|
+
requireSame(
|
|
117
|
+
identity(baseline),
|
|
118
|
+
identity(current),
|
|
119
|
+
"Configured site, provider identity or production hostname changed.",
|
|
120
|
+
);
|
|
121
|
+
if (
|
|
122
|
+
a.observation.provider === "ga" &&
|
|
123
|
+
record.measurementChanges.some((change) => {
|
|
124
|
+
const day = localDay(change.at, timezone);
|
|
125
|
+
return day >= a.report.start && day <= b.report.end;
|
|
126
|
+
})
|
|
127
|
+
)
|
|
128
|
+
throw new Incompatible(
|
|
129
|
+
"A declared measurement change intersects the combined periods; GA deltas are withheld.",
|
|
130
|
+
);
|
|
131
|
+
return {
|
|
132
|
+
...compareObservation(a.observation, b.observation, maxRows, before.context, after.context),
|
|
133
|
+
timezone,
|
|
134
|
+
deploymentDay,
|
|
135
|
+
gapDays: (Date.parse(b.report.start) - Date.parse(a.report.end)) / 86400000 - 1,
|
|
136
|
+
weekdays: {
|
|
137
|
+
baselineStart: new Date(a.report.start).getUTCDay(),
|
|
138
|
+
currentStart: new Date(b.report.start).getUTCDay(),
|
|
139
|
+
},
|
|
140
|
+
};
|
|
141
|
+
} catch (error) {
|
|
142
|
+
return {
|
|
143
|
+
name,
|
|
144
|
+
status: "incompatible",
|
|
145
|
+
reason:
|
|
146
|
+
error instanceof Incompatible
|
|
147
|
+
? error.message
|
|
148
|
+
: "Unsupported report or reporting timezone; no delta calculated.",
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
return {
|
|
153
|
+
evaluationVersion: 1,
|
|
154
|
+
record,
|
|
155
|
+
recordVerification: "User-supplied deployment and change context, not independently verified",
|
|
156
|
+
scope: "Original configured snapshot reports; affectedUrls are annotations and do not filter report rows",
|
|
157
|
+
state: !current
|
|
158
|
+
? observations.some((o) => o.status === "pending")
|
|
159
|
+
? "pending"
|
|
160
|
+
: "unavailable"
|
|
161
|
+
: observations.some((o) => ["compared", "limited"].includes(o.status))
|
|
162
|
+
? "descriptive"
|
|
163
|
+
: "unavailable",
|
|
164
|
+
observations,
|
|
165
|
+
confounded: record.overlappingChanges.length > 0,
|
|
166
|
+
contextChanged: Boolean(
|
|
167
|
+
after &&
|
|
168
|
+
canonical(configSchema.parse(before.context.config)) !== canonical(configSchema.parse(after.context.config)),
|
|
169
|
+
),
|
|
170
|
+
limitations: [
|
|
171
|
+
"Cloudflare, Bing, crawl graphs, HTML, sitemaps and inspection evidence are unsupported; no cross-provider funnel is calculated.",
|
|
172
|
+
"Declared overlapping changes confound attribution. Configuration-context changes remain visible and can alter event interpretation.",
|
|
173
|
+
"Descriptive comparison only. This does not establish causality, statistical significance or the change's effect on rankings.",
|
|
174
|
+
"Deployment day is excluded in each report's timezone. Equal windows can still differ in weekdays, seasonality, demand or search-system changes.",
|
|
175
|
+
"Only declared measurement and overlapping changes are known. An empty list is not proof that no other changes occurred.",
|
|
176
|
+
"Affected URLs do not change property, hostname, channel or row scope. Missing rows remain unknown, never zero.",
|
|
177
|
+
"Collected timestamps do not prove immutable historical data; source hashes identify the supplied artifacts, not their authenticity.",
|
|
178
|
+
],
|
|
179
|
+
};
|
|
180
|
+
},
|
|
181
|
+
);
|
|
182
|
+
const data = result.pages[0]?.response as { observations: Array<{ status: string }> } | undefined;
|
|
183
|
+
if (
|
|
184
|
+
result.status === "ok" &&
|
|
185
|
+
(!current || !data?.observations.length || data.observations.some((o) => o.status !== "compared"))
|
|
186
|
+
)
|
|
187
|
+
result.status = "partial";
|
|
188
|
+
return result;
|
|
189
|
+
}
|
package/src/api/execute.ts
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
import { verifyRender } from "./verify-render.js";
|
|
2
|
+
import { followupChanges } from "./followup-changes.js";
|
|
3
|
+
import { cloudflareAudit } from "./cloudflare.js";
|
|
4
|
+
|
|
5
|
+
import { evaluateChange } from "./evaluate-change.js";
|
|
6
|
+
import { crawl } from "./crawl.js";
|
|
7
|
+
import { technicalChanges } from "./technical-changes.js";
|
|
8
|
+
import { investigate } from "./investigation.js";
|
|
1
9
|
import { assessSnapshot } from "./assessment.js";
|
|
2
10
|
import { opportunities } from "./opportunities.js";
|
|
3
11
|
import { gaRealtime } from "./ga-realtime.js";
|
|
@@ -34,6 +42,12 @@ export async function execute(input: unknown): Promise<Evidence> {
|
|
|
34
42
|
"invalid_input",
|
|
35
43
|
);
|
|
36
44
|
}
|
|
45
|
+
if (
|
|
46
|
+
op.operation === "page.verify" &&
|
|
47
|
+
op.navigation &&
|
|
48
|
+
new URL(op.navigation.fromUrl).origin !== new URL(op.url).origin
|
|
49
|
+
)
|
|
50
|
+
throw new RequestError("Navigation source must share the target origin", null, "invalid_input");
|
|
37
51
|
const result = await dispatch(op);
|
|
38
52
|
if (op.operation.startsWith("ga.")) result.credential = await gaCredentialInfo();
|
|
39
53
|
if (op.operation.startsWith("bing."))
|
|
@@ -48,6 +62,21 @@ export async function execute(input: unknown): Promise<Evidence> {
|
|
|
48
62
|
|
|
49
63
|
async function dispatch(op: ParsedOperation): Promise<Evidence> {
|
|
50
64
|
switch (op.operation) {
|
|
65
|
+
case "page.verify":
|
|
66
|
+
return verifyRender(op);
|
|
67
|
+
case "cloudflare.audit":
|
|
68
|
+
return cloudflareAudit(op);
|
|
69
|
+
|
|
70
|
+
case "change.followup":
|
|
71
|
+
return followupChanges(op.experiments, op.asOf ?? new Date().toISOString(), op.lagDays);
|
|
72
|
+
case "change.evaluate":
|
|
73
|
+
return evaluateChange(op.record, op.baseline, op.current, op.maxRows);
|
|
74
|
+
case "crawl":
|
|
75
|
+
return crawl(op, execute);
|
|
76
|
+
case "technical.compare":
|
|
77
|
+
return technicalChanges(op.baseline, op.current);
|
|
78
|
+
case "investigate":
|
|
79
|
+
return investigate(op, execute);
|
|
51
80
|
case "opportunities":
|
|
52
81
|
return opportunities(op.snapshot, {
|
|
53
82
|
minImpressions: op.minImpressions,
|