pagesight 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -82
- package/docs/credentials.md +54 -0
- package/docs/diagnostics.md +85 -0
- package/docs/measurement.md +178 -0
- package/docs/opportunities.md +78 -0
- package/docs/snapshots.md +130 -0
- package/docs/usage.md +135 -0
- package/package.json +26 -29
- package/src/api/assessment.ts +315 -0
- package/src/api/bing.ts +83 -0
- package/src/api/compare-snapshots.ts +167 -0
- package/src/api/discover.ts +39 -0
- package/src/api/doctor.ts +35 -0
- package/src/api/evidence-schema.ts +78 -0
- package/src/api/evidence.ts +99 -0
- package/src/api/execute.ts +160 -0
- package/src/api/ga-freshness.ts +20 -0
- package/src/api/ga-realtime.ts +40 -0
- package/src/api/index.ts +5 -0
- package/src/api/opportunities.ts +317 -0
- package/src/api/report-table.ts +216 -0
- package/src/api/reports.ts +108 -0
- package/src/api/schema.ts +271 -0
- package/src/api/snapshot.ts +202 -0
- package/src/api/ui-findings.ts +68 -0
- package/src/assessment-text.ts +55 -0
- package/src/cli.ts +217 -0
- package/src/http.ts +39 -0
- package/src/index.ts +8 -25
- package/src/mcp-server.ts +27 -0
- package/src/mcp.ts +5 -0
- package/src/opportunities-text.ts +61 -0
- package/src/providers/bing.ts +48 -0
- package/src/{lib → providers}/crux.ts +4 -9
- package/src/providers/ga.ts +114 -0
- package/src/providers/google-tokens.ts +86 -0
- package/src/providers/gsc-auth.ts +93 -0
- package/src/{lib → providers}/gsc.ts +27 -24
- package/src/{lib/psi.ts → providers/pagespeed.ts} +3 -14
- package/src/shared/dates.ts +32 -0
- package/src/shared/http.ts +63 -0
- package/src/tools/ai.ts +19 -27
- package/src/tools/audit.ts +18 -98
- package/src/tools/observe.ts +28 -0
- package/src/tools/page/analyze.ts +194 -0
- package/src/tools/page/batch.ts +163 -0
- package/src/tools/page/contrast.ts +128 -0
- package/src/tools/page/links.ts +200 -0
- package/src/tools/page/metadata.ts +225 -0
- package/src/tools/page/structured-data.ts +288 -0
- package/src/tools/page/tool.ts +48 -0
- package/src/tools/search/actions.ts +56 -0
- package/src/tools/search/analytics.ts +266 -0
- package/src/tools/search/coverage.ts +247 -0
- package/src/tools/search/gaps.ts +129 -0
- package/src/tools/search/inspection.ts +160 -0
- package/src/tools/search/result.ts +3 -0
- package/src/tools/search/sample.ts +110 -0
- package/src/tools/search/schema.ts +62 -0
- package/src/{lib/sitemap.ts → tools/search/sitemap-sampling.ts} +1 -45
- package/src/tools/search/sites.ts +86 -0
- package/src/tools/search/tool.ts +11 -0
- package/src/tools/setup.ts +1 -1
- package/src/tools/speed/analyze.ts +191 -0
- package/src/tools/speed/batch.ts +265 -0
- package/src/tools/speed/crux.ts +176 -0
- package/src/tools/speed/pagespeed.ts +273 -0
- package/src/tools/speed/schema.ts +39 -0
- package/src/tools/speed/tool.ts +11 -0
- package/src/web/fetch.ts +31 -0
- package/src/web/images.ts +62 -0
- package/src/web/page-observation.ts +69 -0
- package/src/{lib → web}/robots.ts +20 -12
- package/src/web/sitemap-inventory.ts +64 -0
- package/src/web/sitemap-parser.ts +59 -0
- package/src/lib/auth.ts +0 -187
- package/src/tools/page.ts +0 -1241
- package/src/tools/search.ts +0 -1118
- package/src/tools/speed.ts +0 -956
package/src/cli.ts
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { renderAssessment } from "./assessment-text.js";
|
|
2
|
+
import { renderOpportunities } from "./opportunities-text.js";
|
|
3
|
+
import { parseArgs } from "node:util";
|
|
4
|
+
import { defaultDates } from "./shared/dates.js";
|
|
5
|
+
import { execute } from "./api/index.js";
|
|
6
|
+
import { RequestError } from "./shared/http.js";
|
|
7
|
+
|
|
8
|
+
export const help = `Pagesight — read-only site evidence
|
|
9
|
+
|
|
10
|
+
pagesight Show CLI help
|
|
11
|
+
pagesight mcp Start MCP explicitly
|
|
12
|
+
pagesight discover --url https://example.com/ [--providers gsc,ga]
|
|
13
|
+
pagesight doctor --config seo.config.json
|
|
14
|
+
pagesight gsc sites
|
|
15
|
+
pagesight gsc sitemaps --site sc-domain:example.com
|
|
16
|
+
pagesight gsc inspect --site sc-domain:example.com --url https://example.com/
|
|
17
|
+
pagesight gsc report --site sc-domain:example.com --request report.json [--max-pages 4]
|
|
18
|
+
pagesight bing crawl-stats --site https://example.com/
|
|
19
|
+
pagesight bing crawl-issues --site https://example.com/
|
|
20
|
+
pagesight bing url-info --site https://example.com/ --url https://example.com/page
|
|
21
|
+
pagesight bing link-counts --site https://example.com/ [--max-pages 4]
|
|
22
|
+
pagesight bing url-links --site https://example.com/ --url https://example.com/page [--max-pages 4]
|
|
23
|
+
pagesight bing sites
|
|
24
|
+
pagesight bing queries --site https://example.com/
|
|
25
|
+
pagesight bing pages --site https://example.com/
|
|
26
|
+
pagesight bing traffic --site https://example.com/
|
|
27
|
+
pagesight ga realtime --property 123456 --request realtime.json
|
|
28
|
+
pagesight ga accounts
|
|
29
|
+
pagesight ga property --property 123456
|
|
30
|
+
pagesight ga key-events --property 123456
|
|
31
|
+
pagesight ga report --property 123456 --request report.json [--max-pages 4]
|
|
32
|
+
pagesight page --url https://example.com/
|
|
33
|
+
pagesight speed psi --url https://example.com/ [--strategy mobile]
|
|
34
|
+
pagesight speed crux --url https://example.com/ [--origin] [--form-factor PHONE]
|
|
35
|
+
pagesight speed history --url https://example.com/ [--origin]
|
|
36
|
+
pagesight snapshot --config seo.config.json [--start YYYY-MM-DD --end YYYY-MM-DD]
|
|
37
|
+
pagesight assess --snapshot saved.json [--format text] [--max-rows 10]
|
|
38
|
+
pagesight opportunities --snapshot saved.json [--min-impressions 20] [--max-clicks 2] [--max-rows 10] [--format text]
|
|
39
|
+
pagesight compare --baseline before.json --current after.json [--max-rows 100]
|
|
40
|
+
pagesight evidence import --request findings.json
|
|
41
|
+
pagesight api --request operation.json
|
|
42
|
+
pagesight serve [--port 6095] Local HTTP API; requires PAGESIGHT_API_TOKEN
|
|
43
|
+
|
|
44
|
+
Data commands emit JSON by default; assess --format text prints a readable summary. --json is accepted for clarity.
|
|
45
|
+
--out FILE saves the same evidence locally. Exit: 0 success, 1 provider failure,
|
|
46
|
+
2 invalid input, 3 partial evidence. Snapshot defaults to 28 days ending Pacific
|
|
47
|
+
today minus 3 days; max-pages defaults to 4 (ad hoc reports: 1; maximum: 20).
|
|
48
|
+
Credentials: GSC_* env, PAGESIGHT_GA_CREDENTIALS or GOOGLE_APPLICATION_CREDENTIALS (GA), GOOGLE_API_KEY (speed), BING_WEBMASTER_API_KEY (Bing).
|
|
49
|
+
Requests, property metadata, provider limits and errors are preserved in evidence.
|
|
50
|
+
`;
|
|
51
|
+
|
|
52
|
+
async function jsonFile(path: string | undefined, label: string): Promise<unknown> {
|
|
53
|
+
if (!path) throw new RequestError(`Missing --${label}`, null, "invalid_input");
|
|
54
|
+
try {
|
|
55
|
+
return await Bun.file(path).json();
|
|
56
|
+
} catch {
|
|
57
|
+
throw new RequestError(`Cannot read --${label} JSON file`, null, "invalid_input");
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export async function runCli(args: string[]): Promise<number> {
|
|
62
|
+
try {
|
|
63
|
+
const { values, positionals } = parseArgs({
|
|
64
|
+
args,
|
|
65
|
+
allowPositionals: true,
|
|
66
|
+
strict: true,
|
|
67
|
+
options: {
|
|
68
|
+
port: { type: "string" },
|
|
69
|
+
help: { type: "boolean", short: "h" },
|
|
70
|
+
json: { type: "boolean" },
|
|
71
|
+
config: { type: "string" },
|
|
72
|
+
request: { type: "string" },
|
|
73
|
+
site: { type: "string" },
|
|
74
|
+
property: { type: "string" },
|
|
75
|
+
url: { type: "string" },
|
|
76
|
+
out: { type: "string" },
|
|
77
|
+
start: { type: "string" },
|
|
78
|
+
end: { type: "string" },
|
|
79
|
+
"max-pages": { type: "string" },
|
|
80
|
+
strategy: { type: "string" },
|
|
81
|
+
"form-factor": { type: "string" },
|
|
82
|
+
origin: { type: "boolean" },
|
|
83
|
+
providers: { type: "string" },
|
|
84
|
+
snapshot: { type: "string" },
|
|
85
|
+
format: { type: "string" },
|
|
86
|
+
baseline: { type: "string" },
|
|
87
|
+
current: { type: "string" },
|
|
88
|
+
"max-rows": { type: "string" },
|
|
89
|
+
"min-impressions": { type: "string" },
|
|
90
|
+
"max-clicks": { type: "string" },
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
if (args.length === 0 || values.help || positionals[0] === "help") {
|
|
94
|
+
process.stdout.write(help);
|
|
95
|
+
return 0;
|
|
96
|
+
}
|
|
97
|
+
const [family, action] = positionals;
|
|
98
|
+
if (positionals.length > (["gsc", "ga", "bing", "speed", "evidence"].includes(family) ? 2 : 1))
|
|
99
|
+
throw new RequestError("Too many command arguments", null, "invalid_input");
|
|
100
|
+
const operation = ["gsc", "ga", "bing", "speed", "evidence"].includes(family)
|
|
101
|
+
? `${family}.${action ?? ""}`
|
|
102
|
+
: family;
|
|
103
|
+
const flags: Record<string, string[]> = {
|
|
104
|
+
"evidence.import": ["request"],
|
|
105
|
+
assess: ["snapshot", "max-rows", "format"],
|
|
106
|
+
opportunities: ["snapshot", "max-rows", "format", "min-impressions", "max-clicks"],
|
|
107
|
+
discover: ["url", "providers"],
|
|
108
|
+
compare: ["baseline", "current", "max-rows"],
|
|
109
|
+
"bing.crawl-stats": ["site"],
|
|
110
|
+
"bing.crawl-issues": ["site"],
|
|
111
|
+
"bing.url-info": ["site", "url"],
|
|
112
|
+
"bing.link-counts": ["site", "max-pages"],
|
|
113
|
+
"bing.url-links": ["site", "url", "max-pages"],
|
|
114
|
+
"bing.sites": [],
|
|
115
|
+
"bing.queries": ["site"],
|
|
116
|
+
"bing.pages": ["site"],
|
|
117
|
+
"bing.traffic": ["site"],
|
|
118
|
+
"gsc.sites": [],
|
|
119
|
+
"gsc.sitemaps": ["site"],
|
|
120
|
+
"gsc.inspect": ["site", "url"],
|
|
121
|
+
"gsc.report": ["site", "request", "max-pages"],
|
|
122
|
+
"ga.realtime": ["property", "request"],
|
|
123
|
+
"ga.accounts": [],
|
|
124
|
+
"ga.property": ["property"],
|
|
125
|
+
"ga.key-events": ["property"],
|
|
126
|
+
"ga.report": ["property", "request", "max-pages"],
|
|
127
|
+
page: ["url"],
|
|
128
|
+
"speed.psi": ["url", "strategy"],
|
|
129
|
+
"speed.crux": ["url", "origin", "form-factor"],
|
|
130
|
+
"speed.history": ["url", "origin", "form-factor"],
|
|
131
|
+
doctor: ["config"],
|
|
132
|
+
snapshot: ["config", "start", "end", "max-pages"],
|
|
133
|
+
api: ["request"],
|
|
134
|
+
serve: ["port"],
|
|
135
|
+
};
|
|
136
|
+
const allowed = [...(flags[operation] ?? []), ...(operation === "serve" ? [] : ["out", "json"])];
|
|
137
|
+
for (const flag of Object.keys(values))
|
|
138
|
+
if (!allowed.includes(flag))
|
|
139
|
+
throw new RequestError(`--${flag} is not supported for ${operation}`, null, "invalid_input");
|
|
140
|
+
if (operation === "serve") {
|
|
141
|
+
const port = Number(values.port ?? 6095);
|
|
142
|
+
if (!Number.isInteger(port) || port < 1 || port > 65535)
|
|
143
|
+
throw new RequestError("Invalid port", null, "invalid_input");
|
|
144
|
+
const { startHttpApi } = await import("./http.js");
|
|
145
|
+
const server = startHttpApi(process.env.PAGESIGHT_API_TOKEN ?? "", port);
|
|
146
|
+
process.stderr.write(`Pagesight API listening on ${server.url}v1/query\n`);
|
|
147
|
+
return 0;
|
|
148
|
+
}
|
|
149
|
+
if (values.format && !["json", "text"].includes(values.format))
|
|
150
|
+
throw new RequestError("Use --format json or text", null, "invalid_input");
|
|
151
|
+
if (values.json && values.format === "text")
|
|
152
|
+
throw new RequestError("--json and --format text conflict", null, "invalid_input");
|
|
153
|
+
let input: unknown;
|
|
154
|
+
if (operation === "api") input = await jsonFile(values.request, "request");
|
|
155
|
+
else if (operation === "evidence.import")
|
|
156
|
+
input = { operation, document: await jsonFile(values.request, "request") };
|
|
157
|
+
else if (operation === "opportunities")
|
|
158
|
+
input = {
|
|
159
|
+
operation,
|
|
160
|
+
snapshot: await jsonFile(values.snapshot, "snapshot"),
|
|
161
|
+
minImpressions: Number(values["min-impressions"] ?? 20),
|
|
162
|
+
maxClicks: Number(values["max-clicks"] ?? 2),
|
|
163
|
+
maxRows: Number(values["max-rows"] ?? 10),
|
|
164
|
+
};
|
|
165
|
+
else if (operation === "assess")
|
|
166
|
+
input = {
|
|
167
|
+
operation,
|
|
168
|
+
snapshot: await jsonFile(values.snapshot, "snapshot"),
|
|
169
|
+
maxRows: Number(values["max-rows"] ?? 10),
|
|
170
|
+
};
|
|
171
|
+
else if (operation === "compare")
|
|
172
|
+
input = {
|
|
173
|
+
operation,
|
|
174
|
+
baseline: await jsonFile(values.baseline, "baseline"),
|
|
175
|
+
current: await jsonFile(values.current, "current"),
|
|
176
|
+
maxRows: Number(values["max-rows"] ?? 100),
|
|
177
|
+
};
|
|
178
|
+
else if (operation === "snapshot" || operation === "doctor") {
|
|
179
|
+
const config = await jsonFile(values.config, "config");
|
|
180
|
+
if (operation === "doctor") input = { operation, config };
|
|
181
|
+
else {
|
|
182
|
+
if (Boolean(values.start) !== Boolean(values.end))
|
|
183
|
+
throw new RequestError("Supply both --start and --end, or neither", null, "invalid_input");
|
|
184
|
+
const dates = values.start ? { startDate: values.start, endDate: values.end } : defaultDates();
|
|
185
|
+
input = { operation, config, ...dates, maxPages: Number(values["max-pages"] ?? 4) };
|
|
186
|
+
}
|
|
187
|
+
} else {
|
|
188
|
+
input = {
|
|
189
|
+
operation,
|
|
190
|
+
...(values.site ? { site: values.site } : {}),
|
|
191
|
+
...(values.property ? { property: values.property } : {}),
|
|
192
|
+
...(values.url ? { url: values.url } : {}),
|
|
193
|
+
...(values.request ? { request: await jsonFile(values.request, "request") } : {}),
|
|
194
|
+
...(values["max-pages"] ? { maxPages: Number(values["max-pages"]) } : {}),
|
|
195
|
+
...(values.strategy ? { strategy: values.strategy } : {}),
|
|
196
|
+
...(values["form-factor"] ? { formFactor: values["form-factor"] } : {}),
|
|
197
|
+
...(values.origin ? { origin: true } : {}),
|
|
198
|
+
...(values.providers !== undefined ? { providers: values.providers.split(",") } : {}),
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
const result = await execute(input);
|
|
202
|
+
const output =
|
|
203
|
+
values.format === "text"
|
|
204
|
+
? operation === "opportunities"
|
|
205
|
+
? renderOpportunities(result)
|
|
206
|
+
: renderAssessment(result)
|
|
207
|
+
: `${JSON.stringify(result, null, 2)}\n`;
|
|
208
|
+
if (values.out) await Bun.write(values.out, output);
|
|
209
|
+
process.stdout.write(output);
|
|
210
|
+
return result.status === "ok" ? 0 : result.status === "partial" ? 3 : 1;
|
|
211
|
+
} catch (error) {
|
|
212
|
+
process.stderr.write(
|
|
213
|
+
`${JSON.stringify({ error: { code: error instanceof RequestError ? error.code : "invalid_input", message: error instanceof RequestError ? error.message : "Invalid command or request; use --help" } })}\n`,
|
|
214
|
+
);
|
|
215
|
+
return 2;
|
|
216
|
+
}
|
|
217
|
+
}
|
package/src/http.ts
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { timingSafeEqual } from "node:crypto";
|
|
2
|
+
import { execute } from "./api/index.js";
|
|
3
|
+
import { RequestError } from "./shared/http.js";
|
|
4
|
+
|
|
5
|
+
export function startHttpApi(token: string, port = 6095) {
|
|
6
|
+
if (token.length < 24)
|
|
7
|
+
throw new RequestError("Set PAGESIGHT_API_TOKEN to at least 24 characters", null, "invalid_input");
|
|
8
|
+
return Bun.serve({
|
|
9
|
+
hostname: "127.0.0.1",
|
|
10
|
+
port,
|
|
11
|
+
maxRequestBodySize: 32_000_000,
|
|
12
|
+
idleTimeout: 255,
|
|
13
|
+
async fetch(request) {
|
|
14
|
+
const authorization = Buffer.from(request.headers.get("authorization") ?? "");
|
|
15
|
+
const expected = Buffer.from(`Bearer ${token}`);
|
|
16
|
+
if (authorization.length !== expected.length || !timingSafeEqual(authorization, expected))
|
|
17
|
+
return Response.json({ error: { code: "unauthenticated" } }, { status: 401 });
|
|
18
|
+
if (request.method !== "POST" || new URL(request.url).pathname !== "/v1/query")
|
|
19
|
+
return Response.json({ error: { code: "not_found" } }, { status: 404 });
|
|
20
|
+
try {
|
|
21
|
+
const result = await execute(await request.json());
|
|
22
|
+
return Response.json(result, {
|
|
23
|
+
status: result.status === "error" ? 502 : 200,
|
|
24
|
+
headers: { "Cache-Control": "no-store" },
|
|
25
|
+
});
|
|
26
|
+
} catch (error) {
|
|
27
|
+
return Response.json(
|
|
28
|
+
{
|
|
29
|
+
error: {
|
|
30
|
+
code: error instanceof RequestError ? error.code : "invalid_input",
|
|
31
|
+
message: error instanceof RequestError ? error.message : "Invalid JSON operation",
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
{ status: 400 },
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
});
|
|
39
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,26 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
|
-
|
|
3
|
-
import
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
const pkg = await Bun.file(new URL("../package.json", import.meta.url)).json();
|
|
12
|
-
|
|
13
|
-
const server = new McpServer({
|
|
14
|
-
name: "pagesight",
|
|
15
|
-
version: pkg.version,
|
|
16
|
-
});
|
|
17
|
-
|
|
18
|
-
registerAiTool(server);
|
|
19
|
-
registerAuditTool(server);
|
|
20
|
-
registerPageTool(server);
|
|
21
|
-
registerSearchTool(server);
|
|
22
|
-
registerSetupTool(server);
|
|
23
|
-
registerSpeedTool(server);
|
|
24
|
-
|
|
25
|
-
const transport = new StdioServerTransport();
|
|
26
|
-
await server.connect(transport);
|
|
2
|
+
if (Bun.argv.length === 3 && Bun.argv[2] === "mcp") {
|
|
3
|
+
await import("./mcp.js");
|
|
4
|
+
} else {
|
|
5
|
+
const { runCli } = await import("./cli.js");
|
|
6
|
+
process.exitCode = await runCli(Bun.argv.slice(2));
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export {};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import { registerAiTool } from "./tools/ai.js";
|
|
3
|
+
import { registerAuditTool } from "./tools/audit.js";
|
|
4
|
+
import { registerObserveTool } from "./tools/observe.js";
|
|
5
|
+
import { registerPageTool } from "./tools/page/tool.js";
|
|
6
|
+
import { registerSearchTool } from "./tools/search/tool.js";
|
|
7
|
+
import { registerSetupTool } from "./tools/setup.js";
|
|
8
|
+
import { registerSpeedTool } from "./tools/speed/tool.js";
|
|
9
|
+
|
|
10
|
+
const pkg = await Bun.file(new URL("../package.json", import.meta.url)).json();
|
|
11
|
+
|
|
12
|
+
export function createMcpServer() {
|
|
13
|
+
const server = new McpServer({
|
|
14
|
+
name: "pagesight",
|
|
15
|
+
version: pkg.version,
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
registerObserveTool(server);
|
|
19
|
+
registerAiTool(server);
|
|
20
|
+
registerAuditTool(server);
|
|
21
|
+
registerPageTool(server);
|
|
22
|
+
registerSearchTool(server);
|
|
23
|
+
registerSetupTool(server);
|
|
24
|
+
registerSpeedTool(server);
|
|
25
|
+
|
|
26
|
+
return server;
|
|
27
|
+
}
|
package/src/mcp.ts
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { Evidence } from "./api/evidence.js";
|
|
2
|
+
|
|
3
|
+
const text = (value: unknown) =>
|
|
4
|
+
JSON.stringify(value).replace(
|
|
5
|
+
/[\u007f-\u009f\u2028\u2029]/gu,
|
|
6
|
+
(c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`,
|
|
7
|
+
);
|
|
8
|
+
export function renderOpportunities(result: Evidence): string {
|
|
9
|
+
const out = result.pages[0]?.response as
|
|
10
|
+
| {
|
|
11
|
+
site: string;
|
|
12
|
+
policy: unknown;
|
|
13
|
+
observedSearchRows: number | null;
|
|
14
|
+
unusableSearchRows: number;
|
|
15
|
+
qualifyingObservedRows: number | null;
|
|
16
|
+
omittedCandidates: number;
|
|
17
|
+
requestedDates: unknown;
|
|
18
|
+
snapshotSha256: string;
|
|
19
|
+
limitations: string[];
|
|
20
|
+
unassociatedOrganic: unknown[];
|
|
21
|
+
candidates: Array<{
|
|
22
|
+
url: string;
|
|
23
|
+
reason: string;
|
|
24
|
+
search: unknown;
|
|
25
|
+
organic: unknown;
|
|
26
|
+
technical: unknown;
|
|
27
|
+
unknowns: string[];
|
|
28
|
+
nextChecks: string[];
|
|
29
|
+
suggestedRequests: unknown[];
|
|
30
|
+
}>;
|
|
31
|
+
}
|
|
32
|
+
| undefined;
|
|
33
|
+
if (!out) return `${JSON.stringify(result, null, 2)}\n`;
|
|
34
|
+
const lines = [
|
|
35
|
+
`Pagesight opportunities: ${text(out.site)} (${result.status})`,
|
|
36
|
+
`Period: ${text(out.requestedDates)}; supplied snapshot, not reverified.`,
|
|
37
|
+
`Snapshot hash: ${out.snapshotSha256}`,
|
|
38
|
+
`Selection policy: ${text(out.policy)}`,
|
|
39
|
+
`Observed search rows: ${text(out.observedSearchRows)}; unusable: ${out.unusableSearchRows}; qualifying: ${text(out.qualifyingObservedRows)}; omitted candidates: ${out.omittedCandidates}`,
|
|
40
|
+
];
|
|
41
|
+
for (const candidate of out.candidates)
|
|
42
|
+
lines.push(
|
|
43
|
+
"",
|
|
44
|
+
text(candidate.url),
|
|
45
|
+
`Why: ${text(candidate.reason)}`,
|
|
46
|
+
`Search: ${text(candidate.search)}`,
|
|
47
|
+
`Organic associations: ${text(candidate.organic)}`,
|
|
48
|
+
`Technical evidence: ${text(candidate.technical)}`,
|
|
49
|
+
...candidate.unknowns.map((u) => `Unknown: ${text(u)}`),
|
|
50
|
+
...candidate.nextChecks.map((n) => `Next check: ${text(n)}`),
|
|
51
|
+
...candidate.suggestedRequests.map((request) => `Suggested API request: ${text(request)}`),
|
|
52
|
+
);
|
|
53
|
+
lines.push(
|
|
54
|
+
"",
|
|
55
|
+
...out.unassociatedOrganic.map(
|
|
56
|
+
(table) => `Organic evidence not associated with displayed candidates: ${text(table)}`,
|
|
57
|
+
),
|
|
58
|
+
);
|
|
59
|
+
lines.push("", ...out.limitations.map((l) => `Limit: ${text(l)}`));
|
|
60
|
+
return `${lines.join("\n")}\n`;
|
|
61
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { RequestError, requestJson } from "../shared/http.js";
|
|
2
|
+
|
|
3
|
+
export const bingMethods = {
|
|
4
|
+
"crawl-stats": "GetCrawlStats",
|
|
5
|
+
"crawl-issues": "GetCrawlIssues",
|
|
6
|
+
"url-info": "GetUrlInfo",
|
|
7
|
+
"link-counts": "GetLinkCounts",
|
|
8
|
+
"url-links": "GetUrlLinks",
|
|
9
|
+
sites: "GetUserSites",
|
|
10
|
+
queries: "GetQueryStats",
|
|
11
|
+
pages: "GetPageStats",
|
|
12
|
+
traffic: "GetRankAndTrafficStats",
|
|
13
|
+
} as const;
|
|
14
|
+
export type BingAction = keyof typeof bingMethods;
|
|
15
|
+
|
|
16
|
+
export async function bingFetch(
|
|
17
|
+
action: BingAction,
|
|
18
|
+
site?: string,
|
|
19
|
+
parameters: Record<string, string> = {},
|
|
20
|
+
): Promise<Record<string, unknown> & { d: unknown }> {
|
|
21
|
+
const key = process.env.BING_WEBMASTER_API_KEY;
|
|
22
|
+
if (!key) throw new RequestError("Set BING_WEBMASTER_API_KEY", null, "not_configured");
|
|
23
|
+
const url = new URL(`https://ssl.bing.com/webmaster/api.svc/json/${bingMethods[action]}`);
|
|
24
|
+
url.searchParams.set("apikey", key);
|
|
25
|
+
if (site !== undefined) url.searchParams.set("siteUrl", site);
|
|
26
|
+
for (const [name, value] of Object.entries(parameters)) url.searchParams.set(name, value);
|
|
27
|
+
const response = await requestJson<unknown>(url.href, { redirect: "error" });
|
|
28
|
+
if (response && typeof response === "object" && "ErrorCode" in response && Number.isSafeInteger(response.ErrorCode))
|
|
29
|
+
throw new RequestError("Bing returned an API fault", null, `bing_fault_${String(response.ErrorCode)}`);
|
|
30
|
+
if (!response || typeof response !== "object" || !("d" in response))
|
|
31
|
+
throw new RequestError("Bing returned an unexpected response envelope", null, "invalid_response");
|
|
32
|
+
const d = response.d;
|
|
33
|
+
const object = d !== null && typeof d === "object" && !Array.isArray(d);
|
|
34
|
+
let valid: boolean;
|
|
35
|
+
if (action === "url-info") valid = object && "Url" in d && typeof d.Url === "string";
|
|
36
|
+
else if (action === "link-counts" || action === "url-links") {
|
|
37
|
+
const rows = action === "link-counts" ? "Links" : "Details";
|
|
38
|
+
valid =
|
|
39
|
+
object &&
|
|
40
|
+
rows in d &&
|
|
41
|
+
Array.isArray(d[rows as keyof typeof d]) &&
|
|
42
|
+
"TotalPages" in d &&
|
|
43
|
+
Number.isSafeInteger(d.TotalPages) &&
|
|
44
|
+
Number(d.TotalPages) >= 0;
|
|
45
|
+
} else valid = Array.isArray(d);
|
|
46
|
+
if (!valid) throw new RequestError("Bing returned an unexpected response envelope", null, "invalid_response");
|
|
47
|
+
return response as Record<string, unknown> & { d: unknown };
|
|
48
|
+
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import { RequestError, requestJson } from "../shared/http.js";
|
|
2
|
+
|
|
1
3
|
const CRUX_API = "https://chromeuxreport.googleapis.com/v1/records";
|
|
2
4
|
|
|
3
5
|
function getApiKey(): string {
|
|
4
6
|
const key = process.env.GOOGLE_API_KEY;
|
|
5
|
-
if (!key) throw new
|
|
7
|
+
if (!key) throw new RequestError("GOOGLE_API_KEY is required for CrUX API", null, "not_configured");
|
|
6
8
|
return key;
|
|
7
9
|
}
|
|
8
10
|
|
|
@@ -91,18 +93,11 @@ async function cruxFetch<T>(
|
|
|
91
93
|
body.collectionPeriodCount = options.collectionPeriodCount;
|
|
92
94
|
}
|
|
93
95
|
|
|
94
|
-
|
|
96
|
+
return requestJson<T>(`${CRUX_API}:${endpoint}?key=${key}`, {
|
|
95
97
|
method: "POST",
|
|
96
98
|
headers: { "Content-Type": "application/json" },
|
|
97
99
|
body: JSON.stringify(body),
|
|
98
100
|
});
|
|
99
|
-
|
|
100
|
-
if (!res.ok) {
|
|
101
|
-
const err = await res.text();
|
|
102
|
-
throw new Error(`CrUX API error (${res.status}): ${err}`);
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
return res.json() as Promise<T>;
|
|
106
101
|
}
|
|
107
102
|
|
|
108
103
|
// --- CrUX Daily API ---
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { homedir } from "node:os";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { getOAuthToken, getServiceAccountToken } from "./google-tokens.js";
|
|
4
|
+
import { RequestError, requestJson } from "../shared/http.js";
|
|
5
|
+
|
|
6
|
+
const SCOPE = "https://www.googleapis.com/auth/analytics.readonly";
|
|
7
|
+
let cached: { path: string; token: string; expiresAt: number } | undefined;
|
|
8
|
+
let pending: Promise<string> | undefined;
|
|
9
|
+
|
|
10
|
+
export function gaCredentialPath(): string {
|
|
11
|
+
return (
|
|
12
|
+
process.env.PAGESIGHT_GA_CREDENTIALS ||
|
|
13
|
+
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
|
14
|
+
join(homedir(), ".config/gcloud/application_default_credentials.json")
|
|
15
|
+
);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export async function gaAccessToken(): Promise<string> {
|
|
19
|
+
const path = gaCredentialPath();
|
|
20
|
+
if (cached?.path === path && cached.expiresAt > Date.now()) return cached.token;
|
|
21
|
+
if (pending) return pending;
|
|
22
|
+
pending = (async () => {
|
|
23
|
+
if (!(await Bun.file(path).exists()))
|
|
24
|
+
throw new RequestError(
|
|
25
|
+
"Set PAGESIGHT_GA_CREDENTIALS or GOOGLE_APPLICATION_CREDENTIALS to a Google credential JSON file",
|
|
26
|
+
null,
|
|
27
|
+
"not_configured",
|
|
28
|
+
);
|
|
29
|
+
let key: { type?: string; client_id?: string; client_secret?: string; refresh_token?: string };
|
|
30
|
+
try {
|
|
31
|
+
key = await Bun.file(path).json();
|
|
32
|
+
if (!key || typeof key !== "object") throw new Error();
|
|
33
|
+
} catch {
|
|
34
|
+
throw new RequestError("Cannot parse configured GA credential JSON", null, "invalid_credentials");
|
|
35
|
+
}
|
|
36
|
+
let token: string;
|
|
37
|
+
if (key.type === "service_account") token = await getServiceAccountToken(path, SCOPE);
|
|
38
|
+
else if (
|
|
39
|
+
key.type === "authorized_user" &&
|
|
40
|
+
typeof key.client_id === "string" &&
|
|
41
|
+
key.client_id &&
|
|
42
|
+
typeof key.client_secret === "string" &&
|
|
43
|
+
key.client_secret &&
|
|
44
|
+
typeof key.refresh_token === "string" &&
|
|
45
|
+
key.refresh_token
|
|
46
|
+
) {
|
|
47
|
+
token = await getOAuthToken(key.client_id, key.client_secret, key.refresh_token);
|
|
48
|
+
} else
|
|
49
|
+
throw new RequestError(
|
|
50
|
+
"Supported GA credentials: service_account or authorized_user ADC",
|
|
51
|
+
null,
|
|
52
|
+
"unsupported_credentials",
|
|
53
|
+
);
|
|
54
|
+
cached = { path, token, expiresAt: Date.now() + 3_000_000 };
|
|
55
|
+
return token;
|
|
56
|
+
})();
|
|
57
|
+
try {
|
|
58
|
+
return await pending;
|
|
59
|
+
} finally {
|
|
60
|
+
pending = undefined;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export async function gaFetch<T>(path: string, body?: unknown, admin = false): Promise<T> {
|
|
65
|
+
const token = await gaAccessToken();
|
|
66
|
+
const base = admin ? "https://analyticsadmin.googleapis.com/v1beta/" : "https://analyticsdata.googleapis.com/v1beta/";
|
|
67
|
+
try {
|
|
68
|
+
return await requestJson<T>(base + path, {
|
|
69
|
+
method: body ? "POST" : "GET",
|
|
70
|
+
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
|
|
71
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
72
|
+
});
|
|
73
|
+
} catch (error) {
|
|
74
|
+
if (error instanceof RequestError && error.status === 401) cached = undefined;
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function normalizeGaProperty(id: string): string {
|
|
80
|
+
const number = id.replace(/^properties\//, "");
|
|
81
|
+
if (!/^\d+$/.test(number)) throw new RequestError("GA property must be a numeric ID", null, "invalid_input");
|
|
82
|
+
return `properties/${number}`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface GaReport {
|
|
86
|
+
rows?: Array<{ dimensionValues?: Array<{ value?: string }>; metricValues?: Array<{ value?: string }> }>;
|
|
87
|
+
rowCount?: number;
|
|
88
|
+
metadata?: {
|
|
89
|
+
timeZone?: string;
|
|
90
|
+
dataLossFromOtherRow?: boolean;
|
|
91
|
+
subjectToThresholding?: boolean;
|
|
92
|
+
samplingMetadatas?: unknown[];
|
|
93
|
+
[key: string]: unknown;
|
|
94
|
+
};
|
|
95
|
+
[key: string]: unknown;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export async function gaCredentialInfo() {
|
|
99
|
+
const source = process.env.PAGESIGHT_GA_CREDENTIALS
|
|
100
|
+
? "PAGESIGHT_GA_CREDENTIALS"
|
|
101
|
+
: process.env.GOOGLE_APPLICATION_CREDENTIALS
|
|
102
|
+
? "GOOGLE_APPLICATION_CREDENTIALS"
|
|
103
|
+
: "gcloud_adc";
|
|
104
|
+
try {
|
|
105
|
+
const key = await Bun.file(gaCredentialPath()).json();
|
|
106
|
+
return {
|
|
107
|
+
source,
|
|
108
|
+
type: typeof key.type === "string" ? key.type : null,
|
|
109
|
+
clientEmail: typeof key.client_email === "string" ? key.client_email : null,
|
|
110
|
+
};
|
|
111
|
+
} catch {
|
|
112
|
+
return { source, type: null, clientEmail: null };
|
|
113
|
+
}
|
|
114
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { RequestError, requestJson } from "../shared/http.js";
|
|
2
|
+
|
|
3
|
+
export const TOKEN_URL = "https://oauth2.googleapis.com/token";
|
|
4
|
+
|
|
5
|
+
interface TokenResponse {
|
|
6
|
+
access_token: string;
|
|
7
|
+
expires_in: number;
|
|
8
|
+
token_type: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export async function getServiceAccountToken(keyPath: string, scope: string): Promise<string> {
|
|
12
|
+
const keyFile = JSON.parse(await Bun.file(keyPath).text());
|
|
13
|
+
if (!keyFile.client_email || !keyFile.private_key) {
|
|
14
|
+
throw new Error("Service account key file missing client_email or private_key");
|
|
15
|
+
}
|
|
16
|
+
const now = Math.floor(Date.now() / 1000);
|
|
17
|
+
|
|
18
|
+
const header = toBase64Url(JSON.stringify({ alg: "RS256", typ: "JWT" }));
|
|
19
|
+
const payload = toBase64Url(
|
|
20
|
+
JSON.stringify({
|
|
21
|
+
iss: keyFile.client_email,
|
|
22
|
+
scope,
|
|
23
|
+
aud: TOKEN_URL,
|
|
24
|
+
iat: now,
|
|
25
|
+
exp: now + 3600,
|
|
26
|
+
}),
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
const signingInput = `${header}.${payload}`;
|
|
30
|
+
const key = await crypto.subtle.importKey(
|
|
31
|
+
"pkcs8",
|
|
32
|
+
pemToBuffer(keyFile.private_key),
|
|
33
|
+
{ name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" },
|
|
34
|
+
false,
|
|
35
|
+
["sign"],
|
|
36
|
+
);
|
|
37
|
+
const signature = await crypto.subtle.sign("RSASSA-PKCS1-v1_5", key, new TextEncoder().encode(signingInput));
|
|
38
|
+
const jwt = `${signingInput}.${bufferToBase64Url(signature)}`;
|
|
39
|
+
|
|
40
|
+
const data = await requestJson<TokenResponse>(TOKEN_URL, {
|
|
41
|
+
method: "POST",
|
|
42
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
43
|
+
body: new URLSearchParams({ grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", assertion: jwt }),
|
|
44
|
+
});
|
|
45
|
+
if (!data.access_token) throw new RequestError("Token exchange returned no access token", null, "invalid_response");
|
|
46
|
+
return data.access_token;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function pemToBuffer(pem: string): ArrayBuffer {
|
|
50
|
+
const b64 = pem
|
|
51
|
+
.replace(/-----BEGIN PRIVATE KEY-----/g, "")
|
|
52
|
+
.replace(/-----END PRIVATE KEY-----/g, "")
|
|
53
|
+
.replace(/\s/g, "");
|
|
54
|
+
const binary = atob(b64);
|
|
55
|
+
const buf = new Uint8Array(binary.length);
|
|
56
|
+
for (let i = 0; i < binary.length; i++) buf[i] = binary.charCodeAt(i);
|
|
57
|
+
return buf.buffer;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function toBase64Url(input: string): string {
|
|
61
|
+
return btoa(input).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function bufferToBase64Url(buf: ArrayBuffer): string {
|
|
65
|
+
const bytes = new Uint8Array(buf);
|
|
66
|
+
let binary = "";
|
|
67
|
+
for (const b of bytes) binary += String.fromCharCode(b);
|
|
68
|
+
return toBase64Url(binary);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// --- OAuth Refresh Token Auth ---
|
|
72
|
+
|
|
73
|
+
export async function getOAuthToken(clientId: string, clientSecret: string, refreshToken: string): Promise<string> {
|
|
74
|
+
const data = await requestJson<TokenResponse>(TOKEN_URL, {
|
|
75
|
+
method: "POST",
|
|
76
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
77
|
+
body: new URLSearchParams({
|
|
78
|
+
grant_type: "refresh_token",
|
|
79
|
+
client_id: clientId,
|
|
80
|
+
client_secret: clientSecret,
|
|
81
|
+
refresh_token: refreshToken,
|
|
82
|
+
}),
|
|
83
|
+
});
|
|
84
|
+
if (!data.access_token) throw new RequestError("Token refresh returned no access token", null, "invalid_response");
|
|
85
|
+
return data.access_token;
|
|
86
|
+
}
|