@gscdump/cli 3.5.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +217 -54
- package/bin/gscdump.mjs +6 -1
- package/dist/analysis-local.mjs +1 -1
- package/dist/auth-state.mjs +112 -0
- package/dist/auth.mjs +109 -122
- package/dist/bing-auth.mjs +200 -0
- package/dist/bing-data.mjs +160 -0
- package/dist/bing-hosted.mjs +121 -0
- package/dist/cli.d.mts +4 -4
- package/dist/cli.mjs +20 -21
- package/dist/cloud-google.mjs +92 -0
- package/dist/command-meta.mjs +13 -1
- package/dist/command-registry.mjs +5 -2
- package/dist/commands/analyze.mjs +34 -177
- package/dist/commands/auth.mjs +161 -7
- package/dist/commands/bing.mjs +337 -0
- package/dist/commands/config.mjs +36 -43
- package/dist/commands/doctor.mjs +73 -24
- package/dist/commands/dump.mjs +1 -1
- package/dist/commands/entities.mjs +4 -4
- package/dist/commands/indexing.mjs +11 -14
- package/dist/commands/init.mjs +1 -1
- package/dist/commands/inspect.mjs +3 -3
- package/dist/commands/mcp.mjs +16 -2
- package/dist/commands/papercut.mjs +76 -0
- package/dist/commands/profile-selection.mjs +2 -2
- package/dist/commands/profile.mjs +8 -3
- package/dist/commands/query.mjs +86 -40
- package/dist/commands/report.mjs +5 -3
- package/dist/commands/sitemaps.mjs +24 -18
- package/dist/commands/skill.mjs +52 -0
- package/dist/commands/stats.mjs +58 -32
- package/dist/commands/sync.mjs +58 -25
- package/dist/config.mjs +39 -4
- package/dist/context.mjs +13 -8
- package/dist/env-file.mjs +1 -1
- package/dist/local-store.mjs +2 -2
- package/dist/mcp/errors.mjs +8 -0
- package/dist/mcp/handlers/diagnostics.mjs +31 -0
- package/dist/mcp/handlers/reports.mjs +38 -11
- package/dist/mcp/server/index.mjs +9 -10
- package/dist/mcp/types.mjs +8 -3
- package/dist/package.mjs +1 -1
- package/dist/papercut.mjs +99 -0
- package/dist/render/analysis.mjs +98 -0
- package/dist/render/charts.mjs +170 -0
- package/dist/render/layout.mjs +87 -0
- package/dist/render/metrics.mjs +163 -0
- package/dist/render/query.mjs +30 -0
- package/dist/render/report.mjs +69 -0
- package/dist/render/terminal.mjs +25 -0
- package/dist/runtime.d.mts +5 -5
- package/dist/runtime.mjs +1 -1
- package/dist/skill.mjs +45 -0
- package/dist/utils.mjs +10 -38
- package/package.json +14 -12
- package/skills/gscdump/SKILL.md +287 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { getCloudAccount } from "./auth-state.mjs";
|
|
2
|
+
import { logger } from "./utils.mjs";
|
|
3
|
+
import { writeBingDump } from "./bing-data.mjs";
|
|
4
|
+
import { createGscdumpV1Client } from "@gscdump/sdk/v1";
|
|
5
|
+
function hostedBingClient(state) {
|
|
6
|
+
return createGscdumpV1Client({
|
|
7
|
+
apiRoot: state.apiRoot,
|
|
8
|
+
credential: state.apiKey
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
async function listHostedBingSites(state) {
|
|
12
|
+
return loadHostedBingConnections(state, (await getCloudAccount(state)).sites, { tolerateFailures: true });
|
|
13
|
+
}
|
|
14
|
+
async function loadHostedBingConnections(state, sites, options = {}) {
|
|
15
|
+
const client = hostedBingClient(state);
|
|
16
|
+
return (await Promise.all(sites.map(async (site) => {
|
|
17
|
+
const connection = await client.getSiteBingConnection({ params: { siteId: site.siteId } }, { signal: AbortSignal.timeout(3e4) }).then((response) => response.data).catch((error) => {
|
|
18
|
+
if (!options.tolerateFailures) throw error;
|
|
19
|
+
const detail = error instanceof Error ? error.message : "Bing connection request failed.";
|
|
20
|
+
logger.warn(`Bing connection for ${site.siteUrl} is unavailable: ${detail}`);
|
|
21
|
+
return null;
|
|
22
|
+
});
|
|
23
|
+
return connection && {
|
|
24
|
+
siteId: site.siteId,
|
|
25
|
+
siteUrl: site.siteUrl,
|
|
26
|
+
connection
|
|
27
|
+
};
|
|
28
|
+
}))).filter((site) => site !== null);
|
|
29
|
+
}
|
|
30
|
+
async function resolveHostedBingSites(state, input) {
|
|
31
|
+
if (Boolean(input.site) === Boolean(input.allSites)) throw new Error("Choose --site or --all-sites.");
|
|
32
|
+
const account = await getCloudAccount(state);
|
|
33
|
+
const exact = account.sites.filter((site) => site.siteId === input.site || site.siteUrl === input.site);
|
|
34
|
+
const normalize = (value) => URL.parse(value.startsWith("sc-domain:") ? `https://${value.slice(10)}/` : value)?.toString();
|
|
35
|
+
const requested = input.allSites ? account.sites : exact.length ? exact : account.sites.filter((site) => normalize(site.siteUrl) === normalize(input.site) && normalize(input.site) !== void 0);
|
|
36
|
+
if (requested.length === 0) throw new Error("No matching Bing site. Run `gscdump bing sites`.");
|
|
37
|
+
if (!input.allSites && requested.length > 1) throw new Error("Multiple Bing sites match. Use a site ID from `gscdump bing sites`.");
|
|
38
|
+
const connections = await loadHostedBingConnections(state, requested, { tolerateFailures: Boolean(input.allSites) });
|
|
39
|
+
const selected = input.allSites ? connections.filter((site) => site.connection._tag === "connected") : connections;
|
|
40
|
+
if (selected.length === 0) throw new Error("No matching Bing site. Run `gscdump bing sites`.");
|
|
41
|
+
if (input.requireConnected !== false && selected.some((site) => site.connection._tag !== "connected")) throw new Error("Bing is not connected. Run `gscdump bing login --site SITE_ID`.");
|
|
42
|
+
return selected;
|
|
43
|
+
}
|
|
44
|
+
async function dumpHostedBingSite(state, site, outDir, options) {
|
|
45
|
+
if (options.datasets.includes("crawl-issues")) throw new Error("Bing crawl issues require local authentication. Use --mode local.");
|
|
46
|
+
const endDate = options.end ?? (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
47
|
+
const startDate = options.start ?? (/* @__PURE__ */ new Date(Date.parse(endDate) - 316224e5)).toISOString().slice(0, 10);
|
|
48
|
+
if (Date.parse(endDate) - Date.parse(startDate) > 316224e5) throw new Error("Hosted Bing exports support at most 366 days. Use a shorter --start and --end range.");
|
|
49
|
+
const client = hostedBingClient(state);
|
|
50
|
+
const results = [];
|
|
51
|
+
let remoteSiteUrl;
|
|
52
|
+
for (const dataset of options.datasets) {
|
|
53
|
+
if (dataset === "crawl-issues") continue;
|
|
54
|
+
let offset = 0;
|
|
55
|
+
let first;
|
|
56
|
+
const rows = [];
|
|
57
|
+
while (true) {
|
|
58
|
+
if (offset > 1e5) throw new Error("Bing pagination reached the API limit. Export a shorter date range.");
|
|
59
|
+
const { data } = await client.getSiteBingData({
|
|
60
|
+
params: { siteId: site.siteId },
|
|
61
|
+
query: {
|
|
62
|
+
dataset,
|
|
63
|
+
startDate,
|
|
64
|
+
endDate,
|
|
65
|
+
limit: 500,
|
|
66
|
+
offset
|
|
67
|
+
}
|
|
68
|
+
}, { signal: AbortSignal.timeout(3e4) });
|
|
69
|
+
if (data.dataset !== dataset) throw new Error("The hosted API returned a different Bing dataset.");
|
|
70
|
+
if (data.sync._tag !== "ready") throw new Error(`Bing ${dataset} data is ${data.sync._tag}. Retry after the hosted sync succeeds.`);
|
|
71
|
+
if (remoteSiteUrl && data.siteUrl !== remoteSiteUrl) throw new Error("The Bing connection changed during export. Retry the dump.");
|
|
72
|
+
remoteSiteUrl = data.siteUrl;
|
|
73
|
+
if (first && (data.sync.observedAt !== first.observedAt || data.pagination.total !== first.total)) throw new Error("The Bing dataset changed during export. Retry the dump.");
|
|
74
|
+
first ??= {
|
|
75
|
+
observedAt: data.sync.observedAt,
|
|
76
|
+
total: data.pagination.total
|
|
77
|
+
};
|
|
78
|
+
if (data.pagination.offset !== offset || data.pagination.hasMore && data.rows.length === 0 || (data.pagination.hasMore ? offset + data.rows.length >= first.total : offset + data.rows.length !== first.total)) throw new Error("The hosted API returned inconsistent Bing pagination. Retry the dump.");
|
|
79
|
+
rows.push(...data.dataset === "keywords" ? data.rows.map(({ keyword, ...row }) => ({
|
|
80
|
+
...row,
|
|
81
|
+
query: keyword
|
|
82
|
+
})) : data.rows);
|
|
83
|
+
if (!data.pagination.hasMore) {
|
|
84
|
+
results.push({
|
|
85
|
+
dataset,
|
|
86
|
+
rows,
|
|
87
|
+
sync: data.sync,
|
|
88
|
+
semantics: data.semantics
|
|
89
|
+
});
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
offset += data.rows.length;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return writeBingDump(remoteSiteUrl ?? site.siteUrl, outDir, options.format, results);
|
|
96
|
+
}
|
|
97
|
+
async function inspectHostedBingUrl(state, siteId, url) {
|
|
98
|
+
const client = hostedBingClient(state);
|
|
99
|
+
let offset = 0;
|
|
100
|
+
while (true) {
|
|
101
|
+
const { data } = await client.listSiteBingIndexingEvidence({
|
|
102
|
+
params: { siteId },
|
|
103
|
+
query: {
|
|
104
|
+
limit: 500,
|
|
105
|
+
offset
|
|
106
|
+
}
|
|
107
|
+
}, { signal: AbortSignal.timeout(3e4) });
|
|
108
|
+
const evidence = data.indexingEvidence.find((row) => row.url === url);
|
|
109
|
+
if (evidence) return evidence;
|
|
110
|
+
if (!data.pagination.hasMore) return {
|
|
111
|
+
_tag: "unknown",
|
|
112
|
+
searchEngine: "bing",
|
|
113
|
+
url,
|
|
114
|
+
reason: "not-observed",
|
|
115
|
+
observedAt: null
|
|
116
|
+
};
|
|
117
|
+
if (data.pagination.offset !== offset || data.indexingEvidence.length === 0) throw new Error("The hosted API returned inconsistent Bing evidence pagination.");
|
|
118
|
+
offset += data.indexingEvidence.length;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
export { dumpHostedBingSite, hostedBingClient, inspectHostedBingUrl, listHostedBingSites, resolveHostedBingSites };
|
package/dist/cli.d.mts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { CliRuntime, CreateCliRuntimeOptions, createCliRuntime } from "./runtime.mjs";
|
|
2
|
-
declare const main: import("citty").CommandDef<import("citty").ArgsDef>;
|
|
3
|
-
interface RunCliOptions {
|
|
2
|
+
export declare const main: import("citty").CommandDef<import("citty").ArgsDef>;
|
|
3
|
+
export interface RunCliOptions {
|
|
4
4
|
rawArgs?: string[];
|
|
5
5
|
loadEnv?: boolean;
|
|
6
6
|
environment?: Record<string, string | undefined>;
|
|
7
7
|
runtime?: CliRuntime;
|
|
8
8
|
}
|
|
9
|
-
declare function runCli(opts?: RunCliOptions): Promise<void>;
|
|
10
|
-
export { type CliRuntime, type CreateCliRuntimeOptions,
|
|
9
|
+
export declare function runCli(opts?: RunCliOptions): Promise<void>;
|
|
10
|
+
export { type CliRuntime, type CreateCliRuntimeOptions, createCliRuntime };
|
package/dist/cli.mjs
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
|
+
import { createCliRuntime, runWithCliRuntime, useCliRuntime } from "./runtime.mjs";
|
|
2
|
+
import { parseAuthMode } from "./auth-state.mjs";
|
|
1
3
|
import { CLI_SUBCOMMANDS } from "./command-registry.mjs";
|
|
2
|
-
import { createCliRuntime, runWithCliRuntime } from "./runtime.mjs";
|
|
3
4
|
import { resolveCliEnvironment } from "./environment.mjs";
|
|
4
5
|
import { applyProfileFromCli } from "./commands/profile-selection.mjs";
|
|
5
6
|
import { loadEnvFromCwd } from "./env-file.mjs";
|
|
6
|
-
import {
|
|
7
|
+
import { terminalOutputOptions } from "./render/terminal.mjs";
|
|
8
|
+
import { VERSION, setNoColor, showSplash, withConfiguredOutput } from "./utils.mjs";
|
|
7
9
|
import process from "node:process";
|
|
8
10
|
import { defineCommand, runMain } from "citty";
|
|
9
11
|
function shouldShowSplash(rawArgs) {
|
|
10
12
|
if (!process.stdout.isTTY) return false;
|
|
11
|
-
if (rawArgs
|
|
13
|
+
if (rawArgs[0] !== "init" && rawArgs[0] !== "auth") return false;
|
|
12
14
|
for (const flag of [
|
|
13
15
|
"--json",
|
|
14
16
|
"--quiet",
|
|
@@ -22,13 +24,10 @@ function shouldShowSplash(rawArgs) {
|
|
|
22
24
|
}
|
|
23
25
|
function prepareCliArgs(input) {
|
|
24
26
|
const rawArgs = [...input];
|
|
27
|
+
useCliRuntime().authModeOverride = parseAuthMode(pluckArgValue(rawArgs, "--mode") ?? void 0);
|
|
25
28
|
const env = resolveCliEnvironment();
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
forceColor: env.forceColor,
|
|
29
|
-
stderrIsTTY: Boolean(process.stderr.isTTY)
|
|
30
|
-
});
|
|
31
|
-
const profile = pluckArgValue(rawArgs, "--profile");
|
|
29
|
+
setNoColor(!terminalOutputOptions().color);
|
|
30
|
+
const profile = pluckArgValue(rawArgs, "--profile", true);
|
|
32
31
|
const configDir = pluckArgValue(rawArgs, "--config-dir") ?? env.configDir ?? null;
|
|
33
32
|
applyProfileFromCli({
|
|
34
33
|
configDir,
|
|
@@ -41,21 +40,21 @@ function prepareCliArgs(input) {
|
|
|
41
40
|
}
|
|
42
41
|
return rawArgs;
|
|
43
42
|
}
|
|
44
|
-
function pluckArgValue(argv, flag) {
|
|
43
|
+
function pluckArgValue(argv, flag, allowQueryTiming = false) {
|
|
44
|
+
let value = null;
|
|
45
45
|
for (let i = 0; i < argv.length; i++) {
|
|
46
46
|
const a = argv[i];
|
|
47
|
-
if (a ===
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
if (
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
47
|
+
if (a === "--") break;
|
|
48
|
+
if (a !== flag && !a.startsWith(`${flag}=`)) continue;
|
|
49
|
+
const inline = a !== flag;
|
|
50
|
+
const next = inline ? a.slice(flag.length + 1) : argv[i + 1];
|
|
51
|
+
if (!inline && allowQueryTiming && argv.includes("query") && argv.indexOf("query") < i && (next === void 0 || next.startsWith("-"))) continue;
|
|
52
|
+
if (!next || !inline && next.startsWith("-")) throw new Error(`${flag} requires a value. Use ${flag}=VALUE.`);
|
|
53
|
+
value = next;
|
|
54
|
+
argv.splice(i, inline ? 1 : 2);
|
|
55
|
+
i--;
|
|
57
56
|
}
|
|
58
|
-
return
|
|
57
|
+
return value;
|
|
59
58
|
}
|
|
60
59
|
const main = defineCommand({
|
|
61
60
|
meta: {
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { cloudRequest } from "./auth-state.mjs";
|
|
2
|
+
import { googleSearchConsole } from "gscdump/client";
|
|
3
|
+
import { ofetch } from "ofetch";
|
|
4
|
+
function createCloudGoogleClient(state, fetchOptions) {
|
|
5
|
+
const request = async (input, init) => {
|
|
6
|
+
const url = new URL(String(input));
|
|
7
|
+
if (url.origin !== "https://searchconsole.googleapis.com") throw new Error("Google indexing and Site Verification require local authentication. Use --mode local.");
|
|
8
|
+
const method = init?.method ?? "GET";
|
|
9
|
+
const signal = init?.signal;
|
|
10
|
+
const body = typeof init?.body === "string" ? JSON.parse(init.body) : {};
|
|
11
|
+
const post = (route, data) => cloudRequest(state, `/cli/gsc/${route}`, {
|
|
12
|
+
method: "POST",
|
|
13
|
+
body: JSON.stringify(data),
|
|
14
|
+
signal
|
|
15
|
+
});
|
|
16
|
+
let result;
|
|
17
|
+
if (url.pathname === "/webmasters/v3/sites" && method === "GET") result = { siteEntry: await cloudRequest(state, "/cli/gsc/sites", { signal }) };
|
|
18
|
+
else if (url.pathname === "/v1/urlInspection/index:inspect" && method === "POST") result = await post("inspect", {
|
|
19
|
+
siteUrl: body.siteUrl,
|
|
20
|
+
urls: [body.inspectionUrl],
|
|
21
|
+
languageCode: body.languageCode,
|
|
22
|
+
raw: true
|
|
23
|
+
});
|
|
24
|
+
else {
|
|
25
|
+
const match = url.pathname.match(/^\/webmasters\/v3\/sites\/([^/]+)(?:\/(searchAnalytics\/query|sitemaps)(?:\/([^/]+))?)?$/);
|
|
26
|
+
if (!match) throw new Error("This Google operation requires local authentication. Use --mode local.");
|
|
27
|
+
const siteUrl = decodeURIComponent(match[1]);
|
|
28
|
+
if (match[2] === "searchAnalytics/query" && method === "POST") {
|
|
29
|
+
const { type, ...query } = body;
|
|
30
|
+
result = await post("query", {
|
|
31
|
+
...query,
|
|
32
|
+
siteUrl,
|
|
33
|
+
searchType: type,
|
|
34
|
+
raw: true
|
|
35
|
+
});
|
|
36
|
+
} else if (match[2] === "sitemaps") {
|
|
37
|
+
if (match[3]) {
|
|
38
|
+
const action = method === "GET" ? "get" : method === "PUT" ? "submit" : method === "DELETE" ? "delete" : null;
|
|
39
|
+
if (!action) throw new Error("This sitemap operation requires local authentication. Use --mode local.");
|
|
40
|
+
const response = await post("sitemaps", {
|
|
41
|
+
siteUrl,
|
|
42
|
+
sitemapUrl: decodeURIComponent(match[3]),
|
|
43
|
+
action
|
|
44
|
+
});
|
|
45
|
+
result = action === "get" ? response : void 0;
|
|
46
|
+
} else if (method === "GET") {
|
|
47
|
+
const query = new URLSearchParams({
|
|
48
|
+
siteUrl,
|
|
49
|
+
raw: "true"
|
|
50
|
+
});
|
|
51
|
+
const sitemapIndex = url.searchParams.get("sitemapIndex");
|
|
52
|
+
if (sitemapIndex) query.set("sitemapIndex", sitemapIndex);
|
|
53
|
+
result = { sitemap: (await cloudRequest(state, `/cli/gsc/sitemaps?${query}`, { signal })).sitemaps };
|
|
54
|
+
} else throw new Error("This sitemap operation requires local authentication. Use --mode local.");
|
|
55
|
+
} else if (!match[2]) {
|
|
56
|
+
const action = method === "GET" ? "get" : method === "PUT" ? "add" : method === "DELETE" ? "remove" : null;
|
|
57
|
+
if (!action) throw new Error("This Site operation requires local authentication. Use --mode local.");
|
|
58
|
+
const response = await post("sites", {
|
|
59
|
+
siteUrl,
|
|
60
|
+
action
|
|
61
|
+
});
|
|
62
|
+
result = action === "get" ? response : void 0;
|
|
63
|
+
} else throw new Error("This Google operation requires local authentication. Use --mode local.");
|
|
64
|
+
}
|
|
65
|
+
return result === void 0 ? new Response(null, { status: 204 }) : Response.json(result);
|
|
66
|
+
};
|
|
67
|
+
const fetchHosted = (input, init) => request(input, init).catch((error) => {
|
|
68
|
+
if (error && typeof error === "object" && "response" in error && error.response instanceof Response) return error.response;
|
|
69
|
+
throw error;
|
|
70
|
+
});
|
|
71
|
+
return googleSearchConsole("", { fetch: ofetch.create({
|
|
72
|
+
...fetchOptions,
|
|
73
|
+
retry: fetchOptions?.retry ?? 3,
|
|
74
|
+
timeout: fetchOptions?.timeout ?? 3e4,
|
|
75
|
+
async onRequest(context) {
|
|
76
|
+
if (new URL(String(context.request)).origin !== "https://searchconsole.googleapis.com") throw new Error("Google indexing and Site Verification require local authentication. Use --mode local.");
|
|
77
|
+
const hooks = fetchOptions?.onRequest;
|
|
78
|
+
for (const hook of hooks ? Array.isArray(hooks) ? hooks : [hooks] : []) await hook(context);
|
|
79
|
+
},
|
|
80
|
+
retryDelay: fetchOptions?.retryDelay ?? (({ response }) => {
|
|
81
|
+
const header = response?.headers.get("retry-after");
|
|
82
|
+
if (header) {
|
|
83
|
+
const seconds = Number(header);
|
|
84
|
+
if (Number.isFinite(seconds)) return Math.max(0, seconds * 1e3);
|
|
85
|
+
const date = Date.parse(header);
|
|
86
|
+
if (Number.isFinite(date)) return Math.max(0, date - Date.now());
|
|
87
|
+
}
|
|
88
|
+
return 1e3;
|
|
89
|
+
})
|
|
90
|
+
}, { fetch: fetchHosted }) });
|
|
91
|
+
}
|
|
92
|
+
export { createCloudGoogleClient };
|
package/dist/command-meta.mjs
CHANGED
|
@@ -80,4 +80,16 @@ const mcpCommandMeta = {
|
|
|
80
80
|
name: "mcp",
|
|
81
81
|
description: "Start MCP server for AI assistants"
|
|
82
82
|
};
|
|
83
|
-
|
|
83
|
+
const skillCommandMeta = {
|
|
84
|
+
name: "skill",
|
|
85
|
+
description: "Install the packaged coding agent skill"
|
|
86
|
+
};
|
|
87
|
+
const papercutCommandMeta = {
|
|
88
|
+
name: "papercut",
|
|
89
|
+
description: "Report a CLI problem to gscdump.com (anonymous, sanitized)"
|
|
90
|
+
};
|
|
91
|
+
const bingCommandMeta = {
|
|
92
|
+
name: "bing",
|
|
93
|
+
description: "Authenticate Bing, list sites, dump data, and read Indexing Evidence"
|
|
94
|
+
};
|
|
95
|
+
export { analyzeCommandMeta, authCommandMeta, bingCommandMeta, configCommandMeta, doctorCommandMeta, dumpCommandMeta, entitiesCommandMeta, indexingCommandMeta, initCommandMeta, inspectCommandMeta, mcpCommandMeta, papercutCommandMeta, profileCommandMeta, queryCommandMeta, reportCommandMeta, sitemapsCommandMeta, sitesCommandMeta, skillCommandMeta, storeCommandMeta, syncCommandMeta };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { analyzeCommandMeta, authCommandMeta, configCommandMeta, doctorCommandMeta, dumpCommandMeta, entitiesCommandMeta, indexingCommandMeta, initCommandMeta, inspectCommandMeta, mcpCommandMeta, profileCommandMeta, queryCommandMeta, reportCommandMeta, sitemapsCommandMeta, sitesCommandMeta, storeCommandMeta, syncCommandMeta } from "./command-meta.mjs";
|
|
1
|
+
import { analyzeCommandMeta, authCommandMeta, bingCommandMeta, configCommandMeta, doctorCommandMeta, dumpCommandMeta, entitiesCommandMeta, indexingCommandMeta, initCommandMeta, inspectCommandMeta, mcpCommandMeta, papercutCommandMeta, profileCommandMeta, queryCommandMeta, reportCommandMeta, sitemapsCommandMeta, sitesCommandMeta, skillCommandMeta, storeCommandMeta, syncCommandMeta } from "./command-meta.mjs";
|
|
2
2
|
function resolveValue(value) {
|
|
3
3
|
return typeof value === "function" ? value() : value;
|
|
4
4
|
}
|
|
@@ -32,6 +32,7 @@ function shallowCommand(meta, load) {
|
|
|
32
32
|
};
|
|
33
33
|
}
|
|
34
34
|
const CLI_SUBCOMMANDS = {
|
|
35
|
+
bing: shallowCommand(bingCommandMeta, () => import("./commands/bing.mjs").then((module) => module.bingCommand)),
|
|
35
36
|
init: shallowCommand(initCommandMeta, () => import("./commands/init.mjs").then((module) => module.initCommand)),
|
|
36
37
|
dump: shallowCommand(dumpCommandMeta, () => import("./commands/dump.mjs").then((module) => module.dumpCommand)),
|
|
37
38
|
query: shallowCommand(queryCommandMeta, () => import("./commands/query.mjs").then((module) => module.queryCommand)),
|
|
@@ -51,6 +52,8 @@ const CLI_SUBCOMMANDS = {
|
|
|
51
52
|
config: shallowCommand(configCommandMeta, () => import("./commands/config.mjs").then((module) => module.configCommand)),
|
|
52
53
|
profile: shallowCommand(profileCommandMeta, () => import("./commands/profile.mjs").then((module) => module.profileCommand)),
|
|
53
54
|
doctor: shallowCommand(doctorCommandMeta, () => import("./commands/doctor.mjs").then((module) => module.doctorCommand)),
|
|
54
|
-
mcp: shallowCommand(mcpCommandMeta, () => import("./commands/mcp.mjs").then((module) => module.mcpCommand))
|
|
55
|
+
mcp: shallowCommand(mcpCommandMeta, () => import("./commands/mcp.mjs").then((module) => module.mcpCommand)),
|
|
56
|
+
skill: shallowCommand(skillCommandMeta, () => import("./commands/skill.mjs").then((module) => module.skillCommand)),
|
|
57
|
+
papercut: shallowCommand(papercutCommandMeta, () => import("./commands/papercut.mjs").then((module) => module.papercutCommand))
|
|
55
58
|
};
|
|
56
59
|
export { CLI_SUBCOMMANDS };
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { analyzeCommandMeta } from "../command-meta.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { terminalOutputOptions } from "../render/terminal.mjs";
|
|
3
|
+
import { logger, parseIntegerOption, toCSV } from "../utils.mjs";
|
|
3
4
|
import { gscErrorHandler } from "../error-handler.mjs";
|
|
4
5
|
import { resolveAnalysisSource } from "../analysis-local.mjs";
|
|
6
|
+
import { renderAnalysis } from "../render/analysis.mjs";
|
|
5
7
|
import { defineCommand } from "citty";
|
|
6
8
|
import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
|
|
9
|
+
import { periodOf } from "@gscdump/engine/period";
|
|
7
10
|
const ANALYSIS_TOOLS = defaultAnalyzerRegistry.listAnalyzerIds();
|
|
8
11
|
const TOOL_EXTRA_ARGS = {
|
|
9
12
|
brand: { "brand-terms": {
|
|
@@ -62,7 +65,7 @@ function buildParams(tool, args) {
|
|
|
62
65
|
type: tool,
|
|
63
66
|
startDate: args.start ? String(args.start) : void 0,
|
|
64
67
|
endDate: args.end ? String(args.end) : void 0,
|
|
65
|
-
limit: args.limit
|
|
68
|
+
limit: parseIntegerOption(args.limit, "--limit")
|
|
66
69
|
};
|
|
67
70
|
if (args["brand-terms"]) params.brandTerms = String(args["brand-terms"]).split(",").map((t) => t.trim()).filter(Boolean);
|
|
68
71
|
if (args["prev-start"]) params.prevStartDate = String(args["prev-start"]);
|
|
@@ -70,8 +73,10 @@ function buildParams(tool, args) {
|
|
|
70
73
|
if (args.dimension) params.dimension = String(args.dimension);
|
|
71
74
|
if (args.metric) params.metric = String(args.metric);
|
|
72
75
|
if (args["cluster-by"]) params.clusterBy = String(args["cluster-by"]);
|
|
73
|
-
|
|
74
|
-
|
|
76
|
+
const weeks = parseIntegerOption(args.weeks, "--weeks");
|
|
77
|
+
const minWeeks = parseIntegerOption(args["min-weeks"], "--min-weeks");
|
|
78
|
+
if (weeks !== void 0) params.weeks = weeks;
|
|
79
|
+
if (minWeeks !== void 0) params.minWeeksWithData = minWeeks;
|
|
75
80
|
return params;
|
|
76
81
|
}
|
|
77
82
|
function makeToolCommand(tool) {
|
|
@@ -120,190 +125,42 @@ function makeToolCommand(tool) {
|
|
|
120
125
|
...extraArgs
|
|
121
126
|
},
|
|
122
127
|
async run({ args }) {
|
|
123
|
-
|
|
128
|
+
if (!args.json && ![
|
|
129
|
+
"table",
|
|
130
|
+
"json",
|
|
131
|
+
"csv"
|
|
132
|
+
].includes(args.format ?? "table")) throw new Error("Invalid --format. Use table, json, or csv.");
|
|
133
|
+
const params = buildParams(tool, args);
|
|
134
|
+
const { format, runAnalysis, siteUrl } = await resolveAnalysisSource({
|
|
124
135
|
site: args.site,
|
|
125
136
|
live: !!args.live,
|
|
126
137
|
json: !!args.json,
|
|
127
138
|
format: args.format
|
|
128
139
|
});
|
|
129
|
-
logger.
|
|
130
|
-
const result = await runAnalysis(
|
|
140
|
+
logger.debug(`Running ${tool} analysis...`);
|
|
141
|
+
const result = await runAnalysis(params).catch(gscErrorHandler);
|
|
131
142
|
if (format === "json") {
|
|
132
143
|
console.log(JSON.stringify(result, null, 2));
|
|
133
144
|
return;
|
|
134
145
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
}
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
growthRatio: "ratio_to_pct",
|
|
152
|
-
brandShare: "direct",
|
|
153
|
-
topNConcentration: "direct",
|
|
154
|
-
declinePercent: "direct",
|
|
155
|
-
ctr: "direct",
|
|
156
|
-
share: "direct",
|
|
157
|
-
vsAverage: "ratio_to_pct",
|
|
158
|
-
clicksChangePercent: "scaled",
|
|
159
|
-
impressionsChangePercent: "scaled"
|
|
160
|
-
};
|
|
161
|
-
function formatPct(val, style) {
|
|
162
|
-
const pct = style === "ratio_to_pct" ? (val - 1) * 100 : style === "direct" ? val * 100 : val;
|
|
163
|
-
if (!Number.isFinite(pct)) return "";
|
|
164
|
-
return `${pct > 0 ? "+" : ""}${pct.toFixed(0)}%`;
|
|
165
|
-
}
|
|
166
|
-
function isTimeSeries(arr) {
|
|
167
|
-
if (arr.length === 0) return false;
|
|
168
|
-
const first = arr[0];
|
|
169
|
-
if (typeof first !== "object" || first === null) return false;
|
|
170
|
-
const keys = Object.keys(first);
|
|
171
|
-
const hasBucket = keys.includes("week") || keys.includes("date") || keys.includes("month");
|
|
172
|
-
const hasMetric = keys.includes("clicks") || keys.includes("impressions") || keys.includes("value");
|
|
173
|
-
return hasBucket && hasMetric;
|
|
174
|
-
}
|
|
175
|
-
function pickBucketKey(first) {
|
|
176
|
-
if ("week" in first) return "week";
|
|
177
|
-
if ("date" in first) return "date";
|
|
178
|
-
return "month";
|
|
179
|
-
}
|
|
180
|
-
function pickMetricKey(first) {
|
|
181
|
-
if ("clicks" in first) return "clicks";
|
|
182
|
-
if ("impressions" in first) return "impressions";
|
|
183
|
-
return "value";
|
|
184
|
-
}
|
|
185
|
-
function computeAlignedSparklines(results, col) {
|
|
186
|
-
const allBuckets = /* @__PURE__ */ new Set();
|
|
187
|
-
const perRow = [];
|
|
188
|
-
let bucketKey = "week";
|
|
189
|
-
let metricKey = "clicks";
|
|
190
|
-
for (const r of results) {
|
|
191
|
-
const arr = r[col];
|
|
192
|
-
if (!Array.isArray(arr) || !isTimeSeries(arr)) {
|
|
193
|
-
perRow.push(null);
|
|
194
|
-
continue;
|
|
195
|
-
}
|
|
196
|
-
const first = arr[0];
|
|
197
|
-
bucketKey = pickBucketKey(first);
|
|
198
|
-
metricKey = pickMetricKey(first);
|
|
199
|
-
const m = /* @__PURE__ */ new Map();
|
|
200
|
-
for (const item of arr) {
|
|
201
|
-
const rec = item;
|
|
202
|
-
const key = String(rec[bucketKey]);
|
|
203
|
-
const val = Number(rec[metricKey] ?? 0);
|
|
204
|
-
allBuckets.add(key);
|
|
205
|
-
m.set(key, val);
|
|
206
|
-
}
|
|
207
|
-
perRow.push(m);
|
|
208
|
-
}
|
|
209
|
-
const sorted = [...allBuckets].sort();
|
|
210
|
-
return perRow.map((m) => {
|
|
211
|
-
if (!m) return "";
|
|
212
|
-
const values = sorted.map((b) => m.has(b) ? m.get(b) : null);
|
|
213
|
-
const nonNull = values.filter((v) => v != null);
|
|
214
|
-
if (nonNull.length === 0) return SPARK_GAP.repeat(values.length);
|
|
215
|
-
const min = Math.min(...nonNull);
|
|
216
|
-
const range = Math.max(...nonNull) - min;
|
|
217
|
-
return values.map((v) => {
|
|
218
|
-
if (v == null) return SPARK_GAP;
|
|
219
|
-
if (range === 0) return SPARK_CHARS[0];
|
|
220
|
-
const idx = Math.round((v - min) / range * (SPARK_CHARS.length - 1));
|
|
221
|
-
return SPARK_CHARS[idx];
|
|
222
|
-
}).join("");
|
|
223
|
-
});
|
|
224
|
-
}
|
|
225
|
-
function classifyCol(col, values) {
|
|
226
|
-
const firstNonNull = values.find((v) => v != null);
|
|
227
|
-
if (firstNonNull == null) return "text";
|
|
228
|
-
if (Array.isArray(firstNonNull) && isTimeSeries(firstNonNull)) return "series";
|
|
229
|
-
if (col in PERCENT_COLS && values.every((v) => v == null || typeof v === "number")) return "pct";
|
|
230
|
-
if (values.every((v) => v == null || typeof v === "number")) return values.some((v) => typeof v === "number" && !Number.isInteger(v)) ? "float" : "int";
|
|
231
|
-
return "text";
|
|
232
|
-
}
|
|
233
|
-
function formatCellKinded(val, col, kind) {
|
|
234
|
-
if (val == null) return "";
|
|
235
|
-
if (kind === "int") return typeof val === "number" ? String(val) : String(val);
|
|
236
|
-
if (kind === "float") return typeof val === "number" ? val.toFixed(2) : String(val);
|
|
237
|
-
if (kind === "pct") return typeof val === "number" ? formatPct(val, PERCENT_COLS[col]) : String(val);
|
|
238
|
-
if (Array.isArray(val)) return `[${val.length} item${val.length === 1 ? "" : "s"}]`;
|
|
239
|
-
if (typeof val === "object") return JSON.stringify(val);
|
|
240
|
-
return String(val);
|
|
241
|
-
}
|
|
242
|
-
function computeRowSeriesSparkline(results) {
|
|
243
|
-
if (results.length < 2) return null;
|
|
244
|
-
const first = results[0];
|
|
245
|
-
const bucketKey = "week" in first ? "week" : "date" in first ? "date" : "month" in first ? "month" : null;
|
|
246
|
-
if (!bucketKey) return null;
|
|
247
|
-
const metricKey = "value" in first ? "value" : "clicks" in first ? "clicks" : "impressions" in first ? "impressions" : null;
|
|
248
|
-
if (!metricKey) return null;
|
|
249
|
-
for (const r of results) if (!(bucketKey in r) || !(metricKey in r)) return null;
|
|
250
|
-
const values = [...results].sort((a, b) => String(a[bucketKey]).localeCompare(String(b[bucketKey]))).map((r) => Number(r[metricKey] ?? 0));
|
|
251
|
-
const nonNull = values.filter((v) => Number.isFinite(v));
|
|
252
|
-
if (nonNull.length === 0) return null;
|
|
253
|
-
const min = Math.min(...nonNull);
|
|
254
|
-
const range = Math.max(...nonNull) - min;
|
|
255
|
-
return {
|
|
256
|
-
spark: values.map((v) => {
|
|
257
|
-
if (range === 0) return SPARK_CHARS[0];
|
|
258
|
-
const idx = Math.round((v - min) / range * (SPARK_CHARS.length - 1));
|
|
259
|
-
return SPARK_CHARS[idx];
|
|
260
|
-
}).join(""),
|
|
261
|
-
label: `${results.length} ${bucketKey}${results.length === 1 ? "" : "s"} of ${metricKey}`
|
|
262
|
-
};
|
|
263
|
-
}
|
|
264
|
-
function renderResults(results, total, format) {
|
|
265
|
-
if (format === "csv" && results.length > 0) {
|
|
266
|
-
const cols = Object.keys(results[0]);
|
|
267
|
-
console.log(toCSV(results, cols));
|
|
268
|
-
return;
|
|
269
|
-
}
|
|
270
|
-
if (results.length === 0) {
|
|
271
|
-
logger.warn("No results found");
|
|
272
|
-
return;
|
|
273
|
-
}
|
|
274
|
-
const cols = Object.keys(results[0]);
|
|
275
|
-
const kinds = cols.map((c) => classifyCol(c, results.map((r) => r[c])));
|
|
276
|
-
const sparklineByCol = {};
|
|
277
|
-
cols.forEach((c, i) => {
|
|
278
|
-
if (kinds[i] === "series") sparklineByCol[c] = computeAlignedSparklines(results, c);
|
|
279
|
-
});
|
|
280
|
-
const cellText = (row, rowIdx, colIdx) => {
|
|
281
|
-
const c = cols[colIdx];
|
|
282
|
-
const k = kinds[colIdx];
|
|
283
|
-
if (k === "series") return sparklineByCol[c][rowIdx];
|
|
284
|
-
return formatCellKinded(row[c], c, k);
|
|
285
|
-
};
|
|
286
|
-
const widths = cols.map((c, i) => {
|
|
287
|
-
let w = c.length;
|
|
288
|
-
const limit = Math.min(results.length, 20);
|
|
289
|
-
for (let j = 0; j < limit; j++) {
|
|
290
|
-
const len = cellText(results[j], j, i).length;
|
|
291
|
-
if (len > w) w = len;
|
|
146
|
+
if (format === "csv") {
|
|
147
|
+
console.log(toCSV(result.results, Object.keys(result.results[0] ?? {})));
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
const period = periodOf(params);
|
|
151
|
+
console.log(renderAnalysis(result, {
|
|
152
|
+
id: tool,
|
|
153
|
+
site: siteUrl,
|
|
154
|
+
start: period.startDate,
|
|
155
|
+
end: period.endDate,
|
|
156
|
+
...params.prevStartDate && params.prevEndDate ? { previous: {
|
|
157
|
+
start: params.prevStartDate,
|
|
158
|
+
end: params.prevEndDate
|
|
159
|
+
} } : {},
|
|
160
|
+
metric: params.metric
|
|
161
|
+
}, terminalOutputOptions()));
|
|
292
162
|
}
|
|
293
|
-
return w;
|
|
294
163
|
});
|
|
295
|
-
console.log();
|
|
296
|
-
console.log(` ${cols.map((c, i) => c.padEnd(widths[i])).join(" ")}`);
|
|
297
|
-
console.log(` ${cols.map((_, i) => "─".repeat(widths[i])).join(" ")}`);
|
|
298
|
-
for (let r = 0; r < results.length; r++) console.log(` ${cols.map((_, i) => cellText(results[r], r, i).padEnd(widths[i])).join(" ")}`);
|
|
299
|
-
const rowSeriesSparkline = computeRowSeriesSparkline(results);
|
|
300
|
-
if (rowSeriesSparkline) {
|
|
301
|
-
console.log();
|
|
302
|
-
console.log(` trend: ${rowSeriesSparkline.spark} (${rowSeriesSparkline.label})`);
|
|
303
|
-
}
|
|
304
|
-
console.log();
|
|
305
|
-
logger.success(`${results.length} results`);
|
|
306
|
-
if (total > results.length) logger.info(`Total: ${total} (showing ${results.length})`);
|
|
307
164
|
}
|
|
308
165
|
const listCommand = defineCommand({
|
|
309
166
|
meta: {
|