@gscdump/cli 3.8.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -30
- package/bin/gscdump.mjs +21 -1
- package/dist/analysis-local.mjs +172 -53
- package/dist/auth-state.mjs +21 -6
- package/dist/auth.mjs +27 -18
- package/dist/bing-data.mjs +6 -2
- package/dist/bing-hosted.mjs +1 -1
- package/dist/cli-args.mjs +8 -0
- package/dist/cli.d.mts +5 -1
- package/dist/cli.mjs +41 -10
- package/dist/cloud-google.mjs +8 -1
- package/dist/command-meta.mjs +4 -4
- package/dist/command-registry.mjs +86 -10
- package/dist/commands/analyze.mjs +84 -43
- package/dist/commands/auth.mjs +46 -37
- package/dist/commands/bing.mjs +3 -3
- package/dist/commands/compact.mjs +12 -8
- package/dist/commands/config.mjs +14 -14
- package/dist/commands/doctor.mjs +26 -26
- package/dist/commands/dump.mjs +203 -105
- package/dist/commands/entities.mjs +10 -113
- package/dist/commands/gc.mjs +4 -3
- package/dist/commands/indexing-urls.mjs +185 -0
- package/dist/commands/indexing.mjs +12 -8
- package/dist/commands/init.mjs +77 -18
- package/dist/commands/inspect.mjs +97 -110
- package/dist/commands/mcp.mjs +37 -57
- package/dist/commands/papercut.mjs +1 -1
- package/dist/commands/profile.mjs +9 -17
- package/dist/commands/query.mjs +219 -210
- package/dist/commands/report.mjs +68 -85
- package/dist/commands/rollups.mjs +9 -6
- package/dist/commands/sitemaps.mjs +51 -104
- package/dist/commands/sites.mjs +40 -24
- package/dist/commands/stats.mjs +14 -17
- package/dist/commands/store-purge.mjs +9 -5
- package/dist/commands/store.mjs +0 -12
- package/dist/commands/sync.mjs +774 -296
- package/dist/config.mjs +7 -4
- package/dist/context.mjs +51 -21
- package/dist/coverage.mjs +251 -0
- package/dist/dump-bing.mjs +71 -0
- package/dist/dump-writers.mjs +246 -0
- package/dist/error-handler.mjs +108 -38
- package/dist/filters.mjs +125 -0
- package/dist/hosted-site.mjs +91 -0
- package/dist/inspect-urls.mjs +76 -0
- package/dist/inspection-record.mjs +93 -0
- package/dist/local-entities.mjs +658 -0
- package/dist/mcp/errors.mjs +71 -18
- package/dist/mcp/handlers/diagnostics.mjs +15 -12
- package/dist/mcp/handlers/reports.mjs +33 -42
- package/dist/mcp/server/index.mjs +80 -38
- package/dist/mcp/types.mjs +7 -7
- package/dist/package.mjs +1 -1
- package/dist/quota-ledger.mjs +370 -0
- package/dist/render/analysis.mjs +8 -1
- package/dist/render/report.mjs +8 -1
- package/dist/request-pacer.mjs +29 -0
- package/dist/route.mjs +301 -0
- package/dist/sitemap.mjs +4 -0
- package/dist/sql-views.mjs +117 -0
- package/dist/store-sites.mjs +134 -0
- package/dist/sync-plan.mjs +129 -0
- package/dist/sync-run.mjs +96 -0
- package/dist/table-sources.mjs +57 -0
- package/dist/token-info.mjs +54 -0
- package/dist/utils.mjs +28 -19
- package/dist/window.mjs +159 -0
- package/package.json +17 -14
- package/skills/gscdump/SKILL.md +172 -37
- package/dist/commands/export.mjs +0 -76
- package/dist/native-duckdb.mjs +0 -46
package/dist/auth.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { resolveAuthentication } from "./auth-state.mjs";
|
|
1
2
|
import { getConfigDir, loadConfig } from "./config.mjs";
|
|
2
3
|
import { pickCliEnvironmentValue, resolveCliEnvironment } from "./environment.mjs";
|
|
3
4
|
import { getAppliedEnvKeys, getLoadedEnvPath } from "./env-file.mjs";
|
|
@@ -8,12 +9,12 @@ import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
|
|
|
8
9
|
import fs from "node:fs/promises";
|
|
9
10
|
import path from "node:path";
|
|
10
11
|
import { Buffer } from "node:buffer";
|
|
11
|
-
import { err, ok, unwrapResult } from "gscdump/result";
|
|
12
|
-
import { text } from "@clack/prompts";
|
|
13
|
-
import { createAuth } from "gscdump/client";
|
|
14
12
|
import { createServer } from "node:http";
|
|
15
13
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
14
|
+
import { text } from "@clack/prompts";
|
|
16
15
|
import { CodeChallengeMethod, JWT, OAuth2Client } from "google-auth-library";
|
|
16
|
+
import { createAuth } from "gscdump/client";
|
|
17
|
+
import { err, ok, unwrapResult } from "gscdump/result";
|
|
17
18
|
import open from "open";
|
|
18
19
|
function authErrorToException(error) {
|
|
19
20
|
const exception = new Error(error.message);
|
|
@@ -66,6 +67,12 @@ function resolveBYOK(opts = {}) {
|
|
|
66
67
|
function getTokensPath() {
|
|
67
68
|
return path.join(getConfigDir(), "tokens.json");
|
|
68
69
|
}
|
|
70
|
+
const GOOGLE_NOT_CONNECTED = [
|
|
71
|
+
"Google is not connected. Use one of these:",
|
|
72
|
+
" Local: gscdump auth login",
|
|
73
|
+
" Hosted: gscdump auth login --mode cloud --api-key KEY (a gscdump.com API key)",
|
|
74
|
+
" BYOK: set GSC_ACCESS_TOKEN, or GSC_CLIENT_ID, GSC_CLIENT_SECRET and GSC_REFRESH_TOKEN"
|
|
75
|
+
].join("\n");
|
|
69
76
|
async function loadTokens() {
|
|
70
77
|
return fs.readFile(getTokensPath(), "utf-8").then((data) => JSON.parse(data)).catch(() => null);
|
|
71
78
|
}
|
|
@@ -291,14 +298,8 @@ async function authenticate(credentials, interactive, opts = {}) {
|
|
|
291
298
|
}
|
|
292
299
|
}
|
|
293
300
|
if (!interactive) {
|
|
294
|
-
if (refreshFailed) {
|
|
295
|
-
|
|
296
|
-
logger.info("Refresh token may be revoked or expired. Run `gscdump auth login` to re-authenticate.");
|
|
297
|
-
} else {
|
|
298
|
-
logger.error("Not authenticated");
|
|
299
|
-
logger.info("Run `gscdump auth login` (or `gscdump init` for full setup).");
|
|
300
|
-
}
|
|
301
|
-
process.exit(1);
|
|
301
|
+
if (refreshFailed) throw new Error(`Token refresh failed${refreshError ? `: ${refreshError.message}` : ""}. The refresh token may be revoked or expired. Run \`gscdump auth login\` to sign in again.`);
|
|
302
|
+
throw new Error(GOOGLE_NOT_CONNECTED);
|
|
302
303
|
}
|
|
303
304
|
const state = randomBytes(32).toString("base64url");
|
|
304
305
|
const codeVerifier = randomBytes(32).toString("base64url");
|
|
@@ -332,7 +333,7 @@ async function getAuth(opts = {}) {
|
|
|
332
333
|
});
|
|
333
334
|
let tokens = force ? null : await loadTokens();
|
|
334
335
|
if (tokens?.provider !== "gscdump" || !tokens.refresh_token) {
|
|
335
|
-
if (!interactive) throw new Error(
|
|
336
|
+
if (!interactive) throw new Error(GOOGLE_NOT_CONNECTED);
|
|
336
337
|
tokens = await loginWithPlatform({
|
|
337
338
|
force,
|
|
338
339
|
request: fetch,
|
|
@@ -363,8 +364,15 @@ async function getAuth(opts = {}) {
|
|
|
363
364
|
await client.getAccessToken();
|
|
364
365
|
return client;
|
|
365
366
|
}
|
|
367
|
+
function isStaleServiceAccountPointer(error) {
|
|
368
|
+
if (error instanceof SyntaxError) return true;
|
|
369
|
+
return error?.code === "ENOENT";
|
|
370
|
+
}
|
|
366
371
|
async function resolveAuth(opts = {}) {
|
|
367
|
-
const sa = await resolveServiceAccount({ path: opts.serviceAccount })
|
|
372
|
+
const sa = await resolveServiceAccount({ path: opts.serviceAccount }).catch((error) => {
|
|
373
|
+
if (isStaleServiceAccountPointer(error)) return null;
|
|
374
|
+
throw error;
|
|
375
|
+
});
|
|
368
376
|
if (sa) {
|
|
369
377
|
logger.success("Using service-account credentials");
|
|
370
378
|
return sa;
|
|
@@ -492,9 +500,10 @@ async function formatAuthProvenance() {
|
|
|
492
500
|
}
|
|
493
501
|
return lines.join("\n");
|
|
494
502
|
}
|
|
495
|
-
function
|
|
496
|
-
|
|
497
|
-
if (
|
|
498
|
-
|
|
503
|
+
async function probeAuth() {
|
|
504
|
+
if ((await resolveAuthentication())._tag === "Cloud") return "hosted";
|
|
505
|
+
if (resolveBYOK()) return "google";
|
|
506
|
+
if (await resolveServiceAccount().then(Boolean, (error) => !isStaleServiceAccountPointer(error))) return "google";
|
|
507
|
+
return await loadTokens() !== null ? "google" : "none";
|
|
499
508
|
}
|
|
500
|
-
export { authenticate, clearTokens, describeAuthProvenance, formatAuthProvenance, getAuth, getAuthCredentials,
|
|
509
|
+
export { GOOGLE_NOT_CONNECTED, authenticate, clearTokens, describeAuthProvenance, formatAuthProvenance, getAuth, getAuthCredentials, loadServiceAccount, loadServiceAccountResult, loadTokens, probeAuth, resolveAuth, resolveBYOK, resolveServiceAccount, saveTokens };
|
package/dist/bing-data.mjs
CHANGED
|
@@ -22,13 +22,17 @@ const dumpSchema = z.object({
|
|
|
22
22
|
end: dateSchema.optional()
|
|
23
23
|
}).refine((value) => !value.start || !value.end || value.start <= value.end, { message: "--start must precede --end" });
|
|
24
24
|
function parseBingDumpOptions(args) {
|
|
25
|
+
const defaults = args.start || args.end ? datedDatasets : [...BING_DATASETS];
|
|
25
26
|
const parsed = dumpSchema.safeParse({
|
|
26
27
|
...args,
|
|
27
|
-
datasets: args.datasets ? [...new Set(args.datasets.split(",").map((value) => value.trim()))] :
|
|
28
|
+
datasets: args.datasets ? [...new Set(args.datasets.split(",").map((value) => value.trim()))] : defaults
|
|
28
29
|
});
|
|
29
30
|
if (!parsed.success) throw new Error(`Invalid Bing dump options: ${parsed.error.issues.map((issue) => issue.message).join("; ")}.`);
|
|
30
31
|
if (parsed.data.datasets.includes("crawl-issues") && (parsed.data.start || parsed.data.end)) throw new Error("Bing crawl issues have no dates. Export them without --start or --end.");
|
|
31
|
-
return
|
|
32
|
+
return {
|
|
33
|
+
...parsed.data,
|
|
34
|
+
datasetsExplicit: Boolean(args.datasets)
|
|
35
|
+
};
|
|
32
36
|
}
|
|
33
37
|
function unwrapBing(result) {
|
|
34
38
|
if (result.ok) return result.value;
|
package/dist/bing-hosted.mjs
CHANGED
|
@@ -42,7 +42,7 @@ async function resolveHostedBingSites(state, input) {
|
|
|
42
42
|
return selected;
|
|
43
43
|
}
|
|
44
44
|
async function dumpHostedBingSite(state, site, outDir, options) {
|
|
45
|
-
if (options.datasets.includes("crawl-issues")) throw new Error("Bing crawl issues require local authentication. Use --mode local.");
|
|
45
|
+
if (options.datasetsExplicit && options.datasets.includes("crawl-issues")) throw new Error("Bing crawl issues require local authentication. Use --mode local.");
|
|
46
46
|
const endDate = options.end ?? (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
47
47
|
const startDate = options.start ?? (/* @__PURE__ */ new Date(Date.parse(endDate) - 316224e5)).toISOString().slice(0, 10);
|
|
48
48
|
if (Date.parse(endDate) - Date.parse(startDate) > 316224e5) throw new Error("Hosted Bing exports support at most 366 days. Use a shorter --start and --end range.");
|
package/dist/cli-args.mjs
CHANGED
|
@@ -2,6 +2,12 @@ import { parseArgs } from "node:util";
|
|
|
2
2
|
async function resolve(value) {
|
|
3
3
|
return typeof value === "function" ? value() : value;
|
|
4
4
|
}
|
|
5
|
+
const GLOBAL_OPTIONS = /* @__PURE__ */ new Set([
|
|
6
|
+
"help",
|
|
7
|
+
"h",
|
|
8
|
+
"version",
|
|
9
|
+
"no-color"
|
|
10
|
+
]);
|
|
5
11
|
async function checkCliArgs(command, rawArgs, path = "gscdump") {
|
|
6
12
|
const definitions = await resolve(command.args ?? {});
|
|
7
13
|
const options = {
|
|
@@ -56,6 +62,8 @@ async function checkCliArgs(command, rawArgs, path = "gscdump") {
|
|
|
56
62
|
}
|
|
57
63
|
}
|
|
58
64
|
if (!selected) return `Unknown command ${positional.value}. Run ${path} --help.`;
|
|
65
|
+
const early = tokens.find((token) => token.index < boundary && token.kind === "option" && !GLOBAL_OPTIONS.has(token.name));
|
|
66
|
+
if (early?.kind === "option") return `Put ${early.rawName} after the subcommand: ${path} ${positional.value} ${early.rawName}.`;
|
|
59
67
|
const child = await resolve(selected);
|
|
60
68
|
if (child) return checkCliArgs(child, rawArgs.slice(boundary + 1), `${path} ${positional.value}`);
|
|
61
69
|
}
|
package/dist/cli.d.mts
CHANGED
|
@@ -6,5 +6,9 @@ export interface RunCliOptions {
|
|
|
6
6
|
environment?: Record<string, string | undefined>;
|
|
7
7
|
runtime?: CliRuntime;
|
|
8
8
|
}
|
|
9
|
-
|
|
9
|
+
/**
|
|
10
|
+
* Run one CLI invocation. Every failure ends here: the shell prints it once
|
|
11
|
+
* and returns exit code 1. The caller owns `process.exit`.
|
|
12
|
+
*/
|
|
13
|
+
export declare function runCli(opts?: RunCliOptions): Promise<number>;
|
|
10
14
|
export { type CliRuntime, type CreateCliRuntimeOptions, createCliRuntime };
|
package/dist/cli.mjs
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
|
+
import { CLI_SUBCOMMANDS, resolveUsageTarget } from "./command-registry.mjs";
|
|
2
|
+
import { reportCliError } from "./error-handler.mjs";
|
|
1
3
|
import { createCliRuntime, runWithCliRuntime, useCliRuntime } from "./runtime.mjs";
|
|
2
4
|
import { parseAuthMode } from "./auth-state.mjs";
|
|
3
5
|
import { checkCliArgs } from "./cli-args.mjs";
|
|
4
|
-
import { CLI_SUBCOMMANDS } from "./command-registry.mjs";
|
|
5
6
|
import { resolveCliEnvironment } from "./environment.mjs";
|
|
6
7
|
import { applyProfileFromCli } from "./commands/profile-selection.mjs";
|
|
7
8
|
import { loadEnvFromCwd } from "./env-file.mjs";
|
|
8
|
-
import { terminalOutputOptions } from "./render/terminal.mjs";
|
|
9
|
+
import { resolveOutputOptions, terminalOutputOptions } from "./render/terminal.mjs";
|
|
9
10
|
import { VERSION, setNoColor, showSplash, withConfiguredOutput } from "./utils.mjs";
|
|
10
11
|
import process from "node:process";
|
|
11
|
-
import { defineCommand,
|
|
12
|
+
import { defineCommand, renderUsage, runCommand } from "citty";
|
|
12
13
|
function shouldShowSplash(rawArgs) {
|
|
13
14
|
if (!process.stdout.isTTY) return false;
|
|
14
15
|
if (rawArgs[0] !== "init" && rawArgs[0] !== "auth") return false;
|
|
@@ -68,6 +69,25 @@ const main = defineCommand({
|
|
|
68
69
|
if (shouldShowSplash(rawArgs)) showSplash();
|
|
69
70
|
}
|
|
70
71
|
});
|
|
72
|
+
const HELP_FLAGS = /* @__PURE__ */ new Set(["--help", "-h"]);
|
|
73
|
+
async function runMainCommand(rawArgs) {
|
|
74
|
+
if (rawArgs.some((arg) => HELP_FLAGS.has(arg))) {
|
|
75
|
+
console.log(`${await renderUsage(...await resolveUsageTarget(main, rawArgs))}\n`);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
if (rawArgs.length === 1 && rawArgs[0] === "--version") {
|
|
79
|
+
console.log(VERSION);
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
await runCommand(main, { rawArgs });
|
|
83
|
+
}
|
|
84
|
+
function stderrColor(runtime, rawArgs) {
|
|
85
|
+
return resolveOutputOptions({
|
|
86
|
+
isTTY: process.stderr.isTTY,
|
|
87
|
+
environment: runtime.environment,
|
|
88
|
+
noColor: rawArgs.includes("--no-color")
|
|
89
|
+
}).color;
|
|
90
|
+
}
|
|
71
91
|
async function runCli(opts = {}) {
|
|
72
92
|
const input = opts.rawArgs ?? process.argv.slice(2);
|
|
73
93
|
const runtime = opts.runtime ?? createCliRuntime({
|
|
@@ -75,13 +95,24 @@ async function runCli(opts = {}) {
|
|
|
75
95
|
rawArgs: input
|
|
76
96
|
});
|
|
77
97
|
runtime.rawArgs = [...input];
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
98
|
+
return runWithCliRuntime(runtime, async () => {
|
|
99
|
+
let rawArgs = [...input];
|
|
100
|
+
try {
|
|
101
|
+
if (opts.loadEnv !== false) loadEnvFromCwd();
|
|
102
|
+
rawArgs = prepareCliArgs(input);
|
|
103
|
+
runtime.rawArgs = [...rawArgs];
|
|
104
|
+
const argumentError = await checkCliArgs(main, rawArgs);
|
|
105
|
+
if (argumentError) throw new Error(argumentError);
|
|
106
|
+
await withConfiguredOutput(() => runMainCommand(rawArgs));
|
|
107
|
+
return 0;
|
|
108
|
+
} catch (error) {
|
|
109
|
+
await reportCliError(error, {
|
|
110
|
+
color: stderrColor(runtime, rawArgs),
|
|
111
|
+
usage: async () => renderUsage(...await resolveUsageTarget(main, rawArgs)),
|
|
112
|
+
authSources: async () => (await import("./auth.mjs")).formatAuthProvenance()
|
|
113
|
+
});
|
|
114
|
+
return 1;
|
|
115
|
+
}
|
|
85
116
|
});
|
|
86
117
|
}
|
|
87
118
|
export { createCliRuntime, main, runCli };
|
package/dist/cloud-google.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { HOSTED_KEY_REJECTED } from "./error-handler.mjs";
|
|
1
2
|
import { cloudRequest } from "./auth-state.mjs";
|
|
2
3
|
import { googleSearchConsole } from "gscdump/client";
|
|
3
4
|
import { ofetch } from "ofetch";
|
|
@@ -65,7 +66,13 @@ function createCloudGoogleClient(state, fetchOptions) {
|
|
|
65
66
|
return result === void 0 ? new Response(null, { status: 204 }) : Response.json(result);
|
|
66
67
|
};
|
|
67
68
|
const fetchHosted = (input, init) => request(input, init).catch((error) => {
|
|
68
|
-
if (error && typeof error === "object" && "response" in error && error.response instanceof Response)
|
|
69
|
+
if (error && typeof error === "object" && "response" in error && error.response instanceof Response) {
|
|
70
|
+
if (error.response.status === 401) return Response.json({ error: {
|
|
71
|
+
code: 401,
|
|
72
|
+
message: HOSTED_KEY_REJECTED
|
|
73
|
+
} }, { status: 401 });
|
|
74
|
+
return error.response;
|
|
75
|
+
}
|
|
69
76
|
throw error;
|
|
70
77
|
});
|
|
71
78
|
return googleSearchConsole("", { fetch: ofetch.create({
|
package/dist/command-meta.mjs
CHANGED
|
@@ -4,7 +4,7 @@ const initCommandMeta = {
|
|
|
4
4
|
};
|
|
5
5
|
const dumpCommandMeta = {
|
|
6
6
|
name: "dump",
|
|
7
|
-
description: "Export
|
|
7
|
+
description: "Export the local Store, inspections, sitemaps, and Bing data to a directory"
|
|
8
8
|
};
|
|
9
9
|
const queryCommandMeta = {
|
|
10
10
|
name: "query",
|
|
@@ -20,7 +20,7 @@ const sitemapsCommandMeta = {
|
|
|
20
20
|
};
|
|
21
21
|
const syncCommandMeta = {
|
|
22
22
|
name: "sync",
|
|
23
|
-
description: "Sync GSC data to local
|
|
23
|
+
description: "Sync GSC data, sitemaps, and URL Inspection results to the local Store"
|
|
24
24
|
};
|
|
25
25
|
const storeCommandMeta = {
|
|
26
26
|
name: "store",
|
|
@@ -28,11 +28,11 @@ const storeCommandMeta = {
|
|
|
28
28
|
};
|
|
29
29
|
const inspectCommandMeta = {
|
|
30
30
|
name: "inspect",
|
|
31
|
-
description: "Inspect
|
|
31
|
+
description: "Inspect the indexing status of one or more URLs and save the results to the Store"
|
|
32
32
|
};
|
|
33
33
|
const indexingCommandMeta = {
|
|
34
34
|
name: "indexing",
|
|
35
|
-
description: "Notify Google about URL
|
|
35
|
+
description: "Notify Google about URL changes (Indexing API); list hosted URL Inspection results"
|
|
36
36
|
};
|
|
37
37
|
const entitiesCommandMeta = {
|
|
38
38
|
name: "entities",
|
|
@@ -2,12 +2,74 @@ import { analyzeCommandMeta, authCommandMeta, bingCommandMeta, configCommandMeta
|
|
|
2
2
|
function resolveValue(value) {
|
|
3
3
|
return typeof value === "function" ? value() : value;
|
|
4
4
|
}
|
|
5
|
+
const KEBAB_RE = /-([a-z])/g;
|
|
6
|
+
function camelCase(name) {
|
|
7
|
+
return name.replace(KEBAB_RE, (_, letter) => letter.toUpperCase());
|
|
8
|
+
}
|
|
9
|
+
function isValueFlag(flag, argsDef) {
|
|
10
|
+
const name = flag.replace(/^-{1,2}/, "");
|
|
11
|
+
for (const [key, def] of Object.entries(argsDef)) {
|
|
12
|
+
if (def.type !== "string" && def.type !== "enum") continue;
|
|
13
|
+
const aliases = "alias" in def && def.alias ? [def.alias].flat() : [];
|
|
14
|
+
if (camelCase(name) === camelCase(key) || aliases.includes(name)) return true;
|
|
15
|
+
}
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
function findSubCommandIndex(rawArgs, argsDef) {
|
|
19
|
+
for (let i = 0; i < rawArgs.length; i++) {
|
|
20
|
+
const arg = rawArgs[i];
|
|
21
|
+
if (arg === "--") return -1;
|
|
22
|
+
if (arg.startsWith("-")) {
|
|
23
|
+
if (!arg.includes("=") && isValueFlag(arg, argsDef)) i++;
|
|
24
|
+
continue;
|
|
25
|
+
}
|
|
26
|
+
return i;
|
|
27
|
+
}
|
|
28
|
+
return -1;
|
|
29
|
+
}
|
|
30
|
+
function usageError(message, code) {
|
|
31
|
+
return Object.assign(new Error(message), {
|
|
32
|
+
name: "CLIError",
|
|
33
|
+
code
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
function isUsageError(error) {
|
|
37
|
+
return error instanceof Error && error.name === "CLIError";
|
|
38
|
+
}
|
|
39
|
+
function guardedRun(load) {
|
|
40
|
+
return async (context) => {
|
|
41
|
+
const command = await load();
|
|
42
|
+
const subCommands = await resolveValue(command.subCommands) ?? {};
|
|
43
|
+
if (Object.keys(subCommands).length > 0) {
|
|
44
|
+
const argsDef = await resolveValue(command.args) ?? {};
|
|
45
|
+
if (findSubCommandIndex(context.rawArgs, argsDef) >= 0) return;
|
|
46
|
+
if (!command.run) throw usageError("No command specified.", "E_NO_COMMAND");
|
|
47
|
+
}
|
|
48
|
+
return command.run?.({
|
|
49
|
+
...context,
|
|
50
|
+
cmd: command
|
|
51
|
+
});
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function guardedSubCommands(load) {
|
|
55
|
+
return async () => {
|
|
56
|
+
const subCommands = await resolveValue((await load()).subCommands) ?? {};
|
|
57
|
+
return Object.fromEntries(Object.entries(subCommands).map(([name, sub]) => [name, async () => guardCommand(await resolveValue(sub))]));
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
function guardCommand(command) {
|
|
61
|
+
const load = async () => command;
|
|
62
|
+
return {
|
|
63
|
+
...command,
|
|
64
|
+
subCommands: guardedSubCommands(load),
|
|
65
|
+
run: guardedRun(load)
|
|
66
|
+
};
|
|
67
|
+
}
|
|
5
68
|
function shallowCommand(meta, load) {
|
|
6
69
|
return {
|
|
7
70
|
meta,
|
|
8
71
|
args: async () => await resolveValue((await load()).args) ?? {},
|
|
9
|
-
|
|
10
|
-
subCommands: async () => await resolveValue((await load()).subCommands) ?? {},
|
|
72
|
+
subCommands: guardedSubCommands(load),
|
|
11
73
|
async setup(context) {
|
|
12
74
|
const command = await load();
|
|
13
75
|
await command.setup?.({
|
|
@@ -22,15 +84,29 @@ function shallowCommand(meta, load) {
|
|
|
22
84
|
cmd: command
|
|
23
85
|
});
|
|
24
86
|
},
|
|
25
|
-
|
|
26
|
-
const command = await load();
|
|
27
|
-
return command.run?.({
|
|
28
|
-
...context,
|
|
29
|
-
cmd: command
|
|
30
|
-
});
|
|
31
|
-
}
|
|
87
|
+
run: guardedRun(load)
|
|
32
88
|
};
|
|
33
89
|
}
|
|
90
|
+
async function resolveUsageTarget(command, rawArgs, parent) {
|
|
91
|
+
const subCommands = await resolveValue(command.subCommands) ?? {};
|
|
92
|
+
if (Object.keys(subCommands).length > 0) {
|
|
93
|
+
const index = findSubCommandIndex(rawArgs, await resolveValue(command.args) ?? {});
|
|
94
|
+
const name = rawArgs[index];
|
|
95
|
+
if (name !== void 0) {
|
|
96
|
+
const selected = await findSubCommand(subCommands, name);
|
|
97
|
+
if (selected) return resolveUsageTarget(selected, rawArgs.slice(index + 1), command);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return [command, parent];
|
|
101
|
+
}
|
|
102
|
+
async function findSubCommand(subCommands, name) {
|
|
103
|
+
if (name in subCommands) return await resolveValue(subCommands[name]);
|
|
104
|
+
for (const candidate of Object.values(subCommands)) {
|
|
105
|
+
const resolved = await resolveValue(candidate);
|
|
106
|
+
const meta = await resolveValue(resolved?.meta);
|
|
107
|
+
if (meta?.alias && [meta.alias].flat().includes(name)) return resolved;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
34
110
|
const CLI_SUBCOMMANDS = {
|
|
35
111
|
bing: shallowCommand(bingCommandMeta, () => import("./commands/bing.mjs").then((module) => module.bingCommand)),
|
|
36
112
|
init: shallowCommand(initCommandMeta, () => import("./commands/init.mjs").then((module) => module.initCommand)),
|
|
@@ -56,4 +132,4 @@ const CLI_SUBCOMMANDS = {
|
|
|
56
132
|
skill: shallowCommand(skillCommandMeta, () => import("./commands/skill.mjs").then((module) => module.skillCommand)),
|
|
57
133
|
papercut: shallowCommand(papercutCommandMeta, () => import("./commands/papercut.mjs").then((module) => module.papercutCommand))
|
|
58
134
|
};
|
|
59
|
-
export { CLI_SUBCOMMANDS };
|
|
135
|
+
export { CLI_SUBCOMMANDS, findSubCommandIndex, isUsageError, resolveUsageTarget, usageError };
|
|
@@ -1,38 +1,39 @@
|
|
|
1
1
|
import { analyzeCommandMeta } from "../command-meta.mjs";
|
|
2
2
|
import { terminalOutputOptions } from "../render/terminal.mjs";
|
|
3
|
-
import { logger, parseIntegerOption, toCSV } from "../utils.mjs";
|
|
4
|
-
import {
|
|
5
|
-
import { resolveAnalysisSource } from "../analysis-local.mjs";
|
|
6
|
-
import { renderAnalysis } from "../render/analysis.mjs";
|
|
3
|
+
import { logger, parseFetchBudget, parseIntegerOption, toCSV } from "../utils.mjs";
|
|
4
|
+
import { DEFAULT_WINDOW, PERIOD_FLAGS, parseWindowFlags, windowFlagErrorToException } from "../window.mjs";
|
|
5
|
+
import { analysisNeeds, analyzerTables, resolveAnalysisSource } from "../analysis-local.mjs";
|
|
6
|
+
import { coverageWarning, renderAnalysis } from "../render/analysis.mjs";
|
|
7
7
|
import { defineCommand } from "citty";
|
|
8
|
+
import { DEFAULT_FETCH_BUDGET, MAX_FETCH_BUDGET } from "@gscdump/engine/analysis-types";
|
|
9
|
+
import { unwrapResult } from "gscdump/result";
|
|
10
|
+
import { MOVERS_SORT_METRICS } from "@gscdump/analysis";
|
|
8
11
|
import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
|
|
9
|
-
import { periodOf } from "@gscdump/engine/period";
|
|
10
12
|
const ANALYSIS_TOOLS = defaultAnalyzerRegistry.listAnalyzerIds();
|
|
13
|
+
const COMPARISON_ARGS = {
|
|
14
|
+
"prev-start": {
|
|
15
|
+
type: "string",
|
|
16
|
+
description: "Previous period start date (default: the period before the window)"
|
|
17
|
+
},
|
|
18
|
+
"prev-end": {
|
|
19
|
+
type: "string",
|
|
20
|
+
description: "Previous period end date (pass with --prev-start)"
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
const COMPARISON_TOOLS = /* @__PURE__ */ new Set(["movers", "decay"]);
|
|
11
24
|
const TOOL_EXTRA_ARGS = {
|
|
12
25
|
brand: { "brand-terms": {
|
|
13
26
|
type: "string",
|
|
14
27
|
description: "Comma-separated brand terms (required)"
|
|
15
28
|
} },
|
|
16
29
|
movers: {
|
|
17
|
-
|
|
30
|
+
...COMPARISON_ARGS,
|
|
31
|
+
"sort-by": {
|
|
18
32
|
type: "string",
|
|
19
|
-
description: "
|
|
20
|
-
},
|
|
21
|
-
"prev-end": {
|
|
22
|
-
type: "string",
|
|
23
|
-
description: "Previous period end date (required)"
|
|
24
|
-
}
|
|
25
|
-
},
|
|
26
|
-
decay: {
|
|
27
|
-
"prev-start": {
|
|
28
|
-
type: "string",
|
|
29
|
-
description: "Previous period start date (required)"
|
|
30
|
-
},
|
|
31
|
-
"prev-end": {
|
|
32
|
-
type: "string",
|
|
33
|
-
description: "Previous period end date (required)"
|
|
33
|
+
description: `Sort: ${MOVERS_SORT_METRICS.join(", ")} (default: clicksDelta)`
|
|
34
34
|
}
|
|
35
35
|
},
|
|
36
|
+
decay: COMPARISON_ARGS,
|
|
36
37
|
concentration: { dimension: {
|
|
37
38
|
type: "string",
|
|
38
39
|
description: "Dimension: pages or keywords (default: pages)"
|
|
@@ -63,22 +64,47 @@ const TOOL_EXTRA_ARGS = {
|
|
|
63
64
|
function buildParams(tool, args) {
|
|
64
65
|
const params = {
|
|
65
66
|
type: tool,
|
|
66
|
-
startDate: args.start ? String(args.start) : void 0,
|
|
67
|
-
endDate: args.end ? String(args.end) : void 0,
|
|
68
67
|
limit: parseIntegerOption(args.limit, "--limit")
|
|
69
68
|
};
|
|
69
|
+
const fetchBudget = parseFetchBudget(args["fetch-budget"]);
|
|
70
|
+
if (fetchBudget !== void 0) params.fetchBudget = fetchBudget;
|
|
70
71
|
if (args["brand-terms"]) params.brandTerms = String(args["brand-terms"]).split(",").map((t) => t.trim()).filter(Boolean);
|
|
71
|
-
if (args["prev-start"]) params.prevStartDate = String(args["prev-start"]);
|
|
72
|
-
if (args["prev-end"]) params.prevEndDate = String(args["prev-end"]);
|
|
73
72
|
if (args.dimension) params.dimension = String(args.dimension);
|
|
74
73
|
if (args.metric) params.metric = String(args.metric);
|
|
75
74
|
if (args["cluster-by"]) params.clusterBy = String(args["cluster-by"]);
|
|
75
|
+
if (args["sort-by"]) {
|
|
76
|
+
const sortBy = String(args["sort-by"]);
|
|
77
|
+
if (!MOVERS_SORT_METRICS.includes(sortBy)) throw new Error(`Invalid --sort-by "${sortBy}". Use one of: ${MOVERS_SORT_METRICS.join(", ")}.`);
|
|
78
|
+
params.sortBy = sortBy;
|
|
79
|
+
}
|
|
76
80
|
const weeks = parseIntegerOption(args.weeks, "--weeks");
|
|
77
81
|
const minWeeks = parseIntegerOption(args["min-weeks"], "--min-weeks");
|
|
78
82
|
if (weeks !== void 0) params.weeks = weeks;
|
|
79
83
|
if (minWeeks !== void 0) params.minWeeksWithData = minWeeks;
|
|
80
84
|
return params;
|
|
81
85
|
}
|
|
86
|
+
function withWindow(tool, params, args, anchor) {
|
|
87
|
+
const optional = (key) => args[key] ? String(args[key]) : void 0;
|
|
88
|
+
const window = unwrapResult(parseWindowFlags({
|
|
89
|
+
period: optional("period"),
|
|
90
|
+
start: optional("start"),
|
|
91
|
+
end: optional("end"),
|
|
92
|
+
prevStart: optional("prev-start"),
|
|
93
|
+
prevEnd: optional("prev-end")
|
|
94
|
+
}, {
|
|
95
|
+
...DEFAULT_WINDOW,
|
|
96
|
+
comparison: COMPARISON_TOOLS.has(tool) ? "prev-period" : "none"
|
|
97
|
+
}, anchor), windowFlagErrorToException);
|
|
98
|
+
return {
|
|
99
|
+
...params,
|
|
100
|
+
startDate: window.start,
|
|
101
|
+
endDate: window.end,
|
|
102
|
+
...window.comparison ? {
|
|
103
|
+
prevStartDate: window.comparison.start,
|
|
104
|
+
prevEndDate: window.comparison.end
|
|
105
|
+
} : {}
|
|
106
|
+
};
|
|
107
|
+
}
|
|
82
108
|
function makeToolCommand(tool) {
|
|
83
109
|
const extraArgs = TOOL_EXTRA_ARGS[tool] || {};
|
|
84
110
|
return defineCommand({
|
|
@@ -87,37 +113,45 @@ function makeToolCommand(tool) {
|
|
|
87
113
|
description: `Run ${tool} analysis`
|
|
88
114
|
},
|
|
89
115
|
args: {
|
|
90
|
-
site: {
|
|
116
|
+
"site": {
|
|
91
117
|
type: "string",
|
|
92
118
|
alias: "s",
|
|
93
119
|
description: "Site URL"
|
|
94
120
|
},
|
|
95
|
-
|
|
121
|
+
"period": {
|
|
96
122
|
type: "string",
|
|
97
|
-
description: "
|
|
123
|
+
description: `Window: ${PERIOD_FLAGS.join("|")} (default: 28d, ending on the newest synced day)`
|
|
98
124
|
},
|
|
99
|
-
|
|
125
|
+
"start": {
|
|
100
126
|
type: "string",
|
|
101
|
-
description: "
|
|
127
|
+
description: "Start date (YYYY-MM-DD). Implies --period custom"
|
|
102
128
|
},
|
|
103
|
-
|
|
129
|
+
"end": {
|
|
130
|
+
type: "string",
|
|
131
|
+
description: "End date (YYYY-MM-DD). Implies --period custom"
|
|
132
|
+
},
|
|
133
|
+
"limit": {
|
|
104
134
|
type: "string",
|
|
105
135
|
alias: "l",
|
|
106
136
|
default: "100",
|
|
107
|
-
description: "Max results"
|
|
137
|
+
description: "Max results to return"
|
|
138
|
+
},
|
|
139
|
+
"fetch-budget": {
|
|
140
|
+
type: "string",
|
|
141
|
+
description: `Max rows each live fetch reads (default: ${DEFAULT_FETCH_BUDGET}, max: ${MAX_FETCH_BUDGET})`
|
|
108
142
|
},
|
|
109
|
-
format: {
|
|
143
|
+
"format": {
|
|
110
144
|
type: "string",
|
|
111
145
|
alias: "f",
|
|
112
146
|
default: "table",
|
|
113
147
|
description: "Output: table, json, csv"
|
|
114
148
|
},
|
|
115
|
-
json: {
|
|
149
|
+
"json": {
|
|
116
150
|
type: "boolean",
|
|
117
151
|
default: false,
|
|
118
152
|
description: "Output as JSON"
|
|
119
153
|
},
|
|
120
|
-
live: {
|
|
154
|
+
"live": {
|
|
121
155
|
type: "boolean",
|
|
122
156
|
default: false,
|
|
123
157
|
description: "Force live GSC API; bypass local Parquet store"
|
|
@@ -130,29 +164,36 @@ function makeToolCommand(tool) {
|
|
|
130
164
|
"json",
|
|
131
165
|
"csv"
|
|
132
166
|
].includes(args.format ?? "table")) throw new Error("Invalid --format. Use table, json, or csv.");
|
|
133
|
-
const
|
|
134
|
-
const
|
|
167
|
+
const baseParams = buildParams(tool, args);
|
|
168
|
+
const format = args.json ? "json" : String(args.format ?? "table");
|
|
169
|
+
const { runAnalysis, siteUrl, anchor } = await resolveAnalysisSource({
|
|
135
170
|
site: args.site,
|
|
136
171
|
live: !!args.live,
|
|
137
|
-
json:
|
|
138
|
-
|
|
172
|
+
json: format === "json",
|
|
173
|
+
label: `analyze ${tool}`,
|
|
174
|
+
types: [tool],
|
|
175
|
+
anchorTables: analyzerTables(baseParams),
|
|
176
|
+
needs: (anchor) => analysisNeeds(withWindow(tool, baseParams, args, anchor))
|
|
139
177
|
});
|
|
178
|
+
const params = withWindow(tool, baseParams, args, anchor);
|
|
140
179
|
logger.debug(`Running ${tool} analysis...`);
|
|
141
|
-
const result = await runAnalysis(params)
|
|
180
|
+
const result = await runAnalysis(params);
|
|
181
|
+
const warning = coverageWarning(result.meta.coverage);
|
|
142
182
|
if (format === "json") {
|
|
143
183
|
console.log(JSON.stringify(result, null, 2));
|
|
184
|
+
if (warning) logger.warn(warning);
|
|
144
185
|
return;
|
|
145
186
|
}
|
|
146
187
|
if (format === "csv") {
|
|
147
188
|
console.log(toCSV(result.results, Object.keys(result.results[0] ?? {})));
|
|
189
|
+
if (warning) logger.warn(warning);
|
|
148
190
|
return;
|
|
149
191
|
}
|
|
150
|
-
const period = periodOf(params);
|
|
151
192
|
console.log(renderAnalysis(result, {
|
|
152
193
|
id: tool,
|
|
153
194
|
site: siteUrl,
|
|
154
|
-
start:
|
|
155
|
-
end:
|
|
195
|
+
start: params.startDate,
|
|
196
|
+
end: params.endDate,
|
|
156
197
|
...params.prevStartDate && params.prevEndDate ? { previous: {
|
|
157
198
|
start: params.prevStartDate,
|
|
158
199
|
end: params.prevEndDate
|