@gscdump/cli 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -14
- package/dist/THIRD-PARTY-LICENSES.md +155 -0
- package/dist/_chunks/config.mjs +45 -0
- package/dist/_chunks/libs/destr.mjs +42 -0
- package/dist/_chunks/libs/node-fetch-native.mjs +4136 -0
- package/dist/_chunks/libs/ofetch.mjs +417 -0
- package/dist/_chunks/rolldown-runtime.mjs +13 -0
- package/dist/index.mjs +3457 -447
- package/package.json +8 -8
- package/LICENSE +0 -21
package/dist/index.mjs
CHANGED
|
@@ -1,27 +1,32 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { t as __exportAll } from "./_chunks/rolldown-runtime.mjs";
|
|
3
|
+
import { t as ofetch } from "./_chunks/libs/ofetch.mjs";
|
|
4
|
+
import { a as loadConfig, c as setConfigDir, i as getConfigPath, n as defaultDataDir, o as resolveDataDir, r as getConfigDir, s as saveConfig } from "./_chunks/config.mjs";
|
|
2
5
|
import process from "node:process";
|
|
3
6
|
import { defineCommand, runMain } from "citty";
|
|
4
7
|
import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
|
|
5
8
|
import { AnalyzerCapabilityError, analyzeFromSource, createEngineQuerySource } from "@gscdump/analysis";
|
|
6
9
|
import { createGscApiQuerySource } from "@gscdump/engine-gsc-api";
|
|
7
|
-
import { cancel, isCancel, multiselect, select, text } from "@clack/prompts";
|
|
8
|
-
import { daysAgo, fetchSitemap, formatErrorForCli, getDateRange, googleSearchConsole, progressBar } from "gscdump";
|
|
10
|
+
import { cancel, confirm, isCancel, multiselect, select, text } from "@clack/prompts";
|
|
11
|
+
import { addSite, batchInspectUrls, batchRequestIndexing, createAuth, daysAgo, deleteSite, discoverSitemap, fetchSitemap, fetchSitemapUrls, fetchSitesWithSitemaps, formatErrorForCli, getDateRange, getIndexingMetadata, getVerificationToken, getVerifiedSite, googleSearchConsole, listVerifiedSites, progressBar, requestIndexing, runSequentialBatch, siteUrlToVerificationSite, unverifySite, verificationMethodsFor, verifySite } from "gscdump";
|
|
9
12
|
import fs, { readFile, rm } from "node:fs/promises";
|
|
10
13
|
import { createServer } from "node:http";
|
|
11
14
|
import path from "node:path";
|
|
12
|
-
import { OAuth2Client } from "google-auth-library";
|
|
15
|
+
import { JWT, OAuth2Client } from "google-auth-library";
|
|
16
|
+
import { Buffer } from "node:buffer";
|
|
17
|
+
import fs$1 from "node:fs";
|
|
13
18
|
import os from "node:os";
|
|
14
|
-
import {
|
|
19
|
+
import { createConsola } from "consola";
|
|
15
20
|
import { createNodeHarness } from "@gscdump/engine-duckdb-node";
|
|
16
21
|
import { TABLE_DIMS, transformGscRow } from "@gscdump/engine/ingest";
|
|
17
22
|
import { allTables, inferTable } from "@gscdump/engine/schema";
|
|
18
|
-
import {
|
|
23
|
+
import { DuckDBInstance } from "@duckdb/node-api";
|
|
24
|
+
import { sqlEscape } from "@gscdump/engine/sql";
|
|
19
25
|
import { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore } from "@gscdump/engine/entities";
|
|
20
26
|
import { createGscMcpServer } from "@gscdump/mcp/server";
|
|
21
27
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
22
|
-
import { SearchTypes, between, country, date, device, gsc, page, query, searchAppearance } from "gscdump/query";
|
|
23
|
-
import {
|
|
24
|
-
import { sqlEscape } from "@gscdump/engine/sql";
|
|
28
|
+
import { SearchTypes, and, between, contains, country, date, device, eq, gsc, notRegex, page, query, regex, searchAppearance } from "gscdump/query";
|
|
29
|
+
import { inferLegacyTier } from "@gscdump/engine";
|
|
25
30
|
import { DEFAULT_ROLLUPS, rebuildRollups } from "@gscdump/analysis/rollups";
|
|
26
31
|
import { filesystemStats } from "@gscdump/engine/filesystem";
|
|
27
32
|
var LocalStoreUnsupportedError = class extends Error {
|
|
@@ -63,42 +68,103 @@ async function runLiveAnalysis(client, siteUrl, params) {
|
|
|
63
68
|
throw e;
|
|
64
69
|
});
|
|
65
70
|
}
|
|
66
|
-
|
|
67
|
-
function
|
|
68
|
-
|
|
71
|
+
const ENV_LINE_RE$1 = /^([^=]+)=(.*)$/;
|
|
72
|
+
function parseEnvFile(envPath) {
|
|
73
|
+
let content;
|
|
74
|
+
try {
|
|
75
|
+
content = fs$1.readFileSync(envPath, "utf-8");
|
|
76
|
+
} catch {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
const env = {};
|
|
80
|
+
for (const line of content.split("\n")) {
|
|
81
|
+
const trimmed = line.trim();
|
|
82
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
83
|
+
const match = trimmed.match(ENV_LINE_RE$1);
|
|
84
|
+
if (!match) continue;
|
|
85
|
+
const key = match[1].trim();
|
|
86
|
+
let value = match[2].trim();
|
|
87
|
+
if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
|
|
88
|
+
env[key] = value;
|
|
89
|
+
}
|
|
90
|
+
return env;
|
|
91
|
+
}
|
|
92
|
+
const appliedEnvKeys = /* @__PURE__ */ new Set();
|
|
93
|
+
let loadedEnvPath = null;
|
|
94
|
+
function getAppliedEnvKeys() {
|
|
95
|
+
return appliedEnvKeys;
|
|
69
96
|
}
|
|
70
|
-
function
|
|
71
|
-
return
|
|
97
|
+
function getLoadedEnvPath() {
|
|
98
|
+
return loadedEnvPath;
|
|
72
99
|
}
|
|
73
|
-
function
|
|
74
|
-
|
|
100
|
+
function loadEnvFromCwd() {
|
|
101
|
+
const envPath = path.join(process.cwd(), ".env");
|
|
102
|
+
const parsed = parseEnvFile(envPath);
|
|
103
|
+
if (!parsed) return [];
|
|
104
|
+
loadedEnvPath = envPath;
|
|
105
|
+
const applied = [];
|
|
106
|
+
for (const [key, value] of Object.entries(parsed)) if (process.env[key] === void 0) {
|
|
107
|
+
process.env[key] = value;
|
|
108
|
+
applied.push(key);
|
|
109
|
+
appliedEnvKeys.add(key);
|
|
110
|
+
}
|
|
111
|
+
return applied;
|
|
75
112
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
113
|
+
const VERSION = "0.8.2";
|
|
114
|
+
const baseLogger = createConsola({
|
|
115
|
+
stdout: process.stderr,
|
|
116
|
+
stderr: process.stderr
|
|
117
|
+
});
|
|
118
|
+
const logger = baseLogger.withTag("gscdump");
|
|
119
|
+
function setQuiet(quiet) {
|
|
120
|
+
if (quiet) baseLogger.level = 1;
|
|
80
121
|
}
|
|
81
|
-
|
|
82
|
-
|
|
122
|
+
const OUTPUT_ARGS = {
|
|
123
|
+
json: {
|
|
124
|
+
type: "boolean",
|
|
125
|
+
default: false,
|
|
126
|
+
description: "Output as JSON"
|
|
127
|
+
},
|
|
128
|
+
quiet: {
|
|
129
|
+
type: "boolean",
|
|
130
|
+
alias: "q",
|
|
131
|
+
default: false,
|
|
132
|
+
description: "Suppress info/success output"
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
function applyOutputMode(args) {
|
|
136
|
+
const json = Boolean(args.json);
|
|
137
|
+
const quiet = json || Boolean(args.quiet);
|
|
138
|
+
setQuiet(quiet);
|
|
139
|
+
return {
|
|
140
|
+
json,
|
|
141
|
+
quiet
|
|
142
|
+
};
|
|
83
143
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
144
|
+
let colorEnabled = (() => {
|
|
145
|
+
if (process.env.NO_COLOR) return false;
|
|
146
|
+
if (process.argv.includes("--no-color")) return false;
|
|
147
|
+
return Boolean(process.stderr.isTTY) || Boolean(process.env.FORCE_COLOR);
|
|
148
|
+
})();
|
|
149
|
+
const ANSI_RE = /\x1B\[[0-9;]*m/g;
|
|
150
|
+
let stdoutWrapped = false;
|
|
151
|
+
function wrapStdoutForNoColor() {
|
|
152
|
+
if (stdoutWrapped) return;
|
|
153
|
+
stdoutWrapped = true;
|
|
154
|
+
const original = process.stdout.write.bind(process.stdout);
|
|
155
|
+
process.stdout.write = ((chunk, ...rest) => {
|
|
156
|
+
if (typeof chunk === "string") chunk = chunk.replace(ANSI_RE, "");
|
|
157
|
+
else if (chunk instanceof Uint8Array) chunk = Buffer.from(chunk).toString("utf8").replace(ANSI_RE, "");
|
|
158
|
+
return original(chunk, ...rest);
|
|
88
159
|
});
|
|
89
|
-
await fs.writeFile(path.join(configDir, "config.json"), JSON.stringify(config, null, 2), { mode: 384 });
|
|
90
|
-
}
|
|
91
|
-
function getConfigPath() {
|
|
92
|
-
return path.join(configDir, "config.json");
|
|
93
160
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
console.error();
|
|
100
|
-
process.exit(1);
|
|
161
|
+
function setNoColor(disable) {
|
|
162
|
+
if (disable) {
|
|
163
|
+
colorEnabled = false;
|
|
164
|
+
wrapStdoutForNoColor();
|
|
165
|
+
}
|
|
101
166
|
}
|
|
167
|
+
if (!colorEnabled) wrapStdoutForNoColor();
|
|
102
168
|
const gradientColors = [
|
|
103
169
|
(s) => `\x1B[38;2;52;211;153m${s}\x1B[0m`,
|
|
104
170
|
(s) => `\x1B[38;2;45;212;191m${s}\x1B[0m`,
|
|
@@ -120,6 +186,15 @@ function showSplash() {
|
|
|
120
186
|
function clearLine() {
|
|
121
187
|
process.stdout.write("\r\x1B[K");
|
|
122
188
|
}
|
|
189
|
+
function displayPath(absPath) {
|
|
190
|
+
const cwd = process.cwd();
|
|
191
|
+
const home = os.homedir();
|
|
192
|
+
if (absPath === cwd) return ".";
|
|
193
|
+
if (absPath.startsWith(`${cwd}/`)) return absPath.slice(cwd.length + 1);
|
|
194
|
+
if (absPath === home) return "~";
|
|
195
|
+
if (absPath.startsWith(`${home}/`)) return `~/${absPath.slice(home.length + 1)}`;
|
|
196
|
+
return absPath;
|
|
197
|
+
}
|
|
123
198
|
function formatAge(ms) {
|
|
124
199
|
const delta = Date.now() - ms;
|
|
125
200
|
if (delta < 6e4) return "just now";
|
|
@@ -146,6 +221,16 @@ function toCSV(data, columns) {
|
|
|
146
221
|
return str.includes(",") || str.includes("\"") || str.includes("\n") ? `"${str.replace(/"/g, "\"\"")}"` : str;
|
|
147
222
|
}).join(","))].join("\n");
|
|
148
223
|
}
|
|
224
|
+
async function readUrlList$1(args) {
|
|
225
|
+
if (args.file) return (await fs.readFile(String(args.file), "utf-8")).split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
|
|
226
|
+
if (args.urls) return (Array.isArray(args.urls) ? args.urls : [args.urls]).map(String).filter(Boolean);
|
|
227
|
+
if (!process.stdin.isTTY) {
|
|
228
|
+
const chunks = [];
|
|
229
|
+
for await (const chunk of process.stdin) chunks.push(chunk);
|
|
230
|
+
return Buffer.concat(chunks).toString("utf-8").split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
|
|
231
|
+
}
|
|
232
|
+
return [];
|
|
233
|
+
}
|
|
149
234
|
function exportToCSV(output) {
|
|
150
235
|
const sections = [];
|
|
151
236
|
if (output.pages?.data) sections.push(`# Pages\n${toCSV(output.pages.data, [
|
|
@@ -178,6 +263,82 @@ function exportToCSV(output) {
|
|
|
178
263
|
])}`);
|
|
179
264
|
return sections.join("\n\n");
|
|
180
265
|
}
|
|
266
|
+
const SCOPES = [
|
|
267
|
+
"https://www.googleapis.com/auth/webmasters",
|
|
268
|
+
"https://www.googleapis.com/auth/indexing",
|
|
269
|
+
"https://www.googleapis.com/auth/siteverification"
|
|
270
|
+
];
|
|
271
|
+
async function loadServiceAccount(jsonPath) {
|
|
272
|
+
const raw = await fs.readFile(jsonPath, "utf-8");
|
|
273
|
+
const key = JSON.parse(raw);
|
|
274
|
+
if (key.type !== "service_account") throw new Error(`${jsonPath} is not a service-account key (type=${key.type})`);
|
|
275
|
+
return new JWT({
|
|
276
|
+
email: key.client_email,
|
|
277
|
+
key: key.private_key,
|
|
278
|
+
scopes: SCOPES
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
async function resolveServiceAccount(opts = {}) {
|
|
282
|
+
let p = opts.path || process.env.GSC_SERVICE_ACCOUNT_JSON || process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
|
283
|
+
if (!p) p = (await loadConfig().catch(() => null))?.serviceAccountPath;
|
|
284
|
+
if (!p) return null;
|
|
285
|
+
return loadServiceAccount(p);
|
|
286
|
+
}
|
|
287
|
+
async function authenticateDeviceCode(credentials) {
|
|
288
|
+
const init = await ofetch("https://oauth2.googleapis.com/device/code", {
|
|
289
|
+
method: "POST",
|
|
290
|
+
body: new URLSearchParams({
|
|
291
|
+
client_id: credentials.clientId,
|
|
292
|
+
scope: SCOPES.join(" ")
|
|
293
|
+
})
|
|
294
|
+
}).catch((e) => {
|
|
295
|
+
throw new Error(`Device-code request failed: ${e.message}`);
|
|
296
|
+
});
|
|
297
|
+
console.log();
|
|
298
|
+
console.log(` \x1B[1mDevice-code OAuth\x1B[0m`);
|
|
299
|
+
console.log(` 1. On any device, open: \x1B[36m${init.verification_url}\x1B[0m`);
|
|
300
|
+
console.log(` 2. Enter this code: \x1B[1m${init.user_code}\x1B[0m`);
|
|
301
|
+
console.log(` 3. Approve the requested scopes`);
|
|
302
|
+
console.log();
|
|
303
|
+
logger.info(`Polling for completion (expires in ${Math.floor(init.expires_in / 60)}m)...`);
|
|
304
|
+
const intervalMs = init.interval * 1e3;
|
|
305
|
+
const deadline = Date.now() + init.expires_in * 1e3;
|
|
306
|
+
while (Date.now() < deadline) {
|
|
307
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
308
|
+
const res = await ofetch("https://oauth2.googleapis.com/token", {
|
|
309
|
+
method: "POST",
|
|
310
|
+
body: new URLSearchParams({
|
|
311
|
+
client_id: credentials.clientId,
|
|
312
|
+
client_secret: credentials.clientSecret,
|
|
313
|
+
device_code: init.device_code,
|
|
314
|
+
grant_type: "urn:ietf:params:oauth:grant-type:device_code"
|
|
315
|
+
})
|
|
316
|
+
}).catch((e) => e?.data ?? { error: "request_failed" });
|
|
317
|
+
if (res.access_token) return {
|
|
318
|
+
access_token: res.access_token,
|
|
319
|
+
refresh_token: res.refresh_token,
|
|
320
|
+
expiry_date: res.expires_in ? Date.now() + res.expires_in * 1e3 : void 0
|
|
321
|
+
};
|
|
322
|
+
if (res.error === "authorization_pending" || res.error === "slow_down") continue;
|
|
323
|
+
if (res.error === "access_denied") throw new Error("User denied authorization.");
|
|
324
|
+
if (res.error === "expired_token") throw new Error("Device code expired. Re-run `gscdump auth login --no-browser`.");
|
|
325
|
+
if (res.error) throw new Error(`Device-code poll failed: ${res.error_description || res.error}`);
|
|
326
|
+
}
|
|
327
|
+
throw new Error("Device-code flow timed out.");
|
|
328
|
+
}
|
|
329
|
+
function resolveBYOK(opts = {}) {
|
|
330
|
+
const accessToken = opts.accessToken || process.env.GSC_ACCESS_TOKEN || process.env.GOOGLE_ACCESS_TOKEN;
|
|
331
|
+
const clientId = opts.clientId || process.env.GSC_CLIENT_ID || process.env.GOOGLE_CLIENT_ID;
|
|
332
|
+
const clientSecret = opts.clientSecret || process.env.GSC_CLIENT_SECRET || process.env.GOOGLE_CLIENT_SECRET;
|
|
333
|
+
const refreshToken = opts.refreshToken || process.env.GSC_REFRESH_TOKEN || process.env.GOOGLE_REFRESH_TOKEN;
|
|
334
|
+
if (clientId && clientSecret && refreshToken) return createAuth({
|
|
335
|
+
clientId,
|
|
336
|
+
clientSecret,
|
|
337
|
+
refreshToken
|
|
338
|
+
});
|
|
339
|
+
if (accessToken) return accessToken;
|
|
340
|
+
return null;
|
|
341
|
+
}
|
|
181
342
|
const REDIRECT_URI_RE = /redirect_uri=[^&]+/;
|
|
182
343
|
function getTokensPath() {
|
|
183
344
|
return path.join(getConfigDir(), "tokens.json");
|
|
@@ -200,17 +361,22 @@ async function getAuthCredentials(interactive) {
|
|
|
200
361
|
const envClientId = process.env.GOOGLE_CLIENT_ID;
|
|
201
362
|
const envClientSecret = process.env.GOOGLE_CLIENT_SECRET;
|
|
202
363
|
if (envClientId && envClientSecret) {
|
|
203
|
-
logger.
|
|
364
|
+
logger.info("Using OAuth client from env");
|
|
365
|
+
console.log(` \x1B[90m${envClientId}\x1B[0m`);
|
|
204
366
|
return {
|
|
205
367
|
clientId: envClientId,
|
|
206
368
|
clientSecret: envClientSecret
|
|
207
369
|
};
|
|
208
370
|
}
|
|
209
371
|
const config = await loadConfig();
|
|
210
|
-
if (config.clientId && config.clientSecret)
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
372
|
+
if (config.clientId && config.clientSecret) {
|
|
373
|
+
logger.info(`Using OAuth client from ${displayPath(`${getConfigDir()}/config.json`)}`);
|
|
374
|
+
console.log(` \x1B[90m${config.clientId}\x1B[0m`);
|
|
375
|
+
return {
|
|
376
|
+
clientId: config.clientId,
|
|
377
|
+
clientSecret: config.clientSecret
|
|
378
|
+
};
|
|
379
|
+
}
|
|
214
380
|
if (!interactive) {
|
|
215
381
|
logger.error("GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET required for non-interactive mode");
|
|
216
382
|
process.exit(1);
|
|
@@ -246,25 +412,31 @@ async function getAuthCredentials(interactive) {
|
|
|
246
412
|
async function getAuthCodeViaLoopback(authUrl) {
|
|
247
413
|
return new Promise((resolve, reject) => {
|
|
248
414
|
let resolvedRedirectUri = "";
|
|
249
|
-
|
|
415
|
+
let timeoutId;
|
|
416
|
+
let server;
|
|
417
|
+
const settle = (fn) => {
|
|
418
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
419
|
+
server.closeAllConnections?.();
|
|
420
|
+
server.close();
|
|
421
|
+
fn();
|
|
422
|
+
};
|
|
423
|
+
server = createServer((req, res) => {
|
|
250
424
|
const url = new URL(req.url || "", `http://127.0.0.1`);
|
|
251
425
|
const code = url.searchParams.get("code");
|
|
252
426
|
const error = url.searchParams.get("error");
|
|
253
427
|
if (error) {
|
|
254
428
|
res.writeHead(400, { "Content-Type": "text/html" });
|
|
255
429
|
res.end(`<html><body><h1>Authorization Failed</h1><p>${error}</p><p>You can close this window.</p></body></html>`);
|
|
256
|
-
|
|
257
|
-
reject(/* @__PURE__ */ new Error(`OAuth error: ${error}`));
|
|
430
|
+
settle(() => reject(/* @__PURE__ */ new Error(`OAuth error: ${error}`)));
|
|
258
431
|
return;
|
|
259
432
|
}
|
|
260
433
|
if (code) {
|
|
261
434
|
res.writeHead(200, { "Content-Type": "text/html" });
|
|
262
435
|
res.end(`<html><body><h1>Authorization Successful</h1><p>You can close this window and return to the terminal.</p></body></html>`);
|
|
263
|
-
|
|
264
|
-
resolve({
|
|
436
|
+
settle(() => resolve({
|
|
265
437
|
code,
|
|
266
438
|
redirectUri: resolvedRedirectUri
|
|
267
|
-
});
|
|
439
|
+
}));
|
|
268
440
|
return;
|
|
269
441
|
}
|
|
270
442
|
res.writeHead(400, { "Content-Type": "text/html" });
|
|
@@ -273,7 +445,7 @@ async function getAuthCodeViaLoopback(authUrl) {
|
|
|
273
445
|
server.listen(0, "127.0.0.1", () => {
|
|
274
446
|
const addr = server.address();
|
|
275
447
|
if (!addr || typeof addr === "string") {
|
|
276
|
-
reject(/* @__PURE__ */ new Error("Failed to start local server"));
|
|
448
|
+
settle(() => reject(/* @__PURE__ */ new Error("Failed to start local server")));
|
|
277
449
|
return;
|
|
278
450
|
}
|
|
279
451
|
resolvedRedirectUri = `http://127.0.0.1:${addr.port}`;
|
|
@@ -287,17 +459,16 @@ async function getAuthCodeViaLoopback(authUrl) {
|
|
|
287
459
|
logger.warn("Could not open browser automatically");
|
|
288
460
|
});
|
|
289
461
|
});
|
|
290
|
-
server.on("error", reject);
|
|
291
|
-
setTimeout(() => {
|
|
292
|
-
|
|
293
|
-
reject(/* @__PURE__ */ new Error("Authorization timed out"));
|
|
462
|
+
server.on("error", (err) => settle(() => reject(err)));
|
|
463
|
+
timeoutId = setTimeout(() => {
|
|
464
|
+
settle(() => reject(/* @__PURE__ */ new Error("Authorization timed out")));
|
|
294
465
|
}, 300 * 1e3);
|
|
295
466
|
});
|
|
296
467
|
}
|
|
297
|
-
async function authenticate(credentials, interactive) {
|
|
468
|
+
async function authenticate(credentials, interactive, opts = {}) {
|
|
298
469
|
const oauth2Client = new OAuth2Client(credentials.clientId, credentials.clientSecret, "http://127.0.0.1");
|
|
299
|
-
const envAccessToken = process.env.GOOGLE_ACCESS_TOKEN;
|
|
300
|
-
const envRefreshToken = process.env.GOOGLE_REFRESH_TOKEN;
|
|
470
|
+
const envAccessToken = !opts.force ? process.env.GOOGLE_ACCESS_TOKEN : void 0;
|
|
471
|
+
const envRefreshToken = !opts.force ? process.env.GOOGLE_REFRESH_TOKEN : void 0;
|
|
301
472
|
if (envAccessToken || envRefreshToken) {
|
|
302
473
|
oauth2Client.setCredentials({
|
|
303
474
|
access_token: envAccessToken,
|
|
@@ -309,7 +480,7 @@ async function authenticate(credentials, interactive) {
|
|
|
309
480
|
}
|
|
310
481
|
return oauth2Client;
|
|
311
482
|
}
|
|
312
|
-
const existingTokens = await loadTokens();
|
|
483
|
+
const existingTokens = !opts.force ? await loadTokens() : null;
|
|
313
484
|
if (existingTokens) {
|
|
314
485
|
oauth2Client.setCredentials(existingTokens);
|
|
315
486
|
if (existingTokens.expiry_date && existingTokens.expiry_date < Date.now()) {
|
|
@@ -329,9 +500,16 @@ async function authenticate(credentials, interactive) {
|
|
|
329
500
|
logger.error("No saved tokens. Run interactively first to authenticate.");
|
|
330
501
|
process.exit(1);
|
|
331
502
|
}
|
|
503
|
+
if (opts.noBrowser) {
|
|
504
|
+
const tokens = await authenticateDeviceCode(credentials);
|
|
505
|
+
oauth2Client.setCredentials(tokens);
|
|
506
|
+
await saveTokens(tokens);
|
|
507
|
+
logger.success(`Tokens saved to ${displayPath(getTokensPath())}`);
|
|
508
|
+
return oauth2Client;
|
|
509
|
+
}
|
|
332
510
|
const authUrl = oauth2Client.generateAuthUrl({
|
|
333
511
|
access_type: "offline",
|
|
334
|
-
scope:
|
|
512
|
+
scope: SCOPES,
|
|
335
513
|
prompt: "consent"
|
|
336
514
|
});
|
|
337
515
|
logger.info("Waiting for authorization...");
|
|
@@ -339,24 +517,169 @@ async function authenticate(credentials, interactive) {
|
|
|
339
517
|
const { tokens } = await new OAuth2Client(credentials.clientId, credentials.clientSecret, redirectUri).getToken(code);
|
|
340
518
|
oauth2Client.setCredentials(tokens);
|
|
341
519
|
await saveTokens(tokens);
|
|
342
|
-
logger.success(`Tokens saved to ${getTokensPath()}`);
|
|
520
|
+
logger.success(`Tokens saved to ${displayPath(getTokensPath())}`);
|
|
343
521
|
return oauth2Client;
|
|
344
522
|
}
|
|
345
523
|
async function getAuth(opts = {}) {
|
|
346
|
-
const { interactive = true } = opts;
|
|
347
|
-
return authenticate(await getAuthCredentials(interactive), interactive
|
|
524
|
+
const { interactive = true, noBrowser = false, force = false } = opts;
|
|
525
|
+
return authenticate(await getAuthCredentials(interactive), interactive, {
|
|
526
|
+
noBrowser,
|
|
527
|
+
force
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
async function resolveAuth(opts = {}) {
|
|
531
|
+
const sa = await resolveServiceAccount({ path: opts.serviceAccount });
|
|
532
|
+
if (sa) {
|
|
533
|
+
logger.success("Using service-account credentials");
|
|
534
|
+
return sa;
|
|
535
|
+
}
|
|
536
|
+
const byok = resolveBYOK(opts.byok);
|
|
537
|
+
if (byok) {
|
|
538
|
+
if (typeof byok !== "string") logger.success("Using BYOK credentials");
|
|
539
|
+
return byok;
|
|
540
|
+
}
|
|
541
|
+
return getAuth(opts);
|
|
542
|
+
}
|
|
543
|
+
function envSourceLabel(envVar) {
|
|
544
|
+
return getAppliedEnvKeys().has(envVar) ? `.env (${envVar})` : `shell env (${envVar})`;
|
|
545
|
+
}
|
|
546
|
+
function pickEnvSource(...envVars) {
|
|
547
|
+
for (const v of envVars) {
|
|
548
|
+
const value = process.env[v];
|
|
549
|
+
if (value) return {
|
|
550
|
+
envVar: v,
|
|
551
|
+
value
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
return null;
|
|
555
|
+
}
|
|
556
|
+
function redactCred(v, keepTail = 6) {
|
|
557
|
+
if (!v) return "<unset>";
|
|
558
|
+
if (v.length <= keepTail) return "***";
|
|
559
|
+
return `***${v.slice(-keepTail)}`;
|
|
560
|
+
}
|
|
561
|
+
async function describeAuthProvenance() {
|
|
562
|
+
const rows = [];
|
|
563
|
+
const warnings = [];
|
|
564
|
+
const saEnvPath = process.env.GSC_SERVICE_ACCOUNT_JSON || process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
|
565
|
+
const saConfigPath = !saEnvPath ? (await loadConfig().catch(() => null))?.serviceAccountPath : void 0;
|
|
566
|
+
const saPath = saEnvPath || saConfigPath;
|
|
567
|
+
if (saEnvPath) {
|
|
568
|
+
const saEnv = process.env.GSC_SERVICE_ACCOUNT_JSON ? "GSC_SERVICE_ACCOUNT_JSON" : "GOOGLE_APPLICATION_CREDENTIALS";
|
|
569
|
+
rows.push({
|
|
570
|
+
field: "service_account",
|
|
571
|
+
source: envSourceLabel(saEnv),
|
|
572
|
+
value: displayPath(saEnvPath)
|
|
573
|
+
});
|
|
574
|
+
} else if (saConfigPath) rows.push({
|
|
575
|
+
field: "service_account",
|
|
576
|
+
source: `${displayPath(`${getConfigDir()}/config.json`)}`,
|
|
577
|
+
value: displayPath(saConfigPath)
|
|
578
|
+
});
|
|
579
|
+
const clientId = pickEnvSource("GSC_CLIENT_ID", "GOOGLE_CLIENT_ID");
|
|
580
|
+
const clientSecret = pickEnvSource("GSC_CLIENT_SECRET", "GOOGLE_CLIENT_SECRET");
|
|
581
|
+
const config = await loadConfig().catch(() => null);
|
|
582
|
+
if (clientId) rows.push({
|
|
583
|
+
field: "client_id",
|
|
584
|
+
source: envSourceLabel(clientId.envVar),
|
|
585
|
+
value: clientId.value
|
|
586
|
+
});
|
|
587
|
+
else if (config?.clientId) rows.push({
|
|
588
|
+
field: "client_id",
|
|
589
|
+
source: `${displayPath(`${getConfigDir()}/config.json`)}`,
|
|
590
|
+
value: config.clientId
|
|
591
|
+
});
|
|
592
|
+
else rows.push({
|
|
593
|
+
field: "client_id",
|
|
594
|
+
source: "<none>",
|
|
595
|
+
value: null
|
|
596
|
+
});
|
|
597
|
+
if (clientSecret) rows.push({
|
|
598
|
+
field: "client_secret",
|
|
599
|
+
source: envSourceLabel(clientSecret.envVar),
|
|
600
|
+
value: redactCred(clientSecret.value)
|
|
601
|
+
});
|
|
602
|
+
else if (config?.clientSecret) rows.push({
|
|
603
|
+
field: "client_secret",
|
|
604
|
+
source: `${displayPath(`${getConfigDir()}/config.json`)}`,
|
|
605
|
+
value: redactCred(config.clientSecret)
|
|
606
|
+
});
|
|
607
|
+
else rows.push({
|
|
608
|
+
field: "client_secret",
|
|
609
|
+
source: "<none>",
|
|
610
|
+
value: null
|
|
611
|
+
});
|
|
612
|
+
const accessTok = pickEnvSource("GSC_ACCESS_TOKEN", "GOOGLE_ACCESS_TOKEN");
|
|
613
|
+
const refreshTok = pickEnvSource("GSC_REFRESH_TOKEN", "GOOGLE_REFRESH_TOKEN");
|
|
614
|
+
if (accessTok) rows.push({
|
|
615
|
+
field: "access_token",
|
|
616
|
+
source: envSourceLabel(accessTok.envVar),
|
|
617
|
+
value: redactCred(accessTok.value)
|
|
618
|
+
});
|
|
619
|
+
if (refreshTok) rows.push({
|
|
620
|
+
field: "refresh_token",
|
|
621
|
+
source: envSourceLabel(refreshTok.envVar),
|
|
622
|
+
value: redactCred(refreshTok.value)
|
|
623
|
+
});
|
|
624
|
+
const tokens = await loadTokens();
|
|
625
|
+
if (tokens) {
|
|
626
|
+
const expiry = tokens.expiry_date ? new Date(tokens.expiry_date).toISOString() : "no expiry";
|
|
627
|
+
rows.push({
|
|
628
|
+
field: "saved_tokens",
|
|
629
|
+
source: displayPath(getTokensPath()),
|
|
630
|
+
value: `access=${tokens.access_token ? "present" : "missing"}, refresh=${tokens.refresh_token ? "present" : "missing"}, expiry=${expiry}`
|
|
631
|
+
});
|
|
632
|
+
}
|
|
633
|
+
let effective;
|
|
634
|
+
if (saPath) effective = "service-account";
|
|
635
|
+
else if (clientId && clientSecret && refreshTok) effective = "byok-refresh-token";
|
|
636
|
+
else if (accessTok) effective = "byok-access-token";
|
|
637
|
+
else if (tokens) effective = "saved-tokens";
|
|
638
|
+
else effective = "none";
|
|
639
|
+
if ((effective === "byok-refresh-token" || effective === "byok-access-token") && tokens) warnings.push("BYOK env vars shadow saved tokens. If you just ran `auth login --force`, the env tokens are stale; unset them or update them.");
|
|
640
|
+
if (effective === "byok-refresh-token" && clientId && refreshTok) {
|
|
641
|
+
if (getAppliedEnvKeys().has(clientId.envVar) !== getAppliedEnvKeys().has(refreshTok.envVar)) warnings.push(`client_id and refresh_token come from different sources (${envSourceLabel(clientId.envVar)} vs ${envSourceLabel(refreshTok.envVar)}); they may not match.`);
|
|
642
|
+
}
|
|
643
|
+
const envFile = getLoadedEnvPath();
|
|
644
|
+
if (envFile && getAppliedEnvKeys().size > 0) warnings.push(`Loaded ${getAppliedEnvKeys().size} var(s) from ${displayPath(envFile)}.`);
|
|
645
|
+
return {
|
|
646
|
+
rows,
|
|
647
|
+
effective,
|
|
648
|
+
warnings
|
|
649
|
+
};
|
|
650
|
+
}
|
|
651
|
+
async function formatAuthProvenance() {
|
|
652
|
+
const { rows, effective, warnings } = await describeAuthProvenance();
|
|
653
|
+
const lines = [];
|
|
654
|
+
lines.push(` \x1B[1mAuth config sources\x1B[0m \x1B[90m(effective: ${effective})\x1B[0m`);
|
|
655
|
+
for (const r of rows) {
|
|
656
|
+
const val = r.value ? ` \x1B[90m${r.value}\x1B[0m` : "";
|
|
657
|
+
lines.push(` ${r.field.padEnd(16)} \x1B[36m${r.source}\x1B[0m${val}`);
|
|
658
|
+
}
|
|
659
|
+
if (warnings.length > 0) {
|
|
660
|
+
lines.push("");
|
|
661
|
+
for (const w of warnings) lines.push(` \x1B[33m!\x1B[0m ${w}`);
|
|
662
|
+
}
|
|
663
|
+
return lines.join("\n");
|
|
664
|
+
}
|
|
665
|
+
function isAuthError(err) {
|
|
666
|
+
const msg = (err instanceof Error ? err.message : String(err ?? "")).toLowerCase();
|
|
667
|
+
if (!msg) return false;
|
|
668
|
+
return /\b(?:401|403|unauthorized|forbidden|invalid_grant|invalid_token|insufficient.*scope|invalid_client|token has been expired|token has been revoked)\b/.test(msg) || msg.includes("oauth2.googleapis.com/token");
|
|
348
669
|
}
|
|
349
670
|
function createLocalStore(opts) {
|
|
350
671
|
return createNodeHarness(opts);
|
|
351
672
|
}
|
|
673
|
+
var context_exports = /* @__PURE__ */ __exportAll({ createCommandContext: () => createCommandContext });
|
|
352
674
|
async function createCommandContext(opts = {}) {
|
|
353
|
-
const { needsAuth = false, needsStore = false, interactive = false } = opts;
|
|
675
|
+
const { needsAuth = false, needsStore = false, interactive = false, byok, fetchOptions } = opts;
|
|
354
676
|
const config = await loadConfig();
|
|
355
|
-
const auth = needsAuth ? await
|
|
677
|
+
const auth = needsAuth ? await resolveAuth({
|
|
356
678
|
interactive,
|
|
357
|
-
config
|
|
679
|
+
config,
|
|
680
|
+
byok
|
|
358
681
|
}) : null;
|
|
359
|
-
const client = auth ? googleSearchConsole(auth) : null;
|
|
682
|
+
const client = auth ? googleSearchConsole(auth, { fetchOptions }) : null;
|
|
360
683
|
const store = needsStore ? createLocalStore({ dataDir: resolveDataDir(config) }) : null;
|
|
361
684
|
const loadSites = async () => {
|
|
362
685
|
if (!client) throw new Error("loadSites requires needsAuth: true");
|
|
@@ -402,6 +725,16 @@ async function createCommandContext(opts = {}) {
|
|
|
402
725
|
resolveSite
|
|
403
726
|
};
|
|
404
727
|
}
|
|
728
|
+
async function gscErrorHandler(error) {
|
|
729
|
+
console.error();
|
|
730
|
+
console.error(formatErrorForCli(error));
|
|
731
|
+
if (isAuthError(error)) {
|
|
732
|
+
console.error();
|
|
733
|
+
console.error(await formatAuthProvenance());
|
|
734
|
+
}
|
|
735
|
+
console.error();
|
|
736
|
+
process.exit(1);
|
|
737
|
+
}
|
|
405
738
|
const ANALYSIS_TOOLS = defaultAnalyzerRegistry.listAnalyzerIds();
|
|
406
739
|
const TOOL_EXTRA_ARGS = {
|
|
407
740
|
brand: { "brand-terms": {
|
|
@@ -551,10 +884,7 @@ function makeToolCommand(tool) {
|
|
|
551
884
|
renderResults(localResult.results, localResult.results.length, format);
|
|
552
885
|
return;
|
|
553
886
|
}
|
|
554
|
-
const result = await runLiveAnalysis(ctx.client, siteUrl, params).catch(
|
|
555
|
-
logger.error(`Analysis failed: ${e.message}`);
|
|
556
|
-
process.exit(1);
|
|
557
|
-
});
|
|
887
|
+
const result = await runLiveAnalysis(ctx.client, siteUrl, params).catch(gscErrorHandler);
|
|
558
888
|
if (format === "json") {
|
|
559
889
|
console.log(JSON.stringify(result, null, 2));
|
|
560
890
|
return;
|
|
@@ -735,193 +1065,1234 @@ const analyzeCommand = defineCommand({
|
|
|
735
1065
|
name: "analyze",
|
|
736
1066
|
description: "SEO analysis tools"
|
|
737
1067
|
},
|
|
738
|
-
subCommands:
|
|
1068
|
+
subCommands: {
|
|
1069
|
+
list: defineCommand({
|
|
1070
|
+
meta: {
|
|
1071
|
+
name: "list",
|
|
1072
|
+
description: "List available analyzer ids"
|
|
1073
|
+
},
|
|
1074
|
+
args: { json: {
|
|
1075
|
+
type: "boolean",
|
|
1076
|
+
default: false,
|
|
1077
|
+
description: "Output as JSON"
|
|
1078
|
+
} },
|
|
1079
|
+
async run({ args }) {
|
|
1080
|
+
if (args.json) {
|
|
1081
|
+
console.log(JSON.stringify(ANALYSIS_TOOLS, null, 2));
|
|
1082
|
+
return;
|
|
1083
|
+
}
|
|
1084
|
+
for (const id of ANALYSIS_TOOLS) console.log(id);
|
|
1085
|
+
}
|
|
1086
|
+
}),
|
|
1087
|
+
...Object.fromEntries(ANALYSIS_TOOLS.map((tool) => [tool, makeToolCommand(tool)]))
|
|
1088
|
+
}
|
|
739
1089
|
});
|
|
740
|
-
const
|
|
1090
|
+
const ROOT_DIR = path.join(os.homedir(), ".config", "gscdump");
|
|
1091
|
+
const PROFILES_DIR = path.join(ROOT_DIR, "profiles");
|
|
1092
|
+
const ACTIVE_MARKER = path.join(ROOT_DIR, "active-profile");
|
|
1093
|
+
let activeOverride = null;
|
|
1094
|
+
let configDirOverridden = false;
|
|
1095
|
+
function getProfileDir(name) {
|
|
1096
|
+
return path.join(PROFILES_DIR, name);
|
|
1097
|
+
}
|
|
1098
|
+
function readActiveMarkerSync() {
|
|
1099
|
+
if (!fs$1.existsSync(ACTIVE_MARKER)) return null;
|
|
1100
|
+
return fs$1.readFileSync(ACTIVE_MARKER, "utf-8").trim() || null;
|
|
1101
|
+
}
|
|
1102
|
+
function resolveActiveProfile() {
|
|
1103
|
+
return activeOverride ?? process.env.GSCDUMP_PROFILE ?? readActiveMarkerSync();
|
|
1104
|
+
}
|
|
1105
|
+
function applyProfileFromCli(opts) {
|
|
1106
|
+
if (opts.configDir) {
|
|
1107
|
+
setConfigDir(opts.configDir);
|
|
1108
|
+
configDirOverridden = true;
|
|
1109
|
+
return;
|
|
1110
|
+
}
|
|
1111
|
+
if (opts.profile) {
|
|
1112
|
+
activeOverride = opts.profile;
|
|
1113
|
+
setConfigDir(getProfileDir(opts.profile));
|
|
1114
|
+
return;
|
|
1115
|
+
}
|
|
1116
|
+
const envProfile = process.env.GSCDUMP_PROFILE;
|
|
1117
|
+
if (envProfile) {
|
|
1118
|
+
setConfigDir(getProfileDir(envProfile));
|
|
1119
|
+
return;
|
|
1120
|
+
}
|
|
1121
|
+
const marker = readActiveMarkerSync();
|
|
1122
|
+
if (marker) setConfigDir(getProfileDir(marker));
|
|
1123
|
+
}
|
|
1124
|
+
async function setActiveProfile(name) {
|
|
1125
|
+
await fs.mkdir(ROOT_DIR, {
|
|
1126
|
+
recursive: true,
|
|
1127
|
+
mode: 448
|
|
1128
|
+
});
|
|
1129
|
+
if (name == null) {
|
|
1130
|
+
await fs.rm(ACTIVE_MARKER, { force: true }).catch(() => {});
|
|
1131
|
+
return;
|
|
1132
|
+
}
|
|
1133
|
+
await fs.writeFile(ACTIVE_MARKER, name, { mode: 384 });
|
|
1134
|
+
}
|
|
1135
|
+
async function listProfiles() {
|
|
1136
|
+
return fs.readdir(PROFILES_DIR).then((entries) => entries.filter((e) => !e.startsWith(".")).sort()).catch(() => []);
|
|
1137
|
+
}
|
|
1138
|
+
async function createProfile(name) {
|
|
1139
|
+
const dir = getProfileDir(name);
|
|
1140
|
+
await fs.mkdir(dir, {
|
|
1141
|
+
recursive: true,
|
|
1142
|
+
mode: 448
|
|
1143
|
+
});
|
|
1144
|
+
return dir;
|
|
1145
|
+
}
|
|
1146
|
+
function profileNameFromEmail(email) {
|
|
1147
|
+
return email.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
1148
|
+
}
|
|
1149
|
+
async function adoptCurrentConfigAsProfile(name) {
|
|
1150
|
+
if (configDirOverridden) return null;
|
|
1151
|
+
if (resolveActiveProfile()) return null;
|
|
1152
|
+
const currentDir = getConfigDir();
|
|
1153
|
+
const targetDir = getProfileDir(name);
|
|
1154
|
+
if (currentDir === targetDir) return targetDir;
|
|
1155
|
+
await fs.mkdir(targetDir, {
|
|
1156
|
+
recursive: true,
|
|
1157
|
+
mode: 448
|
|
1158
|
+
});
|
|
1159
|
+
for (const f of ["tokens.json", "config.json"]) {
|
|
1160
|
+
const src = path.join(currentDir, f);
|
|
1161
|
+
const dst = path.join(targetDir, f);
|
|
1162
|
+
if (await fs.stat(src).then(() => true).catch(() => false)) await fs.rename(src, dst).catch(() => {});
|
|
1163
|
+
}
|
|
1164
|
+
await setActiveProfile(name);
|
|
1165
|
+
activeOverride = name;
|
|
1166
|
+
setConfigDir(targetDir);
|
|
1167
|
+
return targetDir;
|
|
1168
|
+
}
|
|
1169
|
+
const profileCommand = defineCommand({
|
|
741
1170
|
meta: {
|
|
742
|
-
name: "
|
|
743
|
-
description: "Manage
|
|
1171
|
+
name: "profile",
|
|
1172
|
+
description: "Manage gscdump profiles (per-account token + config dirs)"
|
|
744
1173
|
},
|
|
745
1174
|
subCommands: {
|
|
746
|
-
|
|
1175
|
+
list: defineCommand({
|
|
747
1176
|
meta: {
|
|
748
|
-
name: "
|
|
749
|
-
description: "
|
|
1177
|
+
name: "list",
|
|
1178
|
+
description: "List configured profiles"
|
|
750
1179
|
},
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
1180
|
+
args: { ...OUTPUT_ARGS },
|
|
1181
|
+
async run({ args }) {
|
|
1182
|
+
const { json } = applyOutputMode(args);
|
|
1183
|
+
const names = await listProfiles();
|
|
1184
|
+
const active = resolveActiveProfile();
|
|
1185
|
+
if (json) {
|
|
1186
|
+
console.log(JSON.stringify({
|
|
1187
|
+
active,
|
|
1188
|
+
profiles: names
|
|
1189
|
+
}, null, 2));
|
|
756
1190
|
return;
|
|
757
1191
|
}
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
logger.success("Authenticated");
|
|
763
|
-
console.log();
|
|
764
|
-
console.log(` Access token: ${hasAccess ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
765
|
-
console.log(` Refresh token: ${hasRefresh ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
766
|
-
if (expiry) {
|
|
767
|
-
const status = isExpired ? "\x1B[33mexpired\x1B[0m" : "\x1B[32mvalid\x1B[0m";
|
|
768
|
-
console.log(` Expires: ${expiry.toISOString()} (${status})`);
|
|
1192
|
+
if (names.length === 0) {
|
|
1193
|
+
logger.warn(`No profiles in ${PROFILES_DIR}`);
|
|
1194
|
+
logger.info("Run `gscdump auth login` to create one automatically, or `gscdump profile create <name>`");
|
|
1195
|
+
return;
|
|
769
1196
|
}
|
|
1197
|
+
for (const n of names) console.log(`${n === active ? "*" : " "} ${n}`);
|
|
770
1198
|
}
|
|
771
1199
|
}),
|
|
772
|
-
|
|
1200
|
+
path: defineCommand({
|
|
773
1201
|
meta: {
|
|
774
|
-
name: "
|
|
775
|
-
description: "
|
|
1202
|
+
name: "path",
|
|
1203
|
+
description: "Print the config directory for a profile"
|
|
776
1204
|
},
|
|
777
|
-
|
|
778
|
-
|
|
1205
|
+
args: { name: {
|
|
1206
|
+
type: "positional",
|
|
1207
|
+
required: false,
|
|
1208
|
+
description: "Profile name (default: active)"
|
|
1209
|
+
} },
|
|
1210
|
+
async run({ args }) {
|
|
1211
|
+
const name = args.name ? String(args.name) : resolveActiveProfile();
|
|
1212
|
+
if (!name) {
|
|
1213
|
+
logger.error("No profile specified and none active (set --profile, GSCDUMP_PROFILE, or run `gscdump profile use <name>`)");
|
|
1214
|
+
process.exit(1);
|
|
1215
|
+
}
|
|
1216
|
+
console.log(getProfileDir(name));
|
|
779
1217
|
}
|
|
780
|
-
})
|
|
781
|
-
|
|
782
|
-
});
|
|
783
|
-
const configCommand = defineCommand({
|
|
784
|
-
meta: {
|
|
785
|
-
name: "config",
|
|
786
|
-
description: "Manage configuration"
|
|
787
|
-
},
|
|
788
|
-
subCommands: {
|
|
789
|
-
show: defineCommand({
|
|
1218
|
+
}),
|
|
1219
|
+
current: defineCommand({
|
|
790
1220
|
meta: {
|
|
791
|
-
name: "
|
|
792
|
-
description: "
|
|
1221
|
+
name: "current",
|
|
1222
|
+
description: "Print the active profile name"
|
|
793
1223
|
},
|
|
794
1224
|
async run() {
|
|
795
|
-
const
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
console.log();
|
|
799
|
-
if (Object.keys(config).length === 0) {
|
|
800
|
-
logger.warn("No config set");
|
|
801
|
-
return;
|
|
802
|
-
}
|
|
803
|
-
console.log(JSON.stringify(config, null, 2));
|
|
1225
|
+
const active = resolveActiveProfile();
|
|
1226
|
+
if (!active) process.exit(1);
|
|
1227
|
+
console.log(active);
|
|
804
1228
|
}
|
|
805
1229
|
}),
|
|
806
|
-
|
|
1230
|
+
use: defineCommand({
|
|
807
1231
|
meta: {
|
|
808
|
-
name: "
|
|
809
|
-
description: "Set
|
|
1232
|
+
name: "use",
|
|
1233
|
+
description: "Set the persisted active profile (subsequent commands no longer need --profile)"
|
|
810
1234
|
},
|
|
811
1235
|
args: {
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
description: "Config key (defaultSite, defaultPeriod, defaultFormat, defaultDb)",
|
|
815
|
-
required: true
|
|
816
|
-
},
|
|
817
|
-
value: {
|
|
1236
|
+
...OUTPUT_ARGS,
|
|
1237
|
+
name: {
|
|
818
1238
|
type: "positional",
|
|
819
|
-
|
|
820
|
-
|
|
1239
|
+
required: true,
|
|
1240
|
+
description: "Profile name"
|
|
821
1241
|
}
|
|
822
1242
|
},
|
|
823
1243
|
async run({ args }) {
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
if (!validKeys.includes(args.key)) {
|
|
831
|
-
logger.error(`Invalid key: ${args.key}`);
|
|
832
|
-
logger.info(`Valid keys: ${validKeys.join(", ")}`);
|
|
1244
|
+
applyOutputMode(args);
|
|
1245
|
+
const name = String(args.name);
|
|
1246
|
+
const dir = getProfileDir(name);
|
|
1247
|
+
if (!await fs.stat(dir).then(() => true).catch(() => false)) {
|
|
1248
|
+
logger.error(`Profile not found: ${name}`);
|
|
1249
|
+
logger.info(`Create it with: gscdump profile create ${name}`);
|
|
833
1250
|
process.exit(1);
|
|
834
1251
|
}
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
await saveConfig(config);
|
|
838
|
-
logger.success(`Set ${args.key} = ${args.value}`);
|
|
1252
|
+
await setActiveProfile(name);
|
|
1253
|
+
logger.success(`Active profile: ${name}`);
|
|
839
1254
|
}
|
|
840
1255
|
}),
|
|
841
|
-
|
|
1256
|
+
create: defineCommand({
|
|
842
1257
|
meta: {
|
|
843
|
-
name: "
|
|
844
|
-
description: "
|
|
1258
|
+
name: "create",
|
|
1259
|
+
description: "Create an empty profile directory"
|
|
1260
|
+
},
|
|
1261
|
+
args: {
|
|
1262
|
+
...OUTPUT_ARGS,
|
|
1263
|
+
"name": {
|
|
1264
|
+
type: "positional",
|
|
1265
|
+
required: true,
|
|
1266
|
+
description: "Profile name"
|
|
1267
|
+
},
|
|
1268
|
+
"no-use": {
|
|
1269
|
+
type: "boolean",
|
|
1270
|
+
default: false,
|
|
1271
|
+
description: "Do not mark the new profile as active"
|
|
1272
|
+
}
|
|
845
1273
|
},
|
|
846
|
-
args: { key: {
|
|
847
|
-
type: "positional",
|
|
848
|
-
description: "Config key to remove",
|
|
849
|
-
required: true
|
|
850
|
-
} },
|
|
851
1274
|
async run({ args }) {
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
await
|
|
855
|
-
|
|
1275
|
+
applyOutputMode(args);
|
|
1276
|
+
const name = String(args.name);
|
|
1277
|
+
const dir = await createProfile(name);
|
|
1278
|
+
if (!args["no-use"]) await setActiveProfile(name);
|
|
1279
|
+
logger.success(`Created profile: ${name}${args["no-use"] ? "" : " (active)"}`);
|
|
1280
|
+
logger.info(displayPath(dir));
|
|
856
1281
|
}
|
|
857
1282
|
}),
|
|
858
|
-
|
|
1283
|
+
clear: defineCommand({
|
|
859
1284
|
meta: {
|
|
860
|
-
name: "
|
|
861
|
-
description: "
|
|
1285
|
+
name: "clear",
|
|
1286
|
+
description: "Clear the persisted active profile (commands fall back to root config dir)"
|
|
862
1287
|
},
|
|
863
|
-
|
|
864
|
-
|
|
1288
|
+
args: { ...OUTPUT_ARGS },
|
|
1289
|
+
async run({ args }) {
|
|
1290
|
+
applyOutputMode(args);
|
|
1291
|
+
await setActiveProfile(null);
|
|
1292
|
+
logger.success("Cleared active profile");
|
|
1293
|
+
}
|
|
1294
|
+
}),
|
|
1295
|
+
delete: defineCommand({
|
|
1296
|
+
meta: {
|
|
1297
|
+
name: "delete",
|
|
1298
|
+
description: "Remove a profile directory (tokens + config)"
|
|
1299
|
+
},
|
|
1300
|
+
args: {
|
|
1301
|
+
...OUTPUT_ARGS,
|
|
1302
|
+
name: {
|
|
1303
|
+
type: "positional",
|
|
1304
|
+
required: true,
|
|
1305
|
+
description: "Profile name"
|
|
1306
|
+
},
|
|
1307
|
+
yes: {
|
|
1308
|
+
type: "boolean",
|
|
1309
|
+
alias: "y",
|
|
1310
|
+
default: false,
|
|
1311
|
+
description: "Skip confirmation"
|
|
1312
|
+
}
|
|
1313
|
+
},
|
|
1314
|
+
async run({ args }) {
|
|
1315
|
+
applyOutputMode(args);
|
|
1316
|
+
const name = String(args.name);
|
|
1317
|
+
const dir = getProfileDir(name);
|
|
1318
|
+
if (!await fs.stat(dir).then(() => true).catch(() => false)) {
|
|
1319
|
+
logger.error(`Profile not found: ${name}`);
|
|
1320
|
+
process.exit(1);
|
|
1321
|
+
}
|
|
1322
|
+
if (!args.yes) {
|
|
1323
|
+
const ok = await confirm({
|
|
1324
|
+
message: `Delete profile "${name}" at ${dir}? Tokens and config will be lost.`,
|
|
1325
|
+
initialValue: false
|
|
1326
|
+
});
|
|
1327
|
+
if (isCancel(ok) || !ok) {
|
|
1328
|
+
logger.info("Cancelled");
|
|
1329
|
+
process.exit(0);
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
await fs.rm(dir, {
|
|
1333
|
+
recursive: true,
|
|
1334
|
+
force: true
|
|
1335
|
+
});
|
|
1336
|
+
if (readActiveMarkerSync() === name) await setActiveProfile(null);
|
|
1337
|
+
logger.success(`Removed profile: ${name}`);
|
|
865
1338
|
}
|
|
866
1339
|
})
|
|
867
1340
|
}
|
|
868
1341
|
});
|
|
869
|
-
const
|
|
870
|
-
|
|
1342
|
+
const REQUIRED_SCOPES$1 = [
|
|
1343
|
+
"https://www.googleapis.com/auth/webmasters",
|
|
1344
|
+
"https://www.googleapis.com/auth/indexing",
|
|
1345
|
+
"https://www.googleapis.com/auth/siteverification"
|
|
1346
|
+
];
|
|
1347
|
+
async function fetchTokenInfo(accessToken) {
|
|
1348
|
+
return ofetch("https://oauth2.googleapis.com/tokeninfo", { query: { access_token: accessToken } }).catch(() => null);
|
|
1349
|
+
}
|
|
1350
|
+
async function resolveLiveAuthState() {
|
|
1351
|
+
const tokens = await loadTokens();
|
|
1352
|
+
const byok = resolveBYOK();
|
|
1353
|
+
let liveToken = null;
|
|
1354
|
+
if (typeof byok === "string") liveToken = byok;
|
|
1355
|
+
else if (byok && "getAccessToken" in byok) liveToken = await byok.getAccessToken().then((r) => r.token ?? null).catch(() => null);
|
|
1356
|
+
else if (tokens?.access_token) liveToken = tokens.access_token;
|
|
1357
|
+
const tokenInfo = liveToken ? await fetchTokenInfo(liveToken) : null;
|
|
1358
|
+
const scopes = tokenInfo?.scope ? tokenInfo.scope.split(/\s+/).filter(Boolean) : [];
|
|
1359
|
+
const has = (s) => scopes.includes(s) || scopes.includes(s.replace(".readonly", ""));
|
|
1360
|
+
const missing = REQUIRED_SCOPES$1.filter((s) => !has(s));
|
|
1361
|
+
return {
|
|
1362
|
+
byok,
|
|
1363
|
+
tokens,
|
|
1364
|
+
liveToken,
|
|
1365
|
+
tokenInfo,
|
|
1366
|
+
scopes,
|
|
1367
|
+
missing
|
|
1368
|
+
};
|
|
1369
|
+
}
|
|
1370
|
+
const statusCommand$1 = defineCommand({
|
|
871
1371
|
meta: {
|
|
872
|
-
name: "
|
|
873
|
-
description: "
|
|
1372
|
+
name: "status",
|
|
1373
|
+
description: "Show current authentication status"
|
|
874
1374
|
},
|
|
875
|
-
args: {
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
1375
|
+
args: { ...OUTPUT_ARGS },
|
|
1376
|
+
async run({ args }) {
|
|
1377
|
+
const { json } = applyOutputMode(args);
|
|
1378
|
+
const { byok, tokens, tokenInfo, scopes, missing } = await resolveLiveAuthState();
|
|
1379
|
+
const byokKind = byok ? typeof byok === "string" ? "access-token" : "refresh-token" : null;
|
|
1380
|
+
if (json) {
|
|
1381
|
+
console.log(JSON.stringify({
|
|
1382
|
+
authenticated: !!tokens || !!byok,
|
|
1383
|
+
source: byok ? "byok" : tokens ? "saved-tokens" : null,
|
|
1384
|
+
byokKind,
|
|
1385
|
+
scopes,
|
|
1386
|
+
tokenAccount: tokenInfo?.email ?? null,
|
|
1387
|
+
tokens: tokens ? {
|
|
1388
|
+
hasAccessToken: !!tokens.access_token,
|
|
1389
|
+
hasRefreshToken: !!tokens.refresh_token,
|
|
1390
|
+
expiry: tokens.expiry_date ? new Date(tokens.expiry_date).toISOString() : null,
|
|
1391
|
+
expired: tokens.expiry_date ? tokens.expiry_date < Date.now() : null
|
|
1392
|
+
} : null
|
|
1393
|
+
}, null, 2));
|
|
1394
|
+
return;
|
|
1395
|
+
}
|
|
1396
|
+
const reportScopes = () => {
|
|
1397
|
+
if (scopes.length === 0) return;
|
|
1398
|
+
console.log(` Scopes:`);
|
|
1399
|
+
for (const s of scopes) console.log(` \x1B[90m└─\x1B[0m ${s}`);
|
|
1400
|
+
if (missing.length > 0) {
|
|
1401
|
+
console.log(` \x1B[33mMissing scopes:\x1B[0m`);
|
|
1402
|
+
for (const s of missing) console.log(` \x1B[90m└─\x1B[0m ${s}`);
|
|
1403
|
+
console.log(` \x1B[90mRun \`gscdump auth login --force\` to re-consent.\x1B[0m`);
|
|
1404
|
+
}
|
|
1405
|
+
};
|
|
1406
|
+
if (byok) {
|
|
1407
|
+
logger.success(`Authenticated via BYOK (${byokKind})`);
|
|
1408
|
+
if (tokenInfo?.email) console.log(` Account: ${tokenInfo.email}`);
|
|
1409
|
+
reportScopes();
|
|
1410
|
+
return;
|
|
1411
|
+
}
|
|
1412
|
+
if (!tokens) {
|
|
1413
|
+
logger.warn("Not authenticated");
|
|
1414
|
+
logger.info("Run `gscdump init` (full setup) or `gscdump auth login` (OAuth only)");
|
|
1415
|
+
logger.info("Or set GSC_ACCESS_TOKEN / GSC_CLIENT_ID + GSC_CLIENT_SECRET + GSC_REFRESH_TOKEN env vars");
|
|
1416
|
+
return;
|
|
1417
|
+
}
|
|
1418
|
+
const hasAccess = !!tokens.access_token;
|
|
1419
|
+
const hasRefresh = !!tokens.refresh_token;
|
|
1420
|
+
const expiry = tokens.expiry_date ? new Date(tokens.expiry_date) : null;
|
|
1421
|
+
const isExpired = expiry && expiry < /* @__PURE__ */ new Date();
|
|
1422
|
+
logger.success("Authenticated (saved tokens)");
|
|
1423
|
+
console.log();
|
|
1424
|
+
console.log(` Access token: ${hasAccess ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
1425
|
+
console.log(` Refresh token: ${hasRefresh ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
1426
|
+
if (expiry) {
|
|
1427
|
+
const status = isExpired ? "\x1B[33mexpired\x1B[0m" : "\x1B[32mvalid\x1B[0m";
|
|
1428
|
+
console.log(` Expires: ${expiry.toISOString()} (${status})`);
|
|
1429
|
+
}
|
|
1430
|
+
if (tokenInfo?.email) console.log(` Account: ${tokenInfo.email}`);
|
|
1431
|
+
reportScopes();
|
|
1432
|
+
}
|
|
1433
|
+
});
|
|
1434
|
+
const refreshCommand = defineCommand({
|
|
1435
|
+
meta: {
|
|
1436
|
+
name: "refresh",
|
|
1437
|
+
description: "Force-refresh saved OAuth tokens (no-op for BYOK)"
|
|
1438
|
+
},
|
|
1439
|
+
args: { ...OUTPUT_ARGS },
|
|
1440
|
+
async run({ args }) {
|
|
1441
|
+
applyOutputMode(args);
|
|
1442
|
+
if (resolveBYOK()) {
|
|
1443
|
+
logger.info("BYOK detected; refresh handled per-call by the SDK");
|
|
1444
|
+
return;
|
|
1445
|
+
}
|
|
1446
|
+
const tokens = await loadTokens();
|
|
1447
|
+
if (!tokens?.refresh_token) {
|
|
1448
|
+
logger.error("No saved refresh token. Run `gscdump auth login`.");
|
|
1449
|
+
process.exit(1);
|
|
1450
|
+
}
|
|
1451
|
+
const credentials = await getAuthCredentials(false).catch((e) => {
|
|
1452
|
+
logger.error(`Cannot resolve credentials: ${e.message}`);
|
|
1453
|
+
process.exit(1);
|
|
1454
|
+
});
|
|
1455
|
+
await saveTokens({
|
|
1456
|
+
...tokens,
|
|
1457
|
+
expiry_date: 1
|
|
1458
|
+
});
|
|
1459
|
+
if ((await authenticate(credentials, false).catch((e) => {
|
|
1460
|
+
logger.error(`Refresh failed: ${e.message}`);
|
|
1461
|
+
process.exit(1);
|
|
1462
|
+
})).credentials?.access_token) logger.success("Token refreshed");
|
|
1463
|
+
else logger.warn("Refresh completed but no new access token returned");
|
|
1464
|
+
}
|
|
1465
|
+
});
|
|
1466
|
+
const authCommand = defineCommand({
|
|
1467
|
+
meta: {
|
|
1468
|
+
name: "auth",
|
|
1469
|
+
description: "Manage authentication"
|
|
1470
|
+
},
|
|
1471
|
+
subCommands: {
|
|
1472
|
+
status: statusCommand$1,
|
|
1473
|
+
login: defineCommand({
|
|
1474
|
+
meta: {
|
|
1475
|
+
name: "login",
|
|
1476
|
+
description: "Run OAuth flow and persist tokens (skip if BYOK env vars set)"
|
|
1477
|
+
},
|
|
1478
|
+
args: {
|
|
1479
|
+
...OUTPUT_ARGS,
|
|
1480
|
+
"force": {
|
|
1481
|
+
type: "boolean",
|
|
1482
|
+
alias: "f",
|
|
1483
|
+
default: false,
|
|
1484
|
+
description: "Re-run OAuth even if tokens already exist"
|
|
1485
|
+
},
|
|
1486
|
+
"browser": {
|
|
1487
|
+
type: "boolean",
|
|
1488
|
+
default: true,
|
|
1489
|
+
description: "Use loopback browser flow. Pass --no-browser for device-code (headless)."
|
|
1490
|
+
},
|
|
1491
|
+
"service-account": {
|
|
1492
|
+
type: "string",
|
|
1493
|
+
description: "Path to a service-account JSON key (skips OAuth)"
|
|
1494
|
+
}
|
|
1495
|
+
},
|
|
1496
|
+
async run({ args }) {
|
|
1497
|
+
applyOutputMode(args);
|
|
1498
|
+
if (resolveBYOK() && !args.force) {
|
|
1499
|
+
logger.info("BYOK env vars detected, no login needed (--force to override)");
|
|
1500
|
+
return;
|
|
1501
|
+
}
|
|
1502
|
+
if (args["service-account"]) {
|
|
1503
|
+
const saPath = path.resolve(String(args["service-account"]));
|
|
1504
|
+
const jwt = await loadServiceAccount(saPath).catch((e) => {
|
|
1505
|
+
logger.error(`Service-account load failed: ${e.message}`);
|
|
1506
|
+
process.exit(1);
|
|
1507
|
+
});
|
|
1508
|
+
await jwt.authorize().catch((e) => {
|
|
1509
|
+
logger.error(`Service-account auth failed: ${e.message}`);
|
|
1510
|
+
process.exit(1);
|
|
1511
|
+
});
|
|
1512
|
+
const config = await loadConfig();
|
|
1513
|
+
config.serviceAccountPath = saPath;
|
|
1514
|
+
await saveConfig(config);
|
|
1515
|
+
logger.success(`Service-account verified: ${jwt.email ?? "OK"}`);
|
|
1516
|
+
logger.info(`Saved path to config: ${saPath}`);
|
|
1517
|
+
return;
|
|
1518
|
+
}
|
|
1519
|
+
if (args.force) await clearTokens();
|
|
1520
|
+
await getAuth({
|
|
1521
|
+
interactive: true,
|
|
1522
|
+
noBrowser: args.browser === false,
|
|
1523
|
+
force: Boolean(args.force)
|
|
1524
|
+
}).catch((e) => {
|
|
1525
|
+
logger.error(`Login failed: ${e.message}`);
|
|
1526
|
+
process.exit(1);
|
|
1527
|
+
});
|
|
1528
|
+
logger.success("Logged in");
|
|
1529
|
+
if (!resolveActiveProfile()) {
|
|
1530
|
+
const tokens = await loadTokens();
|
|
1531
|
+
const info = tokens?.access_token ? await fetchTokenInfo(tokens.access_token) : null;
|
|
1532
|
+
if (info?.email) {
|
|
1533
|
+
const name = profileNameFromEmail(info.email);
|
|
1534
|
+
if (await adoptCurrentConfigAsProfile(name).catch(() => null)) logger.success(`Saved as profile "${name}" (active)`);
|
|
1535
|
+
}
|
|
1536
|
+
}
|
|
1537
|
+
if (resolveBYOK()) {
|
|
1538
|
+
console.log();
|
|
1539
|
+
console.log(await formatAuthProvenance());
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
}),
|
|
1543
|
+
logout: defineCommand({
|
|
1544
|
+
meta: {
|
|
1545
|
+
name: "logout",
|
|
1546
|
+
description: "Clear stored OAuth tokens"
|
|
1547
|
+
},
|
|
1548
|
+
args: { ...OUTPUT_ARGS },
|
|
1549
|
+
async run({ args }) {
|
|
1550
|
+
applyOutputMode(args);
|
|
1551
|
+
await clearTokens();
|
|
1552
|
+
const config = await loadConfig();
|
|
1553
|
+
if (config.serviceAccountPath) {
|
|
1554
|
+
delete config.serviceAccountPath;
|
|
1555
|
+
await saveConfig(config);
|
|
1556
|
+
logger.info("Cleared saved service-account path");
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1559
|
+
}),
|
|
1560
|
+
refresh: refreshCommand,
|
|
1561
|
+
scopes: defineCommand({
|
|
1562
|
+
meta: {
|
|
1563
|
+
name: "scopes",
|
|
1564
|
+
description: "Print granted OAuth scopes (one per line); exits 1 if any required scope is missing"
|
|
1565
|
+
},
|
|
1566
|
+
args: { ...OUTPUT_ARGS },
|
|
1567
|
+
async run({ args }) {
|
|
1568
|
+
const { json } = applyOutputMode(args);
|
|
1569
|
+
const { liveToken, scopes, missing } = await resolveLiveAuthState();
|
|
1570
|
+
if (!liveToken) {
|
|
1571
|
+
if (json) console.log(JSON.stringify({
|
|
1572
|
+
scopes: [],
|
|
1573
|
+
missing: null
|
|
1574
|
+
}, null, 2));
|
|
1575
|
+
else logger.error("Not authenticated");
|
|
1576
|
+
process.exit(1);
|
|
1577
|
+
}
|
|
1578
|
+
if (json) console.log(JSON.stringify({
|
|
1579
|
+
scopes,
|
|
1580
|
+
missing
|
|
1581
|
+
}, null, 2));
|
|
1582
|
+
else for (const s of scopes) console.log(s);
|
|
1583
|
+
if (missing.length > 0) process.exit(1);
|
|
1584
|
+
}
|
|
1585
|
+
})
|
|
1586
|
+
}
|
|
1587
|
+
});
|
|
1588
|
+
const showCommand = defineCommand({
|
|
1589
|
+
meta: {
|
|
1590
|
+
name: "show",
|
|
1591
|
+
description: "Show current config"
|
|
1592
|
+
},
|
|
1593
|
+
args: { ...OUTPUT_ARGS },
|
|
1594
|
+
async run({ args }) {
|
|
1595
|
+
const { json } = applyOutputMode(args);
|
|
1596
|
+
const config = await loadConfig();
|
|
1597
|
+
const configPath = getConfigPath();
|
|
1598
|
+
if (json) {
|
|
1599
|
+
console.log(JSON.stringify({
|
|
1600
|
+
path: configPath,
|
|
1601
|
+
config
|
|
1602
|
+
}, null, 2));
|
|
1603
|
+
return;
|
|
1604
|
+
}
|
|
1605
|
+
logger.info(`Config: ${displayPath(configPath)}`);
|
|
1606
|
+
console.log();
|
|
1607
|
+
if (Object.keys(config).length === 0) {
|
|
1608
|
+
logger.warn("No config set");
|
|
1609
|
+
return;
|
|
1610
|
+
}
|
|
1611
|
+
console.log(JSON.stringify(config, null, 2));
|
|
1612
|
+
}
|
|
1613
|
+
});
|
|
1614
|
+
const VALID_KEYS = [
|
|
1615
|
+
"defaultSite",
|
|
1616
|
+
"defaultPeriod",
|
|
1617
|
+
"defaultFormat",
|
|
1618
|
+
"defaultDb",
|
|
1619
|
+
"dataDir",
|
|
1620
|
+
"defaultLimit",
|
|
1621
|
+
"defaultSearchType",
|
|
1622
|
+
"defaultDataState",
|
|
1623
|
+
"serviceAccountPath"
|
|
1624
|
+
];
|
|
1625
|
+
const NUMERIC_KEYS = new Set(["defaultLimit"]);
|
|
1626
|
+
const configCommand = defineCommand({
|
|
1627
|
+
meta: {
|
|
1628
|
+
name: "config",
|
|
1629
|
+
description: "Manage configuration"
|
|
1630
|
+
},
|
|
1631
|
+
subCommands: {
|
|
1632
|
+
show: showCommand,
|
|
1633
|
+
set: defineCommand({
|
|
1634
|
+
meta: {
|
|
1635
|
+
name: "set",
|
|
1636
|
+
description: "Set a config value"
|
|
1637
|
+
},
|
|
1638
|
+
args: {
|
|
1639
|
+
key: {
|
|
1640
|
+
type: "positional",
|
|
1641
|
+
description: `Config key (${VALID_KEYS.join(", ")})`,
|
|
1642
|
+
required: true
|
|
1643
|
+
},
|
|
1644
|
+
value: {
|
|
1645
|
+
type: "positional",
|
|
1646
|
+
description: "Value to set",
|
|
1647
|
+
required: true
|
|
1648
|
+
},
|
|
1649
|
+
...OUTPUT_ARGS
|
|
1650
|
+
},
|
|
1651
|
+
async run({ args }) {
|
|
1652
|
+
applyOutputMode(args);
|
|
1653
|
+
if (!VALID_KEYS.includes(args.key)) {
|
|
1654
|
+
logger.error(`Invalid key: ${args.key}`);
|
|
1655
|
+
logger.info(`Valid keys: ${VALID_KEYS.join(", ")}`);
|
|
1656
|
+
process.exit(1);
|
|
1657
|
+
}
|
|
1658
|
+
const config = await loadConfig();
|
|
1659
|
+
const value = NUMERIC_KEYS.has(args.key) ? Number(args.value) : args.value;
|
|
1660
|
+
if (NUMERIC_KEYS.has(args.key) && !Number.isFinite(value)) {
|
|
1661
|
+
logger.error(`Invalid numeric value for ${args.key}: ${args.value}`);
|
|
1662
|
+
process.exit(1);
|
|
1663
|
+
}
|
|
1664
|
+
config[args.key] = value;
|
|
1665
|
+
await saveConfig(config);
|
|
1666
|
+
logger.success(`Set ${args.key} = ${value}`);
|
|
1667
|
+
}
|
|
1668
|
+
}),
|
|
1669
|
+
unset: defineCommand({
|
|
1670
|
+
meta: {
|
|
1671
|
+
name: "unset",
|
|
1672
|
+
description: "Remove a config value"
|
|
1673
|
+
},
|
|
1674
|
+
args: {
|
|
1675
|
+
key: {
|
|
1676
|
+
type: "positional",
|
|
1677
|
+
description: `Config key to remove (${VALID_KEYS.join(", ")})`,
|
|
1678
|
+
required: true
|
|
1679
|
+
},
|
|
1680
|
+
...OUTPUT_ARGS
|
|
1681
|
+
},
|
|
1682
|
+
async run({ args }) {
|
|
1683
|
+
applyOutputMode(args);
|
|
1684
|
+
if (!VALID_KEYS.includes(args.key)) {
|
|
1685
|
+
logger.error(`Invalid key: ${args.key}`);
|
|
1686
|
+
logger.info(`Valid keys: ${VALID_KEYS.join(", ")}`);
|
|
1687
|
+
process.exit(1);
|
|
1688
|
+
}
|
|
1689
|
+
const config = await loadConfig();
|
|
1690
|
+
delete config[args.key];
|
|
1691
|
+
await saveConfig(config);
|
|
1692
|
+
logger.success(`Removed ${args.key}`);
|
|
1693
|
+
}
|
|
1694
|
+
}),
|
|
1695
|
+
path: defineCommand({
|
|
1696
|
+
meta: {
|
|
1697
|
+
name: "path",
|
|
1698
|
+
description: "Show config file path"
|
|
1699
|
+
},
|
|
1700
|
+
run() {
|
|
1701
|
+
console.log(getConfigPath());
|
|
1702
|
+
}
|
|
1703
|
+
}),
|
|
1704
|
+
validate: defineCommand({
|
|
1705
|
+
meta: {
|
|
1706
|
+
name: "validate",
|
|
1707
|
+
description: "Validate the saved config (defaultSite is verified, dataDir exists/writable)"
|
|
1708
|
+
},
|
|
1709
|
+
args: { ...OUTPUT_ARGS },
|
|
1710
|
+
async run({ args }) {
|
|
1711
|
+
const { json } = applyOutputMode(args);
|
|
1712
|
+
const { resolveDataDir } = await import("./_chunks/config.mjs").then((n) => n.t);
|
|
1713
|
+
const fs = await import("node:fs/promises");
|
|
1714
|
+
const config = await loadConfig();
|
|
1715
|
+
const issues = [];
|
|
1716
|
+
const dataDir = resolveDataDir(config);
|
|
1717
|
+
const dataDirDisplay = displayPath(dataDir);
|
|
1718
|
+
const stat = await fs.stat(dataDir).catch(() => null);
|
|
1719
|
+
if (stat && !stat.isDirectory()) issues.push({
|
|
1720
|
+
key: "dataDir",
|
|
1721
|
+
level: "fail",
|
|
1722
|
+
message: `${dataDirDisplay} is not a directory`
|
|
1723
|
+
});
|
|
1724
|
+
else if (stat) {
|
|
1725
|
+
const probe = `${dataDir}/.gscdump-config-probe`;
|
|
1726
|
+
if (!await fs.writeFile(probe, "").then(() => fs.rm(probe)).then(() => true).catch(() => false)) issues.push({
|
|
1727
|
+
key: "dataDir",
|
|
1728
|
+
level: "fail",
|
|
1729
|
+
message: `${dataDirDisplay} not writable`
|
|
1730
|
+
});
|
|
1731
|
+
} else issues.push({
|
|
1732
|
+
key: "dataDir",
|
|
1733
|
+
level: "warn",
|
|
1734
|
+
message: `${dataDirDisplay} does not exist (will be created on first sync)`
|
|
1735
|
+
});
|
|
1736
|
+
if (config.defaultSite) if (!!config.clientId && !!config.clientSecret) {
|
|
1737
|
+
const { createCommandContext } = await Promise.resolve().then(() => context_exports);
|
|
1738
|
+
const ctx = await createCommandContext({ needsAuth: true }).catch(() => null);
|
|
1739
|
+
if (ctx) {
|
|
1740
|
+
const sites = await ctx.loadSites().catch(() => null);
|
|
1741
|
+
if (sites && !sites.some((s) => s.siteUrl === config.defaultSite || s.siteUrl.includes(String(config.defaultSite)))) issues.push({
|
|
1742
|
+
key: "defaultSite",
|
|
1743
|
+
level: "fail",
|
|
1744
|
+
message: `${config.defaultSite} is not in the verified site list`
|
|
1745
|
+
});
|
|
1746
|
+
}
|
|
1747
|
+
} else issues.push({
|
|
1748
|
+
key: "defaultSite",
|
|
1749
|
+
level: "warn",
|
|
1750
|
+
message: "set, but auth not configured — skipping verification"
|
|
1751
|
+
});
|
|
1752
|
+
if (config.defaultFormat && !["json", "csv"].includes(config.defaultFormat)) issues.push({
|
|
1753
|
+
key: "defaultFormat",
|
|
1754
|
+
level: "fail",
|
|
1755
|
+
message: `unknown format: ${config.defaultFormat}`
|
|
1756
|
+
});
|
|
1757
|
+
const { SearchTypes } = await import("gscdump/query");
|
|
1758
|
+
const allowedSearchTypes = Object.values(SearchTypes);
|
|
1759
|
+
if (config.defaultSearchType && !allowedSearchTypes.includes(config.defaultSearchType)) issues.push({
|
|
1760
|
+
key: "defaultSearchType",
|
|
1761
|
+
level: "fail",
|
|
1762
|
+
message: `unknown search type: ${config.defaultSearchType} (allowed: ${allowedSearchTypes.join(", ")})`
|
|
1763
|
+
});
|
|
1764
|
+
const allowedDataStates = [
|
|
1765
|
+
"all",
|
|
1766
|
+
"final",
|
|
1767
|
+
"hourly_all"
|
|
1768
|
+
];
|
|
1769
|
+
if (config.defaultDataState && !allowedDataStates.includes(config.defaultDataState)) issues.push({
|
|
1770
|
+
key: "defaultDataState",
|
|
1771
|
+
level: "fail",
|
|
1772
|
+
message: `unknown data state: ${config.defaultDataState} (allowed: ${allowedDataStates.join(", ")})`
|
|
1773
|
+
});
|
|
1774
|
+
if (config.serviceAccountPath) {
|
|
1775
|
+
if (!await fs.stat(config.serviceAccountPath).catch(() => null)) issues.push({
|
|
1776
|
+
key: "serviceAccountPath",
|
|
1777
|
+
level: "fail",
|
|
1778
|
+
message: `${displayPath(config.serviceAccountPath)} does not exist`
|
|
1779
|
+
});
|
|
1780
|
+
}
|
|
1781
|
+
if (json) {
|
|
1782
|
+
console.log(JSON.stringify({
|
|
1783
|
+
ok: !issues.some((i) => i.level === "fail"),
|
|
1784
|
+
issues
|
|
1785
|
+
}, null, 2));
|
|
1786
|
+
return;
|
|
1787
|
+
}
|
|
1788
|
+
if (issues.length === 0) {
|
|
1789
|
+
logger.success("Config OK");
|
|
1790
|
+
return;
|
|
1791
|
+
}
|
|
1792
|
+
for (const i of issues) {
|
|
1793
|
+
const prefix = i.level === "fail" ? "\x1B[31m✗\x1B[0m" : "\x1B[33m!\x1B[0m";
|
|
1794
|
+
console.log(` ${prefix} ${i.key}: ${i.message}`);
|
|
1795
|
+
}
|
|
1796
|
+
if (issues.some((i) => i.level === "fail")) process.exit(1);
|
|
1797
|
+
}
|
|
1798
|
+
})
|
|
1799
|
+
}
|
|
1800
|
+
});
|
|
1801
|
+
const REQUIRED_SCOPES = [
|
|
1802
|
+
"https://www.googleapis.com/auth/webmasters",
|
|
1803
|
+
"https://www.googleapis.com/auth/indexing",
|
|
1804
|
+
"https://www.googleapis.com/auth/siteverification"
|
|
1805
|
+
];
|
|
1806
|
+
const FETCH_TIMEOUT_MS = 5e3;
|
|
1807
|
+
const TIME_SKEW_WARN_MS = 5 * 6e4;
|
|
1808
|
+
const WATERMARK_STALE_DAYS_WARN = 7;
|
|
1809
|
+
const RELEVANT_ENV_KEYS = [
|
|
1810
|
+
"GSC_ACCESS_TOKEN",
|
|
1811
|
+
"GSC_CLIENT_ID",
|
|
1812
|
+
"GSC_CLIENT_SECRET",
|
|
1813
|
+
"GSC_REFRESH_TOKEN",
|
|
1814
|
+
"GOOGLE_ACCESS_TOKEN",
|
|
1815
|
+
"GOOGLE_CLIENT_ID",
|
|
1816
|
+
"GOOGLE_CLIENT_SECRET",
|
|
1817
|
+
"GOOGLE_REFRESH_TOKEN",
|
|
1818
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
1819
|
+
"GSC_SERVICE_ACCOUNT_JSON"
|
|
1820
|
+
];
|
|
1821
|
+
function redact(v) {
|
|
1822
|
+
if (!v) return "<missing>";
|
|
1823
|
+
if (v.length <= 6) return "***";
|
|
1824
|
+
return `***${v.slice(-6)}`;
|
|
1825
|
+
}
|
|
1826
|
+
async function checkEnv() {
|
|
1827
|
+
const envPath = path.join(process.cwd(), ".env");
|
|
1828
|
+
const parsed = parseEnvFile(envPath);
|
|
1829
|
+
if (!parsed) return {
|
|
1830
|
+
checks: [{
|
|
1831
|
+
name: "env",
|
|
1832
|
+
status: "info",
|
|
1833
|
+
detail: "no .env (using shell env / saved tokens)"
|
|
1834
|
+
}],
|
|
1835
|
+
envKeys: /* @__PURE__ */ new Set()
|
|
1836
|
+
};
|
|
1837
|
+
const relevant = RELEVANT_ENV_KEYS.filter((k) => parsed[k] !== void 0);
|
|
1838
|
+
const envKeys = new Set(relevant);
|
|
1839
|
+
if (relevant.length === 0) return {
|
|
1840
|
+
checks: [{
|
|
1841
|
+
name: "env",
|
|
1842
|
+
status: "info",
|
|
1843
|
+
detail: `${displayPath(envPath)} found, no auth vars`
|
|
1844
|
+
}],
|
|
1845
|
+
envKeys
|
|
1846
|
+
};
|
|
1847
|
+
const inventory = relevant.map((k) => {
|
|
1848
|
+
const v = parsed[k];
|
|
1849
|
+
if (k.endsWith("CLIENT_ID")) return `${k}=${v}`;
|
|
1850
|
+
return `${k}=${redact(v)}`;
|
|
1851
|
+
});
|
|
1852
|
+
return {
|
|
1853
|
+
checks: [{
|
|
1854
|
+
name: "env",
|
|
1855
|
+
status: "info",
|
|
1856
|
+
detail: `${displayPath(envPath)} → ${inventory.join(", ")} (not validated — see auth)`
|
|
1857
|
+
}],
|
|
1858
|
+
envKeys
|
|
1859
|
+
};
|
|
1860
|
+
}
|
|
1861
|
+
function describeAuthSource(envKeys, byok) {
|
|
1862
|
+
if (!byok) return "saved tokens";
|
|
1863
|
+
const isAccessToken = typeof byok === "string";
|
|
1864
|
+
const driver = isAccessToken ? "GSC_ACCESS_TOKEN" : "GSC_REFRESH_TOKEN";
|
|
1865
|
+
const source = (isAccessToken ? ["GSC_ACCESS_TOKEN", "GOOGLE_ACCESS_TOKEN"] : ["GSC_REFRESH_TOKEN", "GOOGLE_REFRESH_TOKEN"]).some((k) => envKeys.has(k)) ? ".env" : "shell env";
|
|
1866
|
+
return `BYOK ${isAccessToken ? "(access-token)" : "(refresh-token)"} from ${source} via ${driver}`;
|
|
1867
|
+
}
|
|
1868
|
+
async function checkAuth$1(envKeys) {
|
|
1869
|
+
const checks = [];
|
|
1870
|
+
const byok = resolveBYOK();
|
|
1871
|
+
const tokens = await loadTokens();
|
|
1872
|
+
if (!byok && !tokens) {
|
|
1873
|
+
checks.push({
|
|
1874
|
+
name: "auth",
|
|
1875
|
+
status: "fail",
|
|
1876
|
+
detail: "no BYOK env vars and no saved tokens; run `gscdump init`"
|
|
1877
|
+
});
|
|
1878
|
+
return {
|
|
1879
|
+
checks,
|
|
1880
|
+
liveToken: null
|
|
1881
|
+
};
|
|
1882
|
+
}
|
|
1883
|
+
const clientId = process.env.GSC_CLIENT_ID ?? process.env.GOOGLE_CLIENT_ID ?? (await loadConfig()).clientId ?? null;
|
|
1884
|
+
if (clientId) checks.push({
|
|
1885
|
+
name: "auth.client_id",
|
|
1886
|
+
status: "info",
|
|
1887
|
+
detail: clientId
|
|
1888
|
+
});
|
|
1889
|
+
let liveToken = null;
|
|
1890
|
+
let refreshError = null;
|
|
1891
|
+
if (typeof byok === "string") liveToken = byok;
|
|
1892
|
+
else if (byok && "getAccessToken" in byok) liveToken = await byok.getAccessToken().then((r) => r.token ?? null).catch((e) => {
|
|
1893
|
+
refreshError = e?.data?.error_description ?? e?.data?.error ?? e?.message ?? String(e);
|
|
1894
|
+
return null;
|
|
1895
|
+
});
|
|
1896
|
+
else if (tokens?.access_token) liveToken = tokens.access_token;
|
|
1897
|
+
if (!liveToken) {
|
|
1898
|
+
const source = describeAuthSource(envKeys, byok);
|
|
1899
|
+
const detail = refreshError ? `${source} — refresh failed: ${refreshError} (token revoked / invalid — re-run \`gscdump auth login --force\` and update the source above)` : `${source} — no usable access token`;
|
|
1900
|
+
checks.push({
|
|
1901
|
+
name: "auth",
|
|
1902
|
+
status: "fail",
|
|
1903
|
+
detail
|
|
1904
|
+
});
|
|
1905
|
+
return {
|
|
1906
|
+
checks,
|
|
1907
|
+
liveToken: null
|
|
1908
|
+
};
|
|
1909
|
+
}
|
|
1910
|
+
const info = await ofetch("https://oauth2.googleapis.com/tokeninfo", { query: { access_token: liveToken } }).catch((e) => ({ error: e.message }));
|
|
1911
|
+
if ("error" in info) {
|
|
1912
|
+
checks.push({
|
|
1913
|
+
name: "auth",
|
|
1914
|
+
status: "fail",
|
|
1915
|
+
detail: `tokeninfo failed: ${info.error}`
|
|
1916
|
+
});
|
|
1917
|
+
return {
|
|
1918
|
+
checks,
|
|
1919
|
+
liveToken: null
|
|
1920
|
+
};
|
|
1921
|
+
}
|
|
1922
|
+
checks.push({
|
|
1923
|
+
name: "auth",
|
|
1924
|
+
status: "pass",
|
|
1925
|
+
detail: describeAuthSource(envKeys, byok)
|
|
1926
|
+
});
|
|
1927
|
+
if (info.email) checks.push({
|
|
1928
|
+
name: "auth.account",
|
|
1929
|
+
status: "pass",
|
|
1930
|
+
detail: info.email
|
|
1931
|
+
});
|
|
1932
|
+
const scopes = info.scope ? info.scope.split(/\s+/) : [];
|
|
1933
|
+
const missing = REQUIRED_SCOPES.filter((s) => !scopes.includes(s) && !scopes.includes(s.replace(".readonly", "")));
|
|
1934
|
+
if (missing.length > 0) checks.push({
|
|
1935
|
+
name: "auth.scopes",
|
|
1936
|
+
status: "warn",
|
|
1937
|
+
detail: `missing: ${missing.join(", ")} — \`gscdump auth login --force\` to re-consent`
|
|
1938
|
+
});
|
|
1939
|
+
else checks.push({
|
|
1940
|
+
name: "auth.scopes",
|
|
1941
|
+
status: "pass",
|
|
1942
|
+
detail: `${scopes.length} granted`
|
|
1943
|
+
});
|
|
1944
|
+
if (tokens?.expiry_date) {
|
|
1945
|
+
const expiresInMs = tokens.expiry_date - Date.now();
|
|
1946
|
+
if (expiresInMs < 0) checks.push({
|
|
1947
|
+
name: "auth.expiry",
|
|
1948
|
+
status: "warn",
|
|
1949
|
+
detail: `expired ${new Date(tokens.expiry_date).toISOString()} — will refresh on next call`
|
|
1950
|
+
});
|
|
1951
|
+
else checks.push({
|
|
1952
|
+
name: "auth.expiry",
|
|
1953
|
+
status: "pass",
|
|
1954
|
+
detail: `valid for ${Math.floor(expiresInMs / 6e4)}m`
|
|
1955
|
+
});
|
|
1956
|
+
}
|
|
1957
|
+
return {
|
|
1958
|
+
checks,
|
|
1959
|
+
liveToken
|
|
1960
|
+
};
|
|
1961
|
+
}
|
|
1962
|
+
async function checkTimeSkew() {
|
|
1963
|
+
const dateHeader = await ofetch.raw("https://oauth2.googleapis.com/tokeninfo", {
|
|
1964
|
+
method: "GET",
|
|
1965
|
+
timeout: FETCH_TIMEOUT_MS
|
|
1966
|
+
}).then((r) => r.headers.get("date")).catch((e) => e?.response?.headers?.get("date") ?? null);
|
|
1967
|
+
if (!dateHeader) return [{
|
|
1968
|
+
name: "time",
|
|
1969
|
+
status: "warn",
|
|
1970
|
+
detail: "could not probe Google clock (no Date header)"
|
|
1971
|
+
}];
|
|
1972
|
+
const remoteMs = Date.parse(dateHeader);
|
|
1973
|
+
if (!Number.isFinite(remoteMs)) return [{
|
|
1974
|
+
name: "time",
|
|
1975
|
+
status: "warn",
|
|
1976
|
+
detail: `unparseable Date header: ${dateHeader}`
|
|
1977
|
+
}];
|
|
1978
|
+
const skewMs = Date.now() - remoteMs;
|
|
1979
|
+
const human = `${skewMs >= 0 ? "+" : ""}${(skewMs / 1e3).toFixed(1)}s`;
|
|
1980
|
+
if (Math.abs(skewMs) > TIME_SKEW_WARN_MS) return [{
|
|
1981
|
+
name: "time",
|
|
1982
|
+
status: "warn",
|
|
1983
|
+
detail: `local clock ${human} off Google — OAuth refresh may reject; sync your clock`
|
|
1984
|
+
}];
|
|
1985
|
+
return [{
|
|
1986
|
+
name: "time",
|
|
1987
|
+
status: "pass",
|
|
1988
|
+
detail: `in sync (${human})`
|
|
1989
|
+
}];
|
|
1990
|
+
}
|
|
1991
|
+
async function checkDataDir() {
|
|
1992
|
+
const dataDir = resolveDataDir(await loadConfig());
|
|
1993
|
+
const display = displayPath(dataDir);
|
|
1994
|
+
const stat = await fs.stat(dataDir).catch(() => null);
|
|
1995
|
+
if (!stat) return [{
|
|
1996
|
+
name: "dataDir",
|
|
1997
|
+
status: "warn",
|
|
1998
|
+
detail: `${display} does not exist (will be created on first sync)`
|
|
1999
|
+
}];
|
|
2000
|
+
if (!stat.isDirectory()) return [{
|
|
2001
|
+
name: "dataDir",
|
|
2002
|
+
status: "fail",
|
|
2003
|
+
detail: `${display} is not a directory`
|
|
2004
|
+
}];
|
|
2005
|
+
const probe = `${dataDir}/.gscdump-doctor-probe`;
|
|
2006
|
+
return await fs.writeFile(probe, "").then(() => fs.rm(probe)).then(() => true).catch(() => false) ? [{
|
|
2007
|
+
name: "dataDir",
|
|
2008
|
+
status: "pass",
|
|
2009
|
+
detail: display
|
|
2010
|
+
}] : [{
|
|
2011
|
+
name: "dataDir",
|
|
2012
|
+
status: "fail",
|
|
2013
|
+
detail: `${display} not writable`
|
|
2014
|
+
}];
|
|
2015
|
+
}
|
|
2016
|
+
async function checkStoreWatermarks() {
|
|
2017
|
+
const dataDir = resolveDataDir(await loadConfig());
|
|
2018
|
+
if (!(await fs.stat(dataDir).catch(() => null))?.isDirectory()) return [{
|
|
2019
|
+
name: "store.watermarks",
|
|
2020
|
+
status: "pass",
|
|
2021
|
+
detail: "no store yet (run `gscdump sync`)"
|
|
2022
|
+
}];
|
|
2023
|
+
const store = createLocalStore({ dataDir });
|
|
2024
|
+
const watermarks = await store.engine.getWatermarks({ userId: store.userId }).catch(() => null);
|
|
2025
|
+
if (!watermarks || watermarks.length === 0) return [{
|
|
2026
|
+
name: "store.watermarks",
|
|
2027
|
+
status: "pass",
|
|
2028
|
+
detail: "no watermarks (run `gscdump sync`)"
|
|
2029
|
+
}];
|
|
2030
|
+
const bySite = /* @__PURE__ */ new Map();
|
|
2031
|
+
for (const w of watermarks) {
|
|
2032
|
+
const site = w.siteId ?? "(global)";
|
|
2033
|
+
const existing = bySite.get(site);
|
|
2034
|
+
if (!existing || w.newestDateSynced > existing) bySite.set(site, w.newestDateSynced);
|
|
2035
|
+
}
|
|
2036
|
+
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
2037
|
+
const stale = [];
|
|
2038
|
+
let freshest = null;
|
|
2039
|
+
for (const [site, newest] of bySite) {
|
|
2040
|
+
const days = Math.floor((Date.parse(today) - Date.parse(newest)) / 864e5);
|
|
2041
|
+
if (days > WATERMARK_STALE_DAYS_WARN) stale.push({
|
|
2042
|
+
site,
|
|
2043
|
+
days
|
|
2044
|
+
});
|
|
2045
|
+
if (!freshest || days < freshest.days) freshest = {
|
|
2046
|
+
site,
|
|
2047
|
+
days
|
|
2048
|
+
};
|
|
2049
|
+
}
|
|
2050
|
+
if (stale.length > 0) {
|
|
2051
|
+
const sample = stale.slice(0, 3).map((s) => `${s.site} (${s.days}d)`).join(", ");
|
|
2052
|
+
const more = stale.length > 3 ? ` +${stale.length - 3} more` : "";
|
|
2053
|
+
return [{
|
|
2054
|
+
name: "store.watermarks",
|
|
2055
|
+
status: "warn",
|
|
2056
|
+
detail: `${stale.length}/${bySite.size} site(s) stale >${WATERMARK_STALE_DAYS_WARN}d: ${sample}${more}`
|
|
2057
|
+
}];
|
|
2058
|
+
}
|
|
2059
|
+
const tail = freshest ? `, freshest ${freshest.days}d ago` : "";
|
|
2060
|
+
return [{
|
|
2061
|
+
name: "store.watermarks",
|
|
2062
|
+
status: "pass",
|
|
2063
|
+
detail: `${bySite.size} site(s)${tail}`
|
|
2064
|
+
}];
|
|
2065
|
+
}
|
|
2066
|
+
async function checkApiReachable(name, url) {
|
|
2067
|
+
const reachable = await ofetch.raw(url, {
|
|
2068
|
+
method: "GET",
|
|
2069
|
+
timeout: FETCH_TIMEOUT_MS
|
|
2070
|
+
}).then(() => true).catch(() => false);
|
|
2071
|
+
return [{
|
|
2072
|
+
name,
|
|
2073
|
+
status: reachable ? "pass" : "warn",
|
|
2074
|
+
detail: reachable ? "reachable" : `${new URL(url).hostname} unreachable (network/firewall?)`
|
|
2075
|
+
}];
|
|
2076
|
+
}
|
|
2077
|
+
async function checkGscSites() {
|
|
2078
|
+
const config = await loadConfig();
|
|
2079
|
+
const auth = await resolveAuth({
|
|
2080
|
+
interactive: false,
|
|
2081
|
+
config
|
|
2082
|
+
}).catch(() => null);
|
|
2083
|
+
if (!auth) return [{
|
|
2084
|
+
name: "gsc.sites",
|
|
2085
|
+
status: "warn",
|
|
2086
|
+
detail: "skipped (no usable auth)"
|
|
2087
|
+
}];
|
|
2088
|
+
const sites = await googleSearchConsole(auth).sites().catch((e) => e);
|
|
2089
|
+
if (sites instanceof Error) return [{
|
|
2090
|
+
name: "gsc.sites",
|
|
2091
|
+
status: "fail",
|
|
2092
|
+
detail: `sites() failed: ${sites.message}`
|
|
2093
|
+
}];
|
|
2094
|
+
const checks = [];
|
|
2095
|
+
const verified = sites.filter((s) => s.permissionLevel !== "siteUnverifiedUser").length;
|
|
2096
|
+
checks.push({
|
|
2097
|
+
name: "gsc.sites",
|
|
2098
|
+
status: "pass",
|
|
2099
|
+
detail: `${sites.length} site(s) accessible (${verified} verified)`
|
|
2100
|
+
});
|
|
2101
|
+
if (config.defaultSite) {
|
|
2102
|
+
const match = sites.find((s) => s.siteUrl === config.defaultSite || (s.siteUrl ?? "").includes(String(config.defaultSite)));
|
|
2103
|
+
checks.push(match ? {
|
|
2104
|
+
name: "config.defaultSite",
|
|
2105
|
+
status: "pass",
|
|
2106
|
+
detail: `${config.defaultSite} ✓`
|
|
2107
|
+
} : {
|
|
2108
|
+
name: "config.defaultSite",
|
|
2109
|
+
status: "fail",
|
|
2110
|
+
detail: `${config.defaultSite} not in verified site list`
|
|
2111
|
+
});
|
|
2112
|
+
}
|
|
2113
|
+
return checks;
|
|
2114
|
+
}
|
|
2115
|
+
const doctorCommand = defineCommand({
|
|
2116
|
+
meta: {
|
|
2117
|
+
name: "doctor",
|
|
2118
|
+
description: "Run health checks (env, auth, scopes, time, dataDir, store, GSC reachability + ping, defaultSite)"
|
|
2119
|
+
},
|
|
2120
|
+
args: { ...OUTPUT_ARGS },
|
|
2121
|
+
async run({ args }) {
|
|
2122
|
+
const { json } = applyOutputMode(args);
|
|
2123
|
+
const envResult = await checkEnv();
|
|
2124
|
+
const [authResult, timeChecks, dataDirChecks, watermarkChecks, gscApi, indexingApi, siteVerificationApi] = await Promise.all([
|
|
2125
|
+
checkAuth$1(envResult.envKeys),
|
|
2126
|
+
checkTimeSkew(),
|
|
2127
|
+
checkDataDir(),
|
|
2128
|
+
checkStoreWatermarks(),
|
|
2129
|
+
checkApiReachable("gsc.api", "https://searchconsole.googleapis.com/$discovery/rest?version=v1"),
|
|
2130
|
+
checkApiReachable("indexing.api", "https://indexing.googleapis.com/$discovery/rest?version=v3"),
|
|
2131
|
+
checkApiReachable("siteverification.api", "https://www.googleapis.com/discovery/v1/apis/siteVerification/v1/rest")
|
|
2132
|
+
]);
|
|
2133
|
+
const sitesChecks = authResult.liveToken ? await checkGscSites() : [{
|
|
2134
|
+
name: "gsc.sites",
|
|
2135
|
+
status: "warn",
|
|
2136
|
+
detail: "skipped (auth failed)"
|
|
2137
|
+
}];
|
|
2138
|
+
const all = [
|
|
2139
|
+
...envResult.checks,
|
|
2140
|
+
...authResult.checks,
|
|
2141
|
+
...timeChecks,
|
|
2142
|
+
...dataDirChecks,
|
|
2143
|
+
...watermarkChecks,
|
|
2144
|
+
...gscApi,
|
|
2145
|
+
...indexingApi,
|
|
2146
|
+
...siteVerificationApi,
|
|
2147
|
+
...sitesChecks
|
|
2148
|
+
];
|
|
2149
|
+
if (json) {
|
|
2150
|
+
console.log(JSON.stringify({
|
|
2151
|
+
checks: all,
|
|
2152
|
+
ok: all.every((c) => c.status !== "fail")
|
|
2153
|
+
}, null, 2));
|
|
2154
|
+
return;
|
|
2155
|
+
}
|
|
2156
|
+
const ICONS = {
|
|
2157
|
+
pass: "\x1B[32m✓\x1B[0m",
|
|
2158
|
+
warn: "\x1B[33m!\x1B[0m",
|
|
2159
|
+
fail: "\x1B[31m✗\x1B[0m",
|
|
2160
|
+
info: "\x1B[34mℹ\x1B[0m"
|
|
2161
|
+
};
|
|
2162
|
+
console.log();
|
|
2163
|
+
for (const c of all) {
|
|
2164
|
+
const detail = c.detail ? ` \x1B[90m${c.detail}\x1B[0m` : "";
|
|
2165
|
+
console.log(` ${ICONS[c.status]} ${c.name}${detail}`);
|
|
2166
|
+
}
|
|
2167
|
+
console.log();
|
|
2168
|
+
const failed = all.filter((c) => c.status === "fail");
|
|
2169
|
+
if (failed.length > 0) {
|
|
2170
|
+
logger.error(`${failed.length} check(s) failed`);
|
|
2171
|
+
process.exit(1);
|
|
2172
|
+
}
|
|
2173
|
+
const warned = all.filter((c) => c.status === "warn");
|
|
2174
|
+
if (warned.length > 0) logger.warn(`${warned.length} warning(s)`);
|
|
2175
|
+
else logger.success("All checks passed");
|
|
2176
|
+
}
|
|
2177
|
+
});
|
|
2178
|
+
const DEFAULT_OUT = "./gscdump-export";
|
|
2179
|
+
const FORMATS = [
|
|
2180
|
+
"parquet",
|
|
2181
|
+
"json",
|
|
2182
|
+
"ndjson",
|
|
2183
|
+
"csv"
|
|
2184
|
+
];
|
|
2185
|
+
const dumpCommand = defineCommand({
|
|
2186
|
+
meta: {
|
|
2187
|
+
name: "dump",
|
|
2188
|
+
description: "Export live Parquet files from the local store to a directory"
|
|
2189
|
+
},
|
|
2190
|
+
args: {
|
|
2191
|
+
"site": {
|
|
2192
|
+
type: "string",
|
|
2193
|
+
alias: "s",
|
|
2194
|
+
description: "Site URL (e.g., sc-domain:example.com); ignored with --all-sites"
|
|
2195
|
+
},
|
|
2196
|
+
"out": {
|
|
2197
|
+
type: "string",
|
|
883
2198
|
alias: "o",
|
|
884
2199
|
default: DEFAULT_OUT,
|
|
885
2200
|
description: `Output directory (default: ${DEFAULT_OUT})`
|
|
886
2201
|
},
|
|
887
|
-
|
|
2202
|
+
"format": {
|
|
2203
|
+
type: "string",
|
|
2204
|
+
alias: "F",
|
|
2205
|
+
default: "parquet",
|
|
2206
|
+
description: `Output format: ${FORMATS.join(", ")} (default: parquet copies raw files)`
|
|
2207
|
+
},
|
|
2208
|
+
"tables": {
|
|
2209
|
+
type: "string",
|
|
2210
|
+
alias: "t",
|
|
2211
|
+
description: `Comma-separated table list (default: all). Known: ${allTables().join(", ")}`
|
|
2212
|
+
},
|
|
2213
|
+
"all-sites": {
|
|
888
2214
|
type: "boolean",
|
|
889
2215
|
default: false,
|
|
890
|
-
description: "
|
|
2216
|
+
description: "Iterate every site with local data"
|
|
891
2217
|
},
|
|
892
|
-
|
|
2218
|
+
"compact": {
|
|
893
2219
|
type: "boolean",
|
|
894
|
-
alias: "q",
|
|
895
2220
|
default: false,
|
|
896
|
-
description: "
|
|
897
|
-
}
|
|
2221
|
+
description: "Compact every closed month into a single file before exporting"
|
|
2222
|
+
},
|
|
2223
|
+
...OUTPUT_ARGS
|
|
898
2224
|
},
|
|
899
2225
|
async run({ args }) {
|
|
2226
|
+
const { json, quiet } = applyOutputMode(args);
|
|
2227
|
+
const format = String(args.format);
|
|
2228
|
+
if (!FORMATS.includes(format)) {
|
|
2229
|
+
logger.error(`Invalid --format: ${format}. Allowed: ${FORMATS.join(", ")}`);
|
|
2230
|
+
process.exit(1);
|
|
2231
|
+
}
|
|
2232
|
+
const tablesFilter = args.tables ? new Set(String(args.tables).split(",").map((t) => t.trim()).filter(Boolean)) : null;
|
|
900
2233
|
const ctx = await createCommandContext({
|
|
901
|
-
needsAuth:
|
|
2234
|
+
needsAuth: !args["all-sites"],
|
|
902
2235
|
needsStore: true
|
|
903
2236
|
});
|
|
904
|
-
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
905
2237
|
const store = ctx.store;
|
|
906
2238
|
const outDir = path.resolve(String(args.out));
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
logger.warn(`No data for ${siteUrl}. Run \`gscdump sync\` first.`);
|
|
2239
|
+
const targets = args["all-sites"] ? await listSitesWithData(store) : [await ctx.resolveSite(args.site ? String(args.site) : void 0)];
|
|
2240
|
+
if (targets.length === 0) {
|
|
2241
|
+
logger.warn("No sites with local data. Run `gscdump sync` first.");
|
|
911
2242
|
process.exit(0);
|
|
912
2243
|
}
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
for (const
|
|
916
|
-
const
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
2244
|
+
if (args.compact) for (const siteUrl of targets) await compactClosedMonths(store, siteUrl, quiet);
|
|
2245
|
+
const summary = [];
|
|
2246
|
+
for (const siteUrl of targets) {
|
|
2247
|
+
const entries = (await listLiveEntries(store, siteUrl)).filter((e) => !tablesFilter || tablesFilter.has(e.table));
|
|
2248
|
+
if (entries.length === 0) {
|
|
2249
|
+
if (!quiet) logger.warn(`No data for ${siteUrl}; skipping`);
|
|
2250
|
+
continue;
|
|
2251
|
+
}
|
|
2252
|
+
if (format === "parquet") {
|
|
2253
|
+
const written = await dumpParquet(store, entries, outDir);
|
|
2254
|
+
summary.push({
|
|
2255
|
+
site: siteUrl,
|
|
2256
|
+
files: written,
|
|
2257
|
+
rows: 0,
|
|
2258
|
+
format,
|
|
2259
|
+
outPath: outDir
|
|
2260
|
+
});
|
|
2261
|
+
} else {
|
|
2262
|
+
const written = await dumpRowFormat(store, entries, outDir, siteUrl, format);
|
|
2263
|
+
summary.push({
|
|
2264
|
+
site: siteUrl,
|
|
2265
|
+
files: written.files,
|
|
2266
|
+
rows: written.rows,
|
|
2267
|
+
format,
|
|
2268
|
+
outPath: outDir
|
|
2269
|
+
});
|
|
2270
|
+
}
|
|
2271
|
+
}
|
|
2272
|
+
if (json) {
|
|
2273
|
+
console.log(JSON.stringify({
|
|
2274
|
+
outDir,
|
|
2275
|
+
sites: summary
|
|
2276
|
+
}, null, 2));
|
|
2277
|
+
return;
|
|
2278
|
+
}
|
|
2279
|
+
for (const s of summary) {
|
|
2280
|
+
const rows = s.rows ? `, ${s.rows.toLocaleString()} rows` : "";
|
|
2281
|
+
logger.success(`[${s.site}] ${s.files} ${s.format} file(s)${rows} → ${displayPath(s.outPath)}`);
|
|
921
2282
|
}
|
|
922
|
-
if (!args.quiet) logger.success(`Exported ${copied} file(s) to ${outDir}`);
|
|
923
2283
|
}
|
|
924
2284
|
});
|
|
2285
|
+
async function listSitesWithData(store) {
|
|
2286
|
+
const siteIds = /* @__PURE__ */ new Set();
|
|
2287
|
+
for (const table of allTables()) {
|
|
2288
|
+
const entries = await store.engine.listLive({
|
|
2289
|
+
userId: store.userId,
|
|
2290
|
+
table
|
|
2291
|
+
});
|
|
2292
|
+
for (const e of entries) if (e.siteId) siteIds.add(e.siteId);
|
|
2293
|
+
}
|
|
2294
|
+
return Array.from(siteIds);
|
|
2295
|
+
}
|
|
925
2296
|
async function listLiveEntries(store, siteUrl) {
|
|
926
2297
|
const siteId = store.siteIdFor(siteUrl);
|
|
927
2298
|
return (await Promise.all(allTables().map((table) => store.engine.listLive({
|
|
@@ -930,6 +2301,58 @@ async function listLiveEntries(store, siteUrl) {
|
|
|
930
2301
|
table
|
|
931
2302
|
})))).flat();
|
|
932
2303
|
}
|
|
2304
|
+
async function dumpParquet(store, entries, outDir) {
|
|
2305
|
+
await fs.mkdir(outDir, { recursive: true });
|
|
2306
|
+
let copied = 0;
|
|
2307
|
+
for (const entry of entries) {
|
|
2308
|
+
const bytes = await store.engine.readObject(entry.objectKey);
|
|
2309
|
+
const target = path.join(outDir, entry.objectKey);
|
|
2310
|
+
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
2311
|
+
await fs.writeFile(target, Buffer.from(bytes));
|
|
2312
|
+
copied++;
|
|
2313
|
+
}
|
|
2314
|
+
return copied;
|
|
2315
|
+
}
|
|
2316
|
+
async function dumpRowFormat(store, entries, outDir, siteUrl, format) {
|
|
2317
|
+
const byTable = /* @__PURE__ */ new Map();
|
|
2318
|
+
for (const e of entries) {
|
|
2319
|
+
const arr = byTable.get(e.table) ?? [];
|
|
2320
|
+
arr.push(e);
|
|
2321
|
+
byTable.set(e.table, arr);
|
|
2322
|
+
}
|
|
2323
|
+
const safeSite = siteUrl.replace(/[^a-z0-9]+/gi, "_");
|
|
2324
|
+
const siteDir = path.join(outDir, safeSite);
|
|
2325
|
+
await fs.mkdir(siteDir, { recursive: true });
|
|
2326
|
+
let files = 0;
|
|
2327
|
+
let totalRows = 0;
|
|
2328
|
+
for (const [table, tableEntries] of byTable) {
|
|
2329
|
+
const rows = await readTableRows(tableEntries.map((e) => path.join(store.dataDir, e.objectKey)));
|
|
2330
|
+
const ext = format === "csv" ? "csv" : format === "ndjson" ? "ndjson" : "json";
|
|
2331
|
+
const target = path.join(siteDir, `${table}.${ext}`);
|
|
2332
|
+
let body;
|
|
2333
|
+
if (format === "json") body = JSON.stringify(rows, null, 2);
|
|
2334
|
+
else if (format === "ndjson") body = rows.map((r) => JSON.stringify(r)).join("\n");
|
|
2335
|
+
else body = rows.length > 0 ? toCSV(rows, Object.keys(rows[0])) : "";
|
|
2336
|
+
await fs.writeFile(target, body);
|
|
2337
|
+
files++;
|
|
2338
|
+
totalRows += rows.length;
|
|
2339
|
+
}
|
|
2340
|
+
return {
|
|
2341
|
+
files,
|
|
2342
|
+
rows: totalRows
|
|
2343
|
+
};
|
|
2344
|
+
}
|
|
2345
|
+
async function readTableRows(filePaths) {
|
|
2346
|
+
const instance = await DuckDBInstance.create(":memory:");
|
|
2347
|
+
const conn = await instance.connect();
|
|
2348
|
+
try {
|
|
2349
|
+
const fileList = filePaths.map((p) => `'${sqlEscape(p)}'`).join(", ");
|
|
2350
|
+
return (await conn.runAndReadAll(`SELECT * FROM read_parquet([${fileList}], union_by_name=true)`)).getRowObjects();
|
|
2351
|
+
} finally {
|
|
2352
|
+
conn.closeSync();
|
|
2353
|
+
instance.closeSync();
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
933
2356
|
async function compactClosedMonths(store, siteUrl, quiet) {
|
|
934
2357
|
const siteId = store.siteIdFor(siteUrl);
|
|
935
2358
|
for (const table of allTables()) {
|
|
@@ -958,8 +2381,7 @@ const inspectSubCommand = defineCommand({
|
|
|
958
2381
|
site: {
|
|
959
2382
|
type: "string",
|
|
960
2383
|
alias: "s",
|
|
961
|
-
|
|
962
|
-
description: "Site URL (e.g., sc-domain:example.com)"
|
|
2384
|
+
description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
|
|
963
2385
|
},
|
|
964
2386
|
file: {
|
|
965
2387
|
type: "string",
|
|
@@ -976,24 +2398,19 @@ const inspectSubCommand = defineCommand({
|
|
|
976
2398
|
default: "4",
|
|
977
2399
|
description: "Concurrent in-flight inspect calls (default: 4)"
|
|
978
2400
|
},
|
|
979
|
-
|
|
980
|
-
type: "boolean",
|
|
981
|
-
alias: "q",
|
|
982
|
-
default: false,
|
|
983
|
-
description: "Suppress progress output"
|
|
984
|
-
}
|
|
2401
|
+
...OUTPUT_ARGS
|
|
985
2402
|
},
|
|
986
2403
|
async run({ args }) {
|
|
2404
|
+
const { json, quiet } = applyOutputMode(args);
|
|
987
2405
|
const ctx = await createCommandContext({
|
|
988
2406
|
needsAuth: true,
|
|
989
2407
|
needsStore: true
|
|
990
2408
|
});
|
|
991
2409
|
const client = ctx.client;
|
|
992
2410
|
const store = ctx.store;
|
|
993
|
-
const siteUrl = String(args.site);
|
|
2411
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
994
2412
|
const limit = args.limit ? Number.parseInt(String(args.limit), 10) : INSPECTION_QPD_PER_PROPERTY;
|
|
995
2413
|
const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
|
|
996
|
-
const quiet = Boolean(args.quiet);
|
|
997
2414
|
const urls = (await readUrlList({ file: args.file ? String(args.file) : void 0 })).slice(0, limit);
|
|
998
2415
|
if (urls.length === 0) {
|
|
999
2416
|
logger.warn("No URLs to inspect.");
|
|
@@ -1041,7 +2458,14 @@ const inspectSubCommand = defineCommand({
|
|
|
1041
2458
|
userId: store.userId,
|
|
1042
2459
|
siteId: store.siteIdFor(siteUrl)
|
|
1043
2460
|
}, records);
|
|
1044
|
-
if (
|
|
2461
|
+
if (json) console.log(JSON.stringify({
|
|
2462
|
+
site: siteUrl,
|
|
2463
|
+
inspected: records.length,
|
|
2464
|
+
failed,
|
|
2465
|
+
failures,
|
|
2466
|
+
records
|
|
2467
|
+
}, null, 2));
|
|
2468
|
+
else if (!quiet) {
|
|
1045
2469
|
logger.success(`Inspected ${records.length}/${urls.length} URL(s)`);
|
|
1046
2470
|
if (failed > 0) {
|
|
1047
2471
|
logger.warn(`${failed} failed:`);
|
|
@@ -1058,34 +2482,35 @@ const showSubCommand = defineCommand({
|
|
|
1058
2482
|
description: "Print the latest inspection record for a URL from the local entity store"
|
|
1059
2483
|
},
|
|
1060
2484
|
args: {
|
|
2485
|
+
...OUTPUT_ARGS,
|
|
1061
2486
|
site: {
|
|
1062
2487
|
type: "string",
|
|
1063
2488
|
alias: "s",
|
|
1064
|
-
|
|
1065
|
-
description: "Site URL"
|
|
2489
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
1066
2490
|
},
|
|
1067
2491
|
url: {
|
|
1068
2492
|
type: "positional",
|
|
1069
2493
|
required: true,
|
|
1070
2494
|
description: "URL to look up"
|
|
1071
|
-
},
|
|
1072
|
-
json: {
|
|
1073
|
-
type: "boolean",
|
|
1074
|
-
default: false,
|
|
1075
|
-
description: "Output as JSON"
|
|
1076
2495
|
}
|
|
1077
2496
|
},
|
|
1078
2497
|
async run({ args }) {
|
|
1079
|
-
const
|
|
2498
|
+
const { json } = applyOutputMode(args);
|
|
2499
|
+
const ctx = await createCommandContext({
|
|
2500
|
+
needsAuth: true,
|
|
2501
|
+
needsStore: true
|
|
2502
|
+
});
|
|
2503
|
+
const store = ctx.store;
|
|
2504
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1080
2505
|
const record = await createInspectionStore({ dataSource: store.dataSource }).getLatest({
|
|
1081
2506
|
userId: store.userId,
|
|
1082
|
-
siteId: store.siteIdFor(
|
|
2507
|
+
siteId: store.siteIdFor(siteUrl)
|
|
1083
2508
|
}, String(args.url));
|
|
1084
2509
|
if (!record) {
|
|
1085
2510
|
logger.warn(`No inspection record for ${args.url}`);
|
|
1086
2511
|
process.exit(1);
|
|
1087
2512
|
}
|
|
1088
|
-
if (
|
|
2513
|
+
if (json) {
|
|
1089
2514
|
console.log(JSON.stringify(record, null, 2));
|
|
1090
2515
|
return;
|
|
1091
2516
|
}
|
|
@@ -1103,37 +2528,26 @@ const showSubCommand = defineCommand({
|
|
|
1103
2528
|
});
|
|
1104
2529
|
const sitemapsSnapshotSubCommand = defineCommand({
|
|
1105
2530
|
meta: {
|
|
1106
|
-
name: "snapshot",
|
|
1107
|
-
description: "Fetch current sitemap state from GSC and persist to the local entity store"
|
|
1108
|
-
},
|
|
1109
|
-
args: {
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
description: "Site URL (e.g., sc-domain:example.com)"
|
|
1115
|
-
},
|
|
1116
|
-
quiet: {
|
|
1117
|
-
type: "boolean",
|
|
1118
|
-
alias: "q",
|
|
1119
|
-
default: false,
|
|
1120
|
-
description: "Suppress progress output"
|
|
1121
|
-
},
|
|
1122
|
-
json: {
|
|
1123
|
-
type: "boolean",
|
|
1124
|
-
default: false,
|
|
1125
|
-
description: "Emit the snapshot JSON to stdout"
|
|
2531
|
+
name: "snapshot",
|
|
2532
|
+
description: "Fetch current sitemap state from GSC and persist to the local entity store"
|
|
2533
|
+
},
|
|
2534
|
+
args: {
|
|
2535
|
+
...OUTPUT_ARGS,
|
|
2536
|
+
site: {
|
|
2537
|
+
type: "string",
|
|
2538
|
+
alias: "s",
|
|
2539
|
+
description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
|
|
1126
2540
|
}
|
|
1127
2541
|
},
|
|
1128
2542
|
async run({ args }) {
|
|
2543
|
+
const { json, quiet } = applyOutputMode(args);
|
|
1129
2544
|
const ctx = await createCommandContext({
|
|
1130
2545
|
needsAuth: true,
|
|
1131
2546
|
needsStore: true
|
|
1132
2547
|
});
|
|
1133
2548
|
const client = ctx.client;
|
|
1134
2549
|
const store = ctx.store;
|
|
1135
|
-
const siteUrl = String(args.site);
|
|
1136
|
-
const quiet = Boolean(args.quiet);
|
|
2550
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1137
2551
|
const apiSitemaps = await client.sitemaps.list(siteUrl);
|
|
1138
2552
|
const capturedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1139
2553
|
const records = apiSitemaps.filter((s) => typeof s.path === "string").map((s) => ({
|
|
@@ -1157,7 +2571,7 @@ const sitemapsSnapshotSubCommand = defineCommand({
|
|
|
1157
2571
|
userId: store.userId,
|
|
1158
2572
|
siteId: store.siteIdFor(siteUrl)
|
|
1159
2573
|
}, records);
|
|
1160
|
-
if (
|
|
2574
|
+
if (json) {
|
|
1161
2575
|
console.log(JSON.stringify({
|
|
1162
2576
|
site: siteUrl,
|
|
1163
2577
|
capturedAt,
|
|
@@ -1182,34 +2596,35 @@ const sitemapsShowSubCommand = defineCommand({
|
|
|
1182
2596
|
description: "Print the latest captured sitemap state for a feedpath"
|
|
1183
2597
|
},
|
|
1184
2598
|
args: {
|
|
2599
|
+
...OUTPUT_ARGS,
|
|
1185
2600
|
site: {
|
|
1186
2601
|
type: "string",
|
|
1187
2602
|
alias: "s",
|
|
1188
|
-
|
|
1189
|
-
description: "Site URL"
|
|
2603
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
1190
2604
|
},
|
|
1191
2605
|
path: {
|
|
1192
2606
|
type: "positional",
|
|
1193
2607
|
required: true,
|
|
1194
2608
|
description: "Sitemap path (feedpath)"
|
|
1195
|
-
},
|
|
1196
|
-
json: {
|
|
1197
|
-
type: "boolean",
|
|
1198
|
-
default: false,
|
|
1199
|
-
description: "Output as JSON"
|
|
1200
2609
|
}
|
|
1201
2610
|
},
|
|
1202
2611
|
async run({ args }) {
|
|
1203
|
-
const
|
|
2612
|
+
const { json } = applyOutputMode(args);
|
|
2613
|
+
const ctx = await createCommandContext({
|
|
2614
|
+
needsAuth: true,
|
|
2615
|
+
needsStore: true
|
|
2616
|
+
});
|
|
2617
|
+
const store = ctx.store;
|
|
2618
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1204
2619
|
const record = await createSitemapStore({ dataSource: store.dataSource }).getLatest({
|
|
1205
2620
|
userId: store.userId,
|
|
1206
|
-
siteId: store.siteIdFor(
|
|
2621
|
+
siteId: store.siteIdFor(siteUrl)
|
|
1207
2622
|
}, String(args.path));
|
|
1208
2623
|
if (!record) {
|
|
1209
2624
|
logger.warn(`No sitemap record for ${args.path}`);
|
|
1210
2625
|
process.exit(1);
|
|
1211
2626
|
}
|
|
1212
|
-
if (
|
|
2627
|
+
if (json) {
|
|
1213
2628
|
console.log(JSON.stringify(record, null, 2));
|
|
1214
2629
|
return;
|
|
1215
2630
|
}
|
|
@@ -1249,8 +2664,7 @@ const indexingSubCommand = defineCommand({
|
|
|
1249
2664
|
site: {
|
|
1250
2665
|
type: "string",
|
|
1251
2666
|
alias: "s",
|
|
1252
|
-
|
|
1253
|
-
description: "Site URL (e.g., sc-domain:example.com)"
|
|
2667
|
+
description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
|
|
1254
2668
|
},
|
|
1255
2669
|
file: {
|
|
1256
2670
|
type: "string",
|
|
@@ -1263,23 +2677,18 @@ const indexingSubCommand = defineCommand({
|
|
|
1263
2677
|
default: "4",
|
|
1264
2678
|
description: "Concurrent in-flight getMetadata calls (default: 4)"
|
|
1265
2679
|
},
|
|
1266
|
-
|
|
1267
|
-
type: "boolean",
|
|
1268
|
-
alias: "q",
|
|
1269
|
-
default: false,
|
|
1270
|
-
description: "Suppress progress output"
|
|
1271
|
-
}
|
|
2680
|
+
...OUTPUT_ARGS
|
|
1272
2681
|
},
|
|
1273
2682
|
async run({ args }) {
|
|
2683
|
+
const { quiet } = applyOutputMode(args);
|
|
1274
2684
|
const ctx = await createCommandContext({
|
|
1275
2685
|
needsAuth: true,
|
|
1276
2686
|
needsStore: true
|
|
1277
2687
|
});
|
|
1278
2688
|
const client = ctx.client;
|
|
1279
2689
|
const store = ctx.store;
|
|
1280
|
-
const siteUrl = String(args.site);
|
|
2690
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1281
2691
|
const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
|
|
1282
|
-
const quiet = Boolean(args.quiet);
|
|
1283
2692
|
const urls = await readUrlList({ file: args.file ? String(args.file) : void 0 });
|
|
1284
2693
|
if (urls.length === 0) {
|
|
1285
2694
|
logger.warn("No URLs to fetch metadata for.");
|
|
@@ -1346,6 +2755,313 @@ const entitiesCommand = defineCommand({
|
|
|
1346
2755
|
indexing: indexingSubCommand
|
|
1347
2756
|
}
|
|
1348
2757
|
});
|
|
2758
|
+
const RETRIES_ARG = { retries: {
|
|
2759
|
+
type: "string",
|
|
2760
|
+
description: "Override per-call retry count (default: 3)"
|
|
2761
|
+
} };
|
|
2762
|
+
function parseRetries(v) {
|
|
2763
|
+
if (v == null || v === "") return void 0;
|
|
2764
|
+
const n = Number.parseInt(String(v), 10);
|
|
2765
|
+
return Number.isFinite(n) && n >= 0 ? n : void 0;
|
|
2766
|
+
}
|
|
2767
|
+
async function resolveUrlSource(args) {
|
|
2768
|
+
const fromSitemap = args["from-sitemap"];
|
|
2769
|
+
if (fromSitemap) return fetchSitemapUrls(String(fromSitemap)).catch((e) => {
|
|
2770
|
+
logger.error(`Sitemap fetch failed: ${e.message}`);
|
|
2771
|
+
process.exit(1);
|
|
2772
|
+
});
|
|
2773
|
+
return readUrlList$1(args);
|
|
2774
|
+
}
|
|
2775
|
+
const submitCommand$1 = defineCommand({
|
|
2776
|
+
meta: {
|
|
2777
|
+
name: "submit",
|
|
2778
|
+
description: "Notify Google of a new or updated URL (URL_UPDATED)"
|
|
2779
|
+
},
|
|
2780
|
+
args: {
|
|
2781
|
+
url: {
|
|
2782
|
+
type: "positional",
|
|
2783
|
+
required: true,
|
|
2784
|
+
description: "URL to submit"
|
|
2785
|
+
},
|
|
2786
|
+
...OUTPUT_ARGS,
|
|
2787
|
+
...RETRIES_ARG
|
|
2788
|
+
},
|
|
2789
|
+
async run({ args }) {
|
|
2790
|
+
applyOutputMode(args);
|
|
2791
|
+
const result = await requestIndexing((await createCommandContext({
|
|
2792
|
+
needsAuth: true,
|
|
2793
|
+
fetchOptions: { retry: parseRetries(args.retries) }
|
|
2794
|
+
})).client, args.url, { type: "URL_UPDATED" }).catch(gscErrorHandler);
|
|
2795
|
+
if (args.json) {
|
|
2796
|
+
console.log(JSON.stringify(result, null, 2));
|
|
2797
|
+
return;
|
|
2798
|
+
}
|
|
2799
|
+
logger.success(`Submitted: ${result.url}`);
|
|
2800
|
+
if (result.notifyTime) console.log(` Notified: ${result.notifyTime}`);
|
|
2801
|
+
}
|
|
2802
|
+
});
|
|
2803
|
+
const removeCommand = defineCommand({
|
|
2804
|
+
meta: {
|
|
2805
|
+
name: "remove",
|
|
2806
|
+
description: "Notify Google a URL has been removed (URL_DELETED)"
|
|
2807
|
+
},
|
|
2808
|
+
args: {
|
|
2809
|
+
url: {
|
|
2810
|
+
type: "positional",
|
|
2811
|
+
required: true,
|
|
2812
|
+
description: "URL to mark removed"
|
|
2813
|
+
},
|
|
2814
|
+
...OUTPUT_ARGS,
|
|
2815
|
+
...RETRIES_ARG
|
|
2816
|
+
},
|
|
2817
|
+
async run({ args }) {
|
|
2818
|
+
applyOutputMode(args);
|
|
2819
|
+
const result = await requestIndexing((await createCommandContext({
|
|
2820
|
+
needsAuth: true,
|
|
2821
|
+
fetchOptions: { retry: parseRetries(args.retries) }
|
|
2822
|
+
})).client, args.url, { type: "URL_DELETED" }).catch(gscErrorHandler);
|
|
2823
|
+
if (args.json) {
|
|
2824
|
+
console.log(JSON.stringify(result, null, 2));
|
|
2825
|
+
return;
|
|
2826
|
+
}
|
|
2827
|
+
logger.success(`Removed: ${result.url}`);
|
|
2828
|
+
if (result.notifyTime) console.log(` Notified: ${result.notifyTime}`);
|
|
2829
|
+
}
|
|
2830
|
+
});
|
|
2831
|
+
const statusCommand = defineCommand({
|
|
2832
|
+
meta: {
|
|
2833
|
+
name: "status",
|
|
2834
|
+
description: "Show indexing notification metadata for a URL"
|
|
2835
|
+
},
|
|
2836
|
+
args: {
|
|
2837
|
+
url: {
|
|
2838
|
+
type: "positional",
|
|
2839
|
+
required: true,
|
|
2840
|
+
description: "URL to inspect"
|
|
2841
|
+
},
|
|
2842
|
+
json: {
|
|
2843
|
+
type: "boolean",
|
|
2844
|
+
default: false,
|
|
2845
|
+
description: "Output as JSON"
|
|
2846
|
+
},
|
|
2847
|
+
quiet: {
|
|
2848
|
+
type: "boolean",
|
|
2849
|
+
alias: "q",
|
|
2850
|
+
default: false,
|
|
2851
|
+
description: "Suppress info/success output"
|
|
2852
|
+
}
|
|
2853
|
+
},
|
|
2854
|
+
async run({ args }) {
|
|
2855
|
+
applyOutputMode(args);
|
|
2856
|
+
const meta = await getIndexingMetadata((await createCommandContext({ needsAuth: true })).client, args.url).catch(gscErrorHandler);
|
|
2857
|
+
if (args.json) {
|
|
2858
|
+
console.log(JSON.stringify(meta, null, 2));
|
|
2859
|
+
return;
|
|
2860
|
+
}
|
|
2861
|
+
const fmtNotification = (n) => {
|
|
2862
|
+
if (!n?.notifyTime) return "never";
|
|
2863
|
+
return n.type ? `${n.notifyTime} (${n.type})` : n.notifyTime;
|
|
2864
|
+
};
|
|
2865
|
+
const update = meta.latestUpdate;
|
|
2866
|
+
const remove = meta.latestRemove;
|
|
2867
|
+
console.log();
|
|
2868
|
+
console.log(` \x1B[1mURL:\x1B[0m ${meta.url}`);
|
|
2869
|
+
console.log(` Last update notify: ${fmtNotification(update)}`);
|
|
2870
|
+
console.log(` Last remove notify: ${fmtNotification(remove)}`);
|
|
2871
|
+
console.log();
|
|
2872
|
+
}
|
|
2873
|
+
});
|
|
2874
|
+
const INDEXING_DAILY_QUOTA = 200;
|
|
2875
|
+
const INDEXING_PER_MINUTE_QUOTA = 600;
|
|
2876
|
+
const quotaCommand = defineCommand({
|
|
2877
|
+
meta: {
|
|
2878
|
+
name: "quota",
|
|
2879
|
+
description: "Show documented Indexing API quotas (no live counters; quota usage is not exposed by the API)"
|
|
2880
|
+
},
|
|
2881
|
+
args: { ...OUTPUT_ARGS },
|
|
2882
|
+
async run({ args }) {
|
|
2883
|
+
const { json } = applyOutputMode(args);
|
|
2884
|
+
const payload = {
|
|
2885
|
+
perDay: INDEXING_DAILY_QUOTA,
|
|
2886
|
+
perMinute: INDEXING_PER_MINUTE_QUOTA,
|
|
2887
|
+
note: "Documented defaults. Google does not expose live counters; track yours by counting submit calls.",
|
|
2888
|
+
docs: "https://developers.google.com/search/apis/indexing-api/v3/quota-pricing"
|
|
2889
|
+
};
|
|
2890
|
+
if (json) {
|
|
2891
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
2892
|
+
return;
|
|
2893
|
+
}
|
|
2894
|
+
console.log();
|
|
2895
|
+
console.log(` \x1B[1mIndexing API quota\x1B[0m`);
|
|
2896
|
+
console.log(` Per day: ${payload.perDay}`);
|
|
2897
|
+
console.log(` Per minute: ${payload.perMinute}`);
|
|
2898
|
+
console.log();
|
|
2899
|
+
console.log(` \x1B[90m${payload.note}\x1B[0m`);
|
|
2900
|
+
console.log(` \x1B[90mDocs: ${payload.docs}\x1B[0m`);
|
|
2901
|
+
console.log();
|
|
2902
|
+
}
|
|
2903
|
+
});
|
|
2904
|
+
const indexingCommand = defineCommand({
|
|
2905
|
+
meta: {
|
|
2906
|
+
name: "indexing",
|
|
2907
|
+
description: "Notify Google about URL updates/removals (Indexing API)"
|
|
2908
|
+
},
|
|
2909
|
+
subCommands: {
|
|
2910
|
+
"submit": submitCommand$1,
|
|
2911
|
+
"remove": removeCommand,
|
|
2912
|
+
"status": statusCommand,
|
|
2913
|
+
"batch": defineCommand({
|
|
2914
|
+
meta: {
|
|
2915
|
+
name: "batch",
|
|
2916
|
+
description: "Submit many URLs from a file or stdin (one URL per line)"
|
|
2917
|
+
},
|
|
2918
|
+
args: {
|
|
2919
|
+
...OUTPUT_ARGS,
|
|
2920
|
+
"urls": {
|
|
2921
|
+
type: "positional",
|
|
2922
|
+
required: false,
|
|
2923
|
+
description: "URLs (or use --file/--from-sitemap/stdin)"
|
|
2924
|
+
},
|
|
2925
|
+
"file": {
|
|
2926
|
+
type: "string",
|
|
2927
|
+
alias: "f",
|
|
2928
|
+
description: "File with URLs (one per line)"
|
|
2929
|
+
},
|
|
2930
|
+
"from-sitemap": {
|
|
2931
|
+
type: "string",
|
|
2932
|
+
description: "Sitemap URL (or sitemap index) to pull URLs from"
|
|
2933
|
+
},
|
|
2934
|
+
"type": {
|
|
2935
|
+
type: "string",
|
|
2936
|
+
default: "URL_UPDATED",
|
|
2937
|
+
description: "URL_UPDATED or URL_DELETED"
|
|
2938
|
+
},
|
|
2939
|
+
"delay-ms": {
|
|
2940
|
+
type: "string",
|
|
2941
|
+
default: "100",
|
|
2942
|
+
description: "Delay between requests"
|
|
2943
|
+
},
|
|
2944
|
+
"concurrency": {
|
|
2945
|
+
type: "string",
|
|
2946
|
+
alias: "c",
|
|
2947
|
+
default: "1",
|
|
2948
|
+
description: "Concurrent in-flight requests"
|
|
2949
|
+
},
|
|
2950
|
+
"yes": {
|
|
2951
|
+
type: "boolean",
|
|
2952
|
+
alias: "y",
|
|
2953
|
+
default: false,
|
|
2954
|
+
description: "Skip the over-quota confirmation prompt"
|
|
2955
|
+
},
|
|
2956
|
+
"retries": {
|
|
2957
|
+
type: "string",
|
|
2958
|
+
description: "Override per-call retry count (default: 3)"
|
|
2959
|
+
}
|
|
2960
|
+
},
|
|
2961
|
+
async run({ args }) {
|
|
2962
|
+
applyOutputMode(args);
|
|
2963
|
+
const urls = await resolveUrlSource(args);
|
|
2964
|
+
if (urls.length === 0) {
|
|
2965
|
+
logger.error("No URLs provided. Pass URLs as args, --file, --from-sitemap, or stdin.");
|
|
2966
|
+
process.exit(1);
|
|
2967
|
+
}
|
|
2968
|
+
const type = String(args.type);
|
|
2969
|
+
if (type !== "URL_UPDATED" && type !== "URL_DELETED") {
|
|
2970
|
+
logger.error(`Invalid --type: ${type}. Use URL_UPDATED or URL_DELETED.`);
|
|
2971
|
+
process.exit(1);
|
|
2972
|
+
}
|
|
2973
|
+
if (urls.length > INDEXING_DAILY_QUOTA && !args.yes && !args.json) {
|
|
2974
|
+
logger.warn(`Submitting ${urls.length} URLs but the Indexing API daily quota is ${INDEXING_DAILY_QUOTA}/day.`);
|
|
2975
|
+
logger.warn(`Excess submissions will fail with quota errors. Pass --yes to proceed anyway.`);
|
|
2976
|
+
process.exit(1);
|
|
2977
|
+
}
|
|
2978
|
+
if (urls.length > INDEXING_DAILY_QUOTA && !args.json && !args.quiet) logger.warn(`Proceeding with ${urls.length} URLs (over the ${INDEXING_DAILY_QUOTA}/day quota). Excess will fail.`);
|
|
2979
|
+
const ctx = await createCommandContext({
|
|
2980
|
+
needsAuth: true,
|
|
2981
|
+
fetchOptions: { retry: parseRetries(args.retries) }
|
|
2982
|
+
});
|
|
2983
|
+
const delayMs = Number.parseInt(String(args["delay-ms"]), 10);
|
|
2984
|
+
const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 1);
|
|
2985
|
+
if (!args.json && !args.quiet) logger.info(`Submitting ${urls.length} URLs (${type}) ...`);
|
|
2986
|
+
const results = await batchRequestIndexing(ctx.client, urls, {
|
|
2987
|
+
type,
|
|
2988
|
+
delayMs,
|
|
2989
|
+
concurrency,
|
|
2990
|
+
onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url}`)
|
|
2991
|
+
}).catch(gscErrorHandler);
|
|
2992
|
+
if (args.json) {
|
|
2993
|
+
console.log(JSON.stringify(results, null, 2));
|
|
2994
|
+
return;
|
|
2995
|
+
}
|
|
2996
|
+
if (!args.quiet) logger.success(`Submitted ${results.length}/${urls.length} URLs`);
|
|
2997
|
+
}
|
|
2998
|
+
}),
|
|
2999
|
+
"batch-status": defineCommand({
|
|
3000
|
+
meta: {
|
|
3001
|
+
name: "batch-status",
|
|
3002
|
+
description: "Get indexing notification metadata for many URLs"
|
|
3003
|
+
},
|
|
3004
|
+
args: {
|
|
3005
|
+
...OUTPUT_ARGS,
|
|
3006
|
+
"urls": {
|
|
3007
|
+
type: "positional",
|
|
3008
|
+
required: false,
|
|
3009
|
+
description: "URLs (or use --file/--from-sitemap/stdin)"
|
|
3010
|
+
},
|
|
3011
|
+
"file": {
|
|
3012
|
+
type: "string",
|
|
3013
|
+
alias: "f",
|
|
3014
|
+
description: "File with URLs (one per line)"
|
|
3015
|
+
},
|
|
3016
|
+
"from-sitemap": {
|
|
3017
|
+
type: "string",
|
|
3018
|
+
description: "Sitemap URL (or sitemap index) to pull URLs from"
|
|
3019
|
+
},
|
|
3020
|
+
"delay-ms": {
|
|
3021
|
+
type: "string",
|
|
3022
|
+
default: "100",
|
|
3023
|
+
description: "Delay between requests"
|
|
3024
|
+
},
|
|
3025
|
+
"concurrency": {
|
|
3026
|
+
type: "string",
|
|
3027
|
+
alias: "c",
|
|
3028
|
+
default: "1",
|
|
3029
|
+
description: "Concurrent in-flight requests"
|
|
3030
|
+
},
|
|
3031
|
+
"retries": {
|
|
3032
|
+
type: "string",
|
|
3033
|
+
description: "Override per-call retry count (default: 3)"
|
|
3034
|
+
}
|
|
3035
|
+
},
|
|
3036
|
+
async run({ args }) {
|
|
3037
|
+
applyOutputMode(args);
|
|
3038
|
+
const urls = await resolveUrlSource(args);
|
|
3039
|
+
if (urls.length === 0) {
|
|
3040
|
+
logger.error("No URLs provided. Pass URLs as args, --file, --from-sitemap, or stdin.");
|
|
3041
|
+
process.exit(1);
|
|
3042
|
+
}
|
|
3043
|
+
const ctx = await createCommandContext({
|
|
3044
|
+
needsAuth: true,
|
|
3045
|
+
fetchOptions: { retry: parseRetries(args.retries) }
|
|
3046
|
+
});
|
|
3047
|
+
const delayMs = Number.parseInt(String(args["delay-ms"]), 10);
|
|
3048
|
+
const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 1);
|
|
3049
|
+
if (!args.json && !args.quiet) logger.info(`Fetching status for ${urls.length} URLs ...`);
|
|
3050
|
+
const results = await runSequentialBatch(urls, (url) => getIndexingMetadata(ctx.client, url), {
|
|
3051
|
+
delayMs,
|
|
3052
|
+
concurrency,
|
|
3053
|
+
onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url}`)
|
|
3054
|
+
}).catch(gscErrorHandler);
|
|
3055
|
+
if (args.json) {
|
|
3056
|
+
console.log(JSON.stringify(results, null, 2));
|
|
3057
|
+
return;
|
|
3058
|
+
}
|
|
3059
|
+
if (!args.quiet) logger.success(`Fetched ${results.length}/${urls.length} URLs`);
|
|
3060
|
+
}
|
|
3061
|
+
}),
|
|
3062
|
+
"quota": quotaCommand
|
|
3063
|
+
}
|
|
3064
|
+
});
|
|
1349
3065
|
const ENV_LINE_RE = /^([^=]+)=(.*)$/;
|
|
1350
3066
|
async function promptDataDir(existing) {
|
|
1351
3067
|
const fallback = existing ?? defaultDataDir();
|
|
@@ -1380,38 +3096,62 @@ const initCommand = defineCommand({
|
|
|
1380
3096
|
name: "init",
|
|
1381
3097
|
description: "Set up GSCDump authentication"
|
|
1382
3098
|
},
|
|
1383
|
-
args: {
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
3099
|
+
args: {
|
|
3100
|
+
"force": {
|
|
3101
|
+
type: "boolean",
|
|
3102
|
+
alias: "f",
|
|
3103
|
+
description: "Force re-initialization"
|
|
3104
|
+
},
|
|
3105
|
+
"no-store": {
|
|
3106
|
+
type: "boolean",
|
|
3107
|
+
default: false,
|
|
3108
|
+
description: "Skip dataDir prompt (auth-only setup)"
|
|
3109
|
+
},
|
|
3110
|
+
...OUTPUT_ARGS
|
|
3111
|
+
},
|
|
1388
3112
|
async run({ args }) {
|
|
3113
|
+
applyOutputMode(args);
|
|
1389
3114
|
const config = await loadConfig();
|
|
1390
3115
|
if (config.clientId && config.clientSecret && !args.force) {
|
|
1391
3116
|
logger.info("Already configured");
|
|
1392
3117
|
logger.info("Run with --force to reconfigure");
|
|
1393
3118
|
return;
|
|
1394
3119
|
}
|
|
3120
|
+
const byok = resolveBYOK();
|
|
3121
|
+
if (byok) {
|
|
3122
|
+
const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
|
|
3123
|
+
await saveConfig({
|
|
3124
|
+
...config,
|
|
3125
|
+
dataDir: dataDir ?? config.dataDir
|
|
3126
|
+
});
|
|
3127
|
+
logger.success(`BYOK detected (${typeof byok === "string" ? "access-token" : "refresh-token"}) — auth setup skipped`);
|
|
3128
|
+
logger.success("Setup complete! Run gscdump to get started.");
|
|
3129
|
+
return;
|
|
3130
|
+
}
|
|
1395
3131
|
const envFile = await loadEnvFile();
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
3132
|
+
const envCid = envFile?.GSC_CLIENT_ID ?? envFile?.GOOGLE_CLIENT_ID;
|
|
3133
|
+
const envSec = envFile?.GSC_CLIENT_SECRET ?? envFile?.GOOGLE_CLIENT_SECRET;
|
|
3134
|
+
const envRef = envFile?.GSC_REFRESH_TOKEN ?? envFile?.GOOGLE_REFRESH_TOKEN;
|
|
3135
|
+
const envAcc = envFile?.GSC_ACCESS_TOKEN ?? envFile?.GOOGLE_ACCESS_TOKEN;
|
|
3136
|
+
if (envCid && envSec && envRef) {
|
|
3137
|
+
logger.info("Found .env file with credentials");
|
|
3138
|
+
process.env.GOOGLE_CLIENT_ID = envCid;
|
|
3139
|
+
process.env.GOOGLE_CLIENT_SECRET = envSec;
|
|
3140
|
+
process.env.GOOGLE_REFRESH_TOKEN = envRef;
|
|
3141
|
+
if (envAcc) process.env.GOOGLE_ACCESS_TOKEN = envAcc;
|
|
1402
3142
|
await saveConfig({
|
|
1403
3143
|
...config,
|
|
1404
|
-
clientId:
|
|
1405
|
-
clientSecret:
|
|
3144
|
+
clientId: envCid,
|
|
3145
|
+
clientSecret: envSec,
|
|
1406
3146
|
dataDir: config.dataDir ?? defaultDataDir()
|
|
1407
3147
|
});
|
|
1408
3148
|
const creds = (await authenticate({
|
|
1409
|
-
clientId:
|
|
1410
|
-
clientSecret:
|
|
3149
|
+
clientId: envCid,
|
|
3150
|
+
clientSecret: envSec
|
|
1411
3151
|
}, false)).credentials;
|
|
1412
3152
|
if (creds.access_token) await saveTokens({
|
|
1413
3153
|
access_token: creds.access_token,
|
|
1414
|
-
refresh_token: creds.refresh_token ||
|
|
3154
|
+
refresh_token: creds.refresh_token || envRef,
|
|
1415
3155
|
expiry_date: creds.expiry_date
|
|
1416
3156
|
});
|
|
1417
3157
|
console.log();
|
|
@@ -1422,49 +3162,239 @@ const initCommand = defineCommand({
|
|
|
1422
3162
|
console.log(" \x1B[1mWelcome to GSCDump!\x1B[0m");
|
|
1423
3163
|
console.log(" \x1B[90mGoogle Search Console data extraction CLI\x1B[0m");
|
|
1424
3164
|
console.log();
|
|
1425
|
-
const dataDir = await promptDataDir(config.dataDir);
|
|
3165
|
+
const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
|
|
1426
3166
|
const credentials = await getAuthCredentials(true);
|
|
1427
3167
|
await saveConfig({
|
|
1428
3168
|
...config,
|
|
1429
|
-
dataDir,
|
|
3169
|
+
...dataDir ? { dataDir } : {},
|
|
1430
3170
|
clientId: credentials.clientId,
|
|
1431
3171
|
clientSecret: credentials.clientSecret
|
|
1432
3172
|
});
|
|
1433
|
-
await authenticate(credentials, true);
|
|
3173
|
+
await smokeTest(await authenticate(credentials, true));
|
|
3174
|
+
await maybeWriteEnvFile(credentials.clientId, credentials.clientSecret);
|
|
1434
3175
|
console.log();
|
|
1435
3176
|
logger.success("Setup complete! Run gscdump to get started.");
|
|
1436
3177
|
}
|
|
1437
3178
|
});
|
|
3179
|
+
async function smokeTest(oauth) {
|
|
3180
|
+
const sites = await googleSearchConsole(oauth).sites().catch((e) => e);
|
|
3181
|
+
if (sites instanceof Error) {
|
|
3182
|
+
logger.warn(`Smoke test failed: ${sites.message}`);
|
|
3183
|
+
logger.info("Auth saved, but verify scopes via `gscdump auth status` / `gscdump doctor`.");
|
|
3184
|
+
return;
|
|
3185
|
+
}
|
|
3186
|
+
logger.success(`Verified: ${sites.length} GSC site(s) accessible`);
|
|
3187
|
+
}
|
|
3188
|
+
async function maybeWriteEnvFile(clientId, clientSecret) {
|
|
3189
|
+
const tokens = await loadTokens();
|
|
3190
|
+
if (!tokens?.refresh_token) return;
|
|
3191
|
+
const wants = await confirm({
|
|
3192
|
+
message: "Write a `.env` file with these credentials? (handy for CI / other machines)",
|
|
3193
|
+
initialValue: false
|
|
3194
|
+
});
|
|
3195
|
+
if (isCancel(wants) || !wants) return;
|
|
3196
|
+
const envPath = path.join(process.cwd(), ".env");
|
|
3197
|
+
if (await fs.stat(envPath).then(() => true).catch(() => false)) {
|
|
3198
|
+
const overwrite = await confirm({
|
|
3199
|
+
message: `${displayPath(envPath)} exists. Overwrite?`,
|
|
3200
|
+
initialValue: false
|
|
3201
|
+
});
|
|
3202
|
+
if (isCancel(overwrite) || !overwrite) {
|
|
3203
|
+
logger.info(`Skipped — keep credentials at ${displayPath(envPath)} manually if needed`);
|
|
3204
|
+
return;
|
|
3205
|
+
}
|
|
3206
|
+
}
|
|
3207
|
+
const content = [
|
|
3208
|
+
`GSC_CLIENT_ID=${clientId}`,
|
|
3209
|
+
`GSC_CLIENT_SECRET=${clientSecret}`,
|
|
3210
|
+
`GSC_REFRESH_TOKEN=${tokens.refresh_token}`,
|
|
3211
|
+
""
|
|
3212
|
+
].join("\n");
|
|
3213
|
+
await fs.writeFile(envPath, content, { mode: 384 });
|
|
3214
|
+
logger.success(`Wrote ${displayPath(envPath)}`);
|
|
3215
|
+
}
|
|
3216
|
+
function verdictTone(verdict) {
|
|
3217
|
+
if (verdict === "PASS") return "\x1B[32m";
|
|
3218
|
+
if (verdict === "NEUTRAL" || verdict === "PARTIAL") return "\x1B[33m";
|
|
3219
|
+
if (verdict === "FAIL") return "\x1B[31m";
|
|
3220
|
+
return "\x1B[90m";
|
|
3221
|
+
}
|
|
3222
|
+
function colorVerdict(verdict) {
|
|
3223
|
+
return `${verdictTone(verdict)}${verdict || "N/A"}\x1B[0m`;
|
|
3224
|
+
}
|
|
3225
|
+
function printInspection(url, inspection) {
|
|
3226
|
+
const indexStatus = inspection?.indexStatusResult;
|
|
3227
|
+
console.log();
|
|
3228
|
+
console.log(` \x1B[1mURL:\x1B[0m ${url}`);
|
|
3229
|
+
console.log();
|
|
3230
|
+
console.log(` \x1B[1mIndex status\x1B[0m`);
|
|
3231
|
+
console.log(` Verdict: ${colorVerdict(indexStatus?.verdict)}`);
|
|
3232
|
+
if (indexStatus?.coverageState) console.log(` Coverage: ${indexStatus.coverageState}`);
|
|
3233
|
+
if (indexStatus?.indexingState) console.log(` Indexing: ${indexStatus.indexingState}`);
|
|
3234
|
+
if (indexStatus?.lastCrawlTime) console.log(` Last Crawl: ${indexStatus.lastCrawlTime}`);
|
|
3235
|
+
if (indexStatus?.crawledAs) console.log(` Crawled As: ${indexStatus.crawledAs}`);
|
|
3236
|
+
if (indexStatus?.robotsTxtState) console.log(` Robots.txt: ${indexStatus.robotsTxtState}`);
|
|
3237
|
+
if (indexStatus?.pageFetchState) console.log(` Page Fetch: ${indexStatus.pageFetchState}`);
|
|
3238
|
+
if (indexStatus?.googleCanonical) console.log(` Google Canon: ${indexStatus.googleCanonical}`);
|
|
3239
|
+
if (indexStatus?.userCanonical) console.log(` User Canon: ${indexStatus.userCanonical}`);
|
|
3240
|
+
if (indexStatus?.sitemap?.length) {
|
|
3241
|
+
console.log(` Sitemaps:`);
|
|
3242
|
+
for (const sm of indexStatus.sitemap) console.log(` \x1B[90m└─\x1B[0m ${sm}`);
|
|
3243
|
+
}
|
|
3244
|
+
if (indexStatus?.referringUrls?.length) {
|
|
3245
|
+
const shown = indexStatus.referringUrls.slice(0, 5);
|
|
3246
|
+
console.log(` Referring URLs (${indexStatus.referringUrls.length}):`);
|
|
3247
|
+
for (const r of shown) console.log(` \x1B[90m└─\x1B[0m ${r}`);
|
|
3248
|
+
if (indexStatus.referringUrls.length > shown.length) console.log(` \x1B[90m… ${indexStatus.referringUrls.length - shown.length} more\x1B[0m`);
|
|
3249
|
+
}
|
|
3250
|
+
const rich = inspection?.richResultsResult;
|
|
3251
|
+
if (rich) {
|
|
3252
|
+
console.log();
|
|
3253
|
+
console.log(` \x1B[1mRich results\x1B[0m`);
|
|
3254
|
+
console.log(` Verdict: ${colorVerdict(rich.verdict)}`);
|
|
3255
|
+
if (rich.detectedItems?.length) for (const group of rich.detectedItems) {
|
|
3256
|
+
const count = group.items?.length ?? 0;
|
|
3257
|
+
console.log(` ${group.richResultType ?? "unknown"}: ${count} item${count === 1 ? "" : "s"}`);
|
|
3258
|
+
for (const item of group.items ?? []) if (item.issues?.length) for (const issue of item.issues) {
|
|
3259
|
+
const sev = issue.severity === "ERROR" ? "\x1B[31m" : "\x1B[33m";
|
|
3260
|
+
console.log(` ${sev}${issue.severity ?? "?"}\x1B[0m ${item.name ?? ""}: ${issue.issueMessage ?? ""}`);
|
|
3261
|
+
}
|
|
3262
|
+
}
|
|
3263
|
+
}
|
|
3264
|
+
const amp = inspection?.ampResult;
|
|
3265
|
+
if (amp) {
|
|
3266
|
+
console.log();
|
|
3267
|
+
console.log(` \x1B[1mAMP\x1B[0m`);
|
|
3268
|
+
console.log(` Verdict: ${colorVerdict(amp.verdict)}`);
|
|
3269
|
+
if (amp.ampUrl) console.log(` AMP URL: ${amp.ampUrl}`);
|
|
3270
|
+
if (amp.ampIndexStatusVerdict) console.log(` Index Verdict: ${amp.ampIndexStatusVerdict}`);
|
|
3271
|
+
if (amp.indexingState) console.log(` Indexing: ${amp.indexingState}`);
|
|
3272
|
+
if (amp.robotsTxtState) console.log(` Robots.txt: ${amp.robotsTxtState}`);
|
|
3273
|
+
if (amp.pageFetchState) console.log(` Page Fetch: ${amp.pageFetchState}`);
|
|
3274
|
+
if (amp.lastCrawlTime) console.log(` Last Crawl: ${amp.lastCrawlTime}`);
|
|
3275
|
+
if (amp.issues?.length) for (const issue of amp.issues) {
|
|
3276
|
+
const sev = issue.severity === "ERROR" ? "\x1B[31m" : "\x1B[33m";
|
|
3277
|
+
console.log(` ${sev}${issue.severity ?? "?"}\x1B[0m ${issue.issueMessage ?? ""}`);
|
|
3278
|
+
}
|
|
3279
|
+
}
|
|
3280
|
+
const mobile = inspection?.mobileUsabilityResult;
|
|
3281
|
+
if (mobile) {
|
|
3282
|
+
console.log();
|
|
3283
|
+
console.log(` \x1B[1mMobile usability\x1B[0m`);
|
|
3284
|
+
console.log(` Verdict: ${colorVerdict(mobile.verdict)}`);
|
|
3285
|
+
if (mobile.issues?.length) for (const issue of mobile.issues) console.log(` \x1B[33m${issue.issueType ?? "?"}\x1B[0m ${issue.message ?? ""}`);
|
|
3286
|
+
}
|
|
3287
|
+
if (inspection?.inspectionResultLink) {
|
|
3288
|
+
console.log();
|
|
3289
|
+
console.log(` \x1B[90mOpen in Search Console:\x1B[0m \x1B[36m${inspection.inspectionResultLink}\x1B[0m`);
|
|
3290
|
+
}
|
|
3291
|
+
console.log();
|
|
3292
|
+
}
|
|
3293
|
+
const batchCommand = defineCommand({
|
|
3294
|
+
meta: {
|
|
3295
|
+
name: "batch",
|
|
3296
|
+
description: "Inspect many URLs from a file or stdin (one URL per line)"
|
|
3297
|
+
},
|
|
3298
|
+
args: {
|
|
3299
|
+
...OUTPUT_ARGS,
|
|
3300
|
+
"site": {
|
|
3301
|
+
type: "string",
|
|
3302
|
+
alias: "s",
|
|
3303
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
3304
|
+
},
|
|
3305
|
+
"urls": {
|
|
3306
|
+
type: "positional",
|
|
3307
|
+
required: false,
|
|
3308
|
+
description: "URLs (or use --file/--from-sitemap/stdin)"
|
|
3309
|
+
},
|
|
3310
|
+
"file": {
|
|
3311
|
+
type: "string",
|
|
3312
|
+
alias: "f",
|
|
3313
|
+
description: "File with URLs (one per line)"
|
|
3314
|
+
},
|
|
3315
|
+
"from-sitemap": {
|
|
3316
|
+
type: "string",
|
|
3317
|
+
description: "Sitemap URL (or sitemap index) to pull URLs from"
|
|
3318
|
+
},
|
|
3319
|
+
"delay-ms": {
|
|
3320
|
+
type: "string",
|
|
3321
|
+
default: "200",
|
|
3322
|
+
description: "Delay between requests"
|
|
3323
|
+
},
|
|
3324
|
+
"concurrency": {
|
|
3325
|
+
type: "string",
|
|
3326
|
+
alias: "c",
|
|
3327
|
+
default: "1",
|
|
3328
|
+
description: "Concurrent in-flight requests"
|
|
3329
|
+
}
|
|
3330
|
+
},
|
|
3331
|
+
async run({ args }) {
|
|
3332
|
+
const { json, quiet } = applyOutputMode(args);
|
|
3333
|
+
const urls = args["from-sitemap"] ? await fetchSitemapUrls(String(args["from-sitemap"])).catch((e) => {
|
|
3334
|
+
logger.error(`Sitemap fetch failed: ${e.message}`);
|
|
3335
|
+
process.exit(1);
|
|
3336
|
+
}) : await readUrlList$1(args);
|
|
3337
|
+
if (urls.length === 0) {
|
|
3338
|
+
logger.error("No URLs provided. Pass URLs as args, --file, --from-sitemap, or stdin.");
|
|
3339
|
+
process.exit(1);
|
|
3340
|
+
}
|
|
3341
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
3342
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
3343
|
+
const delayMs = Number.parseInt(String(args["delay-ms"]), 10);
|
|
3344
|
+
const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 1);
|
|
3345
|
+
if (!quiet) logger.info(`Inspecting ${urls.length} URLs ...`);
|
|
3346
|
+
const results = await batchInspectUrls(ctx.client, siteUrl, urls, {
|
|
3347
|
+
delayMs,
|
|
3348
|
+
concurrency,
|
|
3349
|
+
onProgress: quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url} ${r.isIndexed ? "PASS" : "FAIL"}`)
|
|
3350
|
+
}).catch(gscErrorHandler);
|
|
3351
|
+
if (json) {
|
|
3352
|
+
const flattened = results.map((r) => {
|
|
3353
|
+
const indexStatus = r.inspection?.indexStatusResult;
|
|
3354
|
+
return {
|
|
3355
|
+
url: r.url,
|
|
3356
|
+
verdict: indexStatus?.verdict || null,
|
|
3357
|
+
coverageState: indexStatus?.coverageState || null,
|
|
3358
|
+
indexingState: indexStatus?.indexingState || null,
|
|
3359
|
+
lastCrawlTime: indexStatus?.lastCrawlTime || null,
|
|
3360
|
+
isIndexed: r.isIndexed,
|
|
3361
|
+
raw: r.inspection
|
|
3362
|
+
};
|
|
3363
|
+
});
|
|
3364
|
+
console.log(JSON.stringify(flattened, null, 2));
|
|
3365
|
+
return;
|
|
3366
|
+
}
|
|
3367
|
+
const indexed = results.filter((r) => r.isIndexed).length;
|
|
3368
|
+
if (!quiet) logger.success(`Inspected ${results.length} URLs (${indexed} indexed, ${results.length - indexed} not)`);
|
|
3369
|
+
}
|
|
3370
|
+
});
|
|
1438
3371
|
const inspectCommand = defineCommand({
|
|
1439
3372
|
meta: {
|
|
1440
3373
|
name: "inspect",
|
|
1441
|
-
description: "Inspect
|
|
3374
|
+
description: "Inspect URL indexing status (single URL; use `inspect batch` for many)"
|
|
1442
3375
|
},
|
|
1443
3376
|
args: {
|
|
3377
|
+
...OUTPUT_ARGS,
|
|
1444
3378
|
site: {
|
|
1445
3379
|
type: "string",
|
|
1446
3380
|
alias: "s",
|
|
1447
|
-
|
|
1448
|
-
description: "Site URL (e.g., sc-domain:example.com)"
|
|
3381
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
1449
3382
|
},
|
|
1450
3383
|
url: {
|
|
1451
3384
|
type: "positional",
|
|
1452
3385
|
required: true,
|
|
1453
3386
|
description: "URL to inspect"
|
|
1454
|
-
},
|
|
1455
|
-
json: {
|
|
1456
|
-
type: "boolean",
|
|
1457
|
-
default: false,
|
|
1458
|
-
description: "Output as JSON"
|
|
1459
3387
|
}
|
|
1460
|
-
},
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
const
|
|
1467
|
-
|
|
3388
|
+
},
|
|
3389
|
+
subCommands: { batch: batchCommand },
|
|
3390
|
+
async run({ args }) {
|
|
3391
|
+
const { json } = applyOutputMode(args);
|
|
3392
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
3393
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
3394
|
+
const result = await ctx.client.inspect(siteUrl, args.url).catch(gscErrorHandler);
|
|
3395
|
+
const inspection = result?.inspectionResult;
|
|
3396
|
+
const indexStatus = inspection?.indexStatusResult;
|
|
3397
|
+
if (json) {
|
|
1468
3398
|
console.log(JSON.stringify({
|
|
1469
3399
|
url: args.url,
|
|
1470
3400
|
verdict: indexStatus?.verdict || null,
|
|
@@ -1476,23 +3406,11 @@ const inspectCommand = defineCommand({
|
|
|
1476
3406
|
}, null, 2));
|
|
1477
3407
|
return;
|
|
1478
3408
|
}
|
|
1479
|
-
|
|
1480
|
-
console.log(` \x1B[1mURL:\x1B[0m ${args.url}`);
|
|
1481
|
-
console.log();
|
|
1482
|
-
const verdictColor = indexStatus?.verdict === "PASS" ? "\x1B[32m" : "\x1B[31m";
|
|
1483
|
-
console.log(` Verdict: ${verdictColor}${indexStatus?.verdict || "N/A"}\x1B[0m`);
|
|
1484
|
-
if (indexStatus?.coverageState) console.log(` Coverage: ${indexStatus.coverageState}`);
|
|
1485
|
-
if (indexStatus?.indexingState) console.log(` Indexing: ${indexStatus.indexingState}`);
|
|
1486
|
-
if (indexStatus?.lastCrawlTime) console.log(` Last Crawl: ${indexStatus.lastCrawlTime}`);
|
|
1487
|
-
if (indexStatus?.robotsTxtState) console.log(` Robots.txt: ${indexStatus.robotsTxtState}`);
|
|
1488
|
-
if (indexStatus?.pageFetchState) console.log(` Page Fetch: ${indexStatus.pageFetchState}`);
|
|
1489
|
-
if (indexStatus?.googleCanonical) console.log(` Google Canon: ${indexStatus.googleCanonical}`);
|
|
1490
|
-
if (indexStatus?.userCanonical) console.log(` User Canon: ${indexStatus.userCanonical}`);
|
|
1491
|
-
console.log();
|
|
3409
|
+
printInspection(args.url, inspection);
|
|
1492
3410
|
}
|
|
1493
3411
|
});
|
|
1494
3412
|
async function checkAuth() {
|
|
1495
|
-
if ((
|
|
3413
|
+
if (resolveBYOK()) return { ok: true };
|
|
1496
3414
|
const config = await loadConfig();
|
|
1497
3415
|
if (!config.clientId && !config.clientSecret) return {
|
|
1498
3416
|
ok: false,
|
|
@@ -1502,7 +3420,8 @@ Run this command to set up authentication:
|
|
|
1502
3420
|
|
|
1503
3421
|
npx @gscdump/cli init
|
|
1504
3422
|
|
|
1505
|
-
Or
|
|
3423
|
+
Or set BYOK env vars: GSC_ACCESS_TOKEN, or GSC_CLIENT_ID + GSC_CLIENT_SECRET + GSC_REFRESH_TOKEN
|
|
3424
|
+
(GOOGLE_* aliases also accepted).
|
|
1506
3425
|
|
|
1507
3426
|
Then restart your MCP client.`
|
|
1508
3427
|
};
|
|
@@ -1512,7 +3431,7 @@ Then restart your MCP client.`
|
|
|
1512
3431
|
|
|
1513
3432
|
Run this command to authenticate:
|
|
1514
3433
|
|
|
1515
|
-
npx @gscdump/cli auth
|
|
3434
|
+
npx @gscdump/cli auth login
|
|
1516
3435
|
|
|
1517
3436
|
Then restart your MCP client.`
|
|
1518
3437
|
};
|
|
@@ -1554,15 +3473,70 @@ const DIM_COLUMNS = {
|
|
|
1554
3473
|
device,
|
|
1555
3474
|
searchAppearance
|
|
1556
3475
|
};
|
|
3476
|
+
const FILTER_DIMS = [
|
|
3477
|
+
"query",
|
|
3478
|
+
"page",
|
|
3479
|
+
"country",
|
|
3480
|
+
"device",
|
|
3481
|
+
"searchAppearance"
|
|
3482
|
+
];
|
|
3483
|
+
const FILTER_COL = {
|
|
3484
|
+
query,
|
|
3485
|
+
page,
|
|
3486
|
+
country,
|
|
3487
|
+
device,
|
|
3488
|
+
searchAppearance
|
|
3489
|
+
};
|
|
3490
|
+
const ALL_SEARCH_TYPES$1 = Object.values(SearchTypes);
|
|
3491
|
+
const DATA_STATES = [
|
|
3492
|
+
"all",
|
|
3493
|
+
"final",
|
|
3494
|
+
"hourly_all"
|
|
3495
|
+
];
|
|
3496
|
+
const AGGREGATION_TYPES = [
|
|
3497
|
+
"auto",
|
|
3498
|
+
"byPage",
|
|
3499
|
+
"byProperty"
|
|
3500
|
+
];
|
|
3501
|
+
function buildFilterFromArg(col, raw) {
|
|
3502
|
+
if (raw.startsWith("!~")) return makeLeaf(col, "notContains", raw.slice(2));
|
|
3503
|
+
if (raw.startsWith("!re:")) return notRegex(col, raw.slice(4));
|
|
3504
|
+
if (raw.startsWith("!")) return makeLeaf(col, "notEquals", raw.slice(1));
|
|
3505
|
+
if (raw.startsWith("~")) return contains(col, raw.slice(1));
|
|
3506
|
+
if (raw.startsWith("re:")) return regex(col, raw.slice(3));
|
|
3507
|
+
if (raw.startsWith("contains:")) return contains(col, raw.slice(9));
|
|
3508
|
+
if (raw.startsWith("eq:")) return eq(col, raw.slice(3));
|
|
3509
|
+
return eq(col, raw);
|
|
3510
|
+
}
|
|
3511
|
+
function makeLeaf(col, operator, value) {
|
|
3512
|
+
return {
|
|
3513
|
+
_constraints: {},
|
|
3514
|
+
_filters: [{
|
|
3515
|
+
dimension: col.dimension,
|
|
3516
|
+
operator,
|
|
3517
|
+
expression: value
|
|
3518
|
+
}]
|
|
3519
|
+
};
|
|
3520
|
+
}
|
|
1557
3521
|
async function runLiveQuery(client, siteUrl, opts) {
|
|
1558
3522
|
const allRows = [];
|
|
1559
3523
|
let startRow = 0;
|
|
3524
|
+
const baseBody = {
|
|
3525
|
+
startDate: opts.startDate,
|
|
3526
|
+
endDate: opts.endDate,
|
|
3527
|
+
dimensions: opts.dimensions,
|
|
3528
|
+
rowLimit: opts.rowLimit
|
|
3529
|
+
};
|
|
3530
|
+
if (opts.searchType) baseBody.searchType = opts.searchType;
|
|
3531
|
+
if (opts.dataState) baseBody.dataState = opts.dataState;
|
|
3532
|
+
if (opts.aggregationType) baseBody.aggregationType = opts.aggregationType;
|
|
3533
|
+
if (opts.dimensionFilter) {
|
|
3534
|
+
const groups = filterToGroups(opts.dimensionFilter);
|
|
3535
|
+
if (groups.length > 0) baseBody.dimensionFilterGroups = groups;
|
|
3536
|
+
}
|
|
1560
3537
|
while (true) {
|
|
1561
3538
|
const rows = ((await client._rawQuery(siteUrl, {
|
|
1562
|
-
|
|
1563
|
-
endDate: opts.endDate,
|
|
1564
|
-
dimensions: opts.dimensions,
|
|
1565
|
-
rowLimit: opts.rowLimit,
|
|
3539
|
+
...baseBody,
|
|
1566
3540
|
startRow
|
|
1567
3541
|
})).rows || []).map((row) => {
|
|
1568
3542
|
const result = {
|
|
@@ -1582,71 +3556,116 @@ async function runLiveQuery(client, siteUrl, opts) {
|
|
|
1582
3556
|
}
|
|
1583
3557
|
return { rows: allRows };
|
|
1584
3558
|
}
|
|
3559
|
+
function filterToGroups(f) {
|
|
3560
|
+
if (f._filters.length === 0) return [];
|
|
3561
|
+
return [{ filters: f._filters.map((leaf) => ({
|
|
3562
|
+
dimension: leaf.dimension,
|
|
3563
|
+
operator: leaf.operator,
|
|
3564
|
+
expression: leaf.expression
|
|
3565
|
+
})) }];
|
|
3566
|
+
}
|
|
1585
3567
|
const queryCommand = defineCommand({
|
|
1586
3568
|
meta: {
|
|
1587
3569
|
name: "query",
|
|
1588
3570
|
description: "Run a search analytics query (local Parquet by default, --live hits GSC API)"
|
|
1589
3571
|
},
|
|
1590
3572
|
args: {
|
|
1591
|
-
site: {
|
|
3573
|
+
"site": {
|
|
1592
3574
|
type: "string",
|
|
1593
3575
|
alias: "s",
|
|
1594
3576
|
description: "Site URL (e.g., sc-domain:example.com)"
|
|
1595
3577
|
},
|
|
1596
|
-
dimensions: {
|
|
3578
|
+
"dimensions": {
|
|
1597
3579
|
type: "string",
|
|
1598
3580
|
alias: "d",
|
|
1599
3581
|
description: `Dimensions: ${DIMENSIONS.join(",")}`
|
|
1600
3582
|
},
|
|
1601
|
-
start: {
|
|
3583
|
+
"start": {
|
|
1602
3584
|
type: "string",
|
|
1603
3585
|
description: "Start date (YYYY-MM-DD)"
|
|
1604
3586
|
},
|
|
1605
|
-
end: {
|
|
3587
|
+
"end": {
|
|
1606
3588
|
type: "string",
|
|
1607
3589
|
description: "End date (YYYY-MM-DD)"
|
|
1608
3590
|
},
|
|
1609
|
-
limit: {
|
|
3591
|
+
"limit": {
|
|
1610
3592
|
type: "string",
|
|
1611
3593
|
alias: "l",
|
|
1612
3594
|
default: "1000",
|
|
1613
3595
|
description: "Max rows (default: 1000)"
|
|
1614
3596
|
},
|
|
1615
|
-
output: {
|
|
3597
|
+
"output": {
|
|
1616
3598
|
type: "string",
|
|
1617
3599
|
alias: "o",
|
|
1618
3600
|
description: "Output file path (default: stdout)"
|
|
1619
3601
|
},
|
|
1620
|
-
format: {
|
|
3602
|
+
"format": {
|
|
1621
3603
|
type: "string",
|
|
1622
3604
|
alias: "f",
|
|
1623
3605
|
default: "json",
|
|
1624
3606
|
description: "Output format: json or csv"
|
|
1625
3607
|
},
|
|
1626
|
-
sql: {
|
|
3608
|
+
"sql": {
|
|
1627
3609
|
type: "string",
|
|
1628
3610
|
description: "Raw DuckDB SQL using {{FILES}} as the file list placeholder (bypasses builder)"
|
|
1629
3611
|
},
|
|
1630
|
-
table: {
|
|
3612
|
+
"table": {
|
|
1631
3613
|
type: "string",
|
|
1632
3614
|
description: "Analytics table for --sql (default: pages)"
|
|
1633
3615
|
},
|
|
1634
|
-
live: {
|
|
3616
|
+
"live": {
|
|
1635
3617
|
type: "boolean",
|
|
1636
3618
|
default: false,
|
|
1637
3619
|
description: "Bypass local store; hit the GSC API directly"
|
|
1638
3620
|
},
|
|
1639
|
-
quiet: {
|
|
3621
|
+
"quiet": {
|
|
1640
3622
|
type: "boolean",
|
|
1641
3623
|
alias: "q",
|
|
1642
3624
|
default: false,
|
|
1643
3625
|
description: "Suppress progress output"
|
|
1644
3626
|
},
|
|
1645
|
-
interactive: {
|
|
3627
|
+
"interactive": {
|
|
1646
3628
|
type: "boolean",
|
|
1647
3629
|
alias: "i",
|
|
1648
3630
|
default: false,
|
|
1649
3631
|
description: "Interactive mode"
|
|
3632
|
+
},
|
|
3633
|
+
"query": {
|
|
3634
|
+
type: "string",
|
|
3635
|
+
description: "Filter by query (prefix: ~contains, !exclude, re:regex, !re:not-regex)"
|
|
3636
|
+
},
|
|
3637
|
+
"page": {
|
|
3638
|
+
type: "string",
|
|
3639
|
+
description: "Filter by page (same prefix syntax as --query)"
|
|
3640
|
+
},
|
|
3641
|
+
"country": {
|
|
3642
|
+
type: "string",
|
|
3643
|
+
description: "Filter by country (ISO-3 lowercase, e.g. usa). Same prefix syntax as --query"
|
|
3644
|
+
},
|
|
3645
|
+
"device": {
|
|
3646
|
+
type: "string",
|
|
3647
|
+
description: "Filter by device (DESKTOP/MOBILE/TABLET). Same prefix syntax"
|
|
3648
|
+
},
|
|
3649
|
+
"search-appearance": {
|
|
3650
|
+
type: "string",
|
|
3651
|
+
description: "Filter by search appearance feature. Same prefix syntax"
|
|
3652
|
+
},
|
|
3653
|
+
"type": {
|
|
3654
|
+
type: "string",
|
|
3655
|
+
description: `Search type (live mode only). One of: ${ALL_SEARCH_TYPES$1.join(",")}`
|
|
3656
|
+
},
|
|
3657
|
+
"data-state": {
|
|
3658
|
+
type: "string",
|
|
3659
|
+
description: `Data state (live mode only). One of: ${DATA_STATES.join(",")} (default: final)`
|
|
3660
|
+
},
|
|
3661
|
+
"aggregation-type": {
|
|
3662
|
+
type: "string",
|
|
3663
|
+
description: `Aggregation type (live mode only). One of: ${AGGREGATION_TYPES.join(",")}`
|
|
3664
|
+
},
|
|
3665
|
+
"explain": {
|
|
3666
|
+
type: "boolean",
|
|
3667
|
+
default: false,
|
|
3668
|
+
description: "Print the request body / planned local SQL and exit without executing"
|
|
1650
3669
|
}
|
|
1651
3670
|
},
|
|
1652
3671
|
async run({ args }) {
|
|
@@ -1660,10 +3679,25 @@ const queryCommand = defineCommand({
|
|
|
1660
3679
|
});
|
|
1661
3680
|
return;
|
|
1662
3681
|
}
|
|
3682
|
+
const ctxConfig = await loadConfig();
|
|
1663
3683
|
const dimNames = await resolveDimensions(args);
|
|
1664
3684
|
const { startDate, endDate } = await resolveRange(args);
|
|
1665
|
-
|
|
3685
|
+
await promptFilters(args);
|
|
3686
|
+
const limitArg = args.limit != null ? String(args.limit) : null;
|
|
3687
|
+
const rowLimit = limitArg != null && limitArg !== "1000" ? Number.parseInt(limitArg, 10) : ctxConfig.defaultLimit ?? 1e3;
|
|
1666
3688
|
const format = String(args.format);
|
|
3689
|
+
const dimensionFilter = buildDimensionFilter(args);
|
|
3690
|
+
const searchType = parseSearchType(args.type ?? ctxConfig.defaultSearchType);
|
|
3691
|
+
const dataState = args["data-state"] ? String(args["data-state"]) : ctxConfig.defaultDataState;
|
|
3692
|
+
const aggregationType = args["aggregation-type"] ? String(args["aggregation-type"]) : void 0;
|
|
3693
|
+
if (dataState && !DATA_STATES.includes(dataState)) {
|
|
3694
|
+
logger.error(`Invalid --data-state: ${dataState}. Allowed: ${DATA_STATES.join(", ")}`);
|
|
3695
|
+
process.exit(1);
|
|
3696
|
+
}
|
|
3697
|
+
if (aggregationType && !AGGREGATION_TYPES.includes(aggregationType)) {
|
|
3698
|
+
logger.error(`Invalid --aggregation-type: ${aggregationType}. Allowed: ${AGGREGATION_TYPES.join(", ")}`);
|
|
3699
|
+
process.exit(1);
|
|
3700
|
+
}
|
|
1667
3701
|
const ctx = await createCommandContext({
|
|
1668
3702
|
needsAuth: true,
|
|
1669
3703
|
needsStore: !args.live,
|
|
@@ -1671,16 +3705,34 @@ const queryCommand = defineCommand({
|
|
|
1671
3705
|
});
|
|
1672
3706
|
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1673
3707
|
if (args.live) {
|
|
3708
|
+
if (args.explain) {
|
|
3709
|
+
const body = {
|
|
3710
|
+
startDate,
|
|
3711
|
+
endDate,
|
|
3712
|
+
dimensions: dimNames,
|
|
3713
|
+
rowLimit
|
|
3714
|
+
};
|
|
3715
|
+
if (searchType) body.searchType = searchType;
|
|
3716
|
+
if (dataState) body.dataState = dataState;
|
|
3717
|
+
if (aggregationType) body.aggregationType = aggregationType;
|
|
3718
|
+
if (dimensionFilter) body.dimensionFilterGroups = filterToGroups(dimensionFilter);
|
|
3719
|
+
console.log(JSON.stringify({
|
|
3720
|
+
siteUrl,
|
|
3721
|
+
body
|
|
3722
|
+
}, null, 2));
|
|
3723
|
+
return;
|
|
3724
|
+
}
|
|
1674
3725
|
if (!args.quiet) logger.info(`Querying ${siteUrl} via live GSC API...`);
|
|
1675
3726
|
const result = await runLiveQuery(ctx.client, siteUrl, {
|
|
1676
3727
|
startDate,
|
|
1677
3728
|
endDate,
|
|
1678
3729
|
dimensions: dimNames,
|
|
1679
|
-
rowLimit
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
3730
|
+
rowLimit,
|
|
3731
|
+
searchType,
|
|
3732
|
+
dataState,
|
|
3733
|
+
aggregationType,
|
|
3734
|
+
dimensionFilter
|
|
3735
|
+
}).catch(gscErrorHandler);
|
|
1684
3736
|
await writeOutput({
|
|
1685
3737
|
output: {
|
|
1686
3738
|
siteUrl,
|
|
@@ -1698,10 +3750,23 @@ const queryCommand = defineCommand({
|
|
|
1698
3750
|
});
|
|
1699
3751
|
return;
|
|
1700
3752
|
}
|
|
3753
|
+
if (searchType && searchType !== "web") {
|
|
3754
|
+
logger.error(`--type=${searchType} requires --live (local store query path is web-only).`);
|
|
3755
|
+
process.exit(1);
|
|
3756
|
+
}
|
|
3757
|
+
if (dataState || aggregationType) logger.warn("--data-state / --aggregation-type are ignored without --live");
|
|
1701
3758
|
if (!args.quiet) logger.info(`Querying ${siteUrl} from local Parquet store...`);
|
|
1702
|
-
const state = buildLocalState(dimNames, startDate, endDate, rowLimit);
|
|
3759
|
+
const state = buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter);
|
|
1703
3760
|
const store = ctx.store;
|
|
1704
3761
|
const table = inferTable(dimNames);
|
|
3762
|
+
if (args.explain) {
|
|
3763
|
+
console.log(JSON.stringify({
|
|
3764
|
+
siteUrl,
|
|
3765
|
+
table,
|
|
3766
|
+
state
|
|
3767
|
+
}, null, 2));
|
|
3768
|
+
return;
|
|
3769
|
+
}
|
|
1705
3770
|
await assertRangeCovered(store, siteUrl, table, startDate, endDate);
|
|
1706
3771
|
const result = await store.engine.query({
|
|
1707
3772
|
userId: store.userId,
|
|
@@ -1779,9 +3844,68 @@ async function resolveRange(args) {
|
|
|
1779
3844
|
endDate: daysAgo(3)
|
|
1780
3845
|
};
|
|
1781
3846
|
}
|
|
1782
|
-
function
|
|
3847
|
+
async function promptFilters(args) {
|
|
3848
|
+
if (!args.interactive) return;
|
|
3849
|
+
for (const dim of FILTER_DIMS) {
|
|
3850
|
+
if (args[dim]) continue;
|
|
3851
|
+
const v = await text({
|
|
3852
|
+
message: `Filter by ${dim} (blank to skip; prefix ~ for contains, ! for not-equals, re: for regex)`,
|
|
3853
|
+
placeholder: ""
|
|
3854
|
+
});
|
|
3855
|
+
if (isCancel(v)) {
|
|
3856
|
+
cancel("Cancelled");
|
|
3857
|
+
process.exit(0);
|
|
3858
|
+
}
|
|
3859
|
+
if (v && String(v).length > 0) args[dim] = String(v);
|
|
3860
|
+
}
|
|
3861
|
+
if (!args.type) {
|
|
3862
|
+
const t = await text({
|
|
3863
|
+
message: `Search type (blank for default web; allowed: ${ALL_SEARCH_TYPES$1.join(", ")})`,
|
|
3864
|
+
placeholder: ""
|
|
3865
|
+
});
|
|
3866
|
+
if (isCancel(t)) {
|
|
3867
|
+
cancel("Cancelled");
|
|
3868
|
+
process.exit(0);
|
|
3869
|
+
}
|
|
3870
|
+
if (t && String(t).length > 0) args.type = String(t);
|
|
3871
|
+
}
|
|
3872
|
+
if (!args["data-state"]) {
|
|
3873
|
+
const ds = await text({
|
|
3874
|
+
message: `Data state (blank for default 'final'; allowed: ${DATA_STATES.join(", ")})`,
|
|
3875
|
+
placeholder: ""
|
|
3876
|
+
});
|
|
3877
|
+
if (isCancel(ds)) {
|
|
3878
|
+
cancel("Cancelled");
|
|
3879
|
+
process.exit(0);
|
|
3880
|
+
}
|
|
3881
|
+
if (ds && String(ds).length > 0) args["data-state"] = String(ds);
|
|
3882
|
+
}
|
|
3883
|
+
}
|
|
3884
|
+
function buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter) {
|
|
1783
3885
|
const dims = dimNames.map((d) => DIM_COLUMNS[d]).filter((c) => Boolean(c));
|
|
1784
|
-
|
|
3886
|
+
const dateFilter = between(date, startDate, endDate);
|
|
3887
|
+
const filter = dimensionFilter ? and(dateFilter, dimensionFilter) : dateFilter;
|
|
3888
|
+
return gsc.select(...dims).where(filter).limit(rowLimit).getState();
|
|
3889
|
+
}
|
|
3890
|
+
function buildDimensionFilter(args) {
|
|
3891
|
+
const leaves = [];
|
|
3892
|
+
for (const dim of FILTER_DIMS) {
|
|
3893
|
+
const raw = args[dim];
|
|
3894
|
+
if (raw == null || raw === "") continue;
|
|
3895
|
+
leaves.push(buildFilterFromArg(FILTER_COL[dim], String(raw)));
|
|
3896
|
+
}
|
|
3897
|
+
if (leaves.length === 0) return void 0;
|
|
3898
|
+
if (leaves.length === 1) return leaves[0];
|
|
3899
|
+
return and(...leaves);
|
|
3900
|
+
}
|
|
3901
|
+
function parseSearchType(value) {
|
|
3902
|
+
if (!value) return void 0;
|
|
3903
|
+
const v = String(value);
|
|
3904
|
+
if (!ALL_SEARCH_TYPES$1.includes(v)) {
|
|
3905
|
+
logger.error(`Invalid --type: ${v}. Allowed: ${ALL_SEARCH_TYPES$1.join(", ")}`);
|
|
3906
|
+
process.exit(1);
|
|
3907
|
+
}
|
|
3908
|
+
return v;
|
|
1785
3909
|
}
|
|
1786
3910
|
async function assertRangeCovered(store, siteUrl, table, startDate, endDate) {
|
|
1787
3911
|
const wm = (await store.engine.getWatermarks({
|
|
@@ -1827,14 +3951,14 @@ async function runRawSqlMode(opts) {
|
|
|
1827
3951
|
total: rows.length,
|
|
1828
3952
|
data: rows
|
|
1829
3953
|
}, null, 2);
|
|
1830
|
-
if (opts.output) {
|
|
3954
|
+
if (opts.output && opts.output !== "-") {
|
|
1831
3955
|
await fs.writeFile(opts.output, payload);
|
|
1832
3956
|
if (!opts.quiet) logger.info(`Written to ${opts.output}`);
|
|
1833
3957
|
} else console.log(payload);
|
|
1834
3958
|
}
|
|
1835
3959
|
async function writeOutput(opts) {
|
|
1836
3960
|
const content = opts.format === "csv" ? exportToCSV(opts.output) : JSON.stringify(opts.output, null, 2);
|
|
1837
|
-
if (opts.path) {
|
|
3961
|
+
if (opts.path && opts.path !== "-") {
|
|
1838
3962
|
await fs.writeFile(opts.path, content);
|
|
1839
3963
|
if (!opts.quiet) logger.info(`Written to ${opts.path}`);
|
|
1840
3964
|
} else console.log(content);
|
|
@@ -1842,13 +3966,6 @@ async function writeOutput(opts) {
|
|
|
1842
3966
|
function isKnownTable$1(name) {
|
|
1843
3967
|
return allTables().includes(name);
|
|
1844
3968
|
}
|
|
1845
|
-
function requireSite(target) {
|
|
1846
|
-
if (!target) {
|
|
1847
|
-
logger.error("Site URL required (-s)");
|
|
1848
|
-
process.exit(1);
|
|
1849
|
-
}
|
|
1850
|
-
return target;
|
|
1851
|
-
}
|
|
1852
3969
|
const sitemapsCommand = defineCommand({
|
|
1853
3970
|
meta: {
|
|
1854
3971
|
name: "sitemaps",
|
|
@@ -1861,32 +3978,39 @@ const sitemapsCommand = defineCommand({
|
|
|
1861
3978
|
description: "List sitemaps for a site"
|
|
1862
3979
|
},
|
|
1863
3980
|
args: {
|
|
3981
|
+
...OUTPUT_ARGS,
|
|
1864
3982
|
site: {
|
|
1865
3983
|
type: "string",
|
|
1866
3984
|
alias: "s",
|
|
1867
3985
|
description: "Site URL (e.g., sc-domain:example.com or https://example.com/)"
|
|
1868
3986
|
},
|
|
1869
|
-
|
|
3987
|
+
pending: {
|
|
3988
|
+
type: "boolean",
|
|
3989
|
+
default: false,
|
|
3990
|
+
description: "Show only sitemaps with isPending=true"
|
|
3991
|
+
},
|
|
3992
|
+
errored: {
|
|
1870
3993
|
type: "boolean",
|
|
1871
3994
|
default: false,
|
|
1872
|
-
description: "
|
|
3995
|
+
description: "Show only sitemaps with errors > 0"
|
|
1873
3996
|
}
|
|
1874
3997
|
},
|
|
1875
3998
|
async run({ args }) {
|
|
1876
|
-
const
|
|
1877
|
-
const
|
|
1878
|
-
const
|
|
1879
|
-
|
|
1880
|
-
process.exit(1);
|
|
1881
|
-
})).map((sm) => ({
|
|
3999
|
+
const { json } = applyOutputMode(args);
|
|
4000
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
4001
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
4002
|
+
let sitemaps = (await ctx.client.sitemaps.list(siteUrl).catch(gscErrorHandler)).map((sm) => ({
|
|
1882
4003
|
path: sm.path,
|
|
1883
4004
|
type: sm.type || void 0,
|
|
1884
4005
|
isPending: sm.isPending || false,
|
|
1885
4006
|
errors: Number(sm.errors) || 0,
|
|
1886
4007
|
warnings: Number(sm.warnings) || 0,
|
|
1887
|
-
lastDownloaded: sm.lastDownloaded || null
|
|
4008
|
+
lastDownloaded: sm.lastDownloaded || null,
|
|
4009
|
+
lastSubmitted: sm.lastSubmitted || null
|
|
1888
4010
|
}));
|
|
1889
|
-
if (args.
|
|
4011
|
+
if (args.pending) sitemaps = sitemaps.filter((sm) => sm.isPending);
|
|
4012
|
+
if (args.errored) sitemaps = sitemaps.filter((sm) => sm.errors > 0);
|
|
4013
|
+
if (json) {
|
|
1890
4014
|
console.log(JSON.stringify(sitemaps, null, 2));
|
|
1891
4015
|
return;
|
|
1892
4016
|
}
|
|
@@ -1900,7 +4024,8 @@ const sitemapsCommand = defineCommand({
|
|
|
1900
4024
|
const pending = sm.isPending ? " \x1B[33m(pending)\x1B[0m" : "";
|
|
1901
4025
|
const errors = sm.errors ? ` \x1B[31m${sm.errors} errors\x1B[0m` : "";
|
|
1902
4026
|
const warnings = sm.warnings ? ` \x1B[33m${sm.warnings} warnings\x1B[0m` : "";
|
|
1903
|
-
|
|
4027
|
+
const submitted = sm.lastSubmitted ? ` \x1B[90msubmitted ${sm.lastSubmitted}\x1B[0m` : "";
|
|
4028
|
+
console.log(` ${sm.path}${pending}${errors}${warnings}${submitted}`);
|
|
1904
4029
|
}
|
|
1905
4030
|
}
|
|
1906
4031
|
}),
|
|
@@ -1910,27 +4035,25 @@ const sitemapsCommand = defineCommand({
|
|
|
1910
4035
|
description: "Get details for a specific sitemap"
|
|
1911
4036
|
},
|
|
1912
4037
|
args: {
|
|
4038
|
+
...OUTPUT_ARGS,
|
|
1913
4039
|
site: {
|
|
1914
4040
|
type: "string",
|
|
1915
4041
|
alias: "s",
|
|
1916
|
-
|
|
1917
|
-
description: "Site URL"
|
|
4042
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
1918
4043
|
},
|
|
1919
4044
|
url: {
|
|
1920
4045
|
type: "positional",
|
|
1921
4046
|
required: true,
|
|
1922
4047
|
description: "Sitemap URL"
|
|
1923
|
-
},
|
|
1924
|
-
json: {
|
|
1925
|
-
type: "boolean",
|
|
1926
|
-
default: false,
|
|
1927
|
-
description: "Output as JSON"
|
|
1928
4048
|
}
|
|
1929
4049
|
},
|
|
1930
4050
|
async run({ args }) {
|
|
1931
|
-
const
|
|
1932
|
-
const
|
|
1933
|
-
|
|
4051
|
+
const { json } = applyOutputMode(args);
|
|
4052
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
4053
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
4054
|
+
const client = ctx.client;
|
|
4055
|
+
const sitemap = await fetchSitemap(client, siteUrl, args.url).catch(gscErrorHandler);
|
|
4056
|
+
if (json) {
|
|
1934
4057
|
console.log(JSON.stringify(sitemap, null, 2));
|
|
1935
4058
|
return;
|
|
1936
4059
|
}
|
|
@@ -1955,11 +4078,11 @@ const sitemapsCommand = defineCommand({
|
|
|
1955
4078
|
description: "Submit a sitemap to GSC"
|
|
1956
4079
|
},
|
|
1957
4080
|
args: {
|
|
4081
|
+
...OUTPUT_ARGS,
|
|
1958
4082
|
site: {
|
|
1959
4083
|
type: "string",
|
|
1960
4084
|
alias: "s",
|
|
1961
|
-
|
|
1962
|
-
description: "Site URL"
|
|
4085
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
1963
4086
|
},
|
|
1964
4087
|
url: {
|
|
1965
4088
|
type: "positional",
|
|
@@ -1968,10 +4091,18 @@ const sitemapsCommand = defineCommand({
|
|
|
1968
4091
|
}
|
|
1969
4092
|
},
|
|
1970
4093
|
async run({ args }) {
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
4094
|
+
const { json } = applyOutputMode(args);
|
|
4095
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
4096
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
4097
|
+
await ctx.client.sitemaps.submit(siteUrl, args.url).catch(gscErrorHandler);
|
|
4098
|
+
if (json) {
|
|
4099
|
+
console.log(JSON.stringify({
|
|
4100
|
+
siteUrl,
|
|
4101
|
+
feedpath: args.url,
|
|
4102
|
+
status: "submitted"
|
|
4103
|
+
}, null, 2));
|
|
4104
|
+
return;
|
|
4105
|
+
}
|
|
1975
4106
|
logger.success(`Submitted sitemap: ${args.url}`);
|
|
1976
4107
|
}
|
|
1977
4108
|
}),
|
|
@@ -1981,11 +4112,11 @@ const sitemapsCommand = defineCommand({
|
|
|
1981
4112
|
description: "Delete a sitemap from GSC"
|
|
1982
4113
|
},
|
|
1983
4114
|
args: {
|
|
4115
|
+
...OUTPUT_ARGS,
|
|
1984
4116
|
site: {
|
|
1985
4117
|
type: "string",
|
|
1986
4118
|
alias: "s",
|
|
1987
|
-
|
|
1988
|
-
description: "Site URL"
|
|
4119
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
1989
4120
|
},
|
|
1990
4121
|
url: {
|
|
1991
4122
|
type: "positional",
|
|
@@ -1994,41 +4125,588 @@ const sitemapsCommand = defineCommand({
|
|
|
1994
4125
|
}
|
|
1995
4126
|
},
|
|
1996
4127
|
async run({ args }) {
|
|
1997
|
-
|
|
1998
|
-
|
|
4128
|
+
const { json } = applyOutputMode(args);
|
|
4129
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
4130
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
4131
|
+
await ctx.client.sitemaps.delete(siteUrl, args.url).catch(gscErrorHandler);
|
|
4132
|
+
if (json) {
|
|
4133
|
+
console.log(JSON.stringify({
|
|
4134
|
+
siteUrl,
|
|
4135
|
+
feedpath: args.url,
|
|
4136
|
+
status: "deleted"
|
|
4137
|
+
}, null, 2));
|
|
4138
|
+
return;
|
|
4139
|
+
}
|
|
4140
|
+
logger.success(`Deleted sitemap: ${args.url}`);
|
|
4141
|
+
}
|
|
4142
|
+
}),
|
|
4143
|
+
discover: defineCommand({
|
|
4144
|
+
meta: {
|
|
4145
|
+
name: "discover",
|
|
4146
|
+
description: "Probe a domain's robots.txt + common paths for an advertised sitemap (no auth needed)"
|
|
4147
|
+
},
|
|
4148
|
+
args: {
|
|
4149
|
+
...OUTPUT_ARGS,
|
|
4150
|
+
domain: {
|
|
4151
|
+
type: "positional",
|
|
4152
|
+
required: true,
|
|
4153
|
+
description: "Domain (e.g., example.com)"
|
|
4154
|
+
}
|
|
4155
|
+
},
|
|
4156
|
+
async run({ args }) {
|
|
4157
|
+
const { json } = applyOutputMode(args);
|
|
4158
|
+
const domain = String(args.domain).replace(/^https?:\/\//, "").replace(/\/.*$/, "");
|
|
4159
|
+
const url = await discoverSitemap(domain).catch(() => null);
|
|
4160
|
+
if (json) {
|
|
4161
|
+
console.log(JSON.stringify({
|
|
4162
|
+
domain,
|
|
4163
|
+
sitemap: url
|
|
4164
|
+
}, null, 2));
|
|
4165
|
+
return;
|
|
4166
|
+
}
|
|
4167
|
+
if (!url) {
|
|
4168
|
+
logger.warn(`No sitemap discovered for ${domain}`);
|
|
4169
|
+
process.exit(1);
|
|
4170
|
+
}
|
|
4171
|
+
logger.success(`Discovered sitemap: ${url}`);
|
|
4172
|
+
}
|
|
4173
|
+
}),
|
|
4174
|
+
urls: defineCommand({
|
|
4175
|
+
meta: {
|
|
4176
|
+
name: "urls",
|
|
4177
|
+
description: "Fetch a sitemap (or sitemap index) and dump its <loc> URLs (no auth needed)"
|
|
4178
|
+
},
|
|
4179
|
+
args: {
|
|
4180
|
+
...OUTPUT_ARGS,
|
|
4181
|
+
"url": {
|
|
4182
|
+
type: "positional",
|
|
4183
|
+
required: true,
|
|
4184
|
+
description: "Sitemap URL (index files are followed)"
|
|
4185
|
+
},
|
|
4186
|
+
"limit": {
|
|
4187
|
+
type: "string",
|
|
4188
|
+
alias: "l",
|
|
4189
|
+
description: "Stop after N URLs across all nested sitemaps"
|
|
4190
|
+
},
|
|
4191
|
+
"max-depth": {
|
|
4192
|
+
type: "string",
|
|
4193
|
+
description: "Max sitemap-index nesting depth (default: 3)"
|
|
4194
|
+
}
|
|
4195
|
+
},
|
|
4196
|
+
async run({ args }) {
|
|
4197
|
+
const { json } = applyOutputMode(args);
|
|
4198
|
+
const limit = args.limit ? Number.parseInt(String(args.limit), 10) : void 0;
|
|
4199
|
+
const maxDepth = args["max-depth"] ? Number.parseInt(String(args["max-depth"]), 10) : void 0;
|
|
4200
|
+
const urls = await fetchSitemapUrls(String(args.url), {
|
|
4201
|
+
limit,
|
|
4202
|
+
maxDepth
|
|
4203
|
+
}).catch((e) => {
|
|
4204
|
+
logger.error(`Sitemap fetch failed: ${e.message}`);
|
|
1999
4205
|
process.exit(1);
|
|
2000
4206
|
});
|
|
2001
|
-
|
|
4207
|
+
if (json) {
|
|
4208
|
+
console.log(JSON.stringify({
|
|
4209
|
+
sitemap: args.url,
|
|
4210
|
+
count: urls.length,
|
|
4211
|
+
urls
|
|
4212
|
+
}, null, 2));
|
|
4213
|
+
return;
|
|
4214
|
+
}
|
|
4215
|
+
for (const u of urls) console.log(u);
|
|
4216
|
+
}
|
|
4217
|
+
})
|
|
4218
|
+
}
|
|
4219
|
+
});
|
|
4220
|
+
const ALL_METHODS = [
|
|
4221
|
+
"META",
|
|
4222
|
+
"FILE",
|
|
4223
|
+
"DNS_TXT",
|
|
4224
|
+
"DNS_CNAME",
|
|
4225
|
+
"ANALYTICS",
|
|
4226
|
+
"TAG_MANAGER"
|
|
4227
|
+
];
|
|
4228
|
+
function pickDefaultMethod(siteUrl) {
|
|
4229
|
+
return siteUrl.startsWith("sc-domain:") ? "DNS_TXT" : "META";
|
|
4230
|
+
}
|
|
4231
|
+
function validateMethod(siteUrl, method) {
|
|
4232
|
+
const upper = method.toUpperCase();
|
|
4233
|
+
if (!ALL_METHODS.includes(upper)) {
|
|
4234
|
+
logger.error(`Invalid --method: ${method}. Valid: ${ALL_METHODS.join(", ")}`);
|
|
4235
|
+
process.exit(1);
|
|
4236
|
+
}
|
|
4237
|
+
const site = siteUrlToVerificationSite(siteUrl);
|
|
4238
|
+
const allowed = verificationMethodsFor(site);
|
|
4239
|
+
if (!allowed.includes(upper)) {
|
|
4240
|
+
logger.error(`Method ${upper} not valid for ${site.type === "INET_DOMAIN" ? "domain" : "URL-prefix"} property "${siteUrl}". Valid: ${allowed.join(", ")}`);
|
|
4241
|
+
process.exit(1);
|
|
4242
|
+
}
|
|
4243
|
+
return upper;
|
|
4244
|
+
}
|
|
4245
|
+
function printPlacementInstructions(method, siteUrl, token) {
|
|
4246
|
+
console.log();
|
|
4247
|
+
console.log(` \x1B[1mPlacement instructions (${method})\x1B[0m`);
|
|
4248
|
+
console.log();
|
|
4249
|
+
switch (method) {
|
|
4250
|
+
case "META":
|
|
4251
|
+
console.log(` Add this tag inside the <head> of \x1B[36m${siteUrl}\x1B[0m:`);
|
|
4252
|
+
console.log();
|
|
4253
|
+
console.log(` \x1B[2m<meta name="google-site-verification" content="${token}" />\x1B[0m`);
|
|
4254
|
+
break;
|
|
4255
|
+
case "FILE":
|
|
4256
|
+
console.log(` Upload a file named \x1B[1m${token}\x1B[0m to the site root, accessible at:`);
|
|
4257
|
+
console.log();
|
|
4258
|
+
console.log(` \x1B[36m${siteUrl.replace(/\/?$/, "/")}${token}\x1B[0m`);
|
|
4259
|
+
break;
|
|
4260
|
+
case "DNS_TXT":
|
|
4261
|
+
console.log(` Add a TXT record on \x1B[1m${siteUrl.replace(/^sc-domain:/, "")}\x1B[0m with value:`);
|
|
4262
|
+
console.log();
|
|
4263
|
+
console.log(` \x1B[2m${token}\x1B[0m`);
|
|
4264
|
+
break;
|
|
4265
|
+
case "DNS_CNAME": {
|
|
4266
|
+
const [host, target] = token.split(/\s+/, 2);
|
|
4267
|
+
console.log(` Add a CNAME record:`);
|
|
4268
|
+
console.log();
|
|
4269
|
+
console.log(` \x1B[2mHost: ${host}\x1B[0m`);
|
|
4270
|
+
console.log(` \x1B[2mTarget: ${target ?? "(see token)"}\x1B[0m`);
|
|
4271
|
+
break;
|
|
4272
|
+
}
|
|
4273
|
+
case "ANALYTICS":
|
|
4274
|
+
console.log(` Make sure the Google Analytics tracking tag is installed on the site.`);
|
|
4275
|
+
console.log(` Expected tracking ID:`);
|
|
4276
|
+
console.log();
|
|
4277
|
+
console.log(` \x1B[2m${token}\x1B[0m`);
|
|
4278
|
+
break;
|
|
4279
|
+
case "TAG_MANAGER":
|
|
4280
|
+
console.log(` Make sure the Google Tag Manager container snippet is installed on the site.`);
|
|
4281
|
+
console.log(` Expected container ID:`);
|
|
4282
|
+
console.log();
|
|
4283
|
+
console.log(` \x1B[2m${token}\x1B[0m`);
|
|
4284
|
+
break;
|
|
4285
|
+
}
|
|
4286
|
+
console.log();
|
|
4287
|
+
console.log(` \x1B[90mThen run:\x1B[0m gscdump sites verify ${siteUrl} --method ${method}`);
|
|
4288
|
+
console.log();
|
|
4289
|
+
}
|
|
4290
|
+
const addCommand = defineCommand({
|
|
4291
|
+
meta: {
|
|
4292
|
+
name: "add",
|
|
4293
|
+
description: "Register a property in Search Console (pass --verify to chain token + verify in one call)"
|
|
4294
|
+
},
|
|
4295
|
+
args: {
|
|
4296
|
+
url: {
|
|
4297
|
+
type: "positional",
|
|
4298
|
+
required: true,
|
|
4299
|
+
description: "Property URL (https://example.com/ or sc-domain:example.com)"
|
|
4300
|
+
},
|
|
4301
|
+
verify: {
|
|
4302
|
+
type: "boolean",
|
|
4303
|
+
default: false,
|
|
4304
|
+
description: "After adding, fetch a verification token and trigger Google's validation"
|
|
4305
|
+
},
|
|
4306
|
+
method: {
|
|
4307
|
+
type: "string",
|
|
4308
|
+
alias: "m",
|
|
4309
|
+
description: "Verification method (used with --verify; default: META for URL-prefix, DNS_TXT for sc-domain:)"
|
|
4310
|
+
},
|
|
4311
|
+
...OUTPUT_ARGS
|
|
4312
|
+
},
|
|
4313
|
+
async run({ args }) {
|
|
4314
|
+
applyOutputMode(args);
|
|
4315
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
4316
|
+
await addSite(ctx.client, args.url).catch(gscErrorHandler);
|
|
4317
|
+
if (!args.verify) {
|
|
4318
|
+
if (args.json) {
|
|
4319
|
+
console.log(JSON.stringify({
|
|
4320
|
+
siteUrl: args.url,
|
|
4321
|
+
status: "added",
|
|
4322
|
+
verified: false
|
|
4323
|
+
}, null, 2));
|
|
4324
|
+
return;
|
|
4325
|
+
}
|
|
4326
|
+
logger.success(`Added: ${args.url}`);
|
|
4327
|
+
logger.info(`Property is in unverified state. Verify ownership next:`);
|
|
4328
|
+
const method = pickDefaultMethod(args.url);
|
|
4329
|
+
console.log(` \x1B[2mgscdump sites verify-token ${args.url} --method ${method}\x1B[0m`);
|
|
4330
|
+
return;
|
|
4331
|
+
}
|
|
4332
|
+
const method = validateMethod(args.url, args.method ?? pickDefaultMethod(args.url));
|
|
4333
|
+
const tokenResult = await getVerificationToken(ctx.client, args.url, method).catch(gscErrorHandler);
|
|
4334
|
+
if (args.json) {
|
|
4335
|
+
console.log(JSON.stringify({
|
|
4336
|
+
siteUrl: args.url,
|
|
4337
|
+
status: "added",
|
|
4338
|
+
method,
|
|
4339
|
+
token: tokenResult.token,
|
|
4340
|
+
site: tokenResult.site,
|
|
4341
|
+
verified: false,
|
|
4342
|
+
next: "Place the token, then run `gscdump sites verify <url> --method <m>`"
|
|
4343
|
+
}, null, 2));
|
|
4344
|
+
return;
|
|
4345
|
+
}
|
|
4346
|
+
logger.success(`Added: ${args.url}`);
|
|
4347
|
+
printPlacementInstructions(method, args.url, tokenResult.token);
|
|
4348
|
+
const ok = await confirm({
|
|
4349
|
+
message: "Token placed? Trigger Google verification now?",
|
|
4350
|
+
initialValue: true
|
|
4351
|
+
});
|
|
4352
|
+
if (isCancel(ok) || !ok) {
|
|
4353
|
+
logger.info("Skipped verification. Run `gscdump sites verify` once the token is live.");
|
|
4354
|
+
return;
|
|
4355
|
+
}
|
|
4356
|
+
const resource = await verifySite(ctx.client, args.url, method).catch(gscErrorHandler);
|
|
4357
|
+
logger.success(`Verified: ${args.url}`);
|
|
4358
|
+
if (resource.owners?.length) {
|
|
4359
|
+
console.log();
|
|
4360
|
+
console.log(` Owners:`);
|
|
4361
|
+
for (const o of resource.owners) console.log(` \x1B[90m└─\x1B[0m ${o}`);
|
|
4362
|
+
}
|
|
4363
|
+
}
|
|
4364
|
+
});
|
|
4365
|
+
const deleteCommand = defineCommand({
|
|
4366
|
+
meta: {
|
|
4367
|
+
name: "delete",
|
|
4368
|
+
description: "Remove a property from Search Console"
|
|
4369
|
+
},
|
|
4370
|
+
args: {
|
|
4371
|
+
url: {
|
|
4372
|
+
type: "positional",
|
|
4373
|
+
required: true,
|
|
4374
|
+
description: "Property URL to remove"
|
|
4375
|
+
},
|
|
4376
|
+
yes: {
|
|
4377
|
+
type: "boolean",
|
|
4378
|
+
alias: "y",
|
|
4379
|
+
default: false,
|
|
4380
|
+
description: "Skip confirmation prompt"
|
|
4381
|
+
},
|
|
4382
|
+
...OUTPUT_ARGS
|
|
4383
|
+
},
|
|
4384
|
+
async run({ args }) {
|
|
4385
|
+
applyOutputMode(args);
|
|
4386
|
+
if (!args.yes && !args.json) {
|
|
4387
|
+
const ok = await confirm({
|
|
4388
|
+
message: `Remove ${args.url} from Search Console? Local synced data is unaffected.`,
|
|
4389
|
+
initialValue: false
|
|
4390
|
+
});
|
|
4391
|
+
if (isCancel(ok) || !ok) {
|
|
4392
|
+
logger.info("Cancelled");
|
|
4393
|
+
process.exit(0);
|
|
4394
|
+
}
|
|
4395
|
+
}
|
|
4396
|
+
await deleteSite((await createCommandContext({ needsAuth: true })).client, args.url).catch(gscErrorHandler);
|
|
4397
|
+
if (args.json) {
|
|
4398
|
+
console.log(JSON.stringify({
|
|
4399
|
+
siteUrl: args.url,
|
|
4400
|
+
status: "deleted"
|
|
4401
|
+
}, null, 2));
|
|
4402
|
+
return;
|
|
4403
|
+
}
|
|
4404
|
+
logger.success(`Removed: ${args.url}`);
|
|
4405
|
+
}
|
|
4406
|
+
});
|
|
4407
|
+
const verifyTokenCommand = defineCommand({
|
|
4408
|
+
meta: {
|
|
4409
|
+
name: "verify-token",
|
|
4410
|
+
description: "Get a verification token to place on the site or in DNS"
|
|
4411
|
+
},
|
|
4412
|
+
args: {
|
|
4413
|
+
url: {
|
|
4414
|
+
type: "positional",
|
|
4415
|
+
required: true,
|
|
4416
|
+
description: "Property URL"
|
|
4417
|
+
},
|
|
4418
|
+
method: {
|
|
4419
|
+
type: "string",
|
|
4420
|
+
alias: "m",
|
|
4421
|
+
description: "META, FILE, DNS_TXT, DNS_CNAME, ANALYTICS, TAG_MANAGER (default: META for URL-prefix, DNS_TXT for sc-domain:)"
|
|
4422
|
+
},
|
|
4423
|
+
...OUTPUT_ARGS
|
|
4424
|
+
},
|
|
4425
|
+
async run({ args }) {
|
|
4426
|
+
applyOutputMode(args);
|
|
4427
|
+
const method = validateMethod(args.url, args.method ?? pickDefaultMethod(args.url));
|
|
4428
|
+
const result = await getVerificationToken((await createCommandContext({ needsAuth: true })).client, args.url, method).catch(gscErrorHandler);
|
|
4429
|
+
if (args.json) {
|
|
4430
|
+
console.log(JSON.stringify({
|
|
4431
|
+
siteUrl: args.url,
|
|
4432
|
+
method,
|
|
4433
|
+
token: result.token,
|
|
4434
|
+
site: result.site
|
|
4435
|
+
}, null, 2));
|
|
4436
|
+
return;
|
|
4437
|
+
}
|
|
4438
|
+
printPlacementInstructions(method, args.url, result.token);
|
|
4439
|
+
}
|
|
4440
|
+
});
|
|
4441
|
+
const verifyCommand = defineCommand({
|
|
4442
|
+
meta: {
|
|
4443
|
+
name: "verify",
|
|
4444
|
+
description: "Trigger verification — Google fetches/validates the token you placed"
|
|
4445
|
+
},
|
|
4446
|
+
args: {
|
|
4447
|
+
url: {
|
|
4448
|
+
type: "positional",
|
|
4449
|
+
required: true,
|
|
4450
|
+
description: "Property URL"
|
|
4451
|
+
},
|
|
4452
|
+
method: {
|
|
4453
|
+
type: "string",
|
|
4454
|
+
alias: "m",
|
|
4455
|
+
description: "Verification method to validate (must match the one used for verify-token)"
|
|
4456
|
+
},
|
|
4457
|
+
...OUTPUT_ARGS
|
|
4458
|
+
},
|
|
4459
|
+
async run({ args }) {
|
|
4460
|
+
applyOutputMode(args);
|
|
4461
|
+
const method = validateMethod(args.url, args.method ?? pickDefaultMethod(args.url));
|
|
4462
|
+
const resource = await verifySite((await createCommandContext({ needsAuth: true })).client, args.url, method).catch(gscErrorHandler);
|
|
4463
|
+
if (args.json) {
|
|
4464
|
+
console.log(JSON.stringify({
|
|
4465
|
+
siteUrl: args.url,
|
|
4466
|
+
method,
|
|
4467
|
+
resource
|
|
4468
|
+
}, null, 2));
|
|
4469
|
+
return;
|
|
4470
|
+
}
|
|
4471
|
+
logger.success(`Verified: ${args.url}`);
|
|
4472
|
+
if (resource.owners?.length) {
|
|
4473
|
+
console.log();
|
|
4474
|
+
console.log(` Owners:`);
|
|
4475
|
+
for (const o of resource.owners) console.log(` \x1B[90m└─\x1B[0m ${o}`);
|
|
4476
|
+
}
|
|
4477
|
+
}
|
|
4478
|
+
});
|
|
4479
|
+
const verifyGetCommand = defineCommand({
|
|
4480
|
+
meta: {
|
|
4481
|
+
name: "verify-get",
|
|
4482
|
+
description: "Get a single verified WebResource by id"
|
|
4483
|
+
},
|
|
4484
|
+
args: {
|
|
4485
|
+
id: {
|
|
4486
|
+
type: "positional",
|
|
4487
|
+
required: true,
|
|
4488
|
+
description: "WebResource id (from `sites verify-list`)"
|
|
4489
|
+
},
|
|
4490
|
+
...OUTPUT_ARGS
|
|
4491
|
+
},
|
|
4492
|
+
async run({ args }) {
|
|
4493
|
+
applyOutputMode(args);
|
|
4494
|
+
const resource = await getVerifiedSite((await createCommandContext({ needsAuth: true })).client, args.id).catch(gscErrorHandler);
|
|
4495
|
+
if (args.json) {
|
|
4496
|
+
console.log(JSON.stringify(resource, null, 2));
|
|
4497
|
+
return;
|
|
4498
|
+
}
|
|
4499
|
+
const ident = resource.site?.identifier ?? resource.id ?? "?";
|
|
4500
|
+
const type = resource.site?.type === "INET_DOMAIN" ? "domain" : "site";
|
|
4501
|
+
console.log();
|
|
4502
|
+
console.log(` \x1B[1m${ident}\x1B[0m \x1B[90m(${type})\x1B[0m`);
|
|
4503
|
+
console.log(` \x1B[90mid:\x1B[0m ${resource.id ?? "?"}`);
|
|
4504
|
+
if (resource.owners?.length) {
|
|
4505
|
+
console.log(` Owners:`);
|
|
4506
|
+
for (const o of resource.owners) console.log(` \x1B[90m└─\x1B[0m ${o}`);
|
|
4507
|
+
}
|
|
4508
|
+
console.log();
|
|
4509
|
+
}
|
|
4510
|
+
});
|
|
4511
|
+
const unverifyCommand = defineCommand({
|
|
4512
|
+
meta: {
|
|
4513
|
+
name: "unverify",
|
|
4514
|
+
description: "Drop your verified ownership of a WebResource (remove the placed token first!)"
|
|
4515
|
+
},
|
|
4516
|
+
args: {
|
|
4517
|
+
id: {
|
|
4518
|
+
type: "positional",
|
|
4519
|
+
required: true,
|
|
4520
|
+
description: "WebResource id (from `sites verify-list`)"
|
|
4521
|
+
},
|
|
4522
|
+
yes: {
|
|
4523
|
+
type: "boolean",
|
|
4524
|
+
alias: "y",
|
|
4525
|
+
default: false,
|
|
4526
|
+
description: "Skip confirmation prompt"
|
|
4527
|
+
},
|
|
4528
|
+
...OUTPUT_ARGS
|
|
4529
|
+
},
|
|
4530
|
+
async run({ args }) {
|
|
4531
|
+
applyOutputMode(args);
|
|
4532
|
+
if (!args.yes && !args.json) {
|
|
4533
|
+
const ok = await confirm({
|
|
4534
|
+
message: `Unverify WebResource ${args.id}? Remove any placed verification token first or Google may re-verify.`,
|
|
4535
|
+
initialValue: false
|
|
4536
|
+
});
|
|
4537
|
+
if (isCancel(ok) || !ok) {
|
|
4538
|
+
logger.info("Cancelled");
|
|
4539
|
+
process.exit(0);
|
|
2002
4540
|
}
|
|
2003
|
-
}
|
|
4541
|
+
}
|
|
4542
|
+
await unverifySite((await createCommandContext({ needsAuth: true })).client, args.id).catch(gscErrorHandler);
|
|
4543
|
+
if (args.json) {
|
|
4544
|
+
console.log(JSON.stringify({
|
|
4545
|
+
id: args.id,
|
|
4546
|
+
status: "unverified"
|
|
4547
|
+
}, null, 2));
|
|
4548
|
+
return;
|
|
4549
|
+
}
|
|
4550
|
+
logger.success(`Unverified: ${args.id}`);
|
|
2004
4551
|
}
|
|
2005
4552
|
});
|
|
2006
|
-
const
|
|
4553
|
+
const verifyListCommand = defineCommand({
|
|
2007
4554
|
meta: {
|
|
2008
|
-
name: "
|
|
2009
|
-
description: "List
|
|
4555
|
+
name: "verify-list",
|
|
4556
|
+
description: "List verified WebResources from the Site Verification API (distinct from Search Console properties)"
|
|
4557
|
+
},
|
|
4558
|
+
args: { ...OUTPUT_ARGS },
|
|
4559
|
+
async run({ args }) {
|
|
4560
|
+
applyOutputMode(args);
|
|
4561
|
+
const resources = await listVerifiedSites((await createCommandContext({ needsAuth: true })).client).catch(gscErrorHandler);
|
|
4562
|
+
if (args.json) {
|
|
4563
|
+
console.log(JSON.stringify(resources, null, 2));
|
|
4564
|
+
return;
|
|
4565
|
+
}
|
|
4566
|
+
if (resources.length === 0) {
|
|
4567
|
+
logger.warn("No verified WebResources found");
|
|
4568
|
+
return;
|
|
4569
|
+
}
|
|
4570
|
+
logger.success(`${resources.length} verified WebResources:`);
|
|
4571
|
+
console.log();
|
|
4572
|
+
for (const r of resources) {
|
|
4573
|
+
const id = r.id ?? "?";
|
|
4574
|
+
const site = r.site;
|
|
4575
|
+
const ident = site?.identifier ?? id;
|
|
4576
|
+
const type = site?.type === "INET_DOMAIN" ? "domain" : "site";
|
|
4577
|
+
console.log(` \x1B[1m${ident}\x1B[0m \x1B[90m(${type})\x1B[0m`);
|
|
4578
|
+
if (r.owners?.length) for (const o of r.owners) console.log(` \x1B[90m└─\x1B[0m ${o}`);
|
|
4579
|
+
}
|
|
4580
|
+
}
|
|
4581
|
+
});
|
|
4582
|
+
const getCommand = defineCommand({
|
|
4583
|
+
meta: {
|
|
4584
|
+
name: "get",
|
|
4585
|
+
description: "Show a single property's permissionLevel from the sites list"
|
|
4586
|
+
},
|
|
4587
|
+
args: {
|
|
4588
|
+
url: {
|
|
4589
|
+
type: "positional",
|
|
4590
|
+
required: true,
|
|
4591
|
+
description: "Property URL"
|
|
4592
|
+
},
|
|
4593
|
+
...OUTPUT_ARGS
|
|
4594
|
+
},
|
|
4595
|
+
async run({ args }) {
|
|
4596
|
+
applyOutputMode(args);
|
|
4597
|
+
const site = (await (await createCommandContext({ needsAuth: true })).loadSites()).find((s) => s.siteUrl === args.url);
|
|
4598
|
+
if (!site) {
|
|
4599
|
+
if (args.json) {
|
|
4600
|
+
console.log(JSON.stringify(null));
|
|
4601
|
+
process.exit(1);
|
|
4602
|
+
}
|
|
4603
|
+
logger.error(`Not found: ${args.url}`);
|
|
4604
|
+
process.exit(1);
|
|
4605
|
+
}
|
|
4606
|
+
if (args.json) {
|
|
4607
|
+
console.log(JSON.stringify(site, null, 2));
|
|
4608
|
+
return;
|
|
4609
|
+
}
|
|
4610
|
+
const perm = site.permissionLevel === "siteOwner" ? "\x1B[32m" : "\x1B[90m";
|
|
4611
|
+
console.log();
|
|
4612
|
+
console.log(` \x1B[1m${site.siteUrl}\x1B[0m`);
|
|
4613
|
+
console.log(` Permission: ${perm}${site.permissionLevel}\x1B[0m`);
|
|
4614
|
+
console.log();
|
|
4615
|
+
}
|
|
4616
|
+
});
|
|
4617
|
+
const LIST_ARGS = {
|
|
4618
|
+
...OUTPUT_ARGS,
|
|
4619
|
+
"with-sitemaps": {
|
|
4620
|
+
type: "boolean",
|
|
4621
|
+
default: false,
|
|
4622
|
+
description: "Include sitemaps for each owned site"
|
|
2010
4623
|
},
|
|
2011
|
-
|
|
4624
|
+
"owner-only": {
|
|
2012
4625
|
type: "boolean",
|
|
2013
4626
|
default: false,
|
|
2014
|
-
description: "
|
|
2015
|
-
}
|
|
2016
|
-
|
|
2017
|
-
|
|
4627
|
+
description: "Filter to permissionLevel=siteOwner"
|
|
4628
|
+
}
|
|
4629
|
+
};
|
|
4630
|
+
async function runListSites(args) {
|
|
4631
|
+
applyOutputMode(args);
|
|
4632
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
4633
|
+
const ownerOnly = Boolean(args["owner-only"]);
|
|
4634
|
+
if (args["with-sitemaps"]) {
|
|
4635
|
+
const all = await fetchSitesWithSitemaps(ctx.client).catch(gscErrorHandler);
|
|
4636
|
+
const sites = ownerOnly ? all.filter((s) => s.permissionLevel === "siteOwner") : all;
|
|
2018
4637
|
if (args.json) {
|
|
2019
|
-
|
|
4638
|
+
const enriched = sites.map((s) => ({
|
|
4639
|
+
...s,
|
|
4640
|
+
sitemapCounts: {
|
|
4641
|
+
total: s.sitemaps.length,
|
|
4642
|
+
pending: s.sitemaps.filter((sm) => sm.isPending).length,
|
|
4643
|
+
errored: s.sitemaps.filter((sm) => Number(sm.errors) > 0).length
|
|
4644
|
+
}
|
|
4645
|
+
}));
|
|
4646
|
+
console.log(JSON.stringify(enriched, null, 2));
|
|
2020
4647
|
return;
|
|
2021
4648
|
}
|
|
2022
4649
|
if (sites.length === 0) {
|
|
2023
|
-
logger.warn("No verified sites found");
|
|
4650
|
+
logger.warn(ownerOnly ? "No owned sites found" : "No verified sites found");
|
|
2024
4651
|
return;
|
|
2025
4652
|
}
|
|
2026
|
-
logger.success(`Found ${sites.length} sites:`);
|
|
4653
|
+
logger.success(`Found ${sites.length} ${ownerOnly ? "owned" : "verified"} sites:`);
|
|
2027
4654
|
console.log();
|
|
2028
4655
|
for (const site of sites) {
|
|
2029
4656
|
const perm = site.permissionLevel === "siteOwner" ? "\x1B[32m" : "\x1B[90m";
|
|
2030
4657
|
console.log(` ${site.siteUrl} ${perm}(${site.permissionLevel})\x1B[0m`);
|
|
4658
|
+
for (const sm of site.sitemaps) {
|
|
4659
|
+
const pending = sm.isPending ? " \x1B[33m(pending)\x1B[0m" : "";
|
|
4660
|
+
console.log(` \x1B[90m└─\x1B[0m ${sm.path}${pending}`);
|
|
4661
|
+
}
|
|
2031
4662
|
}
|
|
4663
|
+
return;
|
|
4664
|
+
}
|
|
4665
|
+
const all = await ctx.loadSites();
|
|
4666
|
+
const sites = ownerOnly ? all.filter((s) => s.permissionLevel === "siteOwner") : all;
|
|
4667
|
+
if (args.json) {
|
|
4668
|
+
console.log(JSON.stringify(sites, null, 2));
|
|
4669
|
+
return;
|
|
4670
|
+
}
|
|
4671
|
+
if (sites.length === 0) {
|
|
4672
|
+
logger.warn(ownerOnly ? "No owned sites found" : "No verified sites found");
|
|
4673
|
+
return;
|
|
4674
|
+
}
|
|
4675
|
+
logger.success(`Found ${sites.length} ${ownerOnly ? "owned " : ""}sites:`);
|
|
4676
|
+
console.log();
|
|
4677
|
+
for (const site of sites) {
|
|
4678
|
+
const perm = site.permissionLevel === "siteOwner" ? "\x1B[32m" : "\x1B[90m";
|
|
4679
|
+
console.log(` ${site.siteUrl} ${perm}(${site.permissionLevel})\x1B[0m`);
|
|
4680
|
+
}
|
|
4681
|
+
}
|
|
4682
|
+
const sitesCommand = defineCommand({
|
|
4683
|
+
meta: {
|
|
4684
|
+
name: "sites",
|
|
4685
|
+
description: "List GSC sites; manage properties (add/delete) and verify ownership"
|
|
4686
|
+
},
|
|
4687
|
+
args: LIST_ARGS,
|
|
4688
|
+
subCommands: {
|
|
4689
|
+
"list": defineCommand({
|
|
4690
|
+
meta: {
|
|
4691
|
+
name: "list",
|
|
4692
|
+
description: "List GSC sites (alias of bare `sites`)"
|
|
4693
|
+
},
|
|
4694
|
+
args: LIST_ARGS,
|
|
4695
|
+
async run({ args }) {
|
|
4696
|
+
await runListSites(args);
|
|
4697
|
+
}
|
|
4698
|
+
}),
|
|
4699
|
+
"add": addCommand,
|
|
4700
|
+
"delete": deleteCommand,
|
|
4701
|
+
"get": getCommand,
|
|
4702
|
+
"verify-token": verifyTokenCommand,
|
|
4703
|
+
"verify": verifyCommand,
|
|
4704
|
+
"verify-list": verifyListCommand,
|
|
4705
|
+
"verify-get": verifyGetCommand,
|
|
4706
|
+
"unverify": unverifyCommand
|
|
4707
|
+
},
|
|
4708
|
+
async run({ args }) {
|
|
4709
|
+
await runListSites(args);
|
|
2032
4710
|
}
|
|
2033
4711
|
});
|
|
2034
4712
|
const compactCommand = defineCommand({
|
|
@@ -2054,21 +4732,51 @@ const compactCommand = defineCommand({
|
|
|
2054
4732
|
type: "string",
|
|
2055
4733
|
description: "Override d30→d90 age threshold in days (default: 90)"
|
|
2056
4734
|
},
|
|
2057
|
-
"
|
|
4735
|
+
"dry-run": {
|
|
2058
4736
|
type: "boolean",
|
|
2059
|
-
alias: "q",
|
|
2060
4737
|
default: false,
|
|
2061
|
-
description: "
|
|
2062
|
-
}
|
|
4738
|
+
description: "Report tier counts per (table, site) without compacting"
|
|
4739
|
+
},
|
|
4740
|
+
...OUTPUT_ARGS
|
|
2063
4741
|
},
|
|
2064
4742
|
async run({ args }) {
|
|
4743
|
+
const { json } = applyOutputMode(args);
|
|
2065
4744
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2066
4745
|
const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
|
|
2067
|
-
const
|
|
4746
|
+
const dryRun = Boolean(args["dry-run"]);
|
|
2068
4747
|
const thresholds = {};
|
|
2069
4748
|
if (args["raw-days"]) thresholds.raw = Number(args["raw-days"]);
|
|
2070
4749
|
if (args["d7-days"]) thresholds.d7 = Number(args["d7-days"]);
|
|
2071
4750
|
if (args["d30-days"]) thresholds.d30 = Number(args["d30-days"]);
|
|
4751
|
+
if (dryRun) {
|
|
4752
|
+
const report = [];
|
|
4753
|
+
for (const table of allTables()) {
|
|
4754
|
+
const bySite = groupBySite(await store.engine.listLive({
|
|
4755
|
+
userId: store.userId,
|
|
4756
|
+
siteId,
|
|
4757
|
+
table
|
|
4758
|
+
}));
|
|
4759
|
+
for (const [s, group] of bySite) report.push({
|
|
4760
|
+
table,
|
|
4761
|
+
siteId: s,
|
|
4762
|
+
...countByTier(group)
|
|
4763
|
+
});
|
|
4764
|
+
}
|
|
4765
|
+
if (json) {
|
|
4766
|
+
console.log(JSON.stringify({
|
|
4767
|
+
thresholds,
|
|
4768
|
+
plan: report
|
|
4769
|
+
}, null, 2));
|
|
4770
|
+
return;
|
|
4771
|
+
}
|
|
4772
|
+
console.log();
|
|
4773
|
+
console.log(` table site raw d7 d30 d90`);
|
|
4774
|
+
for (const r of report) console.log(` ${r.table.padEnd(20)} ${(r.siteId ?? "-").padEnd(20)} ${String(r.raw).padStart(4)} ${String(r.d7).padStart(4)} ${String(r.d30).padStart(4)} ${String(r.d90).padStart(4)}`);
|
|
4775
|
+
console.log();
|
|
4776
|
+
logger.info(`compact --dry-run: ${report.length} (table, site) pair(s) — pass without --dry-run to apply`);
|
|
4777
|
+
return;
|
|
4778
|
+
}
|
|
4779
|
+
const summary = [];
|
|
2072
4780
|
for (const table of allTables()) {
|
|
2073
4781
|
const entries = await store.engine.listLive({
|
|
2074
4782
|
userId: store.userId,
|
|
@@ -2077,17 +4785,56 @@ const compactCommand = defineCommand({
|
|
|
2077
4785
|
});
|
|
2078
4786
|
const siteIds = new Set(entries.map((e) => e.siteId));
|
|
2079
4787
|
for (const targetSite of siteIds) {
|
|
2080
|
-
|
|
4788
|
+
logger.info(`Compacting ${table} [${targetSite ?? "-"}] (raw→d7→d30→d90)`);
|
|
2081
4789
|
await store.engine.compactTiered({
|
|
2082
4790
|
userId: store.userId,
|
|
2083
4791
|
siteId: targetSite,
|
|
2084
4792
|
table
|
|
2085
4793
|
}, thresholds);
|
|
4794
|
+
summary.push({
|
|
4795
|
+
table,
|
|
4796
|
+
siteId: targetSite
|
|
4797
|
+
});
|
|
2086
4798
|
}
|
|
2087
4799
|
}
|
|
2088
|
-
if (
|
|
4800
|
+
if (json) {
|
|
4801
|
+
console.log(JSON.stringify({
|
|
4802
|
+
thresholds,
|
|
4803
|
+
compacted: summary
|
|
4804
|
+
}, null, 2));
|
|
4805
|
+
return;
|
|
4806
|
+
}
|
|
4807
|
+
logger.success(`compact: done`);
|
|
2089
4808
|
}
|
|
2090
4809
|
});
|
|
4810
|
+
function groupBySite(entries) {
|
|
4811
|
+
const m = /* @__PURE__ */ new Map();
|
|
4812
|
+
for (const e of entries) {
|
|
4813
|
+
const arr = m.get(e.siteId) ?? [];
|
|
4814
|
+
arr.push(e);
|
|
4815
|
+
m.set(e.siteId, arr);
|
|
4816
|
+
}
|
|
4817
|
+
return m;
|
|
4818
|
+
}
|
|
4819
|
+
function countByTier(entries) {
|
|
4820
|
+
let raw = 0;
|
|
4821
|
+
let d7 = 0;
|
|
4822
|
+
let d30 = 0;
|
|
4823
|
+
let d90 = 0;
|
|
4824
|
+
for (const e of entries) {
|
|
4825
|
+
const tier = e.tier ?? inferLegacyTier(e) ?? "raw";
|
|
4826
|
+
if (tier === "raw") raw++;
|
|
4827
|
+
else if (tier === "d7") d7++;
|
|
4828
|
+
else if (tier === "d30") d30++;
|
|
4829
|
+
else if (tier === "d90") d90++;
|
|
4830
|
+
}
|
|
4831
|
+
return {
|
|
4832
|
+
raw,
|
|
4833
|
+
d7,
|
|
4834
|
+
d30,
|
|
4835
|
+
d90
|
|
4836
|
+
};
|
|
4837
|
+
}
|
|
2091
4838
|
async function exportToDuckDB(opts) {
|
|
2092
4839
|
const outPath = path.resolve(opts.outPath);
|
|
2093
4840
|
if (opts.force) await rm(outPath, { force: true });
|
|
@@ -2141,9 +4888,11 @@ const exportCommand = defineCommand({
|
|
|
2141
4888
|
type: "boolean",
|
|
2142
4889
|
default: false,
|
|
2143
4890
|
description: "Overwrite the output file if it already exists"
|
|
2144
|
-
}
|
|
4891
|
+
},
|
|
4892
|
+
...OUTPUT_ARGS
|
|
2145
4893
|
},
|
|
2146
4894
|
async run({ args }) {
|
|
4895
|
+
const { json } = applyOutputMode(args);
|
|
2147
4896
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2148
4897
|
const siteId = args.site ? store.siteIdFor(args.site) : void 0;
|
|
2149
4898
|
const result = await exportToDuckDB({
|
|
@@ -2154,12 +4903,16 @@ const exportCommand = defineCommand({
|
|
|
2154
4903
|
outPath: args.out,
|
|
2155
4904
|
force: args.force
|
|
2156
4905
|
});
|
|
4906
|
+
if (json) {
|
|
4907
|
+
console.log(JSON.stringify(result, null, 2));
|
|
4908
|
+
return;
|
|
4909
|
+
}
|
|
2157
4910
|
if (result.tables.length === 0) {
|
|
2158
4911
|
console.log(`\n No data to export. Run \`gscdump sync\` first.`);
|
|
2159
4912
|
return;
|
|
2160
4913
|
}
|
|
2161
4914
|
for (const t of result.tables) console.log(` ${t.table.padEnd(15)} ${String(t.files).padStart(4)} parquet → ${t.table} (${t.rows.toLocaleString()} rows)`);
|
|
2162
|
-
console.log(`\n Exported ${result.tables.length} table(s), ${result.totalRows.toLocaleString()} rows → ${result.outPath}`);
|
|
4915
|
+
console.log(`\n Exported ${result.tables.length} table(s), ${result.totalRows.toLocaleString()} rows → ${displayPath(result.outPath)}`);
|
|
2163
4916
|
console.log(`\n Attach from DuckDB: \x1B[36mATTACH '${result.outPath}' AS gsc (READ_ONLY); SELECT * FROM gsc.pages LIMIT 10;\x1B[0m`);
|
|
2164
4917
|
console.log(` Attach in a browser: use DuckDB-WASM registerFileBuffer + \x1B[36mATTACH 'gsc.duckdb' AS gsc (READ_ONLY)\x1B[0m`);
|
|
2165
4918
|
}
|
|
@@ -2181,23 +4934,60 @@ const gcCommand = defineCommand({
|
|
|
2181
4934
|
alias: "s",
|
|
2182
4935
|
description: "Restrict to a single site (default: all sites)"
|
|
2183
4936
|
},
|
|
2184
|
-
"
|
|
4937
|
+
"dry-run": {
|
|
2185
4938
|
type: "boolean",
|
|
2186
|
-
alias: "q",
|
|
2187
4939
|
default: false,
|
|
2188
|
-
description: "
|
|
2189
|
-
}
|
|
4940
|
+
description: "List retired manifest entries past the grace window without deleting"
|
|
4941
|
+
},
|
|
4942
|
+
...OUTPUT_ARGS
|
|
2190
4943
|
},
|
|
2191
4944
|
async run({ args }) {
|
|
4945
|
+
const { json } = applyOutputMode(args);
|
|
2192
4946
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2193
4947
|
const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
|
|
2194
|
-
const quiet = Boolean(args.quiet);
|
|
2195
4948
|
const graceMs = Number(args["grace-hours"]) * 36e5;
|
|
4949
|
+
if (args["dry-run"]) {
|
|
4950
|
+
const cutoff = Date.now() - graceMs;
|
|
4951
|
+
const candidates = [];
|
|
4952
|
+
for (const table of allTables()) {
|
|
4953
|
+
const all = await store.engine.listAll({
|
|
4954
|
+
userId: store.userId,
|
|
4955
|
+
siteId,
|
|
4956
|
+
table
|
|
4957
|
+
});
|
|
4958
|
+
for (const e of all) if (e.retiredAt && e.retiredAt < cutoff) candidates.push({
|
|
4959
|
+
table,
|
|
4960
|
+
siteId: e.siteId,
|
|
4961
|
+
partition: e.partition,
|
|
4962
|
+
retiredAt: e.retiredAt,
|
|
4963
|
+
objectKey: e.objectKey
|
|
4964
|
+
});
|
|
4965
|
+
}
|
|
4966
|
+
if (json) {
|
|
4967
|
+
console.log(JSON.stringify({
|
|
4968
|
+
graceHours: Number(args["grace-hours"]),
|
|
4969
|
+
candidates
|
|
4970
|
+
}, null, 2));
|
|
4971
|
+
return;
|
|
4972
|
+
}
|
|
4973
|
+
console.log();
|
|
4974
|
+
for (const c of candidates) console.log(` ${c.objectKey} \x1B[90m(retired ${new Date(c.retiredAt).toISOString()})\x1B[0m`);
|
|
4975
|
+
console.log();
|
|
4976
|
+
logger.info(`gc --dry-run: ${candidates.length} retired manifest entry(ies) past ${args["grace-hours"]}h grace; pass without --dry-run to delete`);
|
|
4977
|
+
return;
|
|
4978
|
+
}
|
|
2196
4979
|
const result = await store.engine.gcOrphans({
|
|
2197
4980
|
userId: store.userId,
|
|
2198
4981
|
siteId
|
|
2199
4982
|
}, graceMs);
|
|
2200
|
-
if (
|
|
4983
|
+
if (json) {
|
|
4984
|
+
console.log(JSON.stringify({
|
|
4985
|
+
graceHours: Number(args["grace-hours"]),
|
|
4986
|
+
deleted: result.deleted
|
|
4987
|
+
}, null, 2));
|
|
4988
|
+
return;
|
|
4989
|
+
}
|
|
4990
|
+
logger.success(`gc: deleted ${result.deleted} orphan file(s)`);
|
|
2201
4991
|
}
|
|
2202
4992
|
});
|
|
2203
4993
|
const rollupsCommand = defineCommand({
|
|
@@ -2211,22 +5001,17 @@ const rollupsCommand = defineCommand({
|
|
|
2211
5001
|
description: "Rebuild post-sync rollups (daily totals, weekly totals, top-N tables) for a site"
|
|
2212
5002
|
},
|
|
2213
5003
|
args: {
|
|
5004
|
+
...OUTPUT_ARGS,
|
|
2214
5005
|
site: {
|
|
2215
5006
|
type: "string",
|
|
2216
5007
|
alias: "s",
|
|
2217
5008
|
description: "Restrict to a single site (default: all sites with local data)"
|
|
2218
|
-
},
|
|
2219
|
-
quiet: {
|
|
2220
|
-
type: "boolean",
|
|
2221
|
-
alias: "q",
|
|
2222
|
-
default: false,
|
|
2223
|
-
description: "Suppress progress output"
|
|
2224
5009
|
}
|
|
2225
5010
|
},
|
|
2226
5011
|
async run({ args }) {
|
|
5012
|
+
const { json } = applyOutputMode(args);
|
|
2227
5013
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2228
5014
|
const explicitSiteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
|
|
2229
|
-
const quiet = Boolean(args.quiet);
|
|
2230
5015
|
const allSiteIds = /* @__PURE__ */ new Set();
|
|
2231
5016
|
if (explicitSiteId) allSiteIds.add(explicitSiteId);
|
|
2232
5017
|
else for (const table of allTables()) {
|
|
@@ -2237,12 +5022,17 @@ const rollupsCommand = defineCommand({
|
|
|
2237
5022
|
for (const e of entries) if (e.siteId) allSiteIds.add(e.siteId);
|
|
2238
5023
|
}
|
|
2239
5024
|
if (allSiteIds.size === 0) {
|
|
2240
|
-
|
|
5025
|
+
if (json) console.log(JSON.stringify({
|
|
5026
|
+
sites: [],
|
|
5027
|
+
totalBytes: 0
|
|
5028
|
+
}, null, 2));
|
|
5029
|
+
else logger.warn("No sites with local data. Run `gscdump sync` first.");
|
|
2241
5030
|
return;
|
|
2242
5031
|
}
|
|
5032
|
+
const summary = [];
|
|
2243
5033
|
let totalBytes = 0;
|
|
2244
5034
|
for (const siteId of allSiteIds) {
|
|
2245
|
-
|
|
5035
|
+
logger.info(`Rebuilding rollups for [${siteId}] (${DEFAULT_ROLLUPS.length} rollups)`);
|
|
2246
5036
|
const results = await rebuildRollups({
|
|
2247
5037
|
engine: store.engine,
|
|
2248
5038
|
dataSource: store.dataSource,
|
|
@@ -2252,12 +5042,29 @@ const rollupsCommand = defineCommand({
|
|
|
2252
5042
|
},
|
|
2253
5043
|
defs: DEFAULT_ROLLUPS
|
|
2254
5044
|
});
|
|
5045
|
+
const site = {
|
|
5046
|
+
siteId,
|
|
5047
|
+
rollups: []
|
|
5048
|
+
};
|
|
2255
5049
|
for (const r of results) {
|
|
2256
5050
|
totalBytes += r.bytes;
|
|
2257
|
-
|
|
5051
|
+
site.rollups.push({
|
|
5052
|
+
id: r.id,
|
|
5053
|
+
bytes: r.bytes,
|
|
5054
|
+
objectKey: r.objectKey
|
|
5055
|
+
});
|
|
5056
|
+
if (!json) console.log(` ${r.id.padEnd(20)} ${(r.bytes / 1024).toFixed(1).padStart(8)} KB ${r.objectKey}`);
|
|
2258
5057
|
}
|
|
5058
|
+
summary.push(site);
|
|
5059
|
+
}
|
|
5060
|
+
if (json) {
|
|
5061
|
+
console.log(JSON.stringify({
|
|
5062
|
+
sites: summary,
|
|
5063
|
+
totalBytes
|
|
5064
|
+
}, null, 2));
|
|
5065
|
+
return;
|
|
2259
5066
|
}
|
|
2260
|
-
|
|
5067
|
+
logger.success(`Rebuilt rollups across ${allSiteIds.size} site(s) — total ${(totalBytes / 1024).toFixed(1)} KB`);
|
|
2261
5068
|
}
|
|
2262
5069
|
}) }
|
|
2263
5070
|
});
|
|
@@ -2267,19 +5074,25 @@ const statsCommand = defineCommand({
|
|
|
2267
5074
|
description: "Show row/byte counts per table and on-disk footprint"
|
|
2268
5075
|
},
|
|
2269
5076
|
args: {
|
|
2270
|
-
|
|
2271
|
-
type: "boolean",
|
|
2272
|
-
default: false,
|
|
2273
|
-
description: "Output as JSON"
|
|
2274
|
-
},
|
|
5077
|
+
...OUTPUT_ARGS,
|
|
2275
5078
|
site: {
|
|
2276
5079
|
type: "string",
|
|
2277
5080
|
description: "Limit to one site URL (sc-domain:example.com, https://example.com/, ...)"
|
|
2278
5081
|
}
|
|
2279
5082
|
},
|
|
2280
5083
|
async run({ args }) {
|
|
5084
|
+
const { json } = applyOutputMode(args);
|
|
2281
5085
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2282
|
-
|
|
5086
|
+
let siteId;
|
|
5087
|
+
if (args.site) {
|
|
5088
|
+
const known = await listKnownSiteIds(store);
|
|
5089
|
+
const candidate = store.siteIdFor(args.site);
|
|
5090
|
+
if (!known.has(candidate)) {
|
|
5091
|
+
logger.error(`No local data for --site=${args.site}. Known site IDs: ${known.size === 0 ? "(none — run `gscdump sync` first)" : Array.from(known).join(", ")}`);
|
|
5092
|
+
process.exit(1);
|
|
5093
|
+
}
|
|
5094
|
+
siteId = candidate;
|
|
5095
|
+
}
|
|
2283
5096
|
const perTable = await Promise.all(allTables().map(async (table) => {
|
|
2284
5097
|
const all = await store.engine.listAll({
|
|
2285
5098
|
userId: store.userId,
|
|
@@ -2300,7 +5113,7 @@ const statsCommand = defineCommand({
|
|
|
2300
5113
|
files: 0,
|
|
2301
5114
|
bytes: 0
|
|
2302
5115
|
}));
|
|
2303
|
-
if (
|
|
5116
|
+
if (json) {
|
|
2304
5117
|
const payload = {
|
|
2305
5118
|
dataDir: store.dataDir,
|
|
2306
5119
|
disk,
|
|
@@ -2323,7 +5136,7 @@ const statsCommand = defineCommand({
|
|
|
2323
5136
|
return;
|
|
2324
5137
|
}
|
|
2325
5138
|
console.log();
|
|
2326
|
-
console.log(` \x1B[1m${store.dataDir}\x1B[0m`);
|
|
5139
|
+
console.log(` \x1B[1m${displayPath(store.dataDir)}\x1B[0m`);
|
|
2327
5140
|
console.log(` \x1B[90mDisk: ${disk.files} file(s), ${formatBytes(disk.bytes)}\x1B[0m`);
|
|
2328
5141
|
console.log();
|
|
2329
5142
|
const totalRows = perTable.reduce((acc, t) => acc + sumRows(t.live), 0);
|
|
@@ -2351,6 +5164,17 @@ const statsCommand = defineCommand({
|
|
|
2351
5164
|
console.log();
|
|
2352
5165
|
}
|
|
2353
5166
|
});
|
|
5167
|
+
async function listKnownSiteIds(store) {
|
|
5168
|
+
const ids = /* @__PURE__ */ new Set();
|
|
5169
|
+
for (const table of allTables()) {
|
|
5170
|
+
const entries = await store.engine.listLive({
|
|
5171
|
+
userId: store.userId,
|
|
5172
|
+
table
|
|
5173
|
+
});
|
|
5174
|
+
for (const e of entries) if (e.siteId) ids.add(e.siteId);
|
|
5175
|
+
}
|
|
5176
|
+
return ids;
|
|
5177
|
+
}
|
|
2354
5178
|
function sortWatermarks(ws) {
|
|
2355
5179
|
return [...ws].sort((a, b) => {
|
|
2356
5180
|
if (a.table !== b.table) return a.table.localeCompare(b.table);
|
|
@@ -2375,11 +5199,98 @@ const storeCommand = defineCommand({
|
|
|
2375
5199
|
description: "Manage the local DuckDB/Parquet store"
|
|
2376
5200
|
},
|
|
2377
5201
|
subCommands: {
|
|
2378
|
-
stats: statsCommand,
|
|
2379
|
-
compact: compactCommand,
|
|
2380
|
-
gc: gcCommand,
|
|
2381
|
-
export: exportCommand,
|
|
2382
|
-
rollups: rollupsCommand
|
|
5202
|
+
"stats": statsCommand,
|
|
5203
|
+
"compact": compactCommand,
|
|
5204
|
+
"gc": gcCommand,
|
|
5205
|
+
"export": exportCommand,
|
|
5206
|
+
"rollups": rollupsCommand,
|
|
5207
|
+
"rm-site": defineCommand({
|
|
5208
|
+
meta: {
|
|
5209
|
+
name: "rm-site",
|
|
5210
|
+
description: "Delete every parquet, manifest, watermark, and sync-state record for a single site"
|
|
5211
|
+
},
|
|
5212
|
+
args: {
|
|
5213
|
+
site: {
|
|
5214
|
+
type: "positional",
|
|
5215
|
+
required: true,
|
|
5216
|
+
description: "Site URL (e.g. sc-domain:example.com)"
|
|
5217
|
+
},
|
|
5218
|
+
yes: {
|
|
5219
|
+
type: "boolean",
|
|
5220
|
+
alias: "y",
|
|
5221
|
+
default: false,
|
|
5222
|
+
description: "Skip confirmation prompt"
|
|
5223
|
+
},
|
|
5224
|
+
...OUTPUT_ARGS
|
|
5225
|
+
},
|
|
5226
|
+
async run({ args }) {
|
|
5227
|
+
const { json } = applyOutputMode(args);
|
|
5228
|
+
const store = (await createCommandContext({ needsStore: true })).store;
|
|
5229
|
+
const siteId = store.siteIdFor(String(args.site));
|
|
5230
|
+
if (!args.yes && !json) {
|
|
5231
|
+
const ok = await confirm({
|
|
5232
|
+
message: `Delete ALL local data for ${args.site}? This is irreversible.`,
|
|
5233
|
+
initialValue: false
|
|
5234
|
+
});
|
|
5235
|
+
if (isCancel(ok) || !ok) {
|
|
5236
|
+
logger.info("Cancelled");
|
|
5237
|
+
process.exit(0);
|
|
5238
|
+
}
|
|
5239
|
+
}
|
|
5240
|
+
const result = await store.engine.purgeTenant({
|
|
5241
|
+
userId: store.userId,
|
|
5242
|
+
siteId
|
|
5243
|
+
});
|
|
5244
|
+
if (json) {
|
|
5245
|
+
console.log(JSON.stringify(result, null, 2));
|
|
5246
|
+
return;
|
|
5247
|
+
}
|
|
5248
|
+
logger.success(`Removed local data for ${args.site}`);
|
|
5249
|
+
console.log(` Objects deleted: ${result.objectsDeleted}`);
|
|
5250
|
+
console.log(` Manifest entries: ${result.entriesRemoved}`);
|
|
5251
|
+
console.log(` Watermarks: ${result.watermarksRemoved}`);
|
|
5252
|
+
console.log(` Sync states: ${result.syncStatesRemoved}`);
|
|
5253
|
+
}
|
|
5254
|
+
}),
|
|
5255
|
+
"reset": defineCommand({
|
|
5256
|
+
meta: {
|
|
5257
|
+
name: "reset",
|
|
5258
|
+
description: "Wipe the entire local store (every site, every table). Irreversible."
|
|
5259
|
+
},
|
|
5260
|
+
args: {
|
|
5261
|
+
yes: {
|
|
5262
|
+
type: "boolean",
|
|
5263
|
+
alias: "y",
|
|
5264
|
+
default: false,
|
|
5265
|
+
description: "Skip confirmation prompt"
|
|
5266
|
+
},
|
|
5267
|
+
...OUTPUT_ARGS
|
|
5268
|
+
},
|
|
5269
|
+
async run({ args }) {
|
|
5270
|
+
const { json } = applyOutputMode(args);
|
|
5271
|
+
const store = (await createCommandContext({ needsStore: true })).store;
|
|
5272
|
+
if (!args.yes && !json) {
|
|
5273
|
+
const ok = await confirm({
|
|
5274
|
+
message: `Wipe the entire local store under ${store.dataDir}? This deletes data for ALL sites.`,
|
|
5275
|
+
initialValue: false
|
|
5276
|
+
});
|
|
5277
|
+
if (isCancel(ok) || !ok) {
|
|
5278
|
+
logger.info("Cancelled");
|
|
5279
|
+
process.exit(0);
|
|
5280
|
+
}
|
|
5281
|
+
}
|
|
5282
|
+
const result = await store.engine.purgeTenant({ userId: store.userId });
|
|
5283
|
+
if (json) {
|
|
5284
|
+
console.log(JSON.stringify(result, null, 2));
|
|
5285
|
+
return;
|
|
5286
|
+
}
|
|
5287
|
+
logger.success("Local store reset");
|
|
5288
|
+
console.log(` Objects deleted: ${result.objectsDeleted}`);
|
|
5289
|
+
console.log(` Manifest entries: ${result.entriesRemoved}`);
|
|
5290
|
+
console.log(` Watermarks: ${result.watermarksRemoved}`);
|
|
5291
|
+
console.log(` Sync states: ${result.syncStatesRemoved}`);
|
|
5292
|
+
}
|
|
5293
|
+
})
|
|
2383
5294
|
}
|
|
2384
5295
|
});
|
|
2385
5296
|
const DEFAULT_TABLES = [
|
|
@@ -2555,12 +5466,7 @@ const syncCommand = defineCommand({
|
|
|
2555
5466
|
type: "boolean",
|
|
2556
5467
|
description: "Sync the last 450 days (full GSC history)"
|
|
2557
5468
|
},
|
|
2558
|
-
|
|
2559
|
-
type: "boolean",
|
|
2560
|
-
alias: "q",
|
|
2561
|
-
default: false,
|
|
2562
|
-
description: "Suppress progress output"
|
|
2563
|
-
},
|
|
5469
|
+
...OUTPUT_ARGS,
|
|
2564
5470
|
"force": {
|
|
2565
5471
|
type: "boolean",
|
|
2566
5472
|
default: false,
|
|
@@ -2571,11 +5477,6 @@ const syncCommand = defineCommand({
|
|
|
2571
5477
|
default: false,
|
|
2572
5478
|
description: "Print watermarks + sync-state summary instead of syncing"
|
|
2573
5479
|
},
|
|
2574
|
-
"json": {
|
|
2575
|
-
type: "boolean",
|
|
2576
|
-
default: false,
|
|
2577
|
-
description: "With --status: emit JSON"
|
|
2578
|
-
},
|
|
2579
5480
|
"concurrency": {
|
|
2580
5481
|
type: "string",
|
|
2581
5482
|
alias: "c",
|
|
@@ -2585,11 +5486,22 @@ const syncCommand = defineCommand({
|
|
|
2585
5486
|
type: "boolean",
|
|
2586
5487
|
default: false,
|
|
2587
5488
|
description: "Run tables sequentially (default: run all tables in parallel)"
|
|
5489
|
+
},
|
|
5490
|
+
"retry-failed": {
|
|
5491
|
+
type: "boolean",
|
|
5492
|
+
default: false,
|
|
5493
|
+
description: "Only re-run dates currently in `failed` state (cheaper than --force)"
|
|
5494
|
+
},
|
|
5495
|
+
"dry-run": {
|
|
5496
|
+
type: "boolean",
|
|
5497
|
+
default: false,
|
|
5498
|
+
description: "Print the planned (table, searchType, date) work and exit without hitting the API"
|
|
2588
5499
|
}
|
|
2589
5500
|
},
|
|
2590
5501
|
async run({ args }) {
|
|
5502
|
+
const { json, quiet } = applyOutputMode(args);
|
|
2591
5503
|
if (args.status) {
|
|
2592
|
-
await printSyncStatus(await loadConfig(), args.site ? String(args.site) : void 0,
|
|
5504
|
+
await printSyncStatus(await loadConfig(), args.site ? String(args.site) : void 0, json);
|
|
2593
5505
|
return;
|
|
2594
5506
|
}
|
|
2595
5507
|
const ctx = await createCommandContext({
|
|
@@ -2624,21 +5536,70 @@ const syncCommand = defineCommand({
|
|
|
2624
5536
|
logger.warn(`All requested types (${requestedTypes.join(", ")}) are marked empty for this site. Pass --force-types to re-probe.`);
|
|
2625
5537
|
return;
|
|
2626
5538
|
}
|
|
2627
|
-
if (skippedTypes.length > 0 && !
|
|
5539
|
+
if (skippedTypes.length > 0 && !quiet) logger.info(`Skipping ${skippedTypes.join(", ")} (marked empty for this site; pass --force-types to re-probe).`);
|
|
2628
5540
|
const endDate = args.end ? String(args.end) : daysAgo(DEFAULT_PENDING_DAYS);
|
|
2629
5541
|
let startDate;
|
|
2630
5542
|
if (args.start) startDate = String(args.start);
|
|
2631
5543
|
else if (args.full) startDate = daysAgo(450);
|
|
2632
5544
|
else if (args.days) startDate = daysAgo(Number.parseInt(String(args.days), 10) + DEFAULT_PENDING_DAYS - 1);
|
|
2633
5545
|
else startDate = daysAgo(DEFAULT_PENDING_DAYS + DEFAULT_PENDING_DAYS - 1);
|
|
2634
|
-
|
|
5546
|
+
let dates = getDateRange(startDate, endDate);
|
|
2635
5547
|
if (dates.length === 0) {
|
|
2636
5548
|
logger.error(`No dates to sync (start=${startDate}, end=${endDate})`);
|
|
2637
5549
|
process.exit(1);
|
|
2638
5550
|
}
|
|
2639
5551
|
const store = ctx.store;
|
|
2640
|
-
if (
|
|
2641
|
-
|
|
5552
|
+
if (args["retry-failed"]) {
|
|
5553
|
+
const failedSet = /* @__PURE__ */ new Set();
|
|
5554
|
+
for (const table of tables) for (const type of types) {
|
|
5555
|
+
const states = await store.engine.getSyncStates({
|
|
5556
|
+
userId: store.userId,
|
|
5557
|
+
siteId,
|
|
5558
|
+
table,
|
|
5559
|
+
searchType: type
|
|
5560
|
+
});
|
|
5561
|
+
for (const s of states) if (s.state === "failed" && s.date >= startDate && s.date <= endDate) failedSet.add(s.date);
|
|
5562
|
+
}
|
|
5563
|
+
dates = dates.filter((d) => failedSet.has(d));
|
|
5564
|
+
if (dates.length === 0) {
|
|
5565
|
+
logger.success("No failed dates in range — nothing to retry.");
|
|
5566
|
+
return;
|
|
5567
|
+
}
|
|
5568
|
+
args.force = true;
|
|
5569
|
+
if (!quiet) logger.info(`--retry-failed: ${dates.length} date(s) to retry`);
|
|
5570
|
+
}
|
|
5571
|
+
if (args["dry-run"]) {
|
|
5572
|
+
const plan = [];
|
|
5573
|
+
for (const table of tables) for (const type of types) for (const date of dates) plan.push({
|
|
5574
|
+
table,
|
|
5575
|
+
searchType: type,
|
|
5576
|
+
date
|
|
5577
|
+
});
|
|
5578
|
+
if (json) {
|
|
5579
|
+
console.log(JSON.stringify({
|
|
5580
|
+
siteUrl,
|
|
5581
|
+
range: {
|
|
5582
|
+
start: startDate,
|
|
5583
|
+
end: endDate
|
|
5584
|
+
},
|
|
5585
|
+
tables,
|
|
5586
|
+
types,
|
|
5587
|
+
totalCalls: plan.length,
|
|
5588
|
+
plan
|
|
5589
|
+
}, null, 2));
|
|
5590
|
+
return;
|
|
5591
|
+
}
|
|
5592
|
+
console.log();
|
|
5593
|
+
logger.info(`Plan: ${plan.length} API call(s) for ${siteUrl}`);
|
|
5594
|
+
console.log(` Tables: ${tables.join(", ")}`);
|
|
5595
|
+
console.log(` Types: ${types.join(", ")}`);
|
|
5596
|
+
console.log(` Range: ${startDate} → ${endDate} (${dates.length} days)`);
|
|
5597
|
+
console.log();
|
|
5598
|
+
logger.info("Pass without --dry-run to execute.");
|
|
5599
|
+
return;
|
|
5600
|
+
}
|
|
5601
|
+
if (!quiet) {
|
|
5602
|
+
logger.info(`Syncing ${siteUrl} (${tables.join(", ")}) [${types.join(", ")}] → ${displayPath(store.dataDir)}`);
|
|
2642
5603
|
logger.info(`Range: ${startDate} → ${endDate} (${dates.length} days)`);
|
|
2643
5604
|
}
|
|
2644
5605
|
const concurrency = args.concurrency ? Math.max(1, Number.parseInt(String(args.concurrency), 10) || DEFAULT_CONCURRENCY) : DEFAULT_CONCURRENCY;
|
|
@@ -2654,7 +5615,7 @@ const syncCommand = defineCommand({
|
|
|
2654
5615
|
label
|
|
2655
5616
|
});
|
|
2656
5617
|
}
|
|
2657
|
-
const progress = createProgressTracker(dates.length * jobs.length,
|
|
5618
|
+
const progress = createProgressTracker(dates.length * jobs.length, quiet);
|
|
2658
5619
|
if (serialTables) for (const job of jobs) totals[job.label] = await syncTable(store, siteUrl, job.table, job.type, dates, client, concurrency, args.force, progress);
|
|
2659
5620
|
else {
|
|
2660
5621
|
const results = await Promise.all(jobs.map((job) => syncTable(store, siteUrl, job.table, job.type, dates, client, concurrency, args.force, progress)));
|
|
@@ -2664,7 +5625,7 @@ const syncCommand = defineCommand({
|
|
|
2664
5625
|
}
|
|
2665
5626
|
progress.done();
|
|
2666
5627
|
const seconds = ((Date.now() - start) / 1e3).toFixed(1);
|
|
2667
|
-
if (!
|
|
5628
|
+
if (!quiet) {
|
|
2668
5629
|
logger.success(`Synced ${siteUrl} in ${seconds}s`);
|
|
2669
5630
|
for (const [t, n] of Object.entries(totals)) {
|
|
2670
5631
|
const suffix = [n.skipped > 0 ? `${n.skipped} skipped` : null, n.failed > 0 ? `\x1B[31m${n.failed} failed\x1B[0m` : null].filter(Boolean).join(", ");
|
|
@@ -2693,7 +5654,7 @@ const syncCommand = defineCommand({
|
|
|
2693
5654
|
userId: store.userId,
|
|
2694
5655
|
siteId
|
|
2695
5656
|
}, toMark);
|
|
2696
|
-
if (!
|
|
5657
|
+
if (!quiet) logger.info(`Marked empty for future syncs: ${toMark.join(", ")} (0 rows across ${dates.length} days; pass --force-types to re-probe).`);
|
|
2697
5658
|
}
|
|
2698
5659
|
}
|
|
2699
5660
|
if (forceTypes && emptyTypesDoc.emptyTypes.length > 0) {
|
|
@@ -2704,13 +5665,13 @@ const syncCommand = defineCommand({
|
|
|
2704
5665
|
userId: store.userId,
|
|
2705
5666
|
siteId
|
|
2706
5667
|
}, toClear);
|
|
2707
|
-
if (!
|
|
5668
|
+
if (!quiet) logger.info(`Cleared empty markers for: ${toClear.join(", ")} (re-probe found data).`);
|
|
2708
5669
|
}
|
|
2709
5670
|
}
|
|
2710
5671
|
const noRollups = Boolean(args["no-rollups"]);
|
|
2711
5672
|
const anyRowsSynced = Object.values(totals).some((t) => t.rows > 0);
|
|
2712
5673
|
if (!noRollups && anyRowsSynced) {
|
|
2713
|
-
if (!
|
|
5674
|
+
if (!quiet) logger.info(`Rebuilding rollups for [${siteId}] (${DEFAULT_ROLLUPS.length} rollups)…`);
|
|
2714
5675
|
const rollupStart = Date.now();
|
|
2715
5676
|
const results = await rebuildRollups({
|
|
2716
5677
|
engine: store.engine,
|
|
@@ -2724,7 +5685,7 @@ const syncCommand = defineCommand({
|
|
|
2724
5685
|
logger.warn(`Rollup rebuild failed: ${err.message}`);
|
|
2725
5686
|
return [];
|
|
2726
5687
|
});
|
|
2727
|
-
if (!
|
|
5688
|
+
if (!quiet && results.length > 0) {
|
|
2728
5689
|
const kb = results.reduce((a, r) => a + r.bytes, 0) / 1024;
|
|
2729
5690
|
const ms = Date.now() - rollupStart;
|
|
2730
5691
|
logger.success(`Rebuilt ${results.length} rollup(s) in ${ms}ms — ${kb.toFixed(1)} KB`);
|
|
@@ -2763,7 +5724,7 @@ async function printSyncStatus(config, siteFilter, asJson) {
|
|
|
2763
5724
|
return;
|
|
2764
5725
|
}
|
|
2765
5726
|
console.log();
|
|
2766
|
-
console.log(` \x1B[1m${store.dataDir}\x1B[0m`);
|
|
5727
|
+
console.log(` \x1B[1m${displayPath(store.dataDir)}\x1B[0m`);
|
|
2767
5728
|
if (siteFilter) console.log(` \x1B[90mSite: ${siteFilter}\x1B[0m`);
|
|
2768
5729
|
console.log();
|
|
2769
5730
|
if (watermarks.length === 0) {
|
|
@@ -2794,6 +5755,52 @@ async function printSyncStatus(config, siteFilter, asJson) {
|
|
|
2794
5755
|
}
|
|
2795
5756
|
console.log();
|
|
2796
5757
|
}
|
|
5758
|
+
function shouldShowSplash() {
|
|
5759
|
+
if (!process.stdout.isTTY) return false;
|
|
5760
|
+
const argv = process.argv;
|
|
5761
|
+
if (argv.includes("mcp")) return false;
|
|
5762
|
+
for (const flag of [
|
|
5763
|
+
"--json",
|
|
5764
|
+
"--quiet",
|
|
5765
|
+
"-q",
|
|
5766
|
+
"--version",
|
|
5767
|
+
"-v",
|
|
5768
|
+
"--help",
|
|
5769
|
+
"-h"
|
|
5770
|
+
]) if (argv.includes(flag)) return false;
|
|
5771
|
+
return true;
|
|
5772
|
+
}
|
|
5773
|
+
function applyGlobalArgs() {
|
|
5774
|
+
const argv = process.argv;
|
|
5775
|
+
if (argv.includes("--no-color") || process.env.NO_COLOR) setNoColor(true);
|
|
5776
|
+
const profile = pluckArgValue(argv, "--profile");
|
|
5777
|
+
applyProfileFromCli({
|
|
5778
|
+
configDir: pluckArgValue(argv, "--config-dir") ?? process.env.GSCDUMP_CONFIG_DIR ?? null,
|
|
5779
|
+
profile
|
|
5780
|
+
});
|
|
5781
|
+
if (argv.includes("-v") && !argv.includes("--version")) {
|
|
5782
|
+
const i = argv.indexOf("-v");
|
|
5783
|
+
argv[i] = "--version";
|
|
5784
|
+
}
|
|
5785
|
+
}
|
|
5786
|
+
function pluckArgValue(argv, flag) {
|
|
5787
|
+
for (let i = 0; i < argv.length; i++) {
|
|
5788
|
+
const a = argv[i];
|
|
5789
|
+
if (a === flag && i + 1 < argv.length) {
|
|
5790
|
+
const v = argv[i + 1];
|
|
5791
|
+
argv.splice(i, 2);
|
|
5792
|
+
return v;
|
|
5793
|
+
}
|
|
5794
|
+
if (a.startsWith(`${flag}=`)) {
|
|
5795
|
+
const v = a.slice(flag.length + 1);
|
|
5796
|
+
argv.splice(i, 1);
|
|
5797
|
+
return v;
|
|
5798
|
+
}
|
|
5799
|
+
}
|
|
5800
|
+
return null;
|
|
5801
|
+
}
|
|
5802
|
+
applyGlobalArgs();
|
|
5803
|
+
loadEnvFromCwd();
|
|
2797
5804
|
runMain(defineCommand({
|
|
2798
5805
|
meta: {
|
|
2799
5806
|
name: "gscdump",
|
|
@@ -2809,14 +5816,17 @@ runMain(defineCommand({
|
|
|
2809
5816
|
sync: syncCommand,
|
|
2810
5817
|
store: storeCommand,
|
|
2811
5818
|
inspect: inspectCommand,
|
|
5819
|
+
indexing: indexingCommand,
|
|
2812
5820
|
entities: entitiesCommand,
|
|
2813
5821
|
analyze: analyzeCommand,
|
|
2814
5822
|
auth: authCommand,
|
|
2815
5823
|
config: configCommand,
|
|
5824
|
+
profile: profileCommand,
|
|
5825
|
+
doctor: doctorCommand,
|
|
2816
5826
|
mcp: mcpCommand
|
|
2817
5827
|
},
|
|
2818
5828
|
setup() {
|
|
2819
|
-
if (
|
|
5829
|
+
if (shouldShowSplash()) showSplash();
|
|
2820
5830
|
}
|
|
2821
5831
|
}));
|
|
2822
5832
|
export {};
|