@gscdump/cli 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -14
- package/dist/THIRD-PARTY-LICENSES.md +155 -0
- package/dist/_chunks/config.mjs +45 -0
- package/dist/_chunks/libs/destr.mjs +42 -0
- package/dist/_chunks/libs/node-fetch-native.mjs +4136 -0
- package/dist/_chunks/libs/ofetch.mjs +417 -0
- package/dist/_chunks/rolldown-runtime.mjs +13 -0
- package/dist/index.mjs +2831 -436
- package/package.json +8 -8
- package/LICENSE +0 -21
package/dist/index.mjs
CHANGED
|
@@ -1,27 +1,32 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { t as __exportAll } from "./_chunks/rolldown-runtime.mjs";
|
|
3
|
+
import { t as ofetch } from "./_chunks/libs/ofetch.mjs";
|
|
4
|
+
import { a as loadConfig, c as setConfigDir, i as getConfigPath, n as defaultDataDir, o as resolveDataDir, r as getConfigDir, s as saveConfig } from "./_chunks/config.mjs";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
2
7
|
import process from "node:process";
|
|
3
8
|
import { defineCommand, runMain } from "citty";
|
|
4
9
|
import { defaultAnalyzerRegistry } from "@gscdump/analysis/registry";
|
|
5
10
|
import { AnalyzerCapabilityError, analyzeFromSource, createEngineQuerySource } from "@gscdump/analysis";
|
|
6
11
|
import { createGscApiQuerySource } from "@gscdump/engine-gsc-api";
|
|
7
|
-
import { cancel, isCancel, multiselect, select, text } from "@clack/prompts";
|
|
8
|
-
import { daysAgo, fetchSitemap, formatErrorForCli, getDateRange, googleSearchConsole, progressBar } from "gscdump";
|
|
12
|
+
import { cancel, confirm, isCancel, multiselect, select, text } from "@clack/prompts";
|
|
13
|
+
import { addSite, batchInspectUrls, batchRequestIndexing, createAuth, daysAgo, deleteSite, fetchSitemap, fetchSitesWithSitemaps, formatErrorForCli, getDateRange, getIndexingMetadata, getVerificationToken, googleSearchConsole, progressBar, requestIndexing, siteUrlToVerificationSite, verificationMethodsFor, verifySite } from "gscdump";
|
|
9
14
|
import fs, { readFile, rm } from "node:fs/promises";
|
|
10
15
|
import { createServer } from "node:http";
|
|
11
|
-
import
|
|
12
|
-
import {
|
|
13
|
-
import
|
|
14
|
-
import {
|
|
16
|
+
import { JWT, OAuth2Client } from "google-auth-library";
|
|
17
|
+
import { Buffer } from "node:buffer";
|
|
18
|
+
import fs$1 from "node:fs";
|
|
19
|
+
import { createConsola } from "consola";
|
|
15
20
|
import { createNodeHarness } from "@gscdump/engine-duckdb-node";
|
|
16
21
|
import { TABLE_DIMS, transformGscRow } from "@gscdump/engine/ingest";
|
|
17
22
|
import { allTables, inferTable } from "@gscdump/engine/schema";
|
|
18
|
-
import {
|
|
23
|
+
import { DuckDBInstance } from "@duckdb/node-api";
|
|
24
|
+
import { sqlEscape } from "@gscdump/engine/sql";
|
|
19
25
|
import { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore } from "@gscdump/engine/entities";
|
|
20
26
|
import { createGscMcpServer } from "@gscdump/mcp/server";
|
|
21
27
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
22
|
-
import { SearchTypes, between, country, date, device, gsc, page, query, searchAppearance } from "gscdump/query";
|
|
23
|
-
import {
|
|
24
|
-
import { sqlEscape } from "@gscdump/engine/sql";
|
|
28
|
+
import { SearchTypes, and, between, contains, country, date, device, eq, gsc, notRegex, page, query, regex, searchAppearance } from "gscdump/query";
|
|
29
|
+
import { inferLegacyTier } from "@gscdump/engine";
|
|
25
30
|
import { DEFAULT_ROLLUPS, rebuildRollups } from "@gscdump/analysis/rollups";
|
|
26
31
|
import { filesystemStats } from "@gscdump/engine/filesystem";
|
|
27
32
|
var LocalStoreUnsupportedError = class extends Error {
|
|
@@ -63,42 +68,81 @@ async function runLiveAnalysis(client, siteUrl, params) {
|
|
|
63
68
|
throw e;
|
|
64
69
|
});
|
|
65
70
|
}
|
|
66
|
-
|
|
67
|
-
function
|
|
68
|
-
|
|
71
|
+
const ENV_LINE_RE$1 = /^([^=]+)=(.*)$/;
|
|
72
|
+
function parseEnvFile(envPath) {
|
|
73
|
+
let content;
|
|
74
|
+
try {
|
|
75
|
+
content = fs$1.readFileSync(envPath, "utf-8");
|
|
76
|
+
} catch {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
const env = {};
|
|
80
|
+
for (const line of content.split("\n")) {
|
|
81
|
+
const trimmed = line.trim();
|
|
82
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
83
|
+
const match = trimmed.match(ENV_LINE_RE$1);
|
|
84
|
+
if (!match) continue;
|
|
85
|
+
const key = match[1].trim();
|
|
86
|
+
let value = match[2].trim();
|
|
87
|
+
if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
|
|
88
|
+
env[key] = value;
|
|
89
|
+
}
|
|
90
|
+
return env;
|
|
69
91
|
}
|
|
70
|
-
|
|
71
|
-
|
|
92
|
+
const appliedEnvKeys = /* @__PURE__ */ new Set();
|
|
93
|
+
let loadedEnvPath = null;
|
|
94
|
+
function getAppliedEnvKeys() {
|
|
95
|
+
return appliedEnvKeys;
|
|
72
96
|
}
|
|
73
|
-
function
|
|
74
|
-
return
|
|
97
|
+
function getLoadedEnvPath() {
|
|
98
|
+
return loadedEnvPath;
|
|
75
99
|
}
|
|
76
|
-
function
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
return
|
|
100
|
+
function loadEnvFromCwd() {
|
|
101
|
+
const envPath = path.join(process.cwd(), ".env");
|
|
102
|
+
const parsed = parseEnvFile(envPath);
|
|
103
|
+
if (!parsed) return [];
|
|
104
|
+
loadedEnvPath = envPath;
|
|
105
|
+
const applied = [];
|
|
106
|
+
for (const [key, value] of Object.entries(parsed)) if (process.env[key] === void 0) {
|
|
107
|
+
process.env[key] = value;
|
|
108
|
+
applied.push(key);
|
|
109
|
+
appliedEnvKeys.add(key);
|
|
110
|
+
}
|
|
111
|
+
return applied;
|
|
80
112
|
}
|
|
81
|
-
|
|
82
|
-
|
|
113
|
+
const VERSION = "0.8.1";
|
|
114
|
+
const baseLogger = createConsola({
|
|
115
|
+
stdout: process.stderr,
|
|
116
|
+
stderr: process.stderr
|
|
117
|
+
});
|
|
118
|
+
const logger = baseLogger.withTag("gscdump");
|
|
119
|
+
function setQuiet(quiet) {
|
|
120
|
+
if (quiet) baseLogger.level = 1;
|
|
83
121
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
122
|
+
let colorEnabled = (() => {
|
|
123
|
+
if (process.env.NO_COLOR) return false;
|
|
124
|
+
if (process.argv.includes("--no-color")) return false;
|
|
125
|
+
return Boolean(process.stderr.isTTY) || Boolean(process.env.FORCE_COLOR);
|
|
126
|
+
})();
|
|
127
|
+
const ANSI_RE = /\x1B\[[0-9;]*m/g;
|
|
128
|
+
let stdoutWrapped = false;
|
|
129
|
+
function wrapStdoutForNoColor() {
|
|
130
|
+
if (stdoutWrapped) return;
|
|
131
|
+
stdoutWrapped = true;
|
|
132
|
+
const original = process.stdout.write.bind(process.stdout);
|
|
133
|
+
process.stdout.write = ((chunk, ...rest) => {
|
|
134
|
+
if (typeof chunk === "string") chunk = chunk.replace(ANSI_RE, "");
|
|
135
|
+
else if (chunk instanceof Uint8Array) chunk = Buffer.from(chunk).toString("utf8").replace(ANSI_RE, "");
|
|
136
|
+
return original(chunk, ...rest);
|
|
88
137
|
});
|
|
89
|
-
await fs.writeFile(path.join(configDir, "config.json"), JSON.stringify(config, null, 2), { mode: 384 });
|
|
90
|
-
}
|
|
91
|
-
function getConfigPath() {
|
|
92
|
-
return path.join(configDir, "config.json");
|
|
93
138
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
console.error();
|
|
100
|
-
process.exit(1);
|
|
139
|
+
function setNoColor(disable) {
|
|
140
|
+
if (disable) {
|
|
141
|
+
colorEnabled = false;
|
|
142
|
+
wrapStdoutForNoColor();
|
|
143
|
+
}
|
|
101
144
|
}
|
|
145
|
+
if (!colorEnabled) wrapStdoutForNoColor();
|
|
102
146
|
const gradientColors = [
|
|
103
147
|
(s) => `\x1B[38;2;52;211;153m${s}\x1B[0m`,
|
|
104
148
|
(s) => `\x1B[38;2;45;212;191m${s}\x1B[0m`,
|
|
@@ -120,6 +164,15 @@ function showSplash() {
|
|
|
120
164
|
function clearLine() {
|
|
121
165
|
process.stdout.write("\r\x1B[K");
|
|
122
166
|
}
|
|
167
|
+
function displayPath(absPath) {
|
|
168
|
+
const cwd = process.cwd();
|
|
169
|
+
const home = os.homedir();
|
|
170
|
+
if (absPath === cwd) return ".";
|
|
171
|
+
if (absPath.startsWith(`${cwd}/`)) return absPath.slice(cwd.length + 1);
|
|
172
|
+
if (absPath === home) return "~";
|
|
173
|
+
if (absPath.startsWith(`${home}/`)) return `~/${absPath.slice(home.length + 1)}`;
|
|
174
|
+
return absPath;
|
|
175
|
+
}
|
|
123
176
|
function formatAge(ms) {
|
|
124
177
|
const delta = Date.now() - ms;
|
|
125
178
|
if (delta < 6e4) return "just now";
|
|
@@ -146,6 +199,16 @@ function toCSV(data, columns) {
|
|
|
146
199
|
return str.includes(",") || str.includes("\"") || str.includes("\n") ? `"${str.replace(/"/g, "\"\"")}"` : str;
|
|
147
200
|
}).join(","))].join("\n");
|
|
148
201
|
}
|
|
202
|
+
async function readUrlList$1(args) {
|
|
203
|
+
if (args.file) return (await fs.readFile(String(args.file), "utf-8")).split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
|
|
204
|
+
if (args.urls) return (Array.isArray(args.urls) ? args.urls : [args.urls]).map(String).filter(Boolean);
|
|
205
|
+
if (!process.stdin.isTTY) {
|
|
206
|
+
const chunks = [];
|
|
207
|
+
for await (const chunk of process.stdin) chunks.push(chunk);
|
|
208
|
+
return Buffer.concat(chunks).toString("utf-8").split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
|
|
209
|
+
}
|
|
210
|
+
return [];
|
|
211
|
+
}
|
|
149
212
|
function exportToCSV(output) {
|
|
150
213
|
const sections = [];
|
|
151
214
|
if (output.pages?.data) sections.push(`# Pages\n${toCSV(output.pages.data, [
|
|
@@ -178,6 +241,81 @@ function exportToCSV(output) {
|
|
|
178
241
|
])}`);
|
|
179
242
|
return sections.join("\n\n");
|
|
180
243
|
}
|
|
244
|
+
const SCOPES = [
|
|
245
|
+
"https://www.googleapis.com/auth/webmasters",
|
|
246
|
+
"https://www.googleapis.com/auth/indexing",
|
|
247
|
+
"https://www.googleapis.com/auth/siteverification"
|
|
248
|
+
];
|
|
249
|
+
async function loadServiceAccount(jsonPath) {
|
|
250
|
+
const raw = await fs.readFile(jsonPath, "utf-8");
|
|
251
|
+
const key = JSON.parse(raw);
|
|
252
|
+
if (key.type !== "service_account") throw new Error(`${jsonPath} is not a service-account key (type=${key.type})`);
|
|
253
|
+
return new JWT({
|
|
254
|
+
email: key.client_email,
|
|
255
|
+
key: key.private_key,
|
|
256
|
+
scopes: SCOPES
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
async function resolveServiceAccount(opts = {}) {
|
|
260
|
+
const p = opts.path || process.env.GSC_SERVICE_ACCOUNT_JSON || process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
|
261
|
+
if (!p) return null;
|
|
262
|
+
return loadServiceAccount(p);
|
|
263
|
+
}
|
|
264
|
+
async function authenticateDeviceCode(credentials) {
|
|
265
|
+
const init = await ofetch("https://oauth2.googleapis.com/device/code", {
|
|
266
|
+
method: "POST",
|
|
267
|
+
body: new URLSearchParams({
|
|
268
|
+
client_id: credentials.clientId,
|
|
269
|
+
scope: SCOPES.join(" ")
|
|
270
|
+
})
|
|
271
|
+
}).catch((e) => {
|
|
272
|
+
throw new Error(`Device-code request failed: ${e.message}`);
|
|
273
|
+
});
|
|
274
|
+
console.log();
|
|
275
|
+
console.log(` \x1B[1mDevice-code OAuth\x1B[0m`);
|
|
276
|
+
console.log(` 1. On any device, open: \x1B[36m${init.verification_url}\x1B[0m`);
|
|
277
|
+
console.log(` 2. Enter this code: \x1B[1m${init.user_code}\x1B[0m`);
|
|
278
|
+
console.log(` 3. Approve the requested scopes`);
|
|
279
|
+
console.log();
|
|
280
|
+
logger.info(`Polling for completion (expires in ${Math.floor(init.expires_in / 60)}m)...`);
|
|
281
|
+
const intervalMs = init.interval * 1e3;
|
|
282
|
+
const deadline = Date.now() + init.expires_in * 1e3;
|
|
283
|
+
while (Date.now() < deadline) {
|
|
284
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
285
|
+
const res = await ofetch("https://oauth2.googleapis.com/token", {
|
|
286
|
+
method: "POST",
|
|
287
|
+
body: new URLSearchParams({
|
|
288
|
+
client_id: credentials.clientId,
|
|
289
|
+
client_secret: credentials.clientSecret,
|
|
290
|
+
device_code: init.device_code,
|
|
291
|
+
grant_type: "urn:ietf:params:oauth:grant-type:device_code"
|
|
292
|
+
})
|
|
293
|
+
}).catch((e) => e?.data ?? { error: "request_failed" });
|
|
294
|
+
if (res.access_token) return {
|
|
295
|
+
access_token: res.access_token,
|
|
296
|
+
refresh_token: res.refresh_token,
|
|
297
|
+
expiry_date: res.expires_in ? Date.now() + res.expires_in * 1e3 : void 0
|
|
298
|
+
};
|
|
299
|
+
if (res.error === "authorization_pending" || res.error === "slow_down") continue;
|
|
300
|
+
if (res.error === "access_denied") throw new Error("User denied authorization.");
|
|
301
|
+
if (res.error === "expired_token") throw new Error("Device code expired. Re-run `gscdump auth login --no-browser`.");
|
|
302
|
+
if (res.error) throw new Error(`Device-code poll failed: ${res.error_description || res.error}`);
|
|
303
|
+
}
|
|
304
|
+
throw new Error("Device-code flow timed out.");
|
|
305
|
+
}
|
|
306
|
+
function resolveBYOK(opts = {}) {
|
|
307
|
+
const accessToken = opts.accessToken || process.env.GSC_ACCESS_TOKEN || process.env.GOOGLE_ACCESS_TOKEN;
|
|
308
|
+
const clientId = opts.clientId || process.env.GSC_CLIENT_ID || process.env.GOOGLE_CLIENT_ID;
|
|
309
|
+
const clientSecret = opts.clientSecret || process.env.GSC_CLIENT_SECRET || process.env.GOOGLE_CLIENT_SECRET;
|
|
310
|
+
const refreshToken = opts.refreshToken || process.env.GSC_REFRESH_TOKEN || process.env.GOOGLE_REFRESH_TOKEN;
|
|
311
|
+
if (clientId && clientSecret && refreshToken) return createAuth({
|
|
312
|
+
clientId,
|
|
313
|
+
clientSecret,
|
|
314
|
+
refreshToken
|
|
315
|
+
});
|
|
316
|
+
if (accessToken) return accessToken;
|
|
317
|
+
return null;
|
|
318
|
+
}
|
|
181
319
|
const REDIRECT_URI_RE = /redirect_uri=[^&]+/;
|
|
182
320
|
function getTokensPath() {
|
|
183
321
|
return path.join(getConfigDir(), "tokens.json");
|
|
@@ -200,17 +338,22 @@ async function getAuthCredentials(interactive) {
|
|
|
200
338
|
const envClientId = process.env.GOOGLE_CLIENT_ID;
|
|
201
339
|
const envClientSecret = process.env.GOOGLE_CLIENT_SECRET;
|
|
202
340
|
if (envClientId && envClientSecret) {
|
|
203
|
-
logger.
|
|
341
|
+
logger.info("Using OAuth client from env");
|
|
342
|
+
console.log(` \x1B[90m${envClientId}\x1B[0m`);
|
|
204
343
|
return {
|
|
205
344
|
clientId: envClientId,
|
|
206
345
|
clientSecret: envClientSecret
|
|
207
346
|
};
|
|
208
347
|
}
|
|
209
348
|
const config = await loadConfig();
|
|
210
|
-
if (config.clientId && config.clientSecret)
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
349
|
+
if (config.clientId && config.clientSecret) {
|
|
350
|
+
logger.info(`Using OAuth client from ${displayPath(`${getConfigDir()}/config.json`)}`);
|
|
351
|
+
console.log(` \x1B[90m${config.clientId}\x1B[0m`);
|
|
352
|
+
return {
|
|
353
|
+
clientId: config.clientId,
|
|
354
|
+
clientSecret: config.clientSecret
|
|
355
|
+
};
|
|
356
|
+
}
|
|
214
357
|
if (!interactive) {
|
|
215
358
|
logger.error("GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET required for non-interactive mode");
|
|
216
359
|
process.exit(1);
|
|
@@ -246,25 +389,31 @@ async function getAuthCredentials(interactive) {
|
|
|
246
389
|
async function getAuthCodeViaLoopback(authUrl) {
|
|
247
390
|
return new Promise((resolve, reject) => {
|
|
248
391
|
let resolvedRedirectUri = "";
|
|
249
|
-
|
|
392
|
+
let timeoutId;
|
|
393
|
+
let server;
|
|
394
|
+
const settle = (fn) => {
|
|
395
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
396
|
+
server.closeAllConnections?.();
|
|
397
|
+
server.close();
|
|
398
|
+
fn();
|
|
399
|
+
};
|
|
400
|
+
server = createServer((req, res) => {
|
|
250
401
|
const url = new URL(req.url || "", `http://127.0.0.1`);
|
|
251
402
|
const code = url.searchParams.get("code");
|
|
252
403
|
const error = url.searchParams.get("error");
|
|
253
404
|
if (error) {
|
|
254
405
|
res.writeHead(400, { "Content-Type": "text/html" });
|
|
255
406
|
res.end(`<html><body><h1>Authorization Failed</h1><p>${error}</p><p>You can close this window.</p></body></html>`);
|
|
256
|
-
|
|
257
|
-
reject(/* @__PURE__ */ new Error(`OAuth error: ${error}`));
|
|
407
|
+
settle(() => reject(/* @__PURE__ */ new Error(`OAuth error: ${error}`)));
|
|
258
408
|
return;
|
|
259
409
|
}
|
|
260
410
|
if (code) {
|
|
261
411
|
res.writeHead(200, { "Content-Type": "text/html" });
|
|
262
412
|
res.end(`<html><body><h1>Authorization Successful</h1><p>You can close this window and return to the terminal.</p></body></html>`);
|
|
263
|
-
|
|
264
|
-
resolve({
|
|
413
|
+
settle(() => resolve({
|
|
265
414
|
code,
|
|
266
415
|
redirectUri: resolvedRedirectUri
|
|
267
|
-
});
|
|
416
|
+
}));
|
|
268
417
|
return;
|
|
269
418
|
}
|
|
270
419
|
res.writeHead(400, { "Content-Type": "text/html" });
|
|
@@ -273,7 +422,7 @@ async function getAuthCodeViaLoopback(authUrl) {
|
|
|
273
422
|
server.listen(0, "127.0.0.1", () => {
|
|
274
423
|
const addr = server.address();
|
|
275
424
|
if (!addr || typeof addr === "string") {
|
|
276
|
-
reject(/* @__PURE__ */ new Error("Failed to start local server"));
|
|
425
|
+
settle(() => reject(/* @__PURE__ */ new Error("Failed to start local server")));
|
|
277
426
|
return;
|
|
278
427
|
}
|
|
279
428
|
resolvedRedirectUri = `http://127.0.0.1:${addr.port}`;
|
|
@@ -287,17 +436,16 @@ async function getAuthCodeViaLoopback(authUrl) {
|
|
|
287
436
|
logger.warn("Could not open browser automatically");
|
|
288
437
|
});
|
|
289
438
|
});
|
|
290
|
-
server.on("error", reject);
|
|
291
|
-
setTimeout(() => {
|
|
292
|
-
|
|
293
|
-
reject(/* @__PURE__ */ new Error("Authorization timed out"));
|
|
439
|
+
server.on("error", (err) => settle(() => reject(err)));
|
|
440
|
+
timeoutId = setTimeout(() => {
|
|
441
|
+
settle(() => reject(/* @__PURE__ */ new Error("Authorization timed out")));
|
|
294
442
|
}, 300 * 1e3);
|
|
295
443
|
});
|
|
296
444
|
}
|
|
297
|
-
async function authenticate(credentials, interactive) {
|
|
445
|
+
async function authenticate(credentials, interactive, opts = {}) {
|
|
298
446
|
const oauth2Client = new OAuth2Client(credentials.clientId, credentials.clientSecret, "http://127.0.0.1");
|
|
299
|
-
const envAccessToken = process.env.GOOGLE_ACCESS_TOKEN;
|
|
300
|
-
const envRefreshToken = process.env.GOOGLE_REFRESH_TOKEN;
|
|
447
|
+
const envAccessToken = !opts.force ? process.env.GOOGLE_ACCESS_TOKEN : void 0;
|
|
448
|
+
const envRefreshToken = !opts.force ? process.env.GOOGLE_REFRESH_TOKEN : void 0;
|
|
301
449
|
if (envAccessToken || envRefreshToken) {
|
|
302
450
|
oauth2Client.setCredentials({
|
|
303
451
|
access_token: envAccessToken,
|
|
@@ -309,7 +457,7 @@ async function authenticate(credentials, interactive) {
|
|
|
309
457
|
}
|
|
310
458
|
return oauth2Client;
|
|
311
459
|
}
|
|
312
|
-
const existingTokens = await loadTokens();
|
|
460
|
+
const existingTokens = !opts.force ? await loadTokens() : null;
|
|
313
461
|
if (existingTokens) {
|
|
314
462
|
oauth2Client.setCredentials(existingTokens);
|
|
315
463
|
if (existingTokens.expiry_date && existingTokens.expiry_date < Date.now()) {
|
|
@@ -329,9 +477,16 @@ async function authenticate(credentials, interactive) {
|
|
|
329
477
|
logger.error("No saved tokens. Run interactively first to authenticate.");
|
|
330
478
|
process.exit(1);
|
|
331
479
|
}
|
|
480
|
+
if (opts.noBrowser) {
|
|
481
|
+
const tokens = await authenticateDeviceCode(credentials);
|
|
482
|
+
oauth2Client.setCredentials(tokens);
|
|
483
|
+
await saveTokens(tokens);
|
|
484
|
+
logger.success(`Tokens saved to ${displayPath(getTokensPath())}`);
|
|
485
|
+
return oauth2Client;
|
|
486
|
+
}
|
|
332
487
|
const authUrl = oauth2Client.generateAuthUrl({
|
|
333
488
|
access_type: "offline",
|
|
334
|
-
scope:
|
|
489
|
+
scope: SCOPES,
|
|
335
490
|
prompt: "consent"
|
|
336
491
|
});
|
|
337
492
|
logger.info("Waiting for authorization...");
|
|
@@ -339,24 +494,163 @@ async function authenticate(credentials, interactive) {
|
|
|
339
494
|
const { tokens } = await new OAuth2Client(credentials.clientId, credentials.clientSecret, redirectUri).getToken(code);
|
|
340
495
|
oauth2Client.setCredentials(tokens);
|
|
341
496
|
await saveTokens(tokens);
|
|
342
|
-
logger.success(`Tokens saved to ${getTokensPath()}`);
|
|
497
|
+
logger.success(`Tokens saved to ${displayPath(getTokensPath())}`);
|
|
343
498
|
return oauth2Client;
|
|
344
499
|
}
|
|
345
500
|
async function getAuth(opts = {}) {
|
|
346
|
-
const { interactive = true } = opts;
|
|
347
|
-
return authenticate(await getAuthCredentials(interactive), interactive
|
|
501
|
+
const { interactive = true, noBrowser = false, force = false } = opts;
|
|
502
|
+
return authenticate(await getAuthCredentials(interactive), interactive, {
|
|
503
|
+
noBrowser,
|
|
504
|
+
force
|
|
505
|
+
});
|
|
506
|
+
}
|
|
507
|
+
async function resolveAuth(opts = {}) {
|
|
508
|
+
const sa = await resolveServiceAccount({ path: opts.serviceAccount });
|
|
509
|
+
if (sa) {
|
|
510
|
+
logger.success("Using service-account credentials");
|
|
511
|
+
return sa;
|
|
512
|
+
}
|
|
513
|
+
const byok = resolveBYOK(opts.byok);
|
|
514
|
+
if (byok) {
|
|
515
|
+
if (typeof byok !== "string") logger.success("Using BYOK credentials");
|
|
516
|
+
return byok;
|
|
517
|
+
}
|
|
518
|
+
return getAuth(opts);
|
|
519
|
+
}
|
|
520
|
+
function envSourceLabel(envVar) {
|
|
521
|
+
return getAppliedEnvKeys().has(envVar) ? `.env (${envVar})` : `shell env (${envVar})`;
|
|
522
|
+
}
|
|
523
|
+
function pickEnvSource(...envVars) {
|
|
524
|
+
for (const v of envVars) {
|
|
525
|
+
const value = process.env[v];
|
|
526
|
+
if (value) return {
|
|
527
|
+
envVar: v,
|
|
528
|
+
value
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
return null;
|
|
532
|
+
}
|
|
533
|
+
function redactCred(v, keepTail = 6) {
|
|
534
|
+
if (!v) return "<unset>";
|
|
535
|
+
if (v.length <= keepTail) return "***";
|
|
536
|
+
return `***${v.slice(-keepTail)}`;
|
|
537
|
+
}
|
|
538
|
+
async function describeAuthProvenance() {
|
|
539
|
+
const rows = [];
|
|
540
|
+
const warnings = [];
|
|
541
|
+
const saPath = process.env.GSC_SERVICE_ACCOUNT_JSON || process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
|
542
|
+
if (saPath) {
|
|
543
|
+
const saEnv = process.env.GSC_SERVICE_ACCOUNT_JSON ? "GSC_SERVICE_ACCOUNT_JSON" : "GOOGLE_APPLICATION_CREDENTIALS";
|
|
544
|
+
rows.push({
|
|
545
|
+
field: "service_account",
|
|
546
|
+
source: envSourceLabel(saEnv),
|
|
547
|
+
value: displayPath(saPath)
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
const clientId = pickEnvSource("GSC_CLIENT_ID", "GOOGLE_CLIENT_ID");
|
|
551
|
+
const clientSecret = pickEnvSource("GSC_CLIENT_SECRET", "GOOGLE_CLIENT_SECRET");
|
|
552
|
+
const config = await loadConfig().catch(() => null);
|
|
553
|
+
if (clientId) rows.push({
|
|
554
|
+
field: "client_id",
|
|
555
|
+
source: envSourceLabel(clientId.envVar),
|
|
556
|
+
value: clientId.value
|
|
557
|
+
});
|
|
558
|
+
else if (config?.clientId) rows.push({
|
|
559
|
+
field: "client_id",
|
|
560
|
+
source: `${displayPath(`${getConfigDir()}/config.json`)}`,
|
|
561
|
+
value: config.clientId
|
|
562
|
+
});
|
|
563
|
+
else rows.push({
|
|
564
|
+
field: "client_id",
|
|
565
|
+
source: "<none>",
|
|
566
|
+
value: null
|
|
567
|
+
});
|
|
568
|
+
if (clientSecret) rows.push({
|
|
569
|
+
field: "client_secret",
|
|
570
|
+
source: envSourceLabel(clientSecret.envVar),
|
|
571
|
+
value: redactCred(clientSecret.value)
|
|
572
|
+
});
|
|
573
|
+
else if (config?.clientSecret) rows.push({
|
|
574
|
+
field: "client_secret",
|
|
575
|
+
source: `${displayPath(`${getConfigDir()}/config.json`)}`,
|
|
576
|
+
value: redactCred(config.clientSecret)
|
|
577
|
+
});
|
|
578
|
+
else rows.push({
|
|
579
|
+
field: "client_secret",
|
|
580
|
+
source: "<none>",
|
|
581
|
+
value: null
|
|
582
|
+
});
|
|
583
|
+
const accessTok = pickEnvSource("GSC_ACCESS_TOKEN", "GOOGLE_ACCESS_TOKEN");
|
|
584
|
+
const refreshTok = pickEnvSource("GSC_REFRESH_TOKEN", "GOOGLE_REFRESH_TOKEN");
|
|
585
|
+
if (accessTok) rows.push({
|
|
586
|
+
field: "access_token",
|
|
587
|
+
source: envSourceLabel(accessTok.envVar),
|
|
588
|
+
value: redactCred(accessTok.value)
|
|
589
|
+
});
|
|
590
|
+
if (refreshTok) rows.push({
|
|
591
|
+
field: "refresh_token",
|
|
592
|
+
source: envSourceLabel(refreshTok.envVar),
|
|
593
|
+
value: redactCred(refreshTok.value)
|
|
594
|
+
});
|
|
595
|
+
const tokens = await loadTokens();
|
|
596
|
+
if (tokens) {
|
|
597
|
+
const expiry = tokens.expiry_date ? new Date(tokens.expiry_date).toISOString() : "no expiry";
|
|
598
|
+
rows.push({
|
|
599
|
+
field: "saved_tokens",
|
|
600
|
+
source: displayPath(getTokensPath()),
|
|
601
|
+
value: `access=${tokens.access_token ? "present" : "missing"}, refresh=${tokens.refresh_token ? "present" : "missing"}, expiry=${expiry}`
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
let effective;
|
|
605
|
+
if (saPath) effective = "service-account";
|
|
606
|
+
else if (clientId && clientSecret && refreshTok) effective = "byok-refresh-token";
|
|
607
|
+
else if (accessTok) effective = "byok-access-token";
|
|
608
|
+
else if (tokens) effective = "saved-tokens";
|
|
609
|
+
else effective = "none";
|
|
610
|
+
if ((effective === "byok-refresh-token" || effective === "byok-access-token") && tokens) warnings.push("BYOK env vars shadow saved tokens. If you just ran `auth login --force`, the env tokens are stale; unset them or update them.");
|
|
611
|
+
if (effective === "byok-refresh-token" && clientId && refreshTok) {
|
|
612
|
+
if (getAppliedEnvKeys().has(clientId.envVar) !== getAppliedEnvKeys().has(refreshTok.envVar)) warnings.push(`client_id and refresh_token come from different sources (${envSourceLabel(clientId.envVar)} vs ${envSourceLabel(refreshTok.envVar)}); they may not match.`);
|
|
613
|
+
}
|
|
614
|
+
const envFile = getLoadedEnvPath();
|
|
615
|
+
if (envFile && getAppliedEnvKeys().size > 0) warnings.push(`Loaded ${getAppliedEnvKeys().size} var(s) from ${displayPath(envFile)}.`);
|
|
616
|
+
return {
|
|
617
|
+
rows,
|
|
618
|
+
effective,
|
|
619
|
+
warnings
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
async function formatAuthProvenance() {
|
|
623
|
+
const { rows, effective, warnings } = await describeAuthProvenance();
|
|
624
|
+
const lines = [];
|
|
625
|
+
lines.push(` \x1B[1mAuth config sources\x1B[0m \x1B[90m(effective: ${effective})\x1B[0m`);
|
|
626
|
+
for (const r of rows) {
|
|
627
|
+
const val = r.value ? ` \x1B[90m${r.value}\x1B[0m` : "";
|
|
628
|
+
lines.push(` ${r.field.padEnd(16)} \x1B[36m${r.source}\x1B[0m${val}`);
|
|
629
|
+
}
|
|
630
|
+
if (warnings.length > 0) {
|
|
631
|
+
lines.push("");
|
|
632
|
+
for (const w of warnings) lines.push(` \x1B[33m!\x1B[0m ${w}`);
|
|
633
|
+
}
|
|
634
|
+
return lines.join("\n");
|
|
635
|
+
}
|
|
636
|
+
function isAuthError(err) {
|
|
637
|
+
const msg = (err instanceof Error ? err.message : String(err ?? "")).toLowerCase();
|
|
638
|
+
if (!msg) return false;
|
|
639
|
+
return /\b(?:401|403|unauthorized|forbidden|invalid_grant|invalid_token|insufficient.*scope|invalid_client|token has been expired|token has been revoked)\b/.test(msg) || msg.includes("oauth2.googleapis.com/token");
|
|
348
640
|
}
|
|
349
641
|
function createLocalStore(opts) {
|
|
350
642
|
return createNodeHarness(opts);
|
|
351
643
|
}
|
|
644
|
+
var context_exports = /* @__PURE__ */ __exportAll({ createCommandContext: () => createCommandContext });
|
|
352
645
|
async function createCommandContext(opts = {}) {
|
|
353
|
-
const { needsAuth = false, needsStore = false, interactive = false } = opts;
|
|
646
|
+
const { needsAuth = false, needsStore = false, interactive = false, byok, fetchOptions } = opts;
|
|
354
647
|
const config = await loadConfig();
|
|
355
|
-
const auth = needsAuth ? await
|
|
648
|
+
const auth = needsAuth ? await resolveAuth({
|
|
356
649
|
interactive,
|
|
357
|
-
config
|
|
650
|
+
config,
|
|
651
|
+
byok
|
|
358
652
|
}) : null;
|
|
359
|
-
const client = auth ? googleSearchConsole(auth) : null;
|
|
653
|
+
const client = auth ? googleSearchConsole(auth, { fetchOptions }) : null;
|
|
360
654
|
const store = needsStore ? createLocalStore({ dataDir: resolveDataDir(config) }) : null;
|
|
361
655
|
const loadSites = async () => {
|
|
362
656
|
if (!client) throw new Error("loadSites requires needsAuth: true");
|
|
@@ -402,6 +696,16 @@ async function createCommandContext(opts = {}) {
|
|
|
402
696
|
resolveSite
|
|
403
697
|
};
|
|
404
698
|
}
|
|
699
|
+
async function gscErrorHandler(error) {
|
|
700
|
+
console.error();
|
|
701
|
+
console.error(formatErrorForCli(error));
|
|
702
|
+
if (isAuthError(error)) {
|
|
703
|
+
console.error();
|
|
704
|
+
console.error(await formatAuthProvenance());
|
|
705
|
+
}
|
|
706
|
+
console.error();
|
|
707
|
+
process.exit(1);
|
|
708
|
+
}
|
|
405
709
|
const ANALYSIS_TOOLS = defaultAnalyzerRegistry.listAnalyzerIds();
|
|
406
710
|
const TOOL_EXTRA_ARGS = {
|
|
407
711
|
brand: { "brand-terms": {
|
|
@@ -551,10 +855,7 @@ function makeToolCommand(tool) {
|
|
|
551
855
|
renderResults(localResult.results, localResult.results.length, format);
|
|
552
856
|
return;
|
|
553
857
|
}
|
|
554
|
-
const result = await runLiveAnalysis(ctx.client, siteUrl, params).catch(
|
|
555
|
-
logger.error(`Analysis failed: ${e.message}`);
|
|
556
|
-
process.exit(1);
|
|
557
|
-
});
|
|
858
|
+
const result = await runLiveAnalysis(ctx.client, siteUrl, params).catch(gscErrorHandler);
|
|
558
859
|
if (format === "json") {
|
|
559
860
|
console.log(JSON.stringify(result, null, 2));
|
|
560
861
|
return;
|
|
@@ -737,35 +1038,203 @@ const analyzeCommand = defineCommand({
|
|
|
737
1038
|
},
|
|
738
1039
|
subCommands: Object.fromEntries(ANALYSIS_TOOLS.map((tool) => [tool, makeToolCommand(tool)]))
|
|
739
1040
|
});
|
|
1041
|
+
async function fetchTokenInfo(accessToken) {
|
|
1042
|
+
return ofetch("https://oauth2.googleapis.com/tokeninfo", { query: { access_token: accessToken } }).catch(() => null);
|
|
1043
|
+
}
|
|
1044
|
+
const statusCommand$1 = defineCommand({
|
|
1045
|
+
meta: {
|
|
1046
|
+
name: "status",
|
|
1047
|
+
description: "Show current authentication status"
|
|
1048
|
+
},
|
|
1049
|
+
args: {
|
|
1050
|
+
json: {
|
|
1051
|
+
type: "boolean",
|
|
1052
|
+
default: false,
|
|
1053
|
+
description: "Output as JSON"
|
|
1054
|
+
},
|
|
1055
|
+
quiet: {
|
|
1056
|
+
type: "boolean",
|
|
1057
|
+
alias: "q",
|
|
1058
|
+
default: false,
|
|
1059
|
+
description: "Suppress info/success output"
|
|
1060
|
+
}
|
|
1061
|
+
},
|
|
1062
|
+
async run({ args }) {
|
|
1063
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
1064
|
+
const tokens = await loadTokens();
|
|
1065
|
+
const byok = resolveBYOK();
|
|
1066
|
+
const byokKind = byok ? typeof byok === "string" ? "access-token" : "refresh-token" : null;
|
|
1067
|
+
let liveToken = null;
|
|
1068
|
+
if (typeof byok === "string") liveToken = byok;
|
|
1069
|
+
else if (byok && "getAccessToken" in byok) liveToken = await byok.getAccessToken().then((r) => r.token ?? null).catch(() => null);
|
|
1070
|
+
else if (tokens?.access_token) liveToken = tokens.access_token;
|
|
1071
|
+
const tokenInfo = liveToken ? await fetchTokenInfo(liveToken) : null;
|
|
1072
|
+
const scopes = tokenInfo?.scope ? tokenInfo.scope.split(/\s+/).filter(Boolean) : [];
|
|
1073
|
+
if (args.json) {
|
|
1074
|
+
console.log(JSON.stringify({
|
|
1075
|
+
authenticated: !!tokens || !!byok,
|
|
1076
|
+
source: byok ? "byok" : tokens ? "saved-tokens" : null,
|
|
1077
|
+
byokKind,
|
|
1078
|
+
scopes,
|
|
1079
|
+
tokenAccount: tokenInfo?.email ?? null,
|
|
1080
|
+
tokens: tokens ? {
|
|
1081
|
+
hasAccessToken: !!tokens.access_token,
|
|
1082
|
+
hasRefreshToken: !!tokens.refresh_token,
|
|
1083
|
+
expiry: tokens.expiry_date ? new Date(tokens.expiry_date).toISOString() : null,
|
|
1084
|
+
expired: tokens.expiry_date ? tokens.expiry_date < Date.now() : null
|
|
1085
|
+
} : null
|
|
1086
|
+
}, null, 2));
|
|
1087
|
+
return;
|
|
1088
|
+
}
|
|
1089
|
+
const reportScopes = () => {
|
|
1090
|
+
if (scopes.length === 0) return;
|
|
1091
|
+
console.log(` Scopes:`);
|
|
1092
|
+
const required = [
|
|
1093
|
+
"https://www.googleapis.com/auth/webmasters",
|
|
1094
|
+
"https://www.googleapis.com/auth/indexing",
|
|
1095
|
+
"https://www.googleapis.com/auth/siteverification"
|
|
1096
|
+
];
|
|
1097
|
+
const has = (s) => scopes.includes(s) || scopes.includes(s.replace(".readonly", ""));
|
|
1098
|
+
for (const s of scopes) console.log(` \x1B[90m└─\x1B[0m ${s}`);
|
|
1099
|
+
const missing = required.filter((s) => !has(s));
|
|
1100
|
+
if (missing.length > 0) {
|
|
1101
|
+
console.log(` \x1B[33mMissing scopes:\x1B[0m`);
|
|
1102
|
+
for (const s of missing) console.log(` \x1B[90m└─\x1B[0m ${s}`);
|
|
1103
|
+
console.log(` \x1B[90mRun \`gscdump auth login --force\` to re-consent.\x1B[0m`);
|
|
1104
|
+
}
|
|
1105
|
+
};
|
|
1106
|
+
if (byok) {
|
|
1107
|
+
logger.success(`Authenticated via BYOK (${byokKind})`);
|
|
1108
|
+
if (tokenInfo?.email) console.log(` Account: ${tokenInfo.email}`);
|
|
1109
|
+
reportScopes();
|
|
1110
|
+
return;
|
|
1111
|
+
}
|
|
1112
|
+
if (!tokens) {
|
|
1113
|
+
logger.warn("Not authenticated");
|
|
1114
|
+
logger.info("Run `gscdump init` (full setup) or `gscdump auth login` (OAuth only)");
|
|
1115
|
+
logger.info("Or set GSC_ACCESS_TOKEN / GSC_CLIENT_ID + GSC_CLIENT_SECRET + GSC_REFRESH_TOKEN env vars");
|
|
1116
|
+
return;
|
|
1117
|
+
}
|
|
1118
|
+
const hasAccess = !!tokens.access_token;
|
|
1119
|
+
const hasRefresh = !!tokens.refresh_token;
|
|
1120
|
+
const expiry = tokens.expiry_date ? new Date(tokens.expiry_date) : null;
|
|
1121
|
+
const isExpired = expiry && expiry < /* @__PURE__ */ new Date();
|
|
1122
|
+
logger.success("Authenticated (saved tokens)");
|
|
1123
|
+
console.log();
|
|
1124
|
+
console.log(` Access token: ${hasAccess ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
1125
|
+
console.log(` Refresh token: ${hasRefresh ? "\x1B[32mpresent\x1B[0m" : "\x1B[31mmissing\x1B[0m"}`);
|
|
1126
|
+
if (expiry) {
|
|
1127
|
+
const status = isExpired ? "\x1B[33mexpired\x1B[0m" : "\x1B[32mvalid\x1B[0m";
|
|
1128
|
+
console.log(` Expires: ${expiry.toISOString()} (${status})`);
|
|
1129
|
+
}
|
|
1130
|
+
if (tokenInfo?.email) console.log(` Account: ${tokenInfo.email}`);
|
|
1131
|
+
reportScopes();
|
|
1132
|
+
}
|
|
1133
|
+
});
|
|
1134
|
+
const refreshCommand = defineCommand({
|
|
1135
|
+
meta: {
|
|
1136
|
+
name: "refresh",
|
|
1137
|
+
description: "Force-refresh saved OAuth tokens (no-op for BYOK)"
|
|
1138
|
+
},
|
|
1139
|
+
args: { quiet: {
|
|
1140
|
+
type: "boolean",
|
|
1141
|
+
alias: "q",
|
|
1142
|
+
default: false,
|
|
1143
|
+
description: "Suppress info/success output"
|
|
1144
|
+
} },
|
|
1145
|
+
async run({ args }) {
|
|
1146
|
+
setQuiet(Boolean(args.quiet));
|
|
1147
|
+
if (resolveBYOK()) {
|
|
1148
|
+
logger.info("BYOK detected; refresh handled per-call by the SDK");
|
|
1149
|
+
return;
|
|
1150
|
+
}
|
|
1151
|
+
const tokens = await loadTokens();
|
|
1152
|
+
if (!tokens?.refresh_token) {
|
|
1153
|
+
logger.error("No saved refresh token. Run `gscdump auth login`.");
|
|
1154
|
+
process.exit(1);
|
|
1155
|
+
}
|
|
1156
|
+
const credentials = await getAuthCredentials(false).catch((e) => {
|
|
1157
|
+
logger.error(`Cannot resolve credentials: ${e.message}`);
|
|
1158
|
+
process.exit(1);
|
|
1159
|
+
});
|
|
1160
|
+
await saveTokens({
|
|
1161
|
+
...tokens,
|
|
1162
|
+
expiry_date: 1
|
|
1163
|
+
});
|
|
1164
|
+
if ((await authenticate(credentials, false).catch((e) => {
|
|
1165
|
+
logger.error(`Refresh failed: ${e.message}`);
|
|
1166
|
+
process.exit(1);
|
|
1167
|
+
})).credentials?.access_token) logger.success("Token refreshed");
|
|
1168
|
+
else logger.warn("Refresh completed but no new access token returned");
|
|
1169
|
+
}
|
|
1170
|
+
});
|
|
740
1171
|
const authCommand = defineCommand({
|
|
741
1172
|
meta: {
|
|
742
1173
|
name: "auth",
|
|
743
1174
|
description: "Manage authentication"
|
|
744
1175
|
},
|
|
745
1176
|
subCommands: {
|
|
746
|
-
status:
|
|
1177
|
+
status: statusCommand$1,
|
|
1178
|
+
login: defineCommand({
|
|
747
1179
|
meta: {
|
|
748
|
-
name: "
|
|
749
|
-
description: "
|
|
1180
|
+
name: "login",
|
|
1181
|
+
description: "Run OAuth flow and persist tokens (skip if BYOK env vars set)"
|
|
750
1182
|
},
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
1183
|
+
args: {
|
|
1184
|
+
"force": {
|
|
1185
|
+
type: "boolean",
|
|
1186
|
+
alias: "f",
|
|
1187
|
+
default: false,
|
|
1188
|
+
description: "Re-run OAuth even if tokens already exist"
|
|
1189
|
+
},
|
|
1190
|
+
"browser": {
|
|
1191
|
+
type: "boolean",
|
|
1192
|
+
default: true,
|
|
1193
|
+
description: "Use loopback browser flow. Pass --no-browser for device-code (headless)."
|
|
1194
|
+
},
|
|
1195
|
+
"service-account": {
|
|
1196
|
+
type: "string",
|
|
1197
|
+
description: "Path to a service-account JSON key (skips OAuth)"
|
|
1198
|
+
},
|
|
1199
|
+
"quiet": {
|
|
1200
|
+
type: "boolean",
|
|
1201
|
+
alias: "q",
|
|
1202
|
+
default: false,
|
|
1203
|
+
description: "Suppress info/success output"
|
|
1204
|
+
}
|
|
1205
|
+
},
|
|
1206
|
+
async run({ args }) {
|
|
1207
|
+
setQuiet(Boolean(args.quiet));
|
|
1208
|
+
if (resolveBYOK() && !args.force) {
|
|
1209
|
+
logger.info("BYOK env vars detected, no login needed (--force to override)");
|
|
756
1210
|
return;
|
|
757
1211
|
}
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
1212
|
+
if (args["service-account"]) {
|
|
1213
|
+
const jwt = await loadServiceAccount(String(args["service-account"])).catch((e) => {
|
|
1214
|
+
logger.error(`Service-account load failed: ${e.message}`);
|
|
1215
|
+
process.exit(1);
|
|
1216
|
+
});
|
|
1217
|
+
await jwt.authorize().catch((e) => {
|
|
1218
|
+
logger.error(`Service-account auth failed: ${e.message}`);
|
|
1219
|
+
process.exit(1);
|
|
1220
|
+
});
|
|
1221
|
+
logger.success(`Service-account verified: ${jwt.email ?? "OK"}`);
|
|
1222
|
+
logger.info(`Set GOOGLE_APPLICATION_CREDENTIALS=${args["service-account"]} to use it across sessions.`);
|
|
1223
|
+
return;
|
|
1224
|
+
}
|
|
1225
|
+
if (args.force) await clearTokens();
|
|
1226
|
+
await getAuth({
|
|
1227
|
+
interactive: true,
|
|
1228
|
+
noBrowser: args.browser === false,
|
|
1229
|
+
force: Boolean(args.force)
|
|
1230
|
+
}).catch((e) => {
|
|
1231
|
+
logger.error(`Login failed: ${e.message}`);
|
|
1232
|
+
process.exit(1);
|
|
1233
|
+
});
|
|
1234
|
+
logger.success("Logged in");
|
|
1235
|
+
if (resolveBYOK()) {
|
|
1236
|
+
console.log();
|
|
1237
|
+
console.log(await formatAuthProvenance());
|
|
769
1238
|
}
|
|
770
1239
|
}
|
|
771
1240
|
}),
|
|
@@ -774,35 +1243,76 @@ const authCommand = defineCommand({
|
|
|
774
1243
|
name: "logout",
|
|
775
1244
|
description: "Clear stored OAuth tokens"
|
|
776
1245
|
},
|
|
777
|
-
|
|
1246
|
+
args: { quiet: {
|
|
1247
|
+
type: "boolean",
|
|
1248
|
+
alias: "q",
|
|
1249
|
+
default: false,
|
|
1250
|
+
description: "Suppress info/success output"
|
|
1251
|
+
} },
|
|
1252
|
+
async run({ args }) {
|
|
1253
|
+
setQuiet(Boolean(args.quiet));
|
|
778
1254
|
await clearTokens();
|
|
779
1255
|
}
|
|
780
|
-
})
|
|
1256
|
+
}),
|
|
1257
|
+
refresh: refreshCommand
|
|
1258
|
+
}
|
|
1259
|
+
});
|
|
1260
|
+
const showCommand = defineCommand({
|
|
1261
|
+
meta: {
|
|
1262
|
+
name: "show",
|
|
1263
|
+
description: "Show current config"
|
|
1264
|
+
},
|
|
1265
|
+
args: {
|
|
1266
|
+
json: {
|
|
1267
|
+
type: "boolean",
|
|
1268
|
+
default: false,
|
|
1269
|
+
description: "Output config as a single JSON object (suppresses path header)"
|
|
1270
|
+
},
|
|
1271
|
+
quiet: {
|
|
1272
|
+
type: "boolean",
|
|
1273
|
+
alias: "q",
|
|
1274
|
+
default: false,
|
|
1275
|
+
description: "Suppress info/success output"
|
|
1276
|
+
}
|
|
1277
|
+
},
|
|
1278
|
+
async run({ args }) {
|
|
1279
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
1280
|
+
const config = await loadConfig();
|
|
1281
|
+
const configPath = getConfigPath();
|
|
1282
|
+
if (args.json) {
|
|
1283
|
+
console.log(JSON.stringify({
|
|
1284
|
+
path: configPath,
|
|
1285
|
+
config
|
|
1286
|
+
}, null, 2));
|
|
1287
|
+
return;
|
|
1288
|
+
}
|
|
1289
|
+
logger.info(`Config: ${displayPath(configPath)}`);
|
|
1290
|
+
console.log();
|
|
1291
|
+
if (Object.keys(config).length === 0) {
|
|
1292
|
+
logger.warn("No config set");
|
|
1293
|
+
return;
|
|
1294
|
+
}
|
|
1295
|
+
console.log(JSON.stringify(config, null, 2));
|
|
781
1296
|
}
|
|
782
1297
|
});
|
|
1298
|
+
const VALID_KEYS = [
|
|
1299
|
+
"defaultSite",
|
|
1300
|
+
"defaultPeriod",
|
|
1301
|
+
"defaultFormat",
|
|
1302
|
+
"defaultDb",
|
|
1303
|
+
"dataDir",
|
|
1304
|
+
"defaultLimit",
|
|
1305
|
+
"defaultSearchType",
|
|
1306
|
+
"defaultDataState"
|
|
1307
|
+
];
|
|
1308
|
+
const NUMERIC_KEYS = new Set(["defaultLimit"]);
|
|
783
1309
|
const configCommand = defineCommand({
|
|
784
1310
|
meta: {
|
|
785
1311
|
name: "config",
|
|
786
1312
|
description: "Manage configuration"
|
|
787
1313
|
},
|
|
788
1314
|
subCommands: {
|
|
789
|
-
show:
|
|
790
|
-
meta: {
|
|
791
|
-
name: "show",
|
|
792
|
-
description: "Show current config"
|
|
793
|
-
},
|
|
794
|
-
async run() {
|
|
795
|
-
const config = await loadConfig();
|
|
796
|
-
const configPath = getConfigPath();
|
|
797
|
-
logger.info(`Config: ${configPath}`);
|
|
798
|
-
console.log();
|
|
799
|
-
if (Object.keys(config).length === 0) {
|
|
800
|
-
logger.warn("No config set");
|
|
801
|
-
return;
|
|
802
|
-
}
|
|
803
|
-
console.log(JSON.stringify(config, null, 2));
|
|
804
|
-
}
|
|
805
|
-
}),
|
|
1315
|
+
show: showCommand,
|
|
806
1316
|
set: defineCommand({
|
|
807
1317
|
meta: {
|
|
808
1318
|
name: "set",
|
|
@@ -811,31 +1321,37 @@ const configCommand = defineCommand({
|
|
|
811
1321
|
args: {
|
|
812
1322
|
key: {
|
|
813
1323
|
type: "positional",
|
|
814
|
-
description:
|
|
1324
|
+
description: `Config key (${VALID_KEYS.join(", ")})`,
|
|
815
1325
|
required: true
|
|
816
1326
|
},
|
|
817
1327
|
value: {
|
|
818
1328
|
type: "positional",
|
|
819
1329
|
description: "Value to set",
|
|
820
1330
|
required: true
|
|
1331
|
+
},
|
|
1332
|
+
quiet: {
|
|
1333
|
+
type: "boolean",
|
|
1334
|
+
alias: "q",
|
|
1335
|
+
default: false,
|
|
1336
|
+
description: "Suppress info/success output"
|
|
821
1337
|
}
|
|
822
1338
|
},
|
|
823
1339
|
async run({ args }) {
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
"defaultPeriod",
|
|
827
|
-
"defaultFormat",
|
|
828
|
-
"defaultDb"
|
|
829
|
-
];
|
|
830
|
-
if (!validKeys.includes(args.key)) {
|
|
1340
|
+
setQuiet(Boolean(args.quiet));
|
|
1341
|
+
if (!VALID_KEYS.includes(args.key)) {
|
|
831
1342
|
logger.error(`Invalid key: ${args.key}`);
|
|
832
|
-
logger.info(`Valid keys: ${
|
|
1343
|
+
logger.info(`Valid keys: ${VALID_KEYS.join(", ")}`);
|
|
833
1344
|
process.exit(1);
|
|
834
1345
|
}
|
|
835
1346
|
const config = await loadConfig();
|
|
836
|
-
|
|
1347
|
+
const value = NUMERIC_KEYS.has(args.key) ? Number(args.value) : args.value;
|
|
1348
|
+
if (NUMERIC_KEYS.has(args.key) && !Number.isFinite(value)) {
|
|
1349
|
+
logger.error(`Invalid numeric value for ${args.key}: ${args.value}`);
|
|
1350
|
+
process.exit(1);
|
|
1351
|
+
}
|
|
1352
|
+
config[args.key] = value;
|
|
837
1353
|
await saveConfig(config);
|
|
838
|
-
logger.success(`Set ${args.key} = ${
|
|
1354
|
+
logger.success(`Set ${args.key} = ${value}`);
|
|
839
1355
|
}
|
|
840
1356
|
}),
|
|
841
1357
|
unset: defineCommand({
|
|
@@ -843,12 +1359,26 @@ const configCommand = defineCommand({
|
|
|
843
1359
|
name: "unset",
|
|
844
1360
|
description: "Remove a config value"
|
|
845
1361
|
},
|
|
846
|
-
args: {
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
1362
|
+
args: {
|
|
1363
|
+
key: {
|
|
1364
|
+
type: "positional",
|
|
1365
|
+
description: `Config key to remove (${VALID_KEYS.join(", ")})`,
|
|
1366
|
+
required: true
|
|
1367
|
+
},
|
|
1368
|
+
quiet: {
|
|
1369
|
+
type: "boolean",
|
|
1370
|
+
alias: "q",
|
|
1371
|
+
default: false,
|
|
1372
|
+
description: "Suppress info/success output"
|
|
1373
|
+
}
|
|
1374
|
+
},
|
|
851
1375
|
async run({ args }) {
|
|
1376
|
+
setQuiet(Boolean(args.quiet));
|
|
1377
|
+
if (!VALID_KEYS.includes(args.key)) {
|
|
1378
|
+
logger.error(`Invalid key: ${args.key}`);
|
|
1379
|
+
logger.info(`Valid keys: ${VALID_KEYS.join(", ")}`);
|
|
1380
|
+
process.exit(1);
|
|
1381
|
+
}
|
|
852
1382
|
const config = await loadConfig();
|
|
853
1383
|
delete config[args.key];
|
|
854
1384
|
await saveConfig(config);
|
|
@@ -863,33 +1393,523 @@ const configCommand = defineCommand({
|
|
|
863
1393
|
run() {
|
|
864
1394
|
console.log(getConfigPath());
|
|
865
1395
|
}
|
|
1396
|
+
}),
|
|
1397
|
+
validate: defineCommand({
|
|
1398
|
+
meta: {
|
|
1399
|
+
name: "validate",
|
|
1400
|
+
description: "Validate the saved config (defaultSite is verified, dataDir exists/writable)"
|
|
1401
|
+
},
|
|
1402
|
+
args: {
|
|
1403
|
+
json: {
|
|
1404
|
+
type: "boolean",
|
|
1405
|
+
default: false,
|
|
1406
|
+
description: "Output as JSON"
|
|
1407
|
+
},
|
|
1408
|
+
quiet: {
|
|
1409
|
+
type: "boolean",
|
|
1410
|
+
alias: "q",
|
|
1411
|
+
default: false,
|
|
1412
|
+
description: "Suppress info/success output"
|
|
1413
|
+
}
|
|
1414
|
+
},
|
|
1415
|
+
async run({ args }) {
|
|
1416
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
1417
|
+
const { resolveDataDir } = await import("./_chunks/config.mjs").then((n) => n.t);
|
|
1418
|
+
const fs = await import("node:fs/promises");
|
|
1419
|
+
const config = await loadConfig();
|
|
1420
|
+
const issues = [];
|
|
1421
|
+
const dataDir = resolveDataDir(config);
|
|
1422
|
+
const dataDirDisplay = displayPath(dataDir);
|
|
1423
|
+
const stat = await fs.stat(dataDir).catch(() => null);
|
|
1424
|
+
if (stat && !stat.isDirectory()) issues.push({
|
|
1425
|
+
key: "dataDir",
|
|
1426
|
+
level: "fail",
|
|
1427
|
+
message: `${dataDirDisplay} is not a directory`
|
|
1428
|
+
});
|
|
1429
|
+
else if (stat) {
|
|
1430
|
+
const probe = `${dataDir}/.gscdump-config-probe`;
|
|
1431
|
+
if (!await fs.writeFile(probe, "").then(() => fs.rm(probe)).then(() => true).catch(() => false)) issues.push({
|
|
1432
|
+
key: "dataDir",
|
|
1433
|
+
level: "fail",
|
|
1434
|
+
message: `${dataDirDisplay} not writable`
|
|
1435
|
+
});
|
|
1436
|
+
} else issues.push({
|
|
1437
|
+
key: "dataDir",
|
|
1438
|
+
level: "warn",
|
|
1439
|
+
message: `${dataDirDisplay} does not exist (will be created on first sync)`
|
|
1440
|
+
});
|
|
1441
|
+
if (config.defaultSite) if (!!config.clientId && !!config.clientSecret) {
|
|
1442
|
+
const { createCommandContext } = await Promise.resolve().then(() => context_exports);
|
|
1443
|
+
const ctx = await createCommandContext({ needsAuth: true }).catch(() => null);
|
|
1444
|
+
if (ctx) {
|
|
1445
|
+
const sites = await ctx.loadSites().catch(() => null);
|
|
1446
|
+
if (sites && !sites.some((s) => s.siteUrl === config.defaultSite || s.siteUrl.includes(String(config.defaultSite)))) issues.push({
|
|
1447
|
+
key: "defaultSite",
|
|
1448
|
+
level: "fail",
|
|
1449
|
+
message: `${config.defaultSite} is not in the verified site list`
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1452
|
+
} else issues.push({
|
|
1453
|
+
key: "defaultSite",
|
|
1454
|
+
level: "warn",
|
|
1455
|
+
message: "set, but auth not configured — skipping verification"
|
|
1456
|
+
});
|
|
1457
|
+
if (config.defaultFormat && !["json", "csv"].includes(config.defaultFormat)) issues.push({
|
|
1458
|
+
key: "defaultFormat",
|
|
1459
|
+
level: "fail",
|
|
1460
|
+
message: `unknown format: ${config.defaultFormat}`
|
|
1461
|
+
});
|
|
1462
|
+
if (args.json) {
|
|
1463
|
+
console.log(JSON.stringify({
|
|
1464
|
+
ok: !issues.some((i) => i.level === "fail"),
|
|
1465
|
+
issues
|
|
1466
|
+
}, null, 2));
|
|
1467
|
+
return;
|
|
1468
|
+
}
|
|
1469
|
+
if (issues.length === 0) {
|
|
1470
|
+
logger.success("Config OK");
|
|
1471
|
+
return;
|
|
1472
|
+
}
|
|
1473
|
+
for (const i of issues) {
|
|
1474
|
+
const prefix = i.level === "fail" ? "\x1B[31m✗\x1B[0m" : "\x1B[33m!\x1B[0m";
|
|
1475
|
+
console.log(` ${prefix} ${i.key}: ${i.message}`);
|
|
1476
|
+
}
|
|
1477
|
+
if (issues.some((i) => i.level === "fail")) process.exit(1);
|
|
1478
|
+
}
|
|
866
1479
|
})
|
|
867
1480
|
}
|
|
868
1481
|
});
|
|
869
|
-
const
|
|
870
|
-
|
|
871
|
-
|
|
1482
|
+
const REQUIRED_SCOPES = [
|
|
1483
|
+
"https://www.googleapis.com/auth/webmasters",
|
|
1484
|
+
"https://www.googleapis.com/auth/indexing",
|
|
1485
|
+
"https://www.googleapis.com/auth/siteverification"
|
|
1486
|
+
];
|
|
1487
|
+
const FETCH_TIMEOUT_MS = 5e3;
|
|
1488
|
+
const TIME_SKEW_WARN_MS = 5 * 6e4;
|
|
1489
|
+
const WATERMARK_STALE_DAYS_WARN = 7;
|
|
1490
|
+
const RELEVANT_ENV_KEYS = [
|
|
1491
|
+
"GSC_ACCESS_TOKEN",
|
|
1492
|
+
"GSC_CLIENT_ID",
|
|
1493
|
+
"GSC_CLIENT_SECRET",
|
|
1494
|
+
"GSC_REFRESH_TOKEN",
|
|
1495
|
+
"GOOGLE_ACCESS_TOKEN",
|
|
1496
|
+
"GOOGLE_CLIENT_ID",
|
|
1497
|
+
"GOOGLE_CLIENT_SECRET",
|
|
1498
|
+
"GOOGLE_REFRESH_TOKEN",
|
|
1499
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
1500
|
+
"GSC_SERVICE_ACCOUNT_JSON"
|
|
1501
|
+
];
|
|
1502
|
+
function redact(v) {
|
|
1503
|
+
if (!v) return "<missing>";
|
|
1504
|
+
if (v.length <= 6) return "***";
|
|
1505
|
+
return `***${v.slice(-6)}`;
|
|
1506
|
+
}
|
|
1507
|
+
async function checkEnv() {
|
|
1508
|
+
const envPath = path.join(process.cwd(), ".env");
|
|
1509
|
+
const parsed = parseEnvFile(envPath);
|
|
1510
|
+
if (!parsed) return {
|
|
1511
|
+
checks: [{
|
|
1512
|
+
name: "env",
|
|
1513
|
+
status: "info",
|
|
1514
|
+
detail: "no .env (using shell env / saved tokens)"
|
|
1515
|
+
}],
|
|
1516
|
+
envKeys: /* @__PURE__ */ new Set()
|
|
1517
|
+
};
|
|
1518
|
+
const relevant = RELEVANT_ENV_KEYS.filter((k) => parsed[k] !== void 0);
|
|
1519
|
+
const envKeys = new Set(relevant);
|
|
1520
|
+
if (relevant.length === 0) return {
|
|
1521
|
+
checks: [{
|
|
1522
|
+
name: "env",
|
|
1523
|
+
status: "info",
|
|
1524
|
+
detail: `${displayPath(envPath)} found, no auth vars`
|
|
1525
|
+
}],
|
|
1526
|
+
envKeys
|
|
1527
|
+
};
|
|
1528
|
+
const inventory = relevant.map((k) => {
|
|
1529
|
+
const v = parsed[k];
|
|
1530
|
+
if (k.endsWith("CLIENT_ID")) return `${k}=${v}`;
|
|
1531
|
+
return `${k}=${redact(v)}`;
|
|
1532
|
+
});
|
|
1533
|
+
return {
|
|
1534
|
+
checks: [{
|
|
1535
|
+
name: "env",
|
|
1536
|
+
status: "info",
|
|
1537
|
+
detail: `${displayPath(envPath)} → ${inventory.join(", ")} (not validated — see auth)`
|
|
1538
|
+
}],
|
|
1539
|
+
envKeys
|
|
1540
|
+
};
|
|
1541
|
+
}
|
|
1542
|
+
function describeAuthSource(envKeys, byok) {
|
|
1543
|
+
if (!byok) return "saved tokens";
|
|
1544
|
+
const isAccessToken = typeof byok === "string";
|
|
1545
|
+
const driver = isAccessToken ? "GSC_ACCESS_TOKEN" : "GSC_REFRESH_TOKEN";
|
|
1546
|
+
const source = (isAccessToken ? ["GSC_ACCESS_TOKEN", "GOOGLE_ACCESS_TOKEN"] : ["GSC_REFRESH_TOKEN", "GOOGLE_REFRESH_TOKEN"]).some((k) => envKeys.has(k)) ? ".env" : "shell env";
|
|
1547
|
+
return `BYOK ${isAccessToken ? "(access-token)" : "(refresh-token)"} from ${source} via ${driver}`;
|
|
1548
|
+
}
|
|
1549
|
+
async function checkAuth$1(envKeys) {
|
|
1550
|
+
const checks = [];
|
|
1551
|
+
const byok = resolveBYOK();
|
|
1552
|
+
const tokens = await loadTokens();
|
|
1553
|
+
if (!byok && !tokens) {
|
|
1554
|
+
checks.push({
|
|
1555
|
+
name: "auth",
|
|
1556
|
+
status: "fail",
|
|
1557
|
+
detail: "no BYOK env vars and no saved tokens; run `gscdump init`"
|
|
1558
|
+
});
|
|
1559
|
+
return {
|
|
1560
|
+
checks,
|
|
1561
|
+
liveToken: null
|
|
1562
|
+
};
|
|
1563
|
+
}
|
|
1564
|
+
const clientId = process.env.GSC_CLIENT_ID ?? process.env.GOOGLE_CLIENT_ID ?? (await loadConfig()).clientId ?? null;
|
|
1565
|
+
if (clientId) checks.push({
|
|
1566
|
+
name: "auth.client_id",
|
|
1567
|
+
status: "info",
|
|
1568
|
+
detail: clientId
|
|
1569
|
+
});
|
|
1570
|
+
let liveToken = null;
|
|
1571
|
+
let refreshError = null;
|
|
1572
|
+
if (typeof byok === "string") liveToken = byok;
|
|
1573
|
+
else if (byok && "getAccessToken" in byok) liveToken = await byok.getAccessToken().then((r) => r.token ?? null).catch((e) => {
|
|
1574
|
+
refreshError = e?.data?.error_description ?? e?.data?.error ?? e?.message ?? String(e);
|
|
1575
|
+
return null;
|
|
1576
|
+
});
|
|
1577
|
+
else if (tokens?.access_token) liveToken = tokens.access_token;
|
|
1578
|
+
if (!liveToken) {
|
|
1579
|
+
const source = describeAuthSource(envKeys, byok);
|
|
1580
|
+
const detail = refreshError ? `${source} — refresh failed: ${refreshError} (token revoked / invalid — re-run \`gscdump auth login --force\` and update the source above)` : `${source} — no usable access token`;
|
|
1581
|
+
checks.push({
|
|
1582
|
+
name: "auth",
|
|
1583
|
+
status: "fail",
|
|
1584
|
+
detail
|
|
1585
|
+
});
|
|
1586
|
+
return {
|
|
1587
|
+
checks,
|
|
1588
|
+
liveToken: null
|
|
1589
|
+
};
|
|
1590
|
+
}
|
|
1591
|
+
const info = await ofetch("https://oauth2.googleapis.com/tokeninfo", { query: { access_token: liveToken } }).catch((e) => ({ error: e.message }));
|
|
1592
|
+
if ("error" in info) {
|
|
1593
|
+
checks.push({
|
|
1594
|
+
name: "auth",
|
|
1595
|
+
status: "fail",
|
|
1596
|
+
detail: `tokeninfo failed: ${info.error}`
|
|
1597
|
+
});
|
|
1598
|
+
return {
|
|
1599
|
+
checks,
|
|
1600
|
+
liveToken: null
|
|
1601
|
+
};
|
|
1602
|
+
}
|
|
1603
|
+
checks.push({
|
|
1604
|
+
name: "auth",
|
|
1605
|
+
status: "pass",
|
|
1606
|
+
detail: describeAuthSource(envKeys, byok)
|
|
1607
|
+
});
|
|
1608
|
+
if (info.email) checks.push({
|
|
1609
|
+
name: "auth.account",
|
|
1610
|
+
status: "pass",
|
|
1611
|
+
detail: info.email
|
|
1612
|
+
});
|
|
1613
|
+
const scopes = info.scope ? info.scope.split(/\s+/) : [];
|
|
1614
|
+
const missing = REQUIRED_SCOPES.filter((s) => !scopes.includes(s) && !scopes.includes(s.replace(".readonly", "")));
|
|
1615
|
+
if (missing.length > 0) checks.push({
|
|
1616
|
+
name: "auth.scopes",
|
|
1617
|
+
status: "warn",
|
|
1618
|
+
detail: `missing: ${missing.join(", ")} — \`gscdump auth login --force\` to re-consent`
|
|
1619
|
+
});
|
|
1620
|
+
else checks.push({
|
|
1621
|
+
name: "auth.scopes",
|
|
1622
|
+
status: "pass",
|
|
1623
|
+
detail: `${scopes.length} granted`
|
|
1624
|
+
});
|
|
1625
|
+
if (tokens?.expiry_date) {
|
|
1626
|
+
const expiresInMs = tokens.expiry_date - Date.now();
|
|
1627
|
+
if (expiresInMs < 0) checks.push({
|
|
1628
|
+
name: "auth.expiry",
|
|
1629
|
+
status: "warn",
|
|
1630
|
+
detail: `expired ${new Date(tokens.expiry_date).toISOString()} — will refresh on next call`
|
|
1631
|
+
});
|
|
1632
|
+
else checks.push({
|
|
1633
|
+
name: "auth.expiry",
|
|
1634
|
+
status: "pass",
|
|
1635
|
+
detail: `valid for ${Math.floor(expiresInMs / 6e4)}m`
|
|
1636
|
+
});
|
|
1637
|
+
}
|
|
1638
|
+
return {
|
|
1639
|
+
checks,
|
|
1640
|
+
liveToken
|
|
1641
|
+
};
|
|
1642
|
+
}
|
|
1643
|
+
async function checkTimeSkew() {
|
|
1644
|
+
const dateHeader = await ofetch.raw("https://oauth2.googleapis.com/tokeninfo", {
|
|
1645
|
+
method: "GET",
|
|
1646
|
+
timeout: FETCH_TIMEOUT_MS
|
|
1647
|
+
}).then((r) => r.headers.get("date")).catch((e) => e?.response?.headers?.get("date") ?? null);
|
|
1648
|
+
if (!dateHeader) return [{
|
|
1649
|
+
name: "time",
|
|
1650
|
+
status: "warn",
|
|
1651
|
+
detail: "could not probe Google clock (no Date header)"
|
|
1652
|
+
}];
|
|
1653
|
+
const remoteMs = Date.parse(dateHeader);
|
|
1654
|
+
if (!Number.isFinite(remoteMs)) return [{
|
|
1655
|
+
name: "time",
|
|
1656
|
+
status: "warn",
|
|
1657
|
+
detail: `unparseable Date header: ${dateHeader}`
|
|
1658
|
+
}];
|
|
1659
|
+
const skewMs = Date.now() - remoteMs;
|
|
1660
|
+
const human = `${skewMs >= 0 ? "+" : ""}${(skewMs / 1e3).toFixed(1)}s`;
|
|
1661
|
+
if (Math.abs(skewMs) > TIME_SKEW_WARN_MS) return [{
|
|
1662
|
+
name: "time",
|
|
1663
|
+
status: "warn",
|
|
1664
|
+
detail: `local clock ${human} off Google — OAuth refresh may reject; sync your clock`
|
|
1665
|
+
}];
|
|
1666
|
+
return [{
|
|
1667
|
+
name: "time",
|
|
1668
|
+
status: "pass",
|
|
1669
|
+
detail: `in sync (${human})`
|
|
1670
|
+
}];
|
|
1671
|
+
}
|
|
1672
|
+
async function checkDataDir() {
|
|
1673
|
+
const dataDir = resolveDataDir(await loadConfig());
|
|
1674
|
+
const display = displayPath(dataDir);
|
|
1675
|
+
const stat = await fs.stat(dataDir).catch(() => null);
|
|
1676
|
+
if (!stat) return [{
|
|
1677
|
+
name: "dataDir",
|
|
1678
|
+
status: "warn",
|
|
1679
|
+
detail: `${display} does not exist (will be created on first sync)`
|
|
1680
|
+
}];
|
|
1681
|
+
if (!stat.isDirectory()) return [{
|
|
1682
|
+
name: "dataDir",
|
|
1683
|
+
status: "fail",
|
|
1684
|
+
detail: `${display} is not a directory`
|
|
1685
|
+
}];
|
|
1686
|
+
const probe = `${dataDir}/.gscdump-doctor-probe`;
|
|
1687
|
+
return await fs.writeFile(probe, "").then(() => fs.rm(probe)).then(() => true).catch(() => false) ? [{
|
|
1688
|
+
name: "dataDir",
|
|
1689
|
+
status: "pass",
|
|
1690
|
+
detail: display
|
|
1691
|
+
}] : [{
|
|
1692
|
+
name: "dataDir",
|
|
1693
|
+
status: "fail",
|
|
1694
|
+
detail: `${display} not writable`
|
|
1695
|
+
}];
|
|
1696
|
+
}
|
|
1697
|
+
async function checkStoreWatermarks() {
|
|
1698
|
+
const dataDir = resolveDataDir(await loadConfig());
|
|
1699
|
+
if (!(await fs.stat(dataDir).catch(() => null))?.isDirectory()) return [{
|
|
1700
|
+
name: "store.watermarks",
|
|
1701
|
+
status: "pass",
|
|
1702
|
+
detail: "no store yet (run `gscdump sync`)"
|
|
1703
|
+
}];
|
|
1704
|
+
const store = createLocalStore({ dataDir });
|
|
1705
|
+
const watermarks = await store.engine.getWatermarks({ userId: store.userId }).catch(() => null);
|
|
1706
|
+
if (!watermarks || watermarks.length === 0) return [{
|
|
1707
|
+
name: "store.watermarks",
|
|
1708
|
+
status: "pass",
|
|
1709
|
+
detail: "no watermarks (run `gscdump sync`)"
|
|
1710
|
+
}];
|
|
1711
|
+
const bySite = /* @__PURE__ */ new Map();
|
|
1712
|
+
for (const w of watermarks) {
|
|
1713
|
+
const site = w.siteId ?? "(global)";
|
|
1714
|
+
const existing = bySite.get(site);
|
|
1715
|
+
if (!existing || w.newestDateSynced > existing) bySite.set(site, w.newestDateSynced);
|
|
1716
|
+
}
|
|
1717
|
+
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
1718
|
+
const stale = [];
|
|
1719
|
+
let freshest = null;
|
|
1720
|
+
for (const [site, newest] of bySite) {
|
|
1721
|
+
const days = Math.floor((Date.parse(today) - Date.parse(newest)) / 864e5);
|
|
1722
|
+
if (days > WATERMARK_STALE_DAYS_WARN) stale.push({
|
|
1723
|
+
site,
|
|
1724
|
+
days
|
|
1725
|
+
});
|
|
1726
|
+
if (!freshest || days < freshest.days) freshest = {
|
|
1727
|
+
site,
|
|
1728
|
+
days
|
|
1729
|
+
};
|
|
1730
|
+
}
|
|
1731
|
+
if (stale.length > 0) {
|
|
1732
|
+
const sample = stale.slice(0, 3).map((s) => `${s.site} (${s.days}d)`).join(", ");
|
|
1733
|
+
const more = stale.length > 3 ? ` +${stale.length - 3} more` : "";
|
|
1734
|
+
return [{
|
|
1735
|
+
name: "store.watermarks",
|
|
1736
|
+
status: "warn",
|
|
1737
|
+
detail: `${stale.length}/${bySite.size} site(s) stale >${WATERMARK_STALE_DAYS_WARN}d: ${sample}${more}`
|
|
1738
|
+
}];
|
|
1739
|
+
}
|
|
1740
|
+
const tail = freshest ? `, freshest ${freshest.days}d ago` : "";
|
|
1741
|
+
return [{
|
|
1742
|
+
name: "store.watermarks",
|
|
1743
|
+
status: "pass",
|
|
1744
|
+
detail: `${bySite.size} site(s)${tail}`
|
|
1745
|
+
}];
|
|
1746
|
+
}
|
|
1747
|
+
async function checkApiReachable(name, url) {
|
|
1748
|
+
const reachable = await ofetch.raw(url, {
|
|
1749
|
+
method: "GET",
|
|
1750
|
+
timeout: FETCH_TIMEOUT_MS
|
|
1751
|
+
}).then(() => true).catch(() => false);
|
|
1752
|
+
return [{
|
|
1753
|
+
name,
|
|
1754
|
+
status: reachable ? "pass" : "warn",
|
|
1755
|
+
detail: reachable ? "reachable" : `${new URL(url).hostname} unreachable (network/firewall?)`
|
|
1756
|
+
}];
|
|
1757
|
+
}
|
|
1758
|
+
async function checkGscSites() {
|
|
1759
|
+
const config = await loadConfig();
|
|
1760
|
+
const auth = await resolveAuth({
|
|
1761
|
+
interactive: false,
|
|
1762
|
+
config
|
|
1763
|
+
}).catch(() => null);
|
|
1764
|
+
if (!auth) return [{
|
|
1765
|
+
name: "gsc.sites",
|
|
1766
|
+
status: "warn",
|
|
1767
|
+
detail: "skipped (no usable auth)"
|
|
1768
|
+
}];
|
|
1769
|
+
const sites = await googleSearchConsole(auth).sites().catch((e) => e);
|
|
1770
|
+
if (sites instanceof Error) return [{
|
|
1771
|
+
name: "gsc.sites",
|
|
1772
|
+
status: "fail",
|
|
1773
|
+
detail: `sites() failed: ${sites.message}`
|
|
1774
|
+
}];
|
|
1775
|
+
const checks = [];
|
|
1776
|
+
const verified = sites.filter((s) => s.permissionLevel !== "siteUnverifiedUser").length;
|
|
1777
|
+
checks.push({
|
|
1778
|
+
name: "gsc.sites",
|
|
1779
|
+
status: "pass",
|
|
1780
|
+
detail: `${sites.length} site(s) accessible (${verified} verified)`
|
|
1781
|
+
});
|
|
1782
|
+
if (config.defaultSite) {
|
|
1783
|
+
const match = sites.find((s) => s.siteUrl === config.defaultSite || (s.siteUrl ?? "").includes(String(config.defaultSite)));
|
|
1784
|
+
checks.push(match ? {
|
|
1785
|
+
name: "config.defaultSite",
|
|
1786
|
+
status: "pass",
|
|
1787
|
+
detail: `${config.defaultSite} ✓`
|
|
1788
|
+
} : {
|
|
1789
|
+
name: "config.defaultSite",
|
|
1790
|
+
status: "fail",
|
|
1791
|
+
detail: `${config.defaultSite} not in verified site list`
|
|
1792
|
+
});
|
|
1793
|
+
}
|
|
1794
|
+
return checks;
|
|
1795
|
+
}
|
|
1796
|
+
const doctorCommand = defineCommand({
|
|
1797
|
+
meta: {
|
|
1798
|
+
name: "doctor",
|
|
1799
|
+
description: "Run health checks (env, auth, scopes, time, dataDir, store, GSC reachability + ping, defaultSite)"
|
|
1800
|
+
},
|
|
1801
|
+
args: { json: {
|
|
1802
|
+
type: "boolean",
|
|
1803
|
+
default: false,
|
|
1804
|
+
description: "Output as JSON"
|
|
1805
|
+
} },
|
|
1806
|
+
async run({ args }) {
|
|
1807
|
+
const envResult = await checkEnv();
|
|
1808
|
+
const [authResult, timeChecks, dataDirChecks, watermarkChecks, gscApi, indexingApi, siteVerificationApi] = await Promise.all([
|
|
1809
|
+
checkAuth$1(envResult.envKeys),
|
|
1810
|
+
checkTimeSkew(),
|
|
1811
|
+
checkDataDir(),
|
|
1812
|
+
checkStoreWatermarks(),
|
|
1813
|
+
checkApiReachable("gsc.api", "https://searchconsole.googleapis.com/$discovery/rest?version=v1"),
|
|
1814
|
+
checkApiReachable("indexing.api", "https://indexing.googleapis.com/$discovery/rest?version=v3"),
|
|
1815
|
+
checkApiReachable("siteverification.api", "https://www.googleapis.com/discovery/v1/apis/siteVerification/v1/rest")
|
|
1816
|
+
]);
|
|
1817
|
+
const sitesChecks = authResult.liveToken ? await checkGscSites() : [{
|
|
1818
|
+
name: "gsc.sites",
|
|
1819
|
+
status: "warn",
|
|
1820
|
+
detail: "skipped (auth failed)"
|
|
1821
|
+
}];
|
|
1822
|
+
const all = [
|
|
1823
|
+
...envResult.checks,
|
|
1824
|
+
...authResult.checks,
|
|
1825
|
+
...timeChecks,
|
|
1826
|
+
...dataDirChecks,
|
|
1827
|
+
...watermarkChecks,
|
|
1828
|
+
...gscApi,
|
|
1829
|
+
...indexingApi,
|
|
1830
|
+
...siteVerificationApi,
|
|
1831
|
+
...sitesChecks
|
|
1832
|
+
];
|
|
1833
|
+
if (args.json) {
|
|
1834
|
+
console.log(JSON.stringify({
|
|
1835
|
+
checks: all,
|
|
1836
|
+
ok: all.every((c) => c.status !== "fail")
|
|
1837
|
+
}, null, 2));
|
|
1838
|
+
return;
|
|
1839
|
+
}
|
|
1840
|
+
const ICONS = {
|
|
1841
|
+
pass: "\x1B[32m✓\x1B[0m",
|
|
1842
|
+
warn: "\x1B[33m!\x1B[0m",
|
|
1843
|
+
fail: "\x1B[31m✗\x1B[0m",
|
|
1844
|
+
info: "\x1B[34mℹ\x1B[0m"
|
|
1845
|
+
};
|
|
1846
|
+
console.log();
|
|
1847
|
+
for (const c of all) {
|
|
1848
|
+
const detail = c.detail ? ` \x1B[90m${c.detail}\x1B[0m` : "";
|
|
1849
|
+
console.log(` ${ICONS[c.status]} ${c.name}${detail}`);
|
|
1850
|
+
}
|
|
1851
|
+
console.log();
|
|
1852
|
+
const failed = all.filter((c) => c.status === "fail");
|
|
1853
|
+
if (failed.length > 0) {
|
|
1854
|
+
logger.error(`${failed.length} check(s) failed`);
|
|
1855
|
+
process.exit(1);
|
|
1856
|
+
}
|
|
1857
|
+
const warned = all.filter((c) => c.status === "warn");
|
|
1858
|
+
if (warned.length > 0) logger.warn(`${warned.length} warning(s)`);
|
|
1859
|
+
else logger.success("All checks passed");
|
|
1860
|
+
}
|
|
1861
|
+
});
|
|
1862
|
+
const DEFAULT_OUT = "./gscdump-export";
|
|
1863
|
+
const FORMATS = [
|
|
1864
|
+
"parquet",
|
|
1865
|
+
"json",
|
|
1866
|
+
"ndjson",
|
|
1867
|
+
"csv"
|
|
1868
|
+
];
|
|
1869
|
+
const dumpCommand = defineCommand({
|
|
1870
|
+
meta: {
|
|
872
1871
|
name: "dump",
|
|
873
1872
|
description: "Export live Parquet files from the local store to a directory"
|
|
874
1873
|
},
|
|
875
1874
|
args: {
|
|
876
|
-
site: {
|
|
1875
|
+
"site": {
|
|
877
1876
|
type: "string",
|
|
878
1877
|
alias: "s",
|
|
879
|
-
description: "Site URL (e.g., sc-domain:example.com)"
|
|
1878
|
+
description: "Site URL (e.g., sc-domain:example.com); ignored with --all-sites"
|
|
880
1879
|
},
|
|
881
|
-
out: {
|
|
1880
|
+
"out": {
|
|
882
1881
|
type: "string",
|
|
883
1882
|
alias: "o",
|
|
884
1883
|
default: DEFAULT_OUT,
|
|
885
1884
|
description: `Output directory (default: ${DEFAULT_OUT})`
|
|
886
1885
|
},
|
|
887
|
-
|
|
1886
|
+
"format": {
|
|
1887
|
+
type: "string",
|
|
1888
|
+
alias: "F",
|
|
1889
|
+
default: "parquet",
|
|
1890
|
+
description: `Output format: ${FORMATS.join(", ")} (default: parquet copies raw files)`
|
|
1891
|
+
},
|
|
1892
|
+
"tables": {
|
|
1893
|
+
type: "string",
|
|
1894
|
+
alias: "t",
|
|
1895
|
+
description: `Comma-separated table list (default: all). Known: ${allTables().join(", ")}`
|
|
1896
|
+
},
|
|
1897
|
+
"all-sites": {
|
|
1898
|
+
type: "boolean",
|
|
1899
|
+
default: false,
|
|
1900
|
+
description: "Iterate every site with local data"
|
|
1901
|
+
},
|
|
1902
|
+
"compact": {
|
|
888
1903
|
type: "boolean",
|
|
889
1904
|
default: false,
|
|
890
1905
|
description: "Compact every closed month into a single file before exporting"
|
|
891
1906
|
},
|
|
892
|
-
|
|
1907
|
+
"json": {
|
|
1908
|
+
type: "boolean",
|
|
1909
|
+
default: false,
|
|
1910
|
+
description: "Emit a JSON summary of what was exported"
|
|
1911
|
+
},
|
|
1912
|
+
"quiet": {
|
|
893
1913
|
type: "boolean",
|
|
894
1914
|
alias: "q",
|
|
895
1915
|
default: false,
|
|
@@ -897,31 +1917,76 @@ const dumpCommand = defineCommand({
|
|
|
897
1917
|
}
|
|
898
1918
|
},
|
|
899
1919
|
async run({ args }) {
|
|
1920
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
1921
|
+
const format = String(args.format);
|
|
1922
|
+
if (!FORMATS.includes(format)) {
|
|
1923
|
+
logger.error(`Invalid --format: ${format}. Allowed: ${FORMATS.join(", ")}`);
|
|
1924
|
+
process.exit(1);
|
|
1925
|
+
}
|
|
1926
|
+
const tablesFilter = args.tables ? new Set(String(args.tables).split(",").map((t) => t.trim()).filter(Boolean)) : null;
|
|
900
1927
|
const ctx = await createCommandContext({
|
|
901
|
-
needsAuth:
|
|
1928
|
+
needsAuth: !args["all-sites"],
|
|
902
1929
|
needsStore: true
|
|
903
1930
|
});
|
|
904
|
-
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
905
1931
|
const store = ctx.store;
|
|
906
1932
|
const outDir = path.resolve(String(args.out));
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
logger.warn(`No data for ${siteUrl}. Run \`gscdump sync\` first.`);
|
|
1933
|
+
const targets = args["all-sites"] ? await listSitesWithData(store) : [await ctx.resolveSite(args.site ? String(args.site) : void 0)];
|
|
1934
|
+
if (targets.length === 0) {
|
|
1935
|
+
logger.warn("No sites with local data. Run `gscdump sync` first.");
|
|
911
1936
|
process.exit(0);
|
|
912
1937
|
}
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
for (const
|
|
916
|
-
const
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
1938
|
+
if (args.compact) for (const siteUrl of targets) await compactClosedMonths(store, siteUrl, args.quiet);
|
|
1939
|
+
const summary = [];
|
|
1940
|
+
for (const siteUrl of targets) {
|
|
1941
|
+
const entries = (await listLiveEntries(store, siteUrl)).filter((e) => !tablesFilter || tablesFilter.has(e.table));
|
|
1942
|
+
if (entries.length === 0) {
|
|
1943
|
+
if (!args.json && !args.quiet) logger.warn(`No data for ${siteUrl}; skipping`);
|
|
1944
|
+
continue;
|
|
1945
|
+
}
|
|
1946
|
+
if (format === "parquet") {
|
|
1947
|
+
const written = await dumpParquet(store, entries, outDir);
|
|
1948
|
+
summary.push({
|
|
1949
|
+
site: siteUrl,
|
|
1950
|
+
files: written,
|
|
1951
|
+
rows: 0,
|
|
1952
|
+
format,
|
|
1953
|
+
outPath: outDir
|
|
1954
|
+
});
|
|
1955
|
+
} else {
|
|
1956
|
+
const written = await dumpRowFormat(store, entries, outDir, siteUrl, format);
|
|
1957
|
+
summary.push({
|
|
1958
|
+
site: siteUrl,
|
|
1959
|
+
files: written.files,
|
|
1960
|
+
rows: written.rows,
|
|
1961
|
+
format,
|
|
1962
|
+
outPath: outDir
|
|
1963
|
+
});
|
|
1964
|
+
}
|
|
1965
|
+
}
|
|
1966
|
+
if (args.json) {
|
|
1967
|
+
console.log(JSON.stringify({
|
|
1968
|
+
outDir,
|
|
1969
|
+
sites: summary
|
|
1970
|
+
}, null, 2));
|
|
1971
|
+
return;
|
|
1972
|
+
}
|
|
1973
|
+
for (const s of summary) {
|
|
1974
|
+
const rows = s.rows ? `, ${s.rows.toLocaleString()} rows` : "";
|
|
1975
|
+
logger.success(`[${s.site}] ${s.files} ${s.format} file(s)${rows} → ${displayPath(s.outPath)}`);
|
|
921
1976
|
}
|
|
922
|
-
if (!args.quiet) logger.success(`Exported ${copied} file(s) to ${outDir}`);
|
|
923
1977
|
}
|
|
924
1978
|
});
|
|
1979
|
+
async function listSitesWithData(store) {
|
|
1980
|
+
const siteIds = /* @__PURE__ */ new Set();
|
|
1981
|
+
for (const table of allTables()) {
|
|
1982
|
+
const entries = await store.engine.listLive({
|
|
1983
|
+
userId: store.userId,
|
|
1984
|
+
table
|
|
1985
|
+
});
|
|
1986
|
+
for (const e of entries) if (e.siteId) siteIds.add(e.siteId);
|
|
1987
|
+
}
|
|
1988
|
+
return Array.from(siteIds);
|
|
1989
|
+
}
|
|
925
1990
|
async function listLiveEntries(store, siteUrl) {
|
|
926
1991
|
const siteId = store.siteIdFor(siteUrl);
|
|
927
1992
|
return (await Promise.all(allTables().map((table) => store.engine.listLive({
|
|
@@ -930,6 +1995,58 @@ async function listLiveEntries(store, siteUrl) {
|
|
|
930
1995
|
table
|
|
931
1996
|
})))).flat();
|
|
932
1997
|
}
|
|
1998
|
+
async function dumpParquet(store, entries, outDir) {
|
|
1999
|
+
await fs.mkdir(outDir, { recursive: true });
|
|
2000
|
+
let copied = 0;
|
|
2001
|
+
for (const entry of entries) {
|
|
2002
|
+
const bytes = await store.engine.readObject(entry.objectKey);
|
|
2003
|
+
const target = path.join(outDir, entry.objectKey);
|
|
2004
|
+
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
2005
|
+
await fs.writeFile(target, Buffer.from(bytes));
|
|
2006
|
+
copied++;
|
|
2007
|
+
}
|
|
2008
|
+
return copied;
|
|
2009
|
+
}
|
|
2010
|
+
async function dumpRowFormat(store, entries, outDir, siteUrl, format) {
|
|
2011
|
+
const byTable = /* @__PURE__ */ new Map();
|
|
2012
|
+
for (const e of entries) {
|
|
2013
|
+
const arr = byTable.get(e.table) ?? [];
|
|
2014
|
+
arr.push(e);
|
|
2015
|
+
byTable.set(e.table, arr);
|
|
2016
|
+
}
|
|
2017
|
+
const safeSite = siteUrl.replace(/[^a-z0-9]+/gi, "_");
|
|
2018
|
+
const siteDir = path.join(outDir, safeSite);
|
|
2019
|
+
await fs.mkdir(siteDir, { recursive: true });
|
|
2020
|
+
let files = 0;
|
|
2021
|
+
let totalRows = 0;
|
|
2022
|
+
for (const [table, tableEntries] of byTable) {
|
|
2023
|
+
const rows = await readTableRows(tableEntries.map((e) => path.join(store.dataDir, e.objectKey)));
|
|
2024
|
+
const ext = format === "csv" ? "csv" : format === "ndjson" ? "ndjson" : "json";
|
|
2025
|
+
const target = path.join(siteDir, `${table}.${ext}`);
|
|
2026
|
+
let body;
|
|
2027
|
+
if (format === "json") body = JSON.stringify(rows, null, 2);
|
|
2028
|
+
else if (format === "ndjson") body = rows.map((r) => JSON.stringify(r)).join("\n");
|
|
2029
|
+
else body = rows.length > 0 ? toCSV(rows, Object.keys(rows[0])) : "";
|
|
2030
|
+
await fs.writeFile(target, body);
|
|
2031
|
+
files++;
|
|
2032
|
+
totalRows += rows.length;
|
|
2033
|
+
}
|
|
2034
|
+
return {
|
|
2035
|
+
files,
|
|
2036
|
+
rows: totalRows
|
|
2037
|
+
};
|
|
2038
|
+
}
|
|
2039
|
+
async function readTableRows(filePaths) {
|
|
2040
|
+
const instance = await DuckDBInstance.create(":memory:");
|
|
2041
|
+
const conn = await instance.connect();
|
|
2042
|
+
try {
|
|
2043
|
+
const fileList = filePaths.map((p) => `'${sqlEscape(p)}'`).join(", ");
|
|
2044
|
+
return (await conn.runAndReadAll(`SELECT * FROM read_parquet([${fileList}], union_by_name=true)`)).getRowObjects();
|
|
2045
|
+
} finally {
|
|
2046
|
+
conn.closeSync();
|
|
2047
|
+
instance.closeSync();
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
933
2050
|
async function compactClosedMonths(store, siteUrl, quiet) {
|
|
934
2051
|
const siteId = store.siteIdFor(siteUrl);
|
|
935
2052
|
for (const table of allTables()) {
|
|
@@ -958,8 +2075,7 @@ const inspectSubCommand = defineCommand({
|
|
|
958
2075
|
site: {
|
|
959
2076
|
type: "string",
|
|
960
2077
|
alias: "s",
|
|
961
|
-
|
|
962
|
-
description: "Site URL (e.g., sc-domain:example.com)"
|
|
2078
|
+
description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
|
|
963
2079
|
},
|
|
964
2080
|
file: {
|
|
965
2081
|
type: "string",
|
|
@@ -976,6 +2092,11 @@ const inspectSubCommand = defineCommand({
|
|
|
976
2092
|
default: "4",
|
|
977
2093
|
description: "Concurrent in-flight inspect calls (default: 4)"
|
|
978
2094
|
},
|
|
2095
|
+
json: {
|
|
2096
|
+
type: "boolean",
|
|
2097
|
+
default: false,
|
|
2098
|
+
description: "Emit a JSON summary of inspection results"
|
|
2099
|
+
},
|
|
979
2100
|
quiet: {
|
|
980
2101
|
type: "boolean",
|
|
981
2102
|
alias: "q",
|
|
@@ -984,16 +2105,17 @@ const inspectSubCommand = defineCommand({
|
|
|
984
2105
|
}
|
|
985
2106
|
},
|
|
986
2107
|
async run({ args }) {
|
|
2108
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
987
2109
|
const ctx = await createCommandContext({
|
|
988
2110
|
needsAuth: true,
|
|
989
2111
|
needsStore: true
|
|
990
2112
|
});
|
|
991
2113
|
const client = ctx.client;
|
|
992
2114
|
const store = ctx.store;
|
|
993
|
-
const siteUrl = String(args.site);
|
|
2115
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
994
2116
|
const limit = args.limit ? Number.parseInt(String(args.limit), 10) : INSPECTION_QPD_PER_PROPERTY;
|
|
995
2117
|
const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
|
|
996
|
-
const quiet = Boolean(args.quiet);
|
|
2118
|
+
const quiet = Boolean(args.quiet) || Boolean(args.json);
|
|
997
2119
|
const urls = (await readUrlList({ file: args.file ? String(args.file) : void 0 })).slice(0, limit);
|
|
998
2120
|
if (urls.length === 0) {
|
|
999
2121
|
logger.warn("No URLs to inspect.");
|
|
@@ -1041,7 +2163,14 @@ const inspectSubCommand = defineCommand({
|
|
|
1041
2163
|
userId: store.userId,
|
|
1042
2164
|
siteId: store.siteIdFor(siteUrl)
|
|
1043
2165
|
}, records);
|
|
1044
|
-
if (
|
|
2166
|
+
if (args.json) console.log(JSON.stringify({
|
|
2167
|
+
site: siteUrl,
|
|
2168
|
+
inspected: records.length,
|
|
2169
|
+
failed,
|
|
2170
|
+
failures,
|
|
2171
|
+
records
|
|
2172
|
+
}, null, 2));
|
|
2173
|
+
else if (!quiet) {
|
|
1045
2174
|
logger.success(`Inspected ${records.length}/${urls.length} URL(s)`);
|
|
1046
2175
|
if (failed > 0) {
|
|
1047
2176
|
logger.warn(`${failed} failed:`);
|
|
@@ -1061,8 +2190,7 @@ const showSubCommand = defineCommand({
|
|
|
1061
2190
|
site: {
|
|
1062
2191
|
type: "string",
|
|
1063
2192
|
alias: "s",
|
|
1064
|
-
|
|
1065
|
-
description: "Site URL"
|
|
2193
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
1066
2194
|
},
|
|
1067
2195
|
url: {
|
|
1068
2196
|
type: "positional",
|
|
@@ -1076,10 +2204,15 @@ const showSubCommand = defineCommand({
|
|
|
1076
2204
|
}
|
|
1077
2205
|
},
|
|
1078
2206
|
async run({ args }) {
|
|
1079
|
-
const
|
|
2207
|
+
const ctx = await createCommandContext({
|
|
2208
|
+
needsAuth: true,
|
|
2209
|
+
needsStore: true
|
|
2210
|
+
});
|
|
2211
|
+
const store = ctx.store;
|
|
2212
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1080
2213
|
const record = await createInspectionStore({ dataSource: store.dataSource }).getLatest({
|
|
1081
2214
|
userId: store.userId,
|
|
1082
|
-
siteId: store.siteIdFor(
|
|
2215
|
+
siteId: store.siteIdFor(siteUrl)
|
|
1083
2216
|
}, String(args.url));
|
|
1084
2217
|
if (!record) {
|
|
1085
2218
|
logger.warn(`No inspection record for ${args.url}`);
|
|
@@ -1110,8 +2243,7 @@ const sitemapsSnapshotSubCommand = defineCommand({
|
|
|
1110
2243
|
site: {
|
|
1111
2244
|
type: "string",
|
|
1112
2245
|
alias: "s",
|
|
1113
|
-
|
|
1114
|
-
description: "Site URL (e.g., sc-domain:example.com)"
|
|
2246
|
+
description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
|
|
1115
2247
|
},
|
|
1116
2248
|
quiet: {
|
|
1117
2249
|
type: "boolean",
|
|
@@ -1132,7 +2264,7 @@ const sitemapsSnapshotSubCommand = defineCommand({
|
|
|
1132
2264
|
});
|
|
1133
2265
|
const client = ctx.client;
|
|
1134
2266
|
const store = ctx.store;
|
|
1135
|
-
const siteUrl = String(args.site);
|
|
2267
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1136
2268
|
const quiet = Boolean(args.quiet);
|
|
1137
2269
|
const apiSitemaps = await client.sitemaps.list(siteUrl);
|
|
1138
2270
|
const capturedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -1185,8 +2317,7 @@ const sitemapsShowSubCommand = defineCommand({
|
|
|
1185
2317
|
site: {
|
|
1186
2318
|
type: "string",
|
|
1187
2319
|
alias: "s",
|
|
1188
|
-
|
|
1189
|
-
description: "Site URL"
|
|
2320
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
1190
2321
|
},
|
|
1191
2322
|
path: {
|
|
1192
2323
|
type: "positional",
|
|
@@ -1200,10 +2331,15 @@ const sitemapsShowSubCommand = defineCommand({
|
|
|
1200
2331
|
}
|
|
1201
2332
|
},
|
|
1202
2333
|
async run({ args }) {
|
|
1203
|
-
const
|
|
2334
|
+
const ctx = await createCommandContext({
|
|
2335
|
+
needsAuth: true,
|
|
2336
|
+
needsStore: true
|
|
2337
|
+
});
|
|
2338
|
+
const store = ctx.store;
|
|
2339
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1204
2340
|
const record = await createSitemapStore({ dataSource: store.dataSource }).getLatest({
|
|
1205
2341
|
userId: store.userId,
|
|
1206
|
-
siteId: store.siteIdFor(
|
|
2342
|
+
siteId: store.siteIdFor(siteUrl)
|
|
1207
2343
|
}, String(args.path));
|
|
1208
2344
|
if (!record) {
|
|
1209
2345
|
logger.warn(`No sitemap record for ${args.path}`);
|
|
@@ -1249,8 +2385,7 @@ const indexingSubCommand = defineCommand({
|
|
|
1249
2385
|
site: {
|
|
1250
2386
|
type: "string",
|
|
1251
2387
|
alias: "s",
|
|
1252
|
-
|
|
1253
|
-
description: "Site URL (e.g., sc-domain:example.com)"
|
|
2388
|
+
description: "Site URL (e.g., sc-domain:example.com); defaults to config.defaultSite or prompt"
|
|
1254
2389
|
},
|
|
1255
2390
|
file: {
|
|
1256
2391
|
type: "string",
|
|
@@ -1277,7 +2412,7 @@ const indexingSubCommand = defineCommand({
|
|
|
1277
2412
|
});
|
|
1278
2413
|
const client = ctx.client;
|
|
1279
2414
|
const store = ctx.store;
|
|
1280
|
-
const siteUrl = String(args.site);
|
|
2415
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1281
2416
|
const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 4);
|
|
1282
2417
|
const quiet = Boolean(args.quiet);
|
|
1283
2418
|
const urls = await readUrlList({ file: args.file ? String(args.file) : void 0 });
|
|
@@ -1346,124 +2481,575 @@ const entitiesCommand = defineCommand({
|
|
|
1346
2481
|
indexing: indexingSubCommand
|
|
1347
2482
|
}
|
|
1348
2483
|
});
|
|
1349
|
-
const
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
if (isCancel(answer)) process.exit(1);
|
|
1358
|
-
return String(answer) || fallback;
|
|
1359
|
-
}
|
|
1360
|
-
async function loadEnvFile() {
|
|
1361
|
-
const envPath = path.join(process.cwd(), ".env");
|
|
1362
|
-
const content = await fs.readFile(envPath, "utf-8").catch(() => null);
|
|
1363
|
-
if (!content) return null;
|
|
1364
|
-
const env = {};
|
|
1365
|
-
for (const line of content.split("\n")) {
|
|
1366
|
-
const trimmed = line.trim();
|
|
1367
|
-
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
1368
|
-
const match = trimmed.match(ENV_LINE_RE);
|
|
1369
|
-
if (match) {
|
|
1370
|
-
const key = match[1].trim();
|
|
1371
|
-
let value = match[2].trim();
|
|
1372
|
-
if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
|
|
1373
|
-
env[key] = value;
|
|
1374
|
-
}
|
|
1375
|
-
}
|
|
1376
|
-
return env;
|
|
2484
|
+
const RETRIES_ARG = { retries: {
|
|
2485
|
+
type: "string",
|
|
2486
|
+
description: "Override per-call retry count (default: 3)"
|
|
2487
|
+
} };
|
|
2488
|
+
function parseRetries(v) {
|
|
2489
|
+
if (v == null || v === "") return void 0;
|
|
2490
|
+
const n = Number.parseInt(String(v), 10);
|
|
2491
|
+
return Number.isFinite(n) && n >= 0 ? n : void 0;
|
|
1377
2492
|
}
|
|
1378
|
-
const
|
|
1379
|
-
meta: {
|
|
1380
|
-
name: "init",
|
|
1381
|
-
description: "Set up GSCDump authentication"
|
|
1382
|
-
},
|
|
1383
|
-
args: { force: {
|
|
1384
|
-
type: "boolean",
|
|
1385
|
-
alias: "f",
|
|
1386
|
-
description: "Force re-initialization"
|
|
1387
|
-
} },
|
|
1388
|
-
async run({ args }) {
|
|
1389
|
-
const config = await loadConfig();
|
|
1390
|
-
if (config.clientId && config.clientSecret && !args.force) {
|
|
1391
|
-
logger.info("Already configured");
|
|
1392
|
-
logger.info("Run with --force to reconfigure");
|
|
1393
|
-
return;
|
|
1394
|
-
}
|
|
1395
|
-
const envFile = await loadEnvFile();
|
|
1396
|
-
if (envFile?.GOOGLE_CLIENT_ID && envFile?.GOOGLE_CLIENT_SECRET && envFile?.GOOGLE_REFRESH_TOKEN) {
|
|
1397
|
-
logger.info("Found .env file with Google credentials");
|
|
1398
|
-
process.env.GOOGLE_CLIENT_ID = envFile.GOOGLE_CLIENT_ID;
|
|
1399
|
-
process.env.GOOGLE_CLIENT_SECRET = envFile.GOOGLE_CLIENT_SECRET;
|
|
1400
|
-
process.env.GOOGLE_REFRESH_TOKEN = envFile.GOOGLE_REFRESH_TOKEN;
|
|
1401
|
-
if (envFile.GOOGLE_ACCESS_TOKEN) process.env.GOOGLE_ACCESS_TOKEN = envFile.GOOGLE_ACCESS_TOKEN;
|
|
1402
|
-
await saveConfig({
|
|
1403
|
-
...config,
|
|
1404
|
-
clientId: envFile.GOOGLE_CLIENT_ID,
|
|
1405
|
-
clientSecret: envFile.GOOGLE_CLIENT_SECRET,
|
|
1406
|
-
dataDir: config.dataDir ?? defaultDataDir()
|
|
1407
|
-
});
|
|
1408
|
-
const creds = (await authenticate({
|
|
1409
|
-
clientId: envFile.GOOGLE_CLIENT_ID,
|
|
1410
|
-
clientSecret: envFile.GOOGLE_CLIENT_SECRET
|
|
1411
|
-
}, false)).credentials;
|
|
1412
|
-
if (creds.access_token) await saveTokens({
|
|
1413
|
-
access_token: creds.access_token,
|
|
1414
|
-
refresh_token: creds.refresh_token || envFile.GOOGLE_REFRESH_TOKEN,
|
|
1415
|
-
expiry_date: creds.expiry_date
|
|
1416
|
-
});
|
|
1417
|
-
console.log();
|
|
1418
|
-
logger.success("Setup complete using .env credentials! Run gscdump to get started.");
|
|
1419
|
-
return;
|
|
1420
|
-
}
|
|
1421
|
-
console.log();
|
|
1422
|
-
console.log(" \x1B[1mWelcome to GSCDump!\x1B[0m");
|
|
1423
|
-
console.log(" \x1B[90mGoogle Search Console data extraction CLI\x1B[0m");
|
|
1424
|
-
console.log();
|
|
1425
|
-
const dataDir = await promptDataDir(config.dataDir);
|
|
1426
|
-
const credentials = await getAuthCredentials(true);
|
|
1427
|
-
await saveConfig({
|
|
1428
|
-
...config,
|
|
1429
|
-
dataDir,
|
|
1430
|
-
clientId: credentials.clientId,
|
|
1431
|
-
clientSecret: credentials.clientSecret
|
|
1432
|
-
});
|
|
1433
|
-
await authenticate(credentials, true);
|
|
1434
|
-
console.log();
|
|
1435
|
-
logger.success("Setup complete! Run gscdump to get started.");
|
|
1436
|
-
}
|
|
1437
|
-
});
|
|
1438
|
-
const inspectCommand = defineCommand({
|
|
2493
|
+
const submitCommand$1 = defineCommand({
|
|
1439
2494
|
meta: {
|
|
1440
|
-
name: "
|
|
1441
|
-
description: "
|
|
2495
|
+
name: "submit",
|
|
2496
|
+
description: "Notify Google of a new or updated URL (URL_UPDATED)"
|
|
1442
2497
|
},
|
|
1443
2498
|
args: {
|
|
1444
|
-
site: {
|
|
1445
|
-
type: "string",
|
|
1446
|
-
alias: "s",
|
|
1447
|
-
required: true,
|
|
1448
|
-
description: "Site URL (e.g., sc-domain:example.com)"
|
|
1449
|
-
},
|
|
1450
2499
|
url: {
|
|
1451
2500
|
type: "positional",
|
|
1452
2501
|
required: true,
|
|
1453
|
-
description: "URL to
|
|
2502
|
+
description: "URL to submit"
|
|
1454
2503
|
},
|
|
1455
2504
|
json: {
|
|
1456
2505
|
type: "boolean",
|
|
1457
2506
|
default: false,
|
|
1458
2507
|
description: "Output as JSON"
|
|
1459
|
-
}
|
|
2508
|
+
},
|
|
2509
|
+
quiet: {
|
|
2510
|
+
type: "boolean",
|
|
2511
|
+
alias: "q",
|
|
2512
|
+
default: false,
|
|
2513
|
+
description: "Suppress info/success output"
|
|
2514
|
+
},
|
|
2515
|
+
...RETRIES_ARG
|
|
1460
2516
|
},
|
|
1461
2517
|
async run({ args }) {
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
2518
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
2519
|
+
const result = await requestIndexing((await createCommandContext({
|
|
2520
|
+
needsAuth: true,
|
|
2521
|
+
fetchOptions: { retry: parseRetries(args.retries) }
|
|
2522
|
+
})).client, args.url, { type: "URL_UPDATED" }).catch(gscErrorHandler);
|
|
2523
|
+
if (args.json) {
|
|
2524
|
+
console.log(JSON.stringify(result, null, 2));
|
|
2525
|
+
return;
|
|
2526
|
+
}
|
|
2527
|
+
logger.success(`Submitted: ${result.url}`);
|
|
2528
|
+
if (result.notifyTime) console.log(` Notified: ${result.notifyTime}`);
|
|
2529
|
+
}
|
|
2530
|
+
});
|
|
2531
|
+
const removeCommand = defineCommand({
|
|
2532
|
+
meta: {
|
|
2533
|
+
name: "remove",
|
|
2534
|
+
description: "Notify Google a URL has been removed (URL_DELETED)"
|
|
2535
|
+
},
|
|
2536
|
+
args: {
|
|
2537
|
+
url: {
|
|
2538
|
+
type: "positional",
|
|
2539
|
+
required: true,
|
|
2540
|
+
description: "URL to mark removed"
|
|
2541
|
+
},
|
|
2542
|
+
json: {
|
|
2543
|
+
type: "boolean",
|
|
2544
|
+
default: false,
|
|
2545
|
+
description: "Output as JSON"
|
|
2546
|
+
},
|
|
2547
|
+
quiet: {
|
|
2548
|
+
type: "boolean",
|
|
2549
|
+
alias: "q",
|
|
2550
|
+
default: false,
|
|
2551
|
+
description: "Suppress info/success output"
|
|
2552
|
+
},
|
|
2553
|
+
...RETRIES_ARG
|
|
2554
|
+
},
|
|
2555
|
+
async run({ args }) {
|
|
2556
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
2557
|
+
const result = await requestIndexing((await createCommandContext({
|
|
2558
|
+
needsAuth: true,
|
|
2559
|
+
fetchOptions: { retry: parseRetries(args.retries) }
|
|
2560
|
+
})).client, args.url, { type: "URL_DELETED" }).catch(gscErrorHandler);
|
|
2561
|
+
if (args.json) {
|
|
2562
|
+
console.log(JSON.stringify(result, null, 2));
|
|
2563
|
+
return;
|
|
2564
|
+
}
|
|
2565
|
+
logger.success(`Removed: ${result.url}`);
|
|
2566
|
+
if (result.notifyTime) console.log(` Notified: ${result.notifyTime}`);
|
|
2567
|
+
}
|
|
2568
|
+
});
|
|
2569
|
+
const statusCommand = defineCommand({
|
|
2570
|
+
meta: {
|
|
2571
|
+
name: "status",
|
|
2572
|
+
description: "Show indexing notification metadata for a URL"
|
|
2573
|
+
},
|
|
2574
|
+
args: {
|
|
2575
|
+
url: {
|
|
2576
|
+
type: "positional",
|
|
2577
|
+
required: true,
|
|
2578
|
+
description: "URL to inspect"
|
|
2579
|
+
},
|
|
2580
|
+
json: {
|
|
2581
|
+
type: "boolean",
|
|
2582
|
+
default: false,
|
|
2583
|
+
description: "Output as JSON"
|
|
2584
|
+
},
|
|
2585
|
+
quiet: {
|
|
2586
|
+
type: "boolean",
|
|
2587
|
+
alias: "q",
|
|
2588
|
+
default: false,
|
|
2589
|
+
description: "Suppress info/success output"
|
|
2590
|
+
}
|
|
2591
|
+
},
|
|
2592
|
+
async run({ args }) {
|
|
2593
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
2594
|
+
const meta = await getIndexingMetadata((await createCommandContext({ needsAuth: true })).client, args.url).catch(gscErrorHandler);
|
|
2595
|
+
if (args.json) {
|
|
2596
|
+
console.log(JSON.stringify(meta, null, 2));
|
|
2597
|
+
return;
|
|
2598
|
+
}
|
|
2599
|
+
const fmtNotification = (n) => {
|
|
2600
|
+
if (!n?.notifyTime) return "never";
|
|
2601
|
+
return n.type ? `${n.notifyTime} (${n.type})` : n.notifyTime;
|
|
2602
|
+
};
|
|
2603
|
+
const update = meta.latestUpdate;
|
|
2604
|
+
const remove = meta.latestRemove;
|
|
2605
|
+
console.log();
|
|
2606
|
+
console.log(` \x1B[1mURL:\x1B[0m ${meta.url}`);
|
|
2607
|
+
console.log(` Last update notify: ${fmtNotification(update)}`);
|
|
2608
|
+
console.log(` Last remove notify: ${fmtNotification(remove)}`);
|
|
2609
|
+
console.log();
|
|
2610
|
+
}
|
|
2611
|
+
});
|
|
2612
|
+
const INDEXING_DAILY_QUOTA = 200;
|
|
2613
|
+
const indexingCommand = defineCommand({
|
|
2614
|
+
meta: {
|
|
2615
|
+
name: "indexing",
|
|
2616
|
+
description: "Notify Google about URL updates/removals (Indexing API)"
|
|
2617
|
+
},
|
|
2618
|
+
subCommands: {
|
|
2619
|
+
submit: submitCommand$1,
|
|
2620
|
+
remove: removeCommand,
|
|
2621
|
+
status: statusCommand,
|
|
2622
|
+
batch: defineCommand({
|
|
2623
|
+
meta: {
|
|
2624
|
+
name: "batch",
|
|
2625
|
+
description: "Submit many URLs from a file or stdin (one URL per line)"
|
|
2626
|
+
},
|
|
2627
|
+
args: {
|
|
2628
|
+
"urls": {
|
|
2629
|
+
type: "positional",
|
|
2630
|
+
required: false,
|
|
2631
|
+
description: "URLs (or use --file/stdin)"
|
|
2632
|
+
},
|
|
2633
|
+
"file": {
|
|
2634
|
+
type: "string",
|
|
2635
|
+
alias: "f",
|
|
2636
|
+
description: "File with URLs (one per line)"
|
|
2637
|
+
},
|
|
2638
|
+
"type": {
|
|
2639
|
+
type: "string",
|
|
2640
|
+
default: "URL_UPDATED",
|
|
2641
|
+
description: "URL_UPDATED or URL_DELETED"
|
|
2642
|
+
},
|
|
2643
|
+
"delay-ms": {
|
|
2644
|
+
type: "string",
|
|
2645
|
+
default: "100",
|
|
2646
|
+
description: "Delay between requests"
|
|
2647
|
+
},
|
|
2648
|
+
"concurrency": {
|
|
2649
|
+
type: "string",
|
|
2650
|
+
alias: "c",
|
|
2651
|
+
default: "1",
|
|
2652
|
+
description: "Concurrent in-flight requests"
|
|
2653
|
+
},
|
|
2654
|
+
"quiet": {
|
|
2655
|
+
type: "boolean",
|
|
2656
|
+
alias: "q",
|
|
2657
|
+
default: false,
|
|
2658
|
+
description: "Suppress progress output"
|
|
2659
|
+
},
|
|
2660
|
+
"json": {
|
|
2661
|
+
type: "boolean",
|
|
2662
|
+
default: false,
|
|
2663
|
+
description: "Output as JSON"
|
|
2664
|
+
},
|
|
2665
|
+
"yes": {
|
|
2666
|
+
type: "boolean",
|
|
2667
|
+
alias: "y",
|
|
2668
|
+
default: false,
|
|
2669
|
+
description: "Skip the over-quota confirmation prompt"
|
|
2670
|
+
},
|
|
2671
|
+
"retries": {
|
|
2672
|
+
type: "string",
|
|
2673
|
+
description: "Override per-call retry count (default: 3)"
|
|
2674
|
+
}
|
|
2675
|
+
},
|
|
2676
|
+
async run({ args }) {
|
|
2677
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
2678
|
+
const urls = await readUrlList$1(args);
|
|
2679
|
+
if (urls.length === 0) {
|
|
2680
|
+
logger.error("No URLs provided. Pass URLs as args, --file, or stdin.");
|
|
2681
|
+
process.exit(1);
|
|
2682
|
+
}
|
|
2683
|
+
const type = String(args.type);
|
|
2684
|
+
if (type !== "URL_UPDATED" && type !== "URL_DELETED") {
|
|
2685
|
+
logger.error(`Invalid --type: ${type}. Use URL_UPDATED or URL_DELETED.`);
|
|
2686
|
+
process.exit(1);
|
|
2687
|
+
}
|
|
2688
|
+
if (urls.length > INDEXING_DAILY_QUOTA && !args.yes && !args.json) {
|
|
2689
|
+
logger.warn(`Submitting ${urls.length} URLs but the Indexing API daily quota is ${INDEXING_DAILY_QUOTA}/day.`);
|
|
2690
|
+
logger.warn(`Excess submissions will fail with quota errors. Pass --yes to proceed anyway.`);
|
|
2691
|
+
process.exit(1);
|
|
2692
|
+
}
|
|
2693
|
+
if (urls.length > INDEXING_DAILY_QUOTA && !args.json && !args.quiet) logger.warn(`Proceeding with ${urls.length} URLs (over the ${INDEXING_DAILY_QUOTA}/day quota). Excess will fail.`);
|
|
2694
|
+
const ctx = await createCommandContext({
|
|
2695
|
+
needsAuth: true,
|
|
2696
|
+
fetchOptions: { retry: parseRetries(args.retries) }
|
|
2697
|
+
});
|
|
2698
|
+
const delayMs = Number.parseInt(String(args["delay-ms"]), 10);
|
|
2699
|
+
const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 1);
|
|
2700
|
+
if (!args.json && !args.quiet) logger.info(`Submitting ${urls.length} URLs (${type}) ...`);
|
|
2701
|
+
const results = await batchRequestIndexing(ctx.client, urls, {
|
|
2702
|
+
type,
|
|
2703
|
+
delayMs,
|
|
2704
|
+
concurrency,
|
|
2705
|
+
onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url}`)
|
|
2706
|
+
}).catch(gscErrorHandler);
|
|
2707
|
+
if (args.json) {
|
|
2708
|
+
console.log(JSON.stringify(results, null, 2));
|
|
2709
|
+
return;
|
|
2710
|
+
}
|
|
2711
|
+
if (!args.quiet) logger.success(`Submitted ${results.length}/${urls.length} URLs`);
|
|
2712
|
+
}
|
|
2713
|
+
})
|
|
2714
|
+
}
|
|
2715
|
+
});
|
|
2716
|
+
const ENV_LINE_RE = /^([^=]+)=(.*)$/;
|
|
2717
|
+
async function promptDataDir(existing) {
|
|
2718
|
+
const fallback = existing ?? defaultDataDir();
|
|
2719
|
+
const answer = await text({
|
|
2720
|
+
message: "Where should Parquet data be stored?",
|
|
2721
|
+
placeholder: fallback,
|
|
2722
|
+
defaultValue: fallback
|
|
2723
|
+
});
|
|
2724
|
+
if (isCancel(answer)) process.exit(1);
|
|
2725
|
+
return String(answer) || fallback;
|
|
2726
|
+
}
|
|
2727
|
+
async function loadEnvFile() {
|
|
2728
|
+
const envPath = path.join(process.cwd(), ".env");
|
|
2729
|
+
const content = await fs.readFile(envPath, "utf-8").catch(() => null);
|
|
2730
|
+
if (!content) return null;
|
|
2731
|
+
const env = {};
|
|
2732
|
+
for (const line of content.split("\n")) {
|
|
2733
|
+
const trimmed = line.trim();
|
|
2734
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
2735
|
+
const match = trimmed.match(ENV_LINE_RE);
|
|
2736
|
+
if (match) {
|
|
2737
|
+
const key = match[1].trim();
|
|
2738
|
+
let value = match[2].trim();
|
|
2739
|
+
if (value.startsWith("\"") && value.endsWith("\"") || value.startsWith("'") && value.endsWith("'")) value = value.slice(1, -1);
|
|
2740
|
+
env[key] = value;
|
|
2741
|
+
}
|
|
2742
|
+
}
|
|
2743
|
+
return env;
|
|
2744
|
+
}
|
|
2745
|
+
const initCommand = defineCommand({
|
|
2746
|
+
meta: {
|
|
2747
|
+
name: "init",
|
|
2748
|
+
description: "Set up GSCDump authentication"
|
|
2749
|
+
},
|
|
2750
|
+
args: {
|
|
2751
|
+
"force": {
|
|
2752
|
+
type: "boolean",
|
|
2753
|
+
alias: "f",
|
|
2754
|
+
description: "Force re-initialization"
|
|
2755
|
+
},
|
|
2756
|
+
"no-store": {
|
|
2757
|
+
type: "boolean",
|
|
2758
|
+
default: false,
|
|
2759
|
+
description: "Skip dataDir prompt (auth-only setup)"
|
|
2760
|
+
},
|
|
2761
|
+
"quiet": {
|
|
2762
|
+
type: "boolean",
|
|
2763
|
+
alias: "q",
|
|
2764
|
+
default: false,
|
|
2765
|
+
description: "Suppress info/success output"
|
|
2766
|
+
}
|
|
2767
|
+
},
|
|
2768
|
+
async run({ args }) {
|
|
2769
|
+
setQuiet(Boolean(args.quiet));
|
|
2770
|
+
const config = await loadConfig();
|
|
2771
|
+
if (config.clientId && config.clientSecret && !args.force) {
|
|
2772
|
+
logger.info("Already configured");
|
|
2773
|
+
logger.info("Run with --force to reconfigure");
|
|
2774
|
+
return;
|
|
2775
|
+
}
|
|
2776
|
+
const byok = resolveBYOK();
|
|
2777
|
+
if (byok) {
|
|
2778
|
+
const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
|
|
2779
|
+
await saveConfig({
|
|
2780
|
+
...config,
|
|
2781
|
+
dataDir: dataDir ?? config.dataDir
|
|
2782
|
+
});
|
|
2783
|
+
logger.success(`BYOK detected (${typeof byok === "string" ? "access-token" : "refresh-token"}) — auth setup skipped`);
|
|
2784
|
+
logger.success("Setup complete! Run gscdump to get started.");
|
|
2785
|
+
return;
|
|
2786
|
+
}
|
|
2787
|
+
const envFile = await loadEnvFile();
|
|
2788
|
+
const envCid = envFile?.GSC_CLIENT_ID ?? envFile?.GOOGLE_CLIENT_ID;
|
|
2789
|
+
const envSec = envFile?.GSC_CLIENT_SECRET ?? envFile?.GOOGLE_CLIENT_SECRET;
|
|
2790
|
+
const envRef = envFile?.GSC_REFRESH_TOKEN ?? envFile?.GOOGLE_REFRESH_TOKEN;
|
|
2791
|
+
const envAcc = envFile?.GSC_ACCESS_TOKEN ?? envFile?.GOOGLE_ACCESS_TOKEN;
|
|
2792
|
+
if (envCid && envSec && envRef) {
|
|
2793
|
+
logger.info("Found .env file with credentials");
|
|
2794
|
+
process.env.GOOGLE_CLIENT_ID = envCid;
|
|
2795
|
+
process.env.GOOGLE_CLIENT_SECRET = envSec;
|
|
2796
|
+
process.env.GOOGLE_REFRESH_TOKEN = envRef;
|
|
2797
|
+
if (envAcc) process.env.GOOGLE_ACCESS_TOKEN = envAcc;
|
|
2798
|
+
await saveConfig({
|
|
2799
|
+
...config,
|
|
2800
|
+
clientId: envCid,
|
|
2801
|
+
clientSecret: envSec,
|
|
2802
|
+
dataDir: config.dataDir ?? defaultDataDir()
|
|
2803
|
+
});
|
|
2804
|
+
const creds = (await authenticate({
|
|
2805
|
+
clientId: envCid,
|
|
2806
|
+
clientSecret: envSec
|
|
2807
|
+
}, false)).credentials;
|
|
2808
|
+
if (creds.access_token) await saveTokens({
|
|
2809
|
+
access_token: creds.access_token,
|
|
2810
|
+
refresh_token: creds.refresh_token || envRef,
|
|
2811
|
+
expiry_date: creds.expiry_date
|
|
2812
|
+
});
|
|
2813
|
+
console.log();
|
|
2814
|
+
logger.success("Setup complete using .env credentials! Run gscdump to get started.");
|
|
2815
|
+
return;
|
|
2816
|
+
}
|
|
2817
|
+
console.log();
|
|
2818
|
+
console.log(" \x1B[1mWelcome to GSCDump!\x1B[0m");
|
|
2819
|
+
console.log(" \x1B[90mGoogle Search Console data extraction CLI\x1B[0m");
|
|
2820
|
+
console.log();
|
|
2821
|
+
const dataDir = args["no-store"] ? void 0 : await promptDataDir(config.dataDir);
|
|
2822
|
+
const credentials = await getAuthCredentials(true);
|
|
2823
|
+
await saveConfig({
|
|
2824
|
+
...config,
|
|
2825
|
+
...dataDir ? { dataDir } : {},
|
|
2826
|
+
clientId: credentials.clientId,
|
|
2827
|
+
clientSecret: credentials.clientSecret
|
|
2828
|
+
});
|
|
2829
|
+
await smokeTest(await authenticate(credentials, true));
|
|
2830
|
+
await maybeWriteEnvFile(credentials.clientId, credentials.clientSecret);
|
|
2831
|
+
console.log();
|
|
2832
|
+
logger.success("Setup complete! Run gscdump to get started.");
|
|
2833
|
+
}
|
|
2834
|
+
});
|
|
2835
|
+
async function smokeTest(oauth) {
|
|
2836
|
+
const sites = await googleSearchConsole(oauth).sites().catch((e) => e);
|
|
2837
|
+
if (sites instanceof Error) {
|
|
2838
|
+
logger.warn(`Smoke test failed: ${sites.message}`);
|
|
2839
|
+
logger.info("Auth saved, but verify scopes via `gscdump auth status` / `gscdump doctor`.");
|
|
2840
|
+
return;
|
|
2841
|
+
}
|
|
2842
|
+
logger.success(`Verified: ${sites.length} GSC site(s) accessible`);
|
|
2843
|
+
}
|
|
2844
|
+
async function maybeWriteEnvFile(clientId, clientSecret) {
|
|
2845
|
+
const tokens = await loadTokens();
|
|
2846
|
+
if (!tokens?.refresh_token) return;
|
|
2847
|
+
const wants = await confirm({
|
|
2848
|
+
message: "Write a `.env` file with these credentials? (handy for CI / other machines)",
|
|
2849
|
+
initialValue: false
|
|
2850
|
+
});
|
|
2851
|
+
if (isCancel(wants) || !wants) return;
|
|
2852
|
+
const envPath = path.join(process.cwd(), ".env");
|
|
2853
|
+
if (await fs.stat(envPath).then(() => true).catch(() => false)) {
|
|
2854
|
+
const overwrite = await confirm({
|
|
2855
|
+
message: `${displayPath(envPath)} exists. Overwrite?`,
|
|
2856
|
+
initialValue: false
|
|
2857
|
+
});
|
|
2858
|
+
if (isCancel(overwrite) || !overwrite) {
|
|
2859
|
+
logger.info(`Skipped — keep credentials at ${displayPath(envPath)} manually if needed`);
|
|
2860
|
+
return;
|
|
2861
|
+
}
|
|
2862
|
+
}
|
|
2863
|
+
const content = [
|
|
2864
|
+
`GSC_CLIENT_ID=${clientId}`,
|
|
2865
|
+
`GSC_CLIENT_SECRET=${clientSecret}`,
|
|
2866
|
+
`GSC_REFRESH_TOKEN=${tokens.refresh_token}`,
|
|
2867
|
+
""
|
|
2868
|
+
].join("\n");
|
|
2869
|
+
await fs.writeFile(envPath, content, { mode: 384 });
|
|
2870
|
+
logger.success(`Wrote ${displayPath(envPath)}`);
|
|
2871
|
+
}
|
|
2872
|
+
function verdictTone(verdict) {
|
|
2873
|
+
if (verdict === "PASS") return "\x1B[32m";
|
|
2874
|
+
if (verdict === "NEUTRAL" || verdict === "PARTIAL") return "\x1B[33m";
|
|
2875
|
+
if (verdict === "FAIL") return "\x1B[31m";
|
|
2876
|
+
return "\x1B[90m";
|
|
2877
|
+
}
|
|
2878
|
+
function colorVerdict(verdict) {
|
|
2879
|
+
return `${verdictTone(verdict)}${verdict || "N/A"}\x1B[0m`;
|
|
2880
|
+
}
|
|
2881
|
+
function printInspection(url, inspection) {
|
|
2882
|
+
const indexStatus = inspection?.indexStatusResult;
|
|
2883
|
+
console.log();
|
|
2884
|
+
console.log(` \x1B[1mURL:\x1B[0m ${url}`);
|
|
2885
|
+
console.log();
|
|
2886
|
+
console.log(` \x1B[1mIndex status\x1B[0m`);
|
|
2887
|
+
console.log(` Verdict: ${colorVerdict(indexStatus?.verdict)}`);
|
|
2888
|
+
if (indexStatus?.coverageState) console.log(` Coverage: ${indexStatus.coverageState}`);
|
|
2889
|
+
if (indexStatus?.indexingState) console.log(` Indexing: ${indexStatus.indexingState}`);
|
|
2890
|
+
if (indexStatus?.lastCrawlTime) console.log(` Last Crawl: ${indexStatus.lastCrawlTime}`);
|
|
2891
|
+
if (indexStatus?.crawledAs) console.log(` Crawled As: ${indexStatus.crawledAs}`);
|
|
2892
|
+
if (indexStatus?.robotsTxtState) console.log(` Robots.txt: ${indexStatus.robotsTxtState}`);
|
|
2893
|
+
if (indexStatus?.pageFetchState) console.log(` Page Fetch: ${indexStatus.pageFetchState}`);
|
|
2894
|
+
if (indexStatus?.googleCanonical) console.log(` Google Canon: ${indexStatus.googleCanonical}`);
|
|
2895
|
+
if (indexStatus?.userCanonical) console.log(` User Canon: ${indexStatus.userCanonical}`);
|
|
2896
|
+
if (indexStatus?.sitemap?.length) {
|
|
2897
|
+
console.log(` Sitemaps:`);
|
|
2898
|
+
for (const sm of indexStatus.sitemap) console.log(` \x1B[90m└─\x1B[0m ${sm}`);
|
|
2899
|
+
}
|
|
2900
|
+
if (indexStatus?.referringUrls?.length) {
|
|
2901
|
+
const shown = indexStatus.referringUrls.slice(0, 5);
|
|
2902
|
+
console.log(` Referring URLs (${indexStatus.referringUrls.length}):`);
|
|
2903
|
+
for (const r of shown) console.log(` \x1B[90m└─\x1B[0m ${r}`);
|
|
2904
|
+
if (indexStatus.referringUrls.length > shown.length) console.log(` \x1B[90m… ${indexStatus.referringUrls.length - shown.length} more\x1B[0m`);
|
|
2905
|
+
}
|
|
2906
|
+
const rich = inspection?.richResultsResult;
|
|
2907
|
+
if (rich) {
|
|
2908
|
+
console.log();
|
|
2909
|
+
console.log(` \x1B[1mRich results\x1B[0m`);
|
|
2910
|
+
console.log(` Verdict: ${colorVerdict(rich.verdict)}`);
|
|
2911
|
+
if (rich.detectedItems?.length) for (const group of rich.detectedItems) {
|
|
2912
|
+
const count = group.items?.length ?? 0;
|
|
2913
|
+
console.log(` ${group.richResultType ?? "unknown"}: ${count} item${count === 1 ? "" : "s"}`);
|
|
2914
|
+
for (const item of group.items ?? []) if (item.issues?.length) for (const issue of item.issues) {
|
|
2915
|
+
const sev = issue.severity === "ERROR" ? "\x1B[31m" : "\x1B[33m";
|
|
2916
|
+
console.log(` ${sev}${issue.severity ?? "?"}\x1B[0m ${item.name ?? ""}: ${issue.issueMessage ?? ""}`);
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2919
|
+
}
|
|
2920
|
+
const amp = inspection?.ampResult;
|
|
2921
|
+
if (amp) {
|
|
2922
|
+
console.log();
|
|
2923
|
+
console.log(` \x1B[1mAMP\x1B[0m`);
|
|
2924
|
+
console.log(` Verdict: ${colorVerdict(amp.verdict)}`);
|
|
2925
|
+
if (amp.ampUrl) console.log(` AMP URL: ${amp.ampUrl}`);
|
|
2926
|
+
if (amp.ampIndexStatusVerdict) console.log(` Index Verdict: ${amp.ampIndexStatusVerdict}`);
|
|
2927
|
+
if (amp.indexingState) console.log(` Indexing: ${amp.indexingState}`);
|
|
2928
|
+
if (amp.robotsTxtState) console.log(` Robots.txt: ${amp.robotsTxtState}`);
|
|
2929
|
+
if (amp.pageFetchState) console.log(` Page Fetch: ${amp.pageFetchState}`);
|
|
2930
|
+
if (amp.lastCrawlTime) console.log(` Last Crawl: ${amp.lastCrawlTime}`);
|
|
2931
|
+
if (amp.issues?.length) for (const issue of amp.issues) {
|
|
2932
|
+
const sev = issue.severity === "ERROR" ? "\x1B[31m" : "\x1B[33m";
|
|
2933
|
+
console.log(` ${sev}${issue.severity ?? "?"}\x1B[0m ${issue.issueMessage ?? ""}`);
|
|
2934
|
+
}
|
|
2935
|
+
}
|
|
2936
|
+
const mobile = inspection?.mobileUsabilityResult;
|
|
2937
|
+
if (mobile) {
|
|
2938
|
+
console.log();
|
|
2939
|
+
console.log(` \x1B[1mMobile usability\x1B[0m`);
|
|
2940
|
+
console.log(` Verdict: ${colorVerdict(mobile.verdict)}`);
|
|
2941
|
+
if (mobile.issues?.length) for (const issue of mobile.issues) console.log(` \x1B[33m${issue.issueType ?? "?"}\x1B[0m ${issue.message ?? ""}`);
|
|
2942
|
+
}
|
|
2943
|
+
if (inspection?.inspectionResultLink) {
|
|
2944
|
+
console.log();
|
|
2945
|
+
console.log(` \x1B[90mOpen in Search Console:\x1B[0m \x1B[36m${inspection.inspectionResultLink}\x1B[0m`);
|
|
2946
|
+
}
|
|
2947
|
+
console.log();
|
|
2948
|
+
}
|
|
2949
|
+
const inspectCommand = defineCommand({
|
|
2950
|
+
meta: {
|
|
2951
|
+
name: "inspect",
|
|
2952
|
+
description: "Inspect URL indexing status (single URL; use `inspect batch` for many)"
|
|
2953
|
+
},
|
|
2954
|
+
args: {
|
|
2955
|
+
site: {
|
|
2956
|
+
type: "string",
|
|
2957
|
+
alias: "s",
|
|
2958
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
2959
|
+
},
|
|
2960
|
+
url: {
|
|
2961
|
+
type: "positional",
|
|
2962
|
+
required: true,
|
|
2963
|
+
description: "URL to inspect"
|
|
2964
|
+
},
|
|
2965
|
+
json: {
|
|
2966
|
+
type: "boolean",
|
|
2967
|
+
default: false,
|
|
2968
|
+
description: "Output as JSON"
|
|
2969
|
+
},
|
|
2970
|
+
quiet: {
|
|
2971
|
+
type: "boolean",
|
|
2972
|
+
alias: "q",
|
|
2973
|
+
default: false,
|
|
2974
|
+
description: "Suppress info/success output"
|
|
2975
|
+
}
|
|
2976
|
+
},
|
|
2977
|
+
subCommands: { batch: defineCommand({
|
|
2978
|
+
meta: {
|
|
2979
|
+
name: "batch",
|
|
2980
|
+
description: "Inspect many URLs from a file or stdin (one URL per line)"
|
|
2981
|
+
},
|
|
2982
|
+
args: {
|
|
2983
|
+
"site": {
|
|
2984
|
+
type: "string",
|
|
2985
|
+
alias: "s",
|
|
2986
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
2987
|
+
},
|
|
2988
|
+
"urls": {
|
|
2989
|
+
type: "positional",
|
|
2990
|
+
required: false,
|
|
2991
|
+
description: "URLs (or use --file/stdin)"
|
|
2992
|
+
},
|
|
2993
|
+
"file": {
|
|
2994
|
+
type: "string",
|
|
2995
|
+
alias: "f",
|
|
2996
|
+
description: "File with URLs (one per line)"
|
|
2997
|
+
},
|
|
2998
|
+
"delay-ms": {
|
|
2999
|
+
type: "string",
|
|
3000
|
+
default: "200",
|
|
3001
|
+
description: "Delay between requests"
|
|
3002
|
+
},
|
|
3003
|
+
"concurrency": {
|
|
3004
|
+
type: "string",
|
|
3005
|
+
alias: "c",
|
|
3006
|
+
default: "1",
|
|
3007
|
+
description: "Concurrent in-flight requests"
|
|
3008
|
+
},
|
|
3009
|
+
"quiet": {
|
|
3010
|
+
type: "boolean",
|
|
3011
|
+
alias: "q",
|
|
3012
|
+
default: false,
|
|
3013
|
+
description: "Suppress progress output"
|
|
3014
|
+
},
|
|
3015
|
+
"json": {
|
|
3016
|
+
type: "boolean",
|
|
3017
|
+
default: false,
|
|
3018
|
+
description: "Output as JSON"
|
|
3019
|
+
}
|
|
3020
|
+
},
|
|
3021
|
+
async run({ args }) {
|
|
3022
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
3023
|
+
const urls = await readUrlList$1(args);
|
|
3024
|
+
if (urls.length === 0) {
|
|
3025
|
+
logger.error("No URLs provided. Pass URLs as args, --file, or stdin.");
|
|
3026
|
+
process.exit(1);
|
|
3027
|
+
}
|
|
3028
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
3029
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
3030
|
+
const delayMs = Number.parseInt(String(args["delay-ms"]), 10);
|
|
3031
|
+
const concurrency = Math.max(1, Number.parseInt(String(args.concurrency), 10) || 1);
|
|
3032
|
+
if (!args.json && !args.quiet) logger.info(`Inspecting ${urls.length} URLs ...`);
|
|
3033
|
+
const results = await batchInspectUrls(ctx.client, siteUrl, urls, {
|
|
3034
|
+
delayMs,
|
|
3035
|
+
concurrency,
|
|
3036
|
+
onProgress: args.json || args.quiet ? void 0 : (r, i, total) => logger.info(`[${i + 1}/${total}] ${r.url} ${r.isIndexed ? "PASS" : "FAIL"}`)
|
|
3037
|
+
}).catch(gscErrorHandler);
|
|
3038
|
+
if (args.json) {
|
|
3039
|
+
console.log(JSON.stringify(results, null, 2));
|
|
3040
|
+
return;
|
|
3041
|
+
}
|
|
3042
|
+
const indexed = results.filter((r) => r.isIndexed).length;
|
|
3043
|
+
if (!args.quiet) logger.success(`Inspected ${results.length} URLs (${indexed} indexed, ${results.length - indexed} not)`);
|
|
3044
|
+
}
|
|
3045
|
+
}) },
|
|
3046
|
+
async run({ args }) {
|
|
3047
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
3048
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
3049
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
3050
|
+
const result = await ctx.client.inspect(siteUrl, args.url).catch(gscErrorHandler);
|
|
3051
|
+
const inspection = result?.inspectionResult;
|
|
3052
|
+
const indexStatus = inspection?.indexStatusResult;
|
|
1467
3053
|
if (args.json) {
|
|
1468
3054
|
console.log(JSON.stringify({
|
|
1469
3055
|
url: args.url,
|
|
@@ -1476,23 +3062,11 @@ const inspectCommand = defineCommand({
|
|
|
1476
3062
|
}, null, 2));
|
|
1477
3063
|
return;
|
|
1478
3064
|
}
|
|
1479
|
-
|
|
1480
|
-
console.log(` \x1B[1mURL:\x1B[0m ${args.url}`);
|
|
1481
|
-
console.log();
|
|
1482
|
-
const verdictColor = indexStatus?.verdict === "PASS" ? "\x1B[32m" : "\x1B[31m";
|
|
1483
|
-
console.log(` Verdict: ${verdictColor}${indexStatus?.verdict || "N/A"}\x1B[0m`);
|
|
1484
|
-
if (indexStatus?.coverageState) console.log(` Coverage: ${indexStatus.coverageState}`);
|
|
1485
|
-
if (indexStatus?.indexingState) console.log(` Indexing: ${indexStatus.indexingState}`);
|
|
1486
|
-
if (indexStatus?.lastCrawlTime) console.log(` Last Crawl: ${indexStatus.lastCrawlTime}`);
|
|
1487
|
-
if (indexStatus?.robotsTxtState) console.log(` Robots.txt: ${indexStatus.robotsTxtState}`);
|
|
1488
|
-
if (indexStatus?.pageFetchState) console.log(` Page Fetch: ${indexStatus.pageFetchState}`);
|
|
1489
|
-
if (indexStatus?.googleCanonical) console.log(` Google Canon: ${indexStatus.googleCanonical}`);
|
|
1490
|
-
if (indexStatus?.userCanonical) console.log(` User Canon: ${indexStatus.userCanonical}`);
|
|
1491
|
-
console.log();
|
|
3065
|
+
printInspection(args.url, inspection);
|
|
1492
3066
|
}
|
|
1493
3067
|
});
|
|
1494
3068
|
async function checkAuth() {
|
|
1495
|
-
if ((
|
|
3069
|
+
if (resolveBYOK()) return { ok: true };
|
|
1496
3070
|
const config = await loadConfig();
|
|
1497
3071
|
if (!config.clientId && !config.clientSecret) return {
|
|
1498
3072
|
ok: false,
|
|
@@ -1502,7 +3076,8 @@ Run this command to set up authentication:
|
|
|
1502
3076
|
|
|
1503
3077
|
npx @gscdump/cli init
|
|
1504
3078
|
|
|
1505
|
-
Or
|
|
3079
|
+
Or set BYOK env vars: GSC_ACCESS_TOKEN, or GSC_CLIENT_ID + GSC_CLIENT_SECRET + GSC_REFRESH_TOKEN
|
|
3080
|
+
(GOOGLE_* aliases also accepted).
|
|
1506
3081
|
|
|
1507
3082
|
Then restart your MCP client.`
|
|
1508
3083
|
};
|
|
@@ -1512,7 +3087,7 @@ Then restart your MCP client.`
|
|
|
1512
3087
|
|
|
1513
3088
|
Run this command to authenticate:
|
|
1514
3089
|
|
|
1515
|
-
npx @gscdump/cli auth
|
|
3090
|
+
npx @gscdump/cli auth login
|
|
1516
3091
|
|
|
1517
3092
|
Then restart your MCP client.`
|
|
1518
3093
|
};
|
|
@@ -1546,23 +3121,78 @@ const DIMENSIONS = [
|
|
|
1546
3121
|
"device",
|
|
1547
3122
|
"searchAppearance"
|
|
1548
3123
|
];
|
|
1549
|
-
const DIM_COLUMNS = {
|
|
1550
|
-
page,
|
|
3124
|
+
const DIM_COLUMNS = {
|
|
3125
|
+
page,
|
|
3126
|
+
query,
|
|
3127
|
+
date,
|
|
3128
|
+
country,
|
|
3129
|
+
device,
|
|
3130
|
+
searchAppearance
|
|
3131
|
+
};
|
|
3132
|
+
const FILTER_DIMS = [
|
|
3133
|
+
"query",
|
|
3134
|
+
"page",
|
|
3135
|
+
"country",
|
|
3136
|
+
"device",
|
|
3137
|
+
"searchAppearance"
|
|
3138
|
+
];
|
|
3139
|
+
const FILTER_COL = {
|
|
1551
3140
|
query,
|
|
1552
|
-
|
|
3141
|
+
page,
|
|
1553
3142
|
country,
|
|
1554
3143
|
device,
|
|
1555
3144
|
searchAppearance
|
|
1556
3145
|
};
|
|
3146
|
+
const ALL_SEARCH_TYPES$1 = Object.values(SearchTypes);
|
|
3147
|
+
const DATA_STATES = [
|
|
3148
|
+
"all",
|
|
3149
|
+
"final",
|
|
3150
|
+
"hourly_all"
|
|
3151
|
+
];
|
|
3152
|
+
const AGGREGATION_TYPES = [
|
|
3153
|
+
"auto",
|
|
3154
|
+
"byPage",
|
|
3155
|
+
"byProperty"
|
|
3156
|
+
];
|
|
3157
|
+
function buildFilterFromArg(col, raw) {
|
|
3158
|
+
if (raw.startsWith("!~")) return makeLeaf(col, "notContains", raw.slice(2));
|
|
3159
|
+
if (raw.startsWith("!re:")) return notRegex(col, raw.slice(4));
|
|
3160
|
+
if (raw.startsWith("!")) return makeLeaf(col, "notEquals", raw.slice(1));
|
|
3161
|
+
if (raw.startsWith("~")) return contains(col, raw.slice(1));
|
|
3162
|
+
if (raw.startsWith("re:")) return regex(col, raw.slice(3));
|
|
3163
|
+
if (raw.startsWith("contains:")) return contains(col, raw.slice(9));
|
|
3164
|
+
if (raw.startsWith("eq:")) return eq(col, raw.slice(3));
|
|
3165
|
+
return eq(col, raw);
|
|
3166
|
+
}
|
|
3167
|
+
function makeLeaf(col, operator, value) {
|
|
3168
|
+
return {
|
|
3169
|
+
_constraints: {},
|
|
3170
|
+
_filters: [{
|
|
3171
|
+
dimension: col.dimension,
|
|
3172
|
+
operator,
|
|
3173
|
+
expression: value
|
|
3174
|
+
}]
|
|
3175
|
+
};
|
|
3176
|
+
}
|
|
1557
3177
|
async function runLiveQuery(client, siteUrl, opts) {
|
|
1558
3178
|
const allRows = [];
|
|
1559
3179
|
let startRow = 0;
|
|
3180
|
+
const baseBody = {
|
|
3181
|
+
startDate: opts.startDate,
|
|
3182
|
+
endDate: opts.endDate,
|
|
3183
|
+
dimensions: opts.dimensions,
|
|
3184
|
+
rowLimit: opts.rowLimit
|
|
3185
|
+
};
|
|
3186
|
+
if (opts.searchType) baseBody.searchType = opts.searchType;
|
|
3187
|
+
if (opts.dataState) baseBody.dataState = opts.dataState;
|
|
3188
|
+
if (opts.aggregationType) baseBody.aggregationType = opts.aggregationType;
|
|
3189
|
+
if (opts.dimensionFilter) {
|
|
3190
|
+
const groups = filterToGroups(opts.dimensionFilter);
|
|
3191
|
+
if (groups.length > 0) baseBody.dimensionFilterGroups = groups;
|
|
3192
|
+
}
|
|
1560
3193
|
while (true) {
|
|
1561
3194
|
const rows = ((await client._rawQuery(siteUrl, {
|
|
1562
|
-
|
|
1563
|
-
endDate: opts.endDate,
|
|
1564
|
-
dimensions: opts.dimensions,
|
|
1565
|
-
rowLimit: opts.rowLimit,
|
|
3195
|
+
...baseBody,
|
|
1566
3196
|
startRow
|
|
1567
3197
|
})).rows || []).map((row) => {
|
|
1568
3198
|
const result = {
|
|
@@ -1582,71 +3212,116 @@ async function runLiveQuery(client, siteUrl, opts) {
|
|
|
1582
3212
|
}
|
|
1583
3213
|
return { rows: allRows };
|
|
1584
3214
|
}
|
|
3215
|
+
function filterToGroups(f) {
|
|
3216
|
+
if (f._filters.length === 0) return [];
|
|
3217
|
+
return [{ filters: f._filters.map((leaf) => ({
|
|
3218
|
+
dimension: leaf.dimension,
|
|
3219
|
+
operator: leaf.operator,
|
|
3220
|
+
expression: leaf.expression
|
|
3221
|
+
})) }];
|
|
3222
|
+
}
|
|
1585
3223
|
const queryCommand = defineCommand({
|
|
1586
3224
|
meta: {
|
|
1587
3225
|
name: "query",
|
|
1588
3226
|
description: "Run a search analytics query (local Parquet by default, --live hits GSC API)"
|
|
1589
3227
|
},
|
|
1590
3228
|
args: {
|
|
1591
|
-
site: {
|
|
3229
|
+
"site": {
|
|
1592
3230
|
type: "string",
|
|
1593
3231
|
alias: "s",
|
|
1594
3232
|
description: "Site URL (e.g., sc-domain:example.com)"
|
|
1595
3233
|
},
|
|
1596
|
-
dimensions: {
|
|
3234
|
+
"dimensions": {
|
|
1597
3235
|
type: "string",
|
|
1598
3236
|
alias: "d",
|
|
1599
3237
|
description: `Dimensions: ${DIMENSIONS.join(",")}`
|
|
1600
3238
|
},
|
|
1601
|
-
start: {
|
|
3239
|
+
"start": {
|
|
1602
3240
|
type: "string",
|
|
1603
3241
|
description: "Start date (YYYY-MM-DD)"
|
|
1604
3242
|
},
|
|
1605
|
-
end: {
|
|
3243
|
+
"end": {
|
|
1606
3244
|
type: "string",
|
|
1607
3245
|
description: "End date (YYYY-MM-DD)"
|
|
1608
3246
|
},
|
|
1609
|
-
limit: {
|
|
3247
|
+
"limit": {
|
|
1610
3248
|
type: "string",
|
|
1611
3249
|
alias: "l",
|
|
1612
3250
|
default: "1000",
|
|
1613
3251
|
description: "Max rows (default: 1000)"
|
|
1614
3252
|
},
|
|
1615
|
-
output: {
|
|
3253
|
+
"output": {
|
|
1616
3254
|
type: "string",
|
|
1617
3255
|
alias: "o",
|
|
1618
3256
|
description: "Output file path (default: stdout)"
|
|
1619
3257
|
},
|
|
1620
|
-
format: {
|
|
3258
|
+
"format": {
|
|
1621
3259
|
type: "string",
|
|
1622
3260
|
alias: "f",
|
|
1623
3261
|
default: "json",
|
|
1624
3262
|
description: "Output format: json or csv"
|
|
1625
3263
|
},
|
|
1626
|
-
sql: {
|
|
3264
|
+
"sql": {
|
|
1627
3265
|
type: "string",
|
|
1628
3266
|
description: "Raw DuckDB SQL using {{FILES}} as the file list placeholder (bypasses builder)"
|
|
1629
3267
|
},
|
|
1630
|
-
table: {
|
|
3268
|
+
"table": {
|
|
1631
3269
|
type: "string",
|
|
1632
3270
|
description: "Analytics table for --sql (default: pages)"
|
|
1633
3271
|
},
|
|
1634
|
-
live: {
|
|
3272
|
+
"live": {
|
|
1635
3273
|
type: "boolean",
|
|
1636
3274
|
default: false,
|
|
1637
3275
|
description: "Bypass local store; hit the GSC API directly"
|
|
1638
3276
|
},
|
|
1639
|
-
quiet: {
|
|
3277
|
+
"quiet": {
|
|
1640
3278
|
type: "boolean",
|
|
1641
3279
|
alias: "q",
|
|
1642
3280
|
default: false,
|
|
1643
3281
|
description: "Suppress progress output"
|
|
1644
3282
|
},
|
|
1645
|
-
interactive: {
|
|
3283
|
+
"interactive": {
|
|
1646
3284
|
type: "boolean",
|
|
1647
3285
|
alias: "i",
|
|
1648
3286
|
default: false,
|
|
1649
3287
|
description: "Interactive mode"
|
|
3288
|
+
},
|
|
3289
|
+
"query": {
|
|
3290
|
+
type: "string",
|
|
3291
|
+
description: "Filter by query (prefix: ~contains, !exclude, re:regex, !re:not-regex)"
|
|
3292
|
+
},
|
|
3293
|
+
"page": {
|
|
3294
|
+
type: "string",
|
|
3295
|
+
description: "Filter by page (same prefix syntax as --query)"
|
|
3296
|
+
},
|
|
3297
|
+
"country": {
|
|
3298
|
+
type: "string",
|
|
3299
|
+
description: "Filter by country (ISO-3 lowercase, e.g. usa). Same prefix syntax as --query"
|
|
3300
|
+
},
|
|
3301
|
+
"device": {
|
|
3302
|
+
type: "string",
|
|
3303
|
+
description: "Filter by device (DESKTOP/MOBILE/TABLET). Same prefix syntax"
|
|
3304
|
+
},
|
|
3305
|
+
"search-appearance": {
|
|
3306
|
+
type: "string",
|
|
3307
|
+
description: "Filter by search appearance feature. Same prefix syntax"
|
|
3308
|
+
},
|
|
3309
|
+
"type": {
|
|
3310
|
+
type: "string",
|
|
3311
|
+
description: `Search type (live mode only). One of: ${ALL_SEARCH_TYPES$1.join(",")}`
|
|
3312
|
+
},
|
|
3313
|
+
"data-state": {
|
|
3314
|
+
type: "string",
|
|
3315
|
+
description: `Data state (live mode only). One of: ${DATA_STATES.join(",")} (default: final)`
|
|
3316
|
+
},
|
|
3317
|
+
"aggregation-type": {
|
|
3318
|
+
type: "string",
|
|
3319
|
+
description: `Aggregation type (live mode only). One of: ${AGGREGATION_TYPES.join(",")}`
|
|
3320
|
+
},
|
|
3321
|
+
"explain": {
|
|
3322
|
+
type: "boolean",
|
|
3323
|
+
default: false,
|
|
3324
|
+
description: "Print the request body / planned local SQL and exit without executing"
|
|
1650
3325
|
}
|
|
1651
3326
|
},
|
|
1652
3327
|
async run({ args }) {
|
|
@@ -1660,10 +3335,25 @@ const queryCommand = defineCommand({
|
|
|
1660
3335
|
});
|
|
1661
3336
|
return;
|
|
1662
3337
|
}
|
|
3338
|
+
const ctxConfig = await loadConfig();
|
|
1663
3339
|
const dimNames = await resolveDimensions(args);
|
|
1664
3340
|
const { startDate, endDate } = await resolveRange(args);
|
|
1665
|
-
|
|
3341
|
+
await promptFilters(args);
|
|
3342
|
+
const limitArg = args.limit != null ? String(args.limit) : null;
|
|
3343
|
+
const rowLimit = limitArg != null && limitArg !== "1000" ? Number.parseInt(limitArg, 10) : ctxConfig.defaultLimit ?? 1e3;
|
|
1666
3344
|
const format = String(args.format);
|
|
3345
|
+
const dimensionFilter = buildDimensionFilter(args);
|
|
3346
|
+
const searchType = parseSearchType(args.type ?? ctxConfig.defaultSearchType);
|
|
3347
|
+
const dataState = args["data-state"] ? String(args["data-state"]) : ctxConfig.defaultDataState;
|
|
3348
|
+
const aggregationType = args["aggregation-type"] ? String(args["aggregation-type"]) : void 0;
|
|
3349
|
+
if (dataState && !DATA_STATES.includes(dataState)) {
|
|
3350
|
+
logger.error(`Invalid --data-state: ${dataState}. Allowed: ${DATA_STATES.join(", ")}`);
|
|
3351
|
+
process.exit(1);
|
|
3352
|
+
}
|
|
3353
|
+
if (aggregationType && !AGGREGATION_TYPES.includes(aggregationType)) {
|
|
3354
|
+
logger.error(`Invalid --aggregation-type: ${aggregationType}. Allowed: ${AGGREGATION_TYPES.join(", ")}`);
|
|
3355
|
+
process.exit(1);
|
|
3356
|
+
}
|
|
1667
3357
|
const ctx = await createCommandContext({
|
|
1668
3358
|
needsAuth: true,
|
|
1669
3359
|
needsStore: !args.live,
|
|
@@ -1671,16 +3361,34 @@ const queryCommand = defineCommand({
|
|
|
1671
3361
|
});
|
|
1672
3362
|
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
1673
3363
|
if (args.live) {
|
|
3364
|
+
if (args.explain) {
|
|
3365
|
+
const body = {
|
|
3366
|
+
startDate,
|
|
3367
|
+
endDate,
|
|
3368
|
+
dimensions: dimNames,
|
|
3369
|
+
rowLimit
|
|
3370
|
+
};
|
|
3371
|
+
if (searchType) body.searchType = searchType;
|
|
3372
|
+
if (dataState) body.dataState = dataState;
|
|
3373
|
+
if (aggregationType) body.aggregationType = aggregationType;
|
|
3374
|
+
if (dimensionFilter) body.dimensionFilterGroups = filterToGroups(dimensionFilter);
|
|
3375
|
+
console.log(JSON.stringify({
|
|
3376
|
+
siteUrl,
|
|
3377
|
+
body
|
|
3378
|
+
}, null, 2));
|
|
3379
|
+
return;
|
|
3380
|
+
}
|
|
1674
3381
|
if (!args.quiet) logger.info(`Querying ${siteUrl} via live GSC API...`);
|
|
1675
3382
|
const result = await runLiveQuery(ctx.client, siteUrl, {
|
|
1676
3383
|
startDate,
|
|
1677
3384
|
endDate,
|
|
1678
3385
|
dimensions: dimNames,
|
|
1679
|
-
rowLimit
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
3386
|
+
rowLimit,
|
|
3387
|
+
searchType,
|
|
3388
|
+
dataState,
|
|
3389
|
+
aggregationType,
|
|
3390
|
+
dimensionFilter
|
|
3391
|
+
}).catch(gscErrorHandler);
|
|
1684
3392
|
await writeOutput({
|
|
1685
3393
|
output: {
|
|
1686
3394
|
siteUrl,
|
|
@@ -1698,10 +3406,23 @@ const queryCommand = defineCommand({
|
|
|
1698
3406
|
});
|
|
1699
3407
|
return;
|
|
1700
3408
|
}
|
|
3409
|
+
if (searchType && searchType !== "web") {
|
|
3410
|
+
logger.error(`--type=${searchType} requires --live (local store query path is web-only).`);
|
|
3411
|
+
process.exit(1);
|
|
3412
|
+
}
|
|
3413
|
+
if (dataState || aggregationType) logger.warn("--data-state / --aggregation-type are ignored without --live");
|
|
1701
3414
|
if (!args.quiet) logger.info(`Querying ${siteUrl} from local Parquet store...`);
|
|
1702
|
-
const state = buildLocalState(dimNames, startDate, endDate, rowLimit);
|
|
3415
|
+
const state = buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter);
|
|
1703
3416
|
const store = ctx.store;
|
|
1704
3417
|
const table = inferTable(dimNames);
|
|
3418
|
+
if (args.explain) {
|
|
3419
|
+
console.log(JSON.stringify({
|
|
3420
|
+
siteUrl,
|
|
3421
|
+
table,
|
|
3422
|
+
state
|
|
3423
|
+
}, null, 2));
|
|
3424
|
+
return;
|
|
3425
|
+
}
|
|
1705
3426
|
await assertRangeCovered(store, siteUrl, table, startDate, endDate);
|
|
1706
3427
|
const result = await store.engine.query({
|
|
1707
3428
|
userId: store.userId,
|
|
@@ -1779,9 +3500,68 @@ async function resolveRange(args) {
|
|
|
1779
3500
|
endDate: daysAgo(3)
|
|
1780
3501
|
};
|
|
1781
3502
|
}
|
|
1782
|
-
function
|
|
3503
|
+
async function promptFilters(args) {
|
|
3504
|
+
if (!args.interactive) return;
|
|
3505
|
+
for (const dim of FILTER_DIMS) {
|
|
3506
|
+
if (args[dim]) continue;
|
|
3507
|
+
const v = await text({
|
|
3508
|
+
message: `Filter by ${dim} (blank to skip; prefix ~ for contains, ! for not-equals, re: for regex)`,
|
|
3509
|
+
placeholder: ""
|
|
3510
|
+
});
|
|
3511
|
+
if (isCancel(v)) {
|
|
3512
|
+
cancel("Cancelled");
|
|
3513
|
+
process.exit(0);
|
|
3514
|
+
}
|
|
3515
|
+
if (v && String(v).length > 0) args[dim] = String(v);
|
|
3516
|
+
}
|
|
3517
|
+
if (!args.type) {
|
|
3518
|
+
const t = await text({
|
|
3519
|
+
message: `Search type (blank for default web; allowed: ${ALL_SEARCH_TYPES$1.join(", ")})`,
|
|
3520
|
+
placeholder: ""
|
|
3521
|
+
});
|
|
3522
|
+
if (isCancel(t)) {
|
|
3523
|
+
cancel("Cancelled");
|
|
3524
|
+
process.exit(0);
|
|
3525
|
+
}
|
|
3526
|
+
if (t && String(t).length > 0) args.type = String(t);
|
|
3527
|
+
}
|
|
3528
|
+
if (!args["data-state"]) {
|
|
3529
|
+
const ds = await text({
|
|
3530
|
+
message: `Data state (blank for default 'final'; allowed: ${DATA_STATES.join(", ")})`,
|
|
3531
|
+
placeholder: ""
|
|
3532
|
+
});
|
|
3533
|
+
if (isCancel(ds)) {
|
|
3534
|
+
cancel("Cancelled");
|
|
3535
|
+
process.exit(0);
|
|
3536
|
+
}
|
|
3537
|
+
if (ds && String(ds).length > 0) args["data-state"] = String(ds);
|
|
3538
|
+
}
|
|
3539
|
+
}
|
|
3540
|
+
function buildLocalState(dimNames, startDate, endDate, rowLimit, dimensionFilter) {
|
|
1783
3541
|
const dims = dimNames.map((d) => DIM_COLUMNS[d]).filter((c) => Boolean(c));
|
|
1784
|
-
|
|
3542
|
+
const dateFilter = between(date, startDate, endDate);
|
|
3543
|
+
const filter = dimensionFilter ? and(dateFilter, dimensionFilter) : dateFilter;
|
|
3544
|
+
return gsc.select(...dims).where(filter).limit(rowLimit).getState();
|
|
3545
|
+
}
|
|
3546
|
+
function buildDimensionFilter(args) {
|
|
3547
|
+
const leaves = [];
|
|
3548
|
+
for (const dim of FILTER_DIMS) {
|
|
3549
|
+
const raw = args[dim];
|
|
3550
|
+
if (raw == null || raw === "") continue;
|
|
3551
|
+
leaves.push(buildFilterFromArg(FILTER_COL[dim], String(raw)));
|
|
3552
|
+
}
|
|
3553
|
+
if (leaves.length === 0) return void 0;
|
|
3554
|
+
if (leaves.length === 1) return leaves[0];
|
|
3555
|
+
return and(...leaves);
|
|
3556
|
+
}
|
|
3557
|
+
function parseSearchType(value) {
|
|
3558
|
+
if (!value) return void 0;
|
|
3559
|
+
const v = String(value);
|
|
3560
|
+
if (!ALL_SEARCH_TYPES$1.includes(v)) {
|
|
3561
|
+
logger.error(`Invalid --type: ${v}. Allowed: ${ALL_SEARCH_TYPES$1.join(", ")}`);
|
|
3562
|
+
process.exit(1);
|
|
3563
|
+
}
|
|
3564
|
+
return v;
|
|
1785
3565
|
}
|
|
1786
3566
|
async function assertRangeCovered(store, siteUrl, table, startDate, endDate) {
|
|
1787
3567
|
const wm = (await store.engine.getWatermarks({
|
|
@@ -1827,14 +3607,14 @@ async function runRawSqlMode(opts) {
|
|
|
1827
3607
|
total: rows.length,
|
|
1828
3608
|
data: rows
|
|
1829
3609
|
}, null, 2);
|
|
1830
|
-
if (opts.output) {
|
|
3610
|
+
if (opts.output && opts.output !== "-") {
|
|
1831
3611
|
await fs.writeFile(opts.output, payload);
|
|
1832
3612
|
if (!opts.quiet) logger.info(`Written to ${opts.output}`);
|
|
1833
3613
|
} else console.log(payload);
|
|
1834
3614
|
}
|
|
1835
3615
|
async function writeOutput(opts) {
|
|
1836
3616
|
const content = opts.format === "csv" ? exportToCSV(opts.output) : JSON.stringify(opts.output, null, 2);
|
|
1837
|
-
if (opts.path) {
|
|
3617
|
+
if (opts.path && opts.path !== "-") {
|
|
1838
3618
|
await fs.writeFile(opts.path, content);
|
|
1839
3619
|
if (!opts.quiet) logger.info(`Written to ${opts.path}`);
|
|
1840
3620
|
} else console.log(content);
|
|
@@ -1842,13 +3622,6 @@ async function writeOutput(opts) {
|
|
|
1842
3622
|
function isKnownTable$1(name) {
|
|
1843
3623
|
return allTables().includes(name);
|
|
1844
3624
|
}
|
|
1845
|
-
function requireSite(target) {
|
|
1846
|
-
if (!target) {
|
|
1847
|
-
logger.error("Site URL required (-s)");
|
|
1848
|
-
process.exit(1);
|
|
1849
|
-
}
|
|
1850
|
-
return target;
|
|
1851
|
-
}
|
|
1852
3625
|
const sitemapsCommand = defineCommand({
|
|
1853
3626
|
meta: {
|
|
1854
3627
|
name: "sitemaps",
|
|
@@ -1870,160 +3643,480 @@ const sitemapsCommand = defineCommand({
|
|
|
1870
3643
|
type: "boolean",
|
|
1871
3644
|
default: false,
|
|
1872
3645
|
description: "Output as JSON"
|
|
3646
|
+
},
|
|
3647
|
+
pending: {
|
|
3648
|
+
type: "boolean",
|
|
3649
|
+
default: false,
|
|
3650
|
+
description: "Show only sitemaps with isPending=true"
|
|
3651
|
+
},
|
|
3652
|
+
errored: {
|
|
3653
|
+
type: "boolean",
|
|
3654
|
+
default: false,
|
|
3655
|
+
description: "Show only sitemaps with errors > 0"
|
|
3656
|
+
}
|
|
3657
|
+
},
|
|
3658
|
+
async run({ args }) {
|
|
3659
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
3660
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
3661
|
+
let sitemaps = (await ctx.client.sitemaps.list(siteUrl).catch((e) => {
|
|
3662
|
+
logger.error(`Failed to fetch sitemaps: ${e.message}`);
|
|
3663
|
+
process.exit(1);
|
|
3664
|
+
})).map((sm) => ({
|
|
3665
|
+
path: sm.path,
|
|
3666
|
+
type: sm.type || void 0,
|
|
3667
|
+
isPending: sm.isPending || false,
|
|
3668
|
+
errors: Number(sm.errors) || 0,
|
|
3669
|
+
warnings: Number(sm.warnings) || 0,
|
|
3670
|
+
lastDownloaded: sm.lastDownloaded || null,
|
|
3671
|
+
lastSubmitted: sm.lastSubmitted || null
|
|
3672
|
+
}));
|
|
3673
|
+
if (args.pending) sitemaps = sitemaps.filter((sm) => sm.isPending);
|
|
3674
|
+
if (args.errored) sitemaps = sitemaps.filter((sm) => sm.errors > 0);
|
|
3675
|
+
if (args.json) {
|
|
3676
|
+
console.log(JSON.stringify(sitemaps, null, 2));
|
|
3677
|
+
return;
|
|
3678
|
+
}
|
|
3679
|
+
if (sitemaps.length === 0) {
|
|
3680
|
+
logger.warn("No sitemaps found");
|
|
3681
|
+
return;
|
|
3682
|
+
}
|
|
3683
|
+
logger.success(`Found ${sitemaps.length} sitemaps:`);
|
|
3684
|
+
console.log();
|
|
3685
|
+
for (const sm of sitemaps) {
|
|
3686
|
+
const pending = sm.isPending ? " \x1B[33m(pending)\x1B[0m" : "";
|
|
3687
|
+
const errors = sm.errors ? ` \x1B[31m${sm.errors} errors\x1B[0m` : "";
|
|
3688
|
+
const warnings = sm.warnings ? ` \x1B[33m${sm.warnings} warnings\x1B[0m` : "";
|
|
3689
|
+
const submitted = sm.lastSubmitted ? ` \x1B[90msubmitted ${sm.lastSubmitted}\x1B[0m` : "";
|
|
3690
|
+
console.log(` ${sm.path}${pending}${errors}${warnings}${submitted}`);
|
|
3691
|
+
}
|
|
3692
|
+
}
|
|
3693
|
+
}),
|
|
3694
|
+
get: defineCommand({
|
|
3695
|
+
meta: {
|
|
3696
|
+
name: "get",
|
|
3697
|
+
description: "Get details for a specific sitemap"
|
|
3698
|
+
},
|
|
3699
|
+
args: {
|
|
3700
|
+
site: {
|
|
3701
|
+
type: "string",
|
|
3702
|
+
alias: "s",
|
|
3703
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
3704
|
+
},
|
|
3705
|
+
url: {
|
|
3706
|
+
type: "positional",
|
|
3707
|
+
required: true,
|
|
3708
|
+
description: "Sitemap URL"
|
|
3709
|
+
},
|
|
3710
|
+
json: {
|
|
3711
|
+
type: "boolean",
|
|
3712
|
+
default: false,
|
|
3713
|
+
description: "Output as JSON"
|
|
3714
|
+
}
|
|
3715
|
+
},
|
|
3716
|
+
async run({ args }) {
|
|
3717
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
3718
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
3719
|
+
const client = ctx.client;
|
|
3720
|
+
const sitemap = await fetchSitemap(client, siteUrl, args.url).catch(gscErrorHandler);
|
|
3721
|
+
if (args.json) {
|
|
3722
|
+
console.log(JSON.stringify(sitemap, null, 2));
|
|
3723
|
+
return;
|
|
3724
|
+
}
|
|
3725
|
+
console.log();
|
|
3726
|
+
console.log(` \x1B[1mPath:\x1B[0m ${sitemap.path}`);
|
|
3727
|
+
console.log(` \x1B[1mType:\x1B[0m ${sitemap.type || "sitemap"}`);
|
|
3728
|
+
console.log(` \x1B[1mLast Submitted:\x1B[0m ${sitemap.lastSubmitted || "N/A"}`);
|
|
3729
|
+
console.log(` \x1B[1mLast Downloaded:\x1B[0m ${sitemap.lastDownloaded || "N/A"}`);
|
|
3730
|
+
console.log(` \x1B[1mPending:\x1B[0m ${sitemap.isPending ? "Yes" : "No"}`);
|
|
3731
|
+
console.log(` \x1B[1mErrors:\x1B[0m ${sitemap.errors || 0}`);
|
|
3732
|
+
console.log(` \x1B[1mWarnings:\x1B[0m ${sitemap.warnings || 0}`);
|
|
3733
|
+
if (sitemap.contents?.length) {
|
|
3734
|
+
console.log();
|
|
3735
|
+
console.log(" \x1B[1mContents:\x1B[0m");
|
|
3736
|
+
for (const c of sitemap.contents) console.log(` ${c.type}: ${c.submitted} submitted, ${c.indexed} indexed`);
|
|
3737
|
+
}
|
|
3738
|
+
}
|
|
3739
|
+
}),
|
|
3740
|
+
submit: defineCommand({
|
|
3741
|
+
meta: {
|
|
3742
|
+
name: "submit",
|
|
3743
|
+
description: "Submit a sitemap to GSC"
|
|
3744
|
+
},
|
|
3745
|
+
args: {
|
|
3746
|
+
site: {
|
|
3747
|
+
type: "string",
|
|
3748
|
+
alias: "s",
|
|
3749
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
3750
|
+
},
|
|
3751
|
+
url: {
|
|
3752
|
+
type: "positional",
|
|
3753
|
+
required: true,
|
|
3754
|
+
description: "Sitemap URL to submit"
|
|
3755
|
+
}
|
|
3756
|
+
},
|
|
3757
|
+
async run({ args }) {
|
|
3758
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
3759
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
3760
|
+
await ctx.client.sitemaps.submit(siteUrl, args.url).catch((e) => {
|
|
3761
|
+
logger.error(`Submit failed: ${e.message}`);
|
|
3762
|
+
process.exit(1);
|
|
3763
|
+
});
|
|
3764
|
+
logger.success(`Submitted sitemap: ${args.url}`);
|
|
3765
|
+
}
|
|
3766
|
+
}),
|
|
3767
|
+
delete: defineCommand({
|
|
3768
|
+
meta: {
|
|
3769
|
+
name: "delete",
|
|
3770
|
+
description: "Delete a sitemap from GSC"
|
|
3771
|
+
},
|
|
3772
|
+
args: {
|
|
3773
|
+
site: {
|
|
3774
|
+
type: "string",
|
|
3775
|
+
alias: "s",
|
|
3776
|
+
description: "Site URL (defaults to config.defaultSite or prompt)"
|
|
3777
|
+
},
|
|
3778
|
+
url: {
|
|
3779
|
+
type: "positional",
|
|
3780
|
+
required: true,
|
|
3781
|
+
description: "Sitemap URL to delete"
|
|
3782
|
+
}
|
|
3783
|
+
},
|
|
3784
|
+
async run({ args }) {
|
|
3785
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
3786
|
+
const siteUrl = await ctx.resolveSite(args.site ? String(args.site) : void 0);
|
|
3787
|
+
await ctx.client.sitemaps.delete(siteUrl, args.url).catch((e) => {
|
|
3788
|
+
logger.error(`Delete failed: ${e.message}`);
|
|
3789
|
+
process.exit(1);
|
|
3790
|
+
});
|
|
3791
|
+
logger.success(`Deleted sitemap: ${args.url}`);
|
|
3792
|
+
}
|
|
3793
|
+
})
|
|
3794
|
+
}
|
|
3795
|
+
});
|
|
3796
|
+
const ALL_METHODS = [
|
|
3797
|
+
"META",
|
|
3798
|
+
"FILE",
|
|
3799
|
+
"DNS_TXT",
|
|
3800
|
+
"DNS_CNAME",
|
|
3801
|
+
"ANALYTICS",
|
|
3802
|
+
"TAG_MANAGER"
|
|
3803
|
+
];
|
|
3804
|
+
function pickDefaultMethod(siteUrl) {
|
|
3805
|
+
return siteUrl.startsWith("sc-domain:") ? "DNS_TXT" : "META";
|
|
3806
|
+
}
|
|
3807
|
+
function validateMethod(siteUrl, method) {
|
|
3808
|
+
const upper = method.toUpperCase();
|
|
3809
|
+
if (!ALL_METHODS.includes(upper)) {
|
|
3810
|
+
logger.error(`Invalid --method: ${method}. Valid: ${ALL_METHODS.join(", ")}`);
|
|
3811
|
+
process.exit(1);
|
|
3812
|
+
}
|
|
3813
|
+
const site = siteUrlToVerificationSite(siteUrl);
|
|
3814
|
+
const allowed = verificationMethodsFor(site);
|
|
3815
|
+
if (!allowed.includes(upper)) {
|
|
3816
|
+
logger.error(`Method ${upper} not valid for ${site.type === "INET_DOMAIN" ? "domain" : "URL-prefix"} property "${siteUrl}". Valid: ${allowed.join(", ")}`);
|
|
3817
|
+
process.exit(1);
|
|
3818
|
+
}
|
|
3819
|
+
return upper;
|
|
3820
|
+
}
|
|
3821
|
+
function printPlacementInstructions(method, siteUrl, token) {
|
|
3822
|
+
console.log();
|
|
3823
|
+
console.log(` \x1B[1mPlacement instructions (${method})\x1B[0m`);
|
|
3824
|
+
console.log();
|
|
3825
|
+
switch (method) {
|
|
3826
|
+
case "META":
|
|
3827
|
+
console.log(` Add this tag inside the <head> of \x1B[36m${siteUrl}\x1B[0m:`);
|
|
3828
|
+
console.log();
|
|
3829
|
+
console.log(` \x1B[2m<meta name="google-site-verification" content="${token}" />\x1B[0m`);
|
|
3830
|
+
break;
|
|
3831
|
+
case "FILE":
|
|
3832
|
+
console.log(` Upload a file named \x1B[1m${token}\x1B[0m to the site root, accessible at:`);
|
|
3833
|
+
console.log();
|
|
3834
|
+
console.log(` \x1B[36m${siteUrl.replace(/\/?$/, "/")}${token}\x1B[0m`);
|
|
3835
|
+
break;
|
|
3836
|
+
case "DNS_TXT":
|
|
3837
|
+
console.log(` Add a TXT record on \x1B[1m${siteUrl.replace(/^sc-domain:/, "")}\x1B[0m with value:`);
|
|
3838
|
+
console.log();
|
|
3839
|
+
console.log(` \x1B[2m${token}\x1B[0m`);
|
|
3840
|
+
break;
|
|
3841
|
+
case "DNS_CNAME": {
|
|
3842
|
+
const [host, target] = token.split(/\s+/, 2);
|
|
3843
|
+
console.log(` Add a CNAME record:`);
|
|
3844
|
+
console.log();
|
|
3845
|
+
console.log(` \x1B[2mHost: ${host}\x1B[0m`);
|
|
3846
|
+
console.log(` \x1B[2mTarget: ${target ?? "(see token)"}\x1B[0m`);
|
|
3847
|
+
break;
|
|
3848
|
+
}
|
|
3849
|
+
case "ANALYTICS":
|
|
3850
|
+
console.log(` Make sure your Google Analytics tracking tag is installed on the site, then run \`gscdump sites verify\`.`);
|
|
3851
|
+
break;
|
|
3852
|
+
case "TAG_MANAGER":
|
|
3853
|
+
console.log(` Make sure your Google Tag Manager container snippet is installed on the site, then run \`gscdump sites verify\`.`);
|
|
3854
|
+
break;
|
|
3855
|
+
}
|
|
3856
|
+
console.log();
|
|
3857
|
+
console.log(` \x1B[90mThen run:\x1B[0m gscdump sites verify ${siteUrl} --method ${method}`);
|
|
3858
|
+
console.log();
|
|
3859
|
+
}
|
|
3860
|
+
const sitesCommand = defineCommand({
|
|
3861
|
+
meta: {
|
|
3862
|
+
name: "sites",
|
|
3863
|
+
description: "List GSC sites; manage properties (add/delete) and verify ownership"
|
|
3864
|
+
},
|
|
3865
|
+
args: {
|
|
3866
|
+
"json": {
|
|
3867
|
+
type: "boolean",
|
|
3868
|
+
default: false,
|
|
3869
|
+
description: "Output as JSON for scripting"
|
|
3870
|
+
},
|
|
3871
|
+
"with-sitemaps": {
|
|
3872
|
+
type: "boolean",
|
|
3873
|
+
default: false,
|
|
3874
|
+
description: "Include sitemaps for each owned site"
|
|
3875
|
+
},
|
|
3876
|
+
"owner-only": {
|
|
3877
|
+
type: "boolean",
|
|
3878
|
+
default: false,
|
|
3879
|
+
description: "Filter to permissionLevel=siteOwner"
|
|
3880
|
+
},
|
|
3881
|
+
"quiet": {
|
|
3882
|
+
type: "boolean",
|
|
3883
|
+
alias: "q",
|
|
3884
|
+
default: false,
|
|
3885
|
+
description: "Suppress info/success output"
|
|
3886
|
+
}
|
|
3887
|
+
},
|
|
3888
|
+
subCommands: {
|
|
3889
|
+
"add": defineCommand({
|
|
3890
|
+
meta: {
|
|
3891
|
+
name: "add",
|
|
3892
|
+
description: "Register a property in Search Console (unverified state — verify ownership separately)"
|
|
3893
|
+
},
|
|
3894
|
+
args: {
|
|
3895
|
+
url: {
|
|
3896
|
+
type: "positional",
|
|
3897
|
+
required: true,
|
|
3898
|
+
description: "Property URL (https://example.com/ or sc-domain:example.com)"
|
|
3899
|
+
},
|
|
3900
|
+
json: {
|
|
3901
|
+
type: "boolean",
|
|
3902
|
+
default: false,
|
|
3903
|
+
description: "Output as JSON"
|
|
3904
|
+
},
|
|
3905
|
+
quiet: {
|
|
3906
|
+
type: "boolean",
|
|
3907
|
+
alias: "q",
|
|
3908
|
+
default: false,
|
|
3909
|
+
description: "Suppress info/success output"
|
|
1873
3910
|
}
|
|
1874
3911
|
},
|
|
1875
3912
|
async run({ args }) {
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
const sitemaps = (await (await createCommandContext({ needsAuth: true })).client.sitemaps.list(siteUrl).catch((e) => {
|
|
1879
|
-
logger.error(`Failed to fetch sitemaps: ${e.message}`);
|
|
1880
|
-
process.exit(1);
|
|
1881
|
-
})).map((sm) => ({
|
|
1882
|
-
path: sm.path,
|
|
1883
|
-
type: sm.type || void 0,
|
|
1884
|
-
isPending: sm.isPending || false,
|
|
1885
|
-
errors: Number(sm.errors) || 0,
|
|
1886
|
-
warnings: Number(sm.warnings) || 0,
|
|
1887
|
-
lastDownloaded: sm.lastDownloaded || null
|
|
1888
|
-
}));
|
|
3913
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
3914
|
+
await addSite((await createCommandContext({ needsAuth: true })).client, args.url).catch(gscErrorHandler);
|
|
1889
3915
|
if (args.json) {
|
|
1890
|
-
console.log(JSON.stringify(
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
3916
|
+
console.log(JSON.stringify({
|
|
3917
|
+
siteUrl: args.url,
|
|
3918
|
+
status: "added",
|
|
3919
|
+
verified: false
|
|
3920
|
+
}, null, 2));
|
|
1895
3921
|
return;
|
|
1896
3922
|
}
|
|
1897
|
-
logger.success(`
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
const errors = sm.errors ? ` \x1B[31m${sm.errors} errors\x1B[0m` : "";
|
|
1902
|
-
const warnings = sm.warnings ? ` \x1B[33m${sm.warnings} warnings\x1B[0m` : "";
|
|
1903
|
-
console.log(` ${sm.path}${pending}${errors}${warnings}`);
|
|
1904
|
-
}
|
|
3923
|
+
logger.success(`Added: ${args.url}`);
|
|
3924
|
+
logger.info(`Property is in unverified state. Verify ownership next:`);
|
|
3925
|
+
const method = pickDefaultMethod(args.url);
|
|
3926
|
+
console.log(` \x1B[2mgscdump sites verify-token ${args.url} --method ${method}\x1B[0m`);
|
|
1905
3927
|
}
|
|
1906
3928
|
}),
|
|
1907
|
-
|
|
3929
|
+
"delete": defineCommand({
|
|
1908
3930
|
meta: {
|
|
1909
|
-
name: "
|
|
1910
|
-
description: "
|
|
3931
|
+
name: "delete",
|
|
3932
|
+
description: "Remove a property from Search Console"
|
|
1911
3933
|
},
|
|
1912
3934
|
args: {
|
|
1913
|
-
site: {
|
|
1914
|
-
type: "string",
|
|
1915
|
-
alias: "s",
|
|
1916
|
-
required: true,
|
|
1917
|
-
description: "Site URL"
|
|
1918
|
-
},
|
|
1919
3935
|
url: {
|
|
1920
3936
|
type: "positional",
|
|
1921
3937
|
required: true,
|
|
1922
|
-
description: "
|
|
3938
|
+
description: "Property URL to remove"
|
|
3939
|
+
},
|
|
3940
|
+
yes: {
|
|
3941
|
+
type: "boolean",
|
|
3942
|
+
alias: "y",
|
|
3943
|
+
default: false,
|
|
3944
|
+
description: "Skip confirmation prompt"
|
|
1923
3945
|
},
|
|
1924
3946
|
json: {
|
|
1925
3947
|
type: "boolean",
|
|
1926
3948
|
default: false,
|
|
1927
3949
|
description: "Output as JSON"
|
|
3950
|
+
},
|
|
3951
|
+
quiet: {
|
|
3952
|
+
type: "boolean",
|
|
3953
|
+
alias: "q",
|
|
3954
|
+
default: false,
|
|
3955
|
+
description: "Suppress info/success output"
|
|
1928
3956
|
}
|
|
1929
3957
|
},
|
|
1930
3958
|
async run({ args }) {
|
|
1931
|
-
|
|
1932
|
-
|
|
3959
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
3960
|
+
if (!args.yes && !args.json) {
|
|
3961
|
+
const ok = await confirm({
|
|
3962
|
+
message: `Remove ${args.url} from Search Console? Local synced data is unaffected.`,
|
|
3963
|
+
initialValue: false
|
|
3964
|
+
});
|
|
3965
|
+
if (isCancel(ok) || !ok) {
|
|
3966
|
+
logger.info("Cancelled");
|
|
3967
|
+
process.exit(0);
|
|
3968
|
+
}
|
|
3969
|
+
}
|
|
3970
|
+
await deleteSite((await createCommandContext({ needsAuth: true })).client, args.url).catch(gscErrorHandler);
|
|
1933
3971
|
if (args.json) {
|
|
1934
|
-
console.log(JSON.stringify(
|
|
3972
|
+
console.log(JSON.stringify({
|
|
3973
|
+
siteUrl: args.url,
|
|
3974
|
+
status: "deleted"
|
|
3975
|
+
}, null, 2));
|
|
1935
3976
|
return;
|
|
1936
3977
|
}
|
|
1937
|
-
|
|
1938
|
-
console.log(` \x1B[1mPath:\x1B[0m ${sitemap.path}`);
|
|
1939
|
-
console.log(` \x1B[1mType:\x1B[0m ${sitemap.type || "sitemap"}`);
|
|
1940
|
-
console.log(` \x1B[1mLast Submitted:\x1B[0m ${sitemap.lastSubmitted || "N/A"}`);
|
|
1941
|
-
console.log(` \x1B[1mLast Downloaded:\x1B[0m ${sitemap.lastDownloaded || "N/A"}`);
|
|
1942
|
-
console.log(` \x1B[1mPending:\x1B[0m ${sitemap.isPending ? "Yes" : "No"}`);
|
|
1943
|
-
console.log(` \x1B[1mErrors:\x1B[0m ${sitemap.errors || 0}`);
|
|
1944
|
-
console.log(` \x1B[1mWarnings:\x1B[0m ${sitemap.warnings || 0}`);
|
|
1945
|
-
if (sitemap.contents?.length) {
|
|
1946
|
-
console.log();
|
|
1947
|
-
console.log(" \x1B[1mContents:\x1B[0m");
|
|
1948
|
-
for (const c of sitemap.contents) console.log(` ${c.type}: ${c.submitted} submitted, ${c.indexed} indexed`);
|
|
1949
|
-
}
|
|
3978
|
+
logger.success(`Removed: ${args.url}`);
|
|
1950
3979
|
}
|
|
1951
3980
|
}),
|
|
1952
|
-
|
|
3981
|
+
"verify-token": defineCommand({
|
|
1953
3982
|
meta: {
|
|
1954
|
-
name: "
|
|
1955
|
-
description: "
|
|
3983
|
+
name: "verify-token",
|
|
3984
|
+
description: "Get a verification token to place on the site or in DNS"
|
|
1956
3985
|
},
|
|
1957
3986
|
args: {
|
|
1958
|
-
site: {
|
|
1959
|
-
type: "string",
|
|
1960
|
-
alias: "s",
|
|
1961
|
-
required: true,
|
|
1962
|
-
description: "Site URL"
|
|
1963
|
-
},
|
|
1964
3987
|
url: {
|
|
1965
3988
|
type: "positional",
|
|
1966
3989
|
required: true,
|
|
1967
|
-
description: "
|
|
3990
|
+
description: "Property URL"
|
|
3991
|
+
},
|
|
3992
|
+
method: {
|
|
3993
|
+
type: "string",
|
|
3994
|
+
alias: "m",
|
|
3995
|
+
description: "META, FILE, DNS_TXT, DNS_CNAME, ANALYTICS, TAG_MANAGER (default: META for URL-prefix, DNS_TXT for sc-domain:)"
|
|
3996
|
+
},
|
|
3997
|
+
json: {
|
|
3998
|
+
type: "boolean",
|
|
3999
|
+
default: false,
|
|
4000
|
+
description: "Output as JSON"
|
|
4001
|
+
},
|
|
4002
|
+
quiet: {
|
|
4003
|
+
type: "boolean",
|
|
4004
|
+
alias: "q",
|
|
4005
|
+
default: false,
|
|
4006
|
+
description: "Suppress info/success output"
|
|
1968
4007
|
}
|
|
1969
4008
|
},
|
|
1970
4009
|
async run({ args }) {
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
4010
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
4011
|
+
const method = validateMethod(args.url, args.method ?? pickDefaultMethod(args.url));
|
|
4012
|
+
const result = await getVerificationToken((await createCommandContext({ needsAuth: true })).client, args.url, method).catch(gscErrorHandler);
|
|
4013
|
+
if (args.json) {
|
|
4014
|
+
console.log(JSON.stringify({
|
|
4015
|
+
siteUrl: args.url,
|
|
4016
|
+
method,
|
|
4017
|
+
token: result.token,
|
|
4018
|
+
site: result.site
|
|
4019
|
+
}, null, 2));
|
|
4020
|
+
return;
|
|
4021
|
+
}
|
|
4022
|
+
printPlacementInstructions(method, args.url, result.token);
|
|
1976
4023
|
}
|
|
1977
4024
|
}),
|
|
1978
|
-
|
|
4025
|
+
"verify": defineCommand({
|
|
1979
4026
|
meta: {
|
|
1980
|
-
name: "
|
|
1981
|
-
description: "
|
|
4027
|
+
name: "verify",
|
|
4028
|
+
description: "Trigger verification — Google fetches/validates the token you placed"
|
|
1982
4029
|
},
|
|
1983
4030
|
args: {
|
|
1984
|
-
site: {
|
|
1985
|
-
type: "string",
|
|
1986
|
-
alias: "s",
|
|
1987
|
-
required: true,
|
|
1988
|
-
description: "Site URL"
|
|
1989
|
-
},
|
|
1990
4031
|
url: {
|
|
1991
4032
|
type: "positional",
|
|
1992
4033
|
required: true,
|
|
1993
|
-
description: "
|
|
4034
|
+
description: "Property URL"
|
|
4035
|
+
},
|
|
4036
|
+
method: {
|
|
4037
|
+
type: "string",
|
|
4038
|
+
alias: "m",
|
|
4039
|
+
description: "Verification method to validate (must match the one used for verify-token)"
|
|
4040
|
+
},
|
|
4041
|
+
json: {
|
|
4042
|
+
type: "boolean",
|
|
4043
|
+
default: false,
|
|
4044
|
+
description: "Output as JSON"
|
|
4045
|
+
},
|
|
4046
|
+
quiet: {
|
|
4047
|
+
type: "boolean",
|
|
4048
|
+
alias: "q",
|
|
4049
|
+
default: false,
|
|
4050
|
+
description: "Suppress info/success output"
|
|
1994
4051
|
}
|
|
1995
4052
|
},
|
|
1996
4053
|
async run({ args }) {
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
4054
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
4055
|
+
const method = validateMethod(args.url, args.method ?? pickDefaultMethod(args.url));
|
|
4056
|
+
const resource = await verifySite((await createCommandContext({ needsAuth: true })).client, args.url, method).catch(gscErrorHandler);
|
|
4057
|
+
if (args.json) {
|
|
4058
|
+
console.log(JSON.stringify({
|
|
4059
|
+
siteUrl: args.url,
|
|
4060
|
+
method,
|
|
4061
|
+
resource
|
|
4062
|
+
}, null, 2));
|
|
4063
|
+
return;
|
|
4064
|
+
}
|
|
4065
|
+
logger.success(`Verified: ${args.url}`);
|
|
4066
|
+
if (resource.owners?.length) {
|
|
4067
|
+
console.log();
|
|
4068
|
+
console.log(` Owners:`);
|
|
4069
|
+
for (const o of resource.owners) console.log(` \x1B[90m└─\x1B[0m ${o}`);
|
|
4070
|
+
}
|
|
2002
4071
|
}
|
|
2003
4072
|
})
|
|
2004
|
-
}
|
|
2005
|
-
});
|
|
2006
|
-
const sitesCommand = defineCommand({
|
|
2007
|
-
meta: {
|
|
2008
|
-
name: "sites",
|
|
2009
|
-
description: "List available GSC sites"
|
|
2010
4073
|
},
|
|
2011
|
-
args: { json: {
|
|
2012
|
-
type: "boolean",
|
|
2013
|
-
default: false,
|
|
2014
|
-
description: "Output as JSON for scripting"
|
|
2015
|
-
} },
|
|
2016
4074
|
async run({ args }) {
|
|
2017
|
-
|
|
4075
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
4076
|
+
const ctx = await createCommandContext({ needsAuth: true });
|
|
4077
|
+
const ownerOnly = Boolean(args["owner-only"]);
|
|
4078
|
+
if (args["with-sitemaps"]) {
|
|
4079
|
+
const all = await fetchSitesWithSitemaps(ctx.client).catch(gscErrorHandler);
|
|
4080
|
+
const sites = ownerOnly ? all.filter((s) => s.permissionLevel === "siteOwner") : all;
|
|
4081
|
+
if (args.json) {
|
|
4082
|
+
const enriched = sites.map((s) => ({
|
|
4083
|
+
...s,
|
|
4084
|
+
sitemapCounts: {
|
|
4085
|
+
total: s.sitemaps.length,
|
|
4086
|
+
pending: s.sitemaps.filter((sm) => sm.isPending).length,
|
|
4087
|
+
errored: s.sitemaps.filter((sm) => Number(sm.errors) > 0).length
|
|
4088
|
+
}
|
|
4089
|
+
}));
|
|
4090
|
+
console.log(JSON.stringify(enriched, null, 2));
|
|
4091
|
+
return;
|
|
4092
|
+
}
|
|
4093
|
+
if (sites.length === 0) {
|
|
4094
|
+
logger.warn(ownerOnly ? "No owned sites found" : "No verified sites found");
|
|
4095
|
+
return;
|
|
4096
|
+
}
|
|
4097
|
+
logger.success(`Found ${sites.length} ${ownerOnly ? "owned" : "verified"} sites:`);
|
|
4098
|
+
console.log();
|
|
4099
|
+
for (const site of sites) {
|
|
4100
|
+
const perm = site.permissionLevel === "siteOwner" ? "\x1B[32m" : "\x1B[90m";
|
|
4101
|
+
console.log(` ${site.siteUrl} ${perm}(${site.permissionLevel})\x1B[0m`);
|
|
4102
|
+
for (const sm of site.sitemaps) {
|
|
4103
|
+
const pending = sm.isPending ? " \x1B[33m(pending)\x1B[0m" : "";
|
|
4104
|
+
console.log(` \x1B[90m└─\x1B[0m ${sm.path}${pending}`);
|
|
4105
|
+
}
|
|
4106
|
+
}
|
|
4107
|
+
return;
|
|
4108
|
+
}
|
|
4109
|
+
const all = await ctx.loadSites();
|
|
4110
|
+
const sites = ownerOnly ? all.filter((s) => s.permissionLevel === "siteOwner") : all;
|
|
2018
4111
|
if (args.json) {
|
|
2019
4112
|
console.log(JSON.stringify(sites, null, 2));
|
|
2020
4113
|
return;
|
|
2021
4114
|
}
|
|
2022
4115
|
if (sites.length === 0) {
|
|
2023
|
-
logger.warn("No verified sites found");
|
|
4116
|
+
logger.warn(ownerOnly ? "No owned sites found" : "No verified sites found");
|
|
2024
4117
|
return;
|
|
2025
4118
|
}
|
|
2026
|
-
logger.success(`Found ${sites.length} sites:`);
|
|
4119
|
+
logger.success(`Found ${sites.length} ${ownerOnly ? "owned " : ""}sites:`);
|
|
2027
4120
|
console.log();
|
|
2028
4121
|
for (const site of sites) {
|
|
2029
4122
|
const perm = site.permissionLevel === "siteOwner" ? "\x1B[32m" : "\x1B[90m";
|
|
@@ -2054,6 +4147,16 @@ const compactCommand = defineCommand({
|
|
|
2054
4147
|
type: "string",
|
|
2055
4148
|
description: "Override d30→d90 age threshold in days (default: 90)"
|
|
2056
4149
|
},
|
|
4150
|
+
"dry-run": {
|
|
4151
|
+
type: "boolean",
|
|
4152
|
+
default: false,
|
|
4153
|
+
description: "Report tier counts per (table, site) without compacting"
|
|
4154
|
+
},
|
|
4155
|
+
"json": {
|
|
4156
|
+
type: "boolean",
|
|
4157
|
+
default: false,
|
|
4158
|
+
description: "Output a JSON summary"
|
|
4159
|
+
},
|
|
2057
4160
|
"quiet": {
|
|
2058
4161
|
type: "boolean",
|
|
2059
4162
|
alias: "q",
|
|
@@ -2062,13 +4165,43 @@ const compactCommand = defineCommand({
|
|
|
2062
4165
|
}
|
|
2063
4166
|
},
|
|
2064
4167
|
async run({ args }) {
|
|
4168
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
2065
4169
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2066
4170
|
const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
|
|
2067
|
-
const
|
|
4171
|
+
const dryRun = Boolean(args["dry-run"]);
|
|
2068
4172
|
const thresholds = {};
|
|
2069
4173
|
if (args["raw-days"]) thresholds.raw = Number(args["raw-days"]);
|
|
2070
4174
|
if (args["d7-days"]) thresholds.d7 = Number(args["d7-days"]);
|
|
2071
4175
|
if (args["d30-days"]) thresholds.d30 = Number(args["d30-days"]);
|
|
4176
|
+
if (dryRun) {
|
|
4177
|
+
const report = [];
|
|
4178
|
+
for (const table of allTables()) {
|
|
4179
|
+
const bySite = groupBySite(await store.engine.listLive({
|
|
4180
|
+
userId: store.userId,
|
|
4181
|
+
siteId,
|
|
4182
|
+
table
|
|
4183
|
+
}));
|
|
4184
|
+
for (const [s, group] of bySite) report.push({
|
|
4185
|
+
table,
|
|
4186
|
+
siteId: s,
|
|
4187
|
+
...countByTier(group)
|
|
4188
|
+
});
|
|
4189
|
+
}
|
|
4190
|
+
if (args.json) {
|
|
4191
|
+
console.log(JSON.stringify({
|
|
4192
|
+
thresholds,
|
|
4193
|
+
plan: report
|
|
4194
|
+
}, null, 2));
|
|
4195
|
+
return;
|
|
4196
|
+
}
|
|
4197
|
+
console.log();
|
|
4198
|
+
console.log(` table site raw d7 d30 d90`);
|
|
4199
|
+
for (const r of report) console.log(` ${r.table.padEnd(20)} ${(r.siteId ?? "-").padEnd(20)} ${String(r.raw).padStart(4)} ${String(r.d7).padStart(4)} ${String(r.d30).padStart(4)} ${String(r.d90).padStart(4)}`);
|
|
4200
|
+
console.log();
|
|
4201
|
+
logger.info(`compact --dry-run: ${report.length} (table, site) pair(s) — pass without --dry-run to apply`);
|
|
4202
|
+
return;
|
|
4203
|
+
}
|
|
4204
|
+
const summary = [];
|
|
2072
4205
|
for (const table of allTables()) {
|
|
2073
4206
|
const entries = await store.engine.listLive({
|
|
2074
4207
|
userId: store.userId,
|
|
@@ -2077,17 +4210,56 @@ const compactCommand = defineCommand({
|
|
|
2077
4210
|
});
|
|
2078
4211
|
const siteIds = new Set(entries.map((e) => e.siteId));
|
|
2079
4212
|
for (const targetSite of siteIds) {
|
|
2080
|
-
|
|
4213
|
+
logger.info(`Compacting ${table} [${targetSite ?? "-"}] (raw→d7→d30→d90)`);
|
|
2081
4214
|
await store.engine.compactTiered({
|
|
2082
4215
|
userId: store.userId,
|
|
2083
4216
|
siteId: targetSite,
|
|
2084
4217
|
table
|
|
2085
4218
|
}, thresholds);
|
|
4219
|
+
summary.push({
|
|
4220
|
+
table,
|
|
4221
|
+
siteId: targetSite
|
|
4222
|
+
});
|
|
2086
4223
|
}
|
|
2087
4224
|
}
|
|
2088
|
-
if (
|
|
4225
|
+
if (args.json) {
|
|
4226
|
+
console.log(JSON.stringify({
|
|
4227
|
+
thresholds,
|
|
4228
|
+
compacted: summary
|
|
4229
|
+
}, null, 2));
|
|
4230
|
+
return;
|
|
4231
|
+
}
|
|
4232
|
+
logger.success(`compact: done`);
|
|
2089
4233
|
}
|
|
2090
4234
|
});
|
|
4235
|
+
function groupBySite(entries) {
|
|
4236
|
+
const m = /* @__PURE__ */ new Map();
|
|
4237
|
+
for (const e of entries) {
|
|
4238
|
+
const arr = m.get(e.siteId) ?? [];
|
|
4239
|
+
arr.push(e);
|
|
4240
|
+
m.set(e.siteId, arr);
|
|
4241
|
+
}
|
|
4242
|
+
return m;
|
|
4243
|
+
}
|
|
4244
|
+
function countByTier(entries) {
|
|
4245
|
+
let raw = 0;
|
|
4246
|
+
let d7 = 0;
|
|
4247
|
+
let d30 = 0;
|
|
4248
|
+
let d90 = 0;
|
|
4249
|
+
for (const e of entries) {
|
|
4250
|
+
const tier = e.tier ?? inferLegacyTier(e) ?? "raw";
|
|
4251
|
+
if (tier === "raw") raw++;
|
|
4252
|
+
else if (tier === "d7") d7++;
|
|
4253
|
+
else if (tier === "d30") d30++;
|
|
4254
|
+
else if (tier === "d90") d90++;
|
|
4255
|
+
}
|
|
4256
|
+
return {
|
|
4257
|
+
raw,
|
|
4258
|
+
d7,
|
|
4259
|
+
d30,
|
|
4260
|
+
d90
|
|
4261
|
+
};
|
|
4262
|
+
}
|
|
2091
4263
|
async function exportToDuckDB(opts) {
|
|
2092
4264
|
const outPath = path.resolve(opts.outPath);
|
|
2093
4265
|
if (opts.force) await rm(outPath, { force: true });
|
|
@@ -2141,9 +4313,21 @@ const exportCommand = defineCommand({
|
|
|
2141
4313
|
type: "boolean",
|
|
2142
4314
|
default: false,
|
|
2143
4315
|
description: "Overwrite the output file if it already exists"
|
|
4316
|
+
},
|
|
4317
|
+
json: {
|
|
4318
|
+
type: "boolean",
|
|
4319
|
+
default: false,
|
|
4320
|
+
description: "Output a JSON summary instead of formatted text"
|
|
4321
|
+
},
|
|
4322
|
+
quiet: {
|
|
4323
|
+
type: "boolean",
|
|
4324
|
+
alias: "q",
|
|
4325
|
+
default: false,
|
|
4326
|
+
description: "Suppress info/success output"
|
|
2144
4327
|
}
|
|
2145
4328
|
},
|
|
2146
4329
|
async run({ args }) {
|
|
4330
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
2147
4331
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2148
4332
|
const siteId = args.site ? store.siteIdFor(args.site) : void 0;
|
|
2149
4333
|
const result = await exportToDuckDB({
|
|
@@ -2154,12 +4338,16 @@ const exportCommand = defineCommand({
|
|
|
2154
4338
|
outPath: args.out,
|
|
2155
4339
|
force: args.force
|
|
2156
4340
|
});
|
|
4341
|
+
if (args.json) {
|
|
4342
|
+
console.log(JSON.stringify(result, null, 2));
|
|
4343
|
+
return;
|
|
4344
|
+
}
|
|
2157
4345
|
if (result.tables.length === 0) {
|
|
2158
4346
|
console.log(`\n No data to export. Run \`gscdump sync\` first.`);
|
|
2159
4347
|
return;
|
|
2160
4348
|
}
|
|
2161
4349
|
for (const t of result.tables) console.log(` ${t.table.padEnd(15)} ${String(t.files).padStart(4)} parquet → ${t.table} (${t.rows.toLocaleString()} rows)`);
|
|
2162
|
-
console.log(`\n Exported ${result.tables.length} table(s), ${result.totalRows.toLocaleString()} rows → ${result.outPath}`);
|
|
4350
|
+
console.log(`\n Exported ${result.tables.length} table(s), ${result.totalRows.toLocaleString()} rows → ${displayPath(result.outPath)}`);
|
|
2163
4351
|
console.log(`\n Attach from DuckDB: \x1B[36mATTACH '${result.outPath}' AS gsc (READ_ONLY); SELECT * FROM gsc.pages LIMIT 10;\x1B[0m`);
|
|
2164
4352
|
console.log(` Attach in a browser: use DuckDB-WASM registerFileBuffer + \x1B[36mATTACH 'gsc.duckdb' AS gsc (READ_ONLY)\x1B[0m`);
|
|
2165
4353
|
}
|
|
@@ -2181,6 +4369,16 @@ const gcCommand = defineCommand({
|
|
|
2181
4369
|
alias: "s",
|
|
2182
4370
|
description: "Restrict to a single site (default: all sites)"
|
|
2183
4371
|
},
|
|
4372
|
+
"dry-run": {
|
|
4373
|
+
type: "boolean",
|
|
4374
|
+
default: false,
|
|
4375
|
+
description: "List retired manifest entries past the grace window without deleting"
|
|
4376
|
+
},
|
|
4377
|
+
"json": {
|
|
4378
|
+
type: "boolean",
|
|
4379
|
+
default: false,
|
|
4380
|
+
description: "Output a JSON summary"
|
|
4381
|
+
},
|
|
2184
4382
|
"quiet": {
|
|
2185
4383
|
type: "boolean",
|
|
2186
4384
|
alias: "q",
|
|
@@ -2189,15 +4387,52 @@ const gcCommand = defineCommand({
|
|
|
2189
4387
|
}
|
|
2190
4388
|
},
|
|
2191
4389
|
async run({ args }) {
|
|
4390
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
2192
4391
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2193
4392
|
const siteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
|
|
2194
|
-
const quiet = Boolean(args.quiet);
|
|
2195
4393
|
const graceMs = Number(args["grace-hours"]) * 36e5;
|
|
4394
|
+
if (args["dry-run"]) {
|
|
4395
|
+
const cutoff = Date.now() - graceMs;
|
|
4396
|
+
const candidates = [];
|
|
4397
|
+
for (const table of allTables()) {
|
|
4398
|
+
const all = await store.engine.listAll({
|
|
4399
|
+
userId: store.userId,
|
|
4400
|
+
siteId,
|
|
4401
|
+
table
|
|
4402
|
+
});
|
|
4403
|
+
for (const e of all) if (e.retiredAt && e.retiredAt < cutoff) candidates.push({
|
|
4404
|
+
table,
|
|
4405
|
+
siteId: e.siteId,
|
|
4406
|
+
partition: e.partition,
|
|
4407
|
+
retiredAt: e.retiredAt,
|
|
4408
|
+
objectKey: e.objectKey
|
|
4409
|
+
});
|
|
4410
|
+
}
|
|
4411
|
+
if (args.json) {
|
|
4412
|
+
console.log(JSON.stringify({
|
|
4413
|
+
graceHours: Number(args["grace-hours"]),
|
|
4414
|
+
candidates
|
|
4415
|
+
}, null, 2));
|
|
4416
|
+
return;
|
|
4417
|
+
}
|
|
4418
|
+
console.log();
|
|
4419
|
+
for (const c of candidates) console.log(` ${c.objectKey} \x1B[90m(retired ${new Date(c.retiredAt).toISOString()})\x1B[0m`);
|
|
4420
|
+
console.log();
|
|
4421
|
+
logger.info(`gc --dry-run: ${candidates.length} retired manifest entry(ies) past ${args["grace-hours"]}h grace; pass without --dry-run to delete`);
|
|
4422
|
+
return;
|
|
4423
|
+
}
|
|
2196
4424
|
const result = await store.engine.gcOrphans({
|
|
2197
4425
|
userId: store.userId,
|
|
2198
4426
|
siteId
|
|
2199
4427
|
}, graceMs);
|
|
2200
|
-
if (
|
|
4428
|
+
if (args.json) {
|
|
4429
|
+
console.log(JSON.stringify({
|
|
4430
|
+
graceHours: Number(args["grace-hours"]),
|
|
4431
|
+
deleted: result.deleted
|
|
4432
|
+
}, null, 2));
|
|
4433
|
+
return;
|
|
4434
|
+
}
|
|
4435
|
+
logger.success(`gc: deleted ${result.deleted} orphan file(s)`);
|
|
2201
4436
|
}
|
|
2202
4437
|
});
|
|
2203
4438
|
const rollupsCommand = defineCommand({
|
|
@@ -2216,6 +4451,11 @@ const rollupsCommand = defineCommand({
|
|
|
2216
4451
|
alias: "s",
|
|
2217
4452
|
description: "Restrict to a single site (default: all sites with local data)"
|
|
2218
4453
|
},
|
|
4454
|
+
json: {
|
|
4455
|
+
type: "boolean",
|
|
4456
|
+
default: false,
|
|
4457
|
+
description: "Output a JSON summary"
|
|
4458
|
+
},
|
|
2219
4459
|
quiet: {
|
|
2220
4460
|
type: "boolean",
|
|
2221
4461
|
alias: "q",
|
|
@@ -2224,9 +4464,9 @@ const rollupsCommand = defineCommand({
|
|
|
2224
4464
|
}
|
|
2225
4465
|
},
|
|
2226
4466
|
async run({ args }) {
|
|
4467
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
2227
4468
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2228
4469
|
const explicitSiteId = args.site ? store.siteIdFor(String(args.site)) : void 0;
|
|
2229
|
-
const quiet = Boolean(args.quiet);
|
|
2230
4470
|
const allSiteIds = /* @__PURE__ */ new Set();
|
|
2231
4471
|
if (explicitSiteId) allSiteIds.add(explicitSiteId);
|
|
2232
4472
|
else for (const table of allTables()) {
|
|
@@ -2237,12 +4477,17 @@ const rollupsCommand = defineCommand({
|
|
|
2237
4477
|
for (const e of entries) if (e.siteId) allSiteIds.add(e.siteId);
|
|
2238
4478
|
}
|
|
2239
4479
|
if (allSiteIds.size === 0) {
|
|
2240
|
-
|
|
4480
|
+
if (args.json) console.log(JSON.stringify({
|
|
4481
|
+
sites: [],
|
|
4482
|
+
totalBytes: 0
|
|
4483
|
+
}, null, 2));
|
|
4484
|
+
else logger.warn("No sites with local data. Run `gscdump sync` first.");
|
|
2241
4485
|
return;
|
|
2242
4486
|
}
|
|
4487
|
+
const summary = [];
|
|
2243
4488
|
let totalBytes = 0;
|
|
2244
4489
|
for (const siteId of allSiteIds) {
|
|
2245
|
-
|
|
4490
|
+
logger.info(`Rebuilding rollups for [${siteId}] (${DEFAULT_ROLLUPS.length} rollups)`);
|
|
2246
4491
|
const results = await rebuildRollups({
|
|
2247
4492
|
engine: store.engine,
|
|
2248
4493
|
dataSource: store.dataSource,
|
|
@@ -2252,12 +4497,29 @@ const rollupsCommand = defineCommand({
|
|
|
2252
4497
|
},
|
|
2253
4498
|
defs: DEFAULT_ROLLUPS
|
|
2254
4499
|
});
|
|
4500
|
+
const site = {
|
|
4501
|
+
siteId,
|
|
4502
|
+
rollups: []
|
|
4503
|
+
};
|
|
2255
4504
|
for (const r of results) {
|
|
2256
4505
|
totalBytes += r.bytes;
|
|
2257
|
-
|
|
4506
|
+
site.rollups.push({
|
|
4507
|
+
id: r.id,
|
|
4508
|
+
bytes: r.bytes,
|
|
4509
|
+
objectKey: r.objectKey
|
|
4510
|
+
});
|
|
4511
|
+
if (!args.json) console.log(` ${r.id.padEnd(20)} ${(r.bytes / 1024).toFixed(1).padStart(8)} KB ${r.objectKey}`);
|
|
2258
4512
|
}
|
|
4513
|
+
summary.push(site);
|
|
4514
|
+
}
|
|
4515
|
+
if (args.json) {
|
|
4516
|
+
console.log(JSON.stringify({
|
|
4517
|
+
sites: summary,
|
|
4518
|
+
totalBytes
|
|
4519
|
+
}, null, 2));
|
|
4520
|
+
return;
|
|
2259
4521
|
}
|
|
2260
|
-
|
|
4522
|
+
logger.success(`Rebuilt rollups across ${allSiteIds.size} site(s) — total ${(totalBytes / 1024).toFixed(1)} KB`);
|
|
2261
4523
|
}
|
|
2262
4524
|
}) }
|
|
2263
4525
|
});
|
|
@@ -2275,11 +4537,27 @@ const statsCommand = defineCommand({
|
|
|
2275
4537
|
site: {
|
|
2276
4538
|
type: "string",
|
|
2277
4539
|
description: "Limit to one site URL (sc-domain:example.com, https://example.com/, ...)"
|
|
4540
|
+
},
|
|
4541
|
+
quiet: {
|
|
4542
|
+
type: "boolean",
|
|
4543
|
+
alias: "q",
|
|
4544
|
+
default: false,
|
|
4545
|
+
description: "Suppress info/success output"
|
|
2278
4546
|
}
|
|
2279
4547
|
},
|
|
2280
4548
|
async run({ args }) {
|
|
4549
|
+
setQuiet(Boolean(args.quiet) || Boolean(args.json));
|
|
2281
4550
|
const store = (await createCommandContext({ needsStore: true })).store;
|
|
2282
|
-
|
|
4551
|
+
let siteId;
|
|
4552
|
+
if (args.site) {
|
|
4553
|
+
const known = await listKnownSiteIds(store);
|
|
4554
|
+
const candidate = store.siteIdFor(args.site);
|
|
4555
|
+
if (!known.has(candidate)) {
|
|
4556
|
+
logger.error(`No local data for --site=${args.site}. Known site IDs: ${known.size === 0 ? "(none — run `gscdump sync` first)" : Array.from(known).join(", ")}`);
|
|
4557
|
+
process.exit(1);
|
|
4558
|
+
}
|
|
4559
|
+
siteId = candidate;
|
|
4560
|
+
}
|
|
2283
4561
|
const perTable = await Promise.all(allTables().map(async (table) => {
|
|
2284
4562
|
const all = await store.engine.listAll({
|
|
2285
4563
|
userId: store.userId,
|
|
@@ -2323,7 +4601,7 @@ const statsCommand = defineCommand({
|
|
|
2323
4601
|
return;
|
|
2324
4602
|
}
|
|
2325
4603
|
console.log();
|
|
2326
|
-
console.log(` \x1B[1m${store.dataDir}\x1B[0m`);
|
|
4604
|
+
console.log(` \x1B[1m${displayPath(store.dataDir)}\x1B[0m`);
|
|
2327
4605
|
console.log(` \x1B[90mDisk: ${disk.files} file(s), ${formatBytes(disk.bytes)}\x1B[0m`);
|
|
2328
4606
|
console.log();
|
|
2329
4607
|
const totalRows = perTable.reduce((acc, t) => acc + sumRows(t.live), 0);
|
|
@@ -2351,6 +4629,17 @@ const statsCommand = defineCommand({
|
|
|
2351
4629
|
console.log();
|
|
2352
4630
|
}
|
|
2353
4631
|
});
|
|
4632
|
+
async function listKnownSiteIds(store) {
|
|
4633
|
+
const ids = /* @__PURE__ */ new Set();
|
|
4634
|
+
for (const table of allTables()) {
|
|
4635
|
+
const entries = await store.engine.listLive({
|
|
4636
|
+
userId: store.userId,
|
|
4637
|
+
table
|
|
4638
|
+
});
|
|
4639
|
+
for (const e of entries) if (e.siteId) ids.add(e.siteId);
|
|
4640
|
+
}
|
|
4641
|
+
return ids;
|
|
4642
|
+
}
|
|
2354
4643
|
function sortWatermarks(ws) {
|
|
2355
4644
|
return [...ws].sort((a, b) => {
|
|
2356
4645
|
if (a.table !== b.table) return a.table.localeCompare(b.table);
|
|
@@ -2585,6 +4874,16 @@ const syncCommand = defineCommand({
|
|
|
2585
4874
|
type: "boolean",
|
|
2586
4875
|
default: false,
|
|
2587
4876
|
description: "Run tables sequentially (default: run all tables in parallel)"
|
|
4877
|
+
},
|
|
4878
|
+
"retry-failed": {
|
|
4879
|
+
type: "boolean",
|
|
4880
|
+
default: false,
|
|
4881
|
+
description: "Only re-run dates currently in `failed` state (cheaper than --force)"
|
|
4882
|
+
},
|
|
4883
|
+
"dry-run": {
|
|
4884
|
+
type: "boolean",
|
|
4885
|
+
default: false,
|
|
4886
|
+
description: "Print the planned (table, searchType, date) work and exit without hitting the API"
|
|
2588
4887
|
}
|
|
2589
4888
|
},
|
|
2590
4889
|
async run({ args }) {
|
|
@@ -2631,14 +4930,63 @@ const syncCommand = defineCommand({
|
|
|
2631
4930
|
else if (args.full) startDate = daysAgo(450);
|
|
2632
4931
|
else if (args.days) startDate = daysAgo(Number.parseInt(String(args.days), 10) + DEFAULT_PENDING_DAYS - 1);
|
|
2633
4932
|
else startDate = daysAgo(DEFAULT_PENDING_DAYS + DEFAULT_PENDING_DAYS - 1);
|
|
2634
|
-
|
|
4933
|
+
let dates = getDateRange(startDate, endDate);
|
|
2635
4934
|
if (dates.length === 0) {
|
|
2636
4935
|
logger.error(`No dates to sync (start=${startDate}, end=${endDate})`);
|
|
2637
4936
|
process.exit(1);
|
|
2638
4937
|
}
|
|
2639
4938
|
const store = ctx.store;
|
|
4939
|
+
if (args["retry-failed"]) {
|
|
4940
|
+
const failedSet = /* @__PURE__ */ new Set();
|
|
4941
|
+
for (const table of tables) for (const type of types) {
|
|
4942
|
+
const states = await store.engine.getSyncStates({
|
|
4943
|
+
userId: store.userId,
|
|
4944
|
+
siteId,
|
|
4945
|
+
table,
|
|
4946
|
+
searchType: type
|
|
4947
|
+
});
|
|
4948
|
+
for (const s of states) if (s.state === "failed" && s.date >= startDate && s.date <= endDate) failedSet.add(s.date);
|
|
4949
|
+
}
|
|
4950
|
+
dates = dates.filter((d) => failedSet.has(d));
|
|
4951
|
+
if (dates.length === 0) {
|
|
4952
|
+
logger.success("No failed dates in range — nothing to retry.");
|
|
4953
|
+
return;
|
|
4954
|
+
}
|
|
4955
|
+
args.force = true;
|
|
4956
|
+
if (!args.quiet) logger.info(`--retry-failed: ${dates.length} date(s) to retry`);
|
|
4957
|
+
}
|
|
4958
|
+
if (args["dry-run"]) {
|
|
4959
|
+
const plan = [];
|
|
4960
|
+
for (const table of tables) for (const type of types) for (const date of dates) plan.push({
|
|
4961
|
+
table,
|
|
4962
|
+
searchType: type,
|
|
4963
|
+
date
|
|
4964
|
+
});
|
|
4965
|
+
if (args.json) {
|
|
4966
|
+
console.log(JSON.stringify({
|
|
4967
|
+
siteUrl,
|
|
4968
|
+
range: {
|
|
4969
|
+
start: startDate,
|
|
4970
|
+
end: endDate
|
|
4971
|
+
},
|
|
4972
|
+
tables,
|
|
4973
|
+
types,
|
|
4974
|
+
totalCalls: plan.length,
|
|
4975
|
+
plan
|
|
4976
|
+
}, null, 2));
|
|
4977
|
+
return;
|
|
4978
|
+
}
|
|
4979
|
+
console.log();
|
|
4980
|
+
logger.info(`Plan: ${plan.length} API call(s) for ${siteUrl}`);
|
|
4981
|
+
console.log(` Tables: ${tables.join(", ")}`);
|
|
4982
|
+
console.log(` Types: ${types.join(", ")}`);
|
|
4983
|
+
console.log(` Range: ${startDate} → ${endDate} (${dates.length} days)`);
|
|
4984
|
+
console.log();
|
|
4985
|
+
logger.info("Pass without --dry-run to execute.");
|
|
4986
|
+
return;
|
|
4987
|
+
}
|
|
2640
4988
|
if (!args.quiet) {
|
|
2641
|
-
logger.info(`Syncing ${siteUrl} (${tables.join(", ")}) [${types.join(", ")}] → ${store.dataDir}`);
|
|
4989
|
+
logger.info(`Syncing ${siteUrl} (${tables.join(", ")}) [${types.join(", ")}] → ${displayPath(store.dataDir)}`);
|
|
2642
4990
|
logger.info(`Range: ${startDate} → ${endDate} (${dates.length} days)`);
|
|
2643
4991
|
}
|
|
2644
4992
|
const concurrency = args.concurrency ? Math.max(1, Number.parseInt(String(args.concurrency), 10) || DEFAULT_CONCURRENCY) : DEFAULT_CONCURRENCY;
|
|
@@ -2763,7 +5111,7 @@ async function printSyncStatus(config, siteFilter, asJson) {
|
|
|
2763
5111
|
return;
|
|
2764
5112
|
}
|
|
2765
5113
|
console.log();
|
|
2766
|
-
console.log(` \x1B[1m${store.dataDir}\x1B[0m`);
|
|
5114
|
+
console.log(` \x1B[1m${displayPath(store.dataDir)}\x1B[0m`);
|
|
2767
5115
|
if (siteFilter) console.log(` \x1B[90mSite: ${siteFilter}\x1B[0m`);
|
|
2768
5116
|
console.log();
|
|
2769
5117
|
if (watermarks.length === 0) {
|
|
@@ -2794,6 +5142,51 @@ async function printSyncStatus(config, siteFilter, asJson) {
|
|
|
2794
5142
|
}
|
|
2795
5143
|
console.log();
|
|
2796
5144
|
}
|
|
5145
|
+
function shouldShowSplash() {
|
|
5146
|
+
if (!process.stdout.isTTY) return false;
|
|
5147
|
+
const argv = process.argv;
|
|
5148
|
+
if (argv.includes("mcp")) return false;
|
|
5149
|
+
for (const flag of [
|
|
5150
|
+
"--json",
|
|
5151
|
+
"--quiet",
|
|
5152
|
+
"-q",
|
|
5153
|
+
"--version",
|
|
5154
|
+
"-v",
|
|
5155
|
+
"--help",
|
|
5156
|
+
"-h"
|
|
5157
|
+
]) if (argv.includes(flag)) return false;
|
|
5158
|
+
return true;
|
|
5159
|
+
}
|
|
5160
|
+
function applyGlobalArgs() {
|
|
5161
|
+
const argv = process.argv;
|
|
5162
|
+
if (argv.includes("--no-color") || process.env.NO_COLOR) setNoColor(true);
|
|
5163
|
+
const profile = pluckArgValue(argv, "--profile") ?? process.env.GSCDUMP_PROFILE;
|
|
5164
|
+
const configDir = pluckArgValue(argv, "--config-dir") ?? process.env.GSCDUMP_CONFIG_DIR;
|
|
5165
|
+
if (configDir) setConfigDir(configDir);
|
|
5166
|
+
else if (profile) setConfigDir(path.join(os.homedir(), ".config", "gscdump", "profiles", profile));
|
|
5167
|
+
if (argv.includes("-v") && !argv.includes("--version")) {
|
|
5168
|
+
const i = argv.indexOf("-v");
|
|
5169
|
+
argv[i] = "--version";
|
|
5170
|
+
}
|
|
5171
|
+
}
|
|
5172
|
+
function pluckArgValue(argv, flag) {
|
|
5173
|
+
for (let i = 0; i < argv.length; i++) {
|
|
5174
|
+
const a = argv[i];
|
|
5175
|
+
if (a === flag && i + 1 < argv.length) {
|
|
5176
|
+
const v = argv[i + 1];
|
|
5177
|
+
argv.splice(i, 2);
|
|
5178
|
+
return v;
|
|
5179
|
+
}
|
|
5180
|
+
if (a.startsWith(`${flag}=`)) {
|
|
5181
|
+
const v = a.slice(flag.length + 1);
|
|
5182
|
+
argv.splice(i, 1);
|
|
5183
|
+
return v;
|
|
5184
|
+
}
|
|
5185
|
+
}
|
|
5186
|
+
return null;
|
|
5187
|
+
}
|
|
5188
|
+
applyGlobalArgs();
|
|
5189
|
+
loadEnvFromCwd();
|
|
2797
5190
|
runMain(defineCommand({
|
|
2798
5191
|
meta: {
|
|
2799
5192
|
name: "gscdump",
|
|
@@ -2809,14 +5202,16 @@ runMain(defineCommand({
|
|
|
2809
5202
|
sync: syncCommand,
|
|
2810
5203
|
store: storeCommand,
|
|
2811
5204
|
inspect: inspectCommand,
|
|
5205
|
+
indexing: indexingCommand,
|
|
2812
5206
|
entities: entitiesCommand,
|
|
2813
5207
|
analyze: analyzeCommand,
|
|
2814
5208
|
auth: authCommand,
|
|
2815
5209
|
config: configCommand,
|
|
5210
|
+
doctor: doctorCommand,
|
|
2816
5211
|
mcp: mcpCommand
|
|
2817
5212
|
},
|
|
2818
5213
|
setup() {
|
|
2819
|
-
if (
|
|
5214
|
+
if (shouldShowSplash()) showSplash();
|
|
2820
5215
|
}
|
|
2821
5216
|
}));
|
|
2822
5217
|
export {};
|